@speckle/objectloader2 2.24.0 → 2.25.0

package/src/helpers/defermentManager.ts

@@ -1,25 +1,141 @@
 import { DeferredBase } from './deferredBase.js'
-import { Base, Item } from '../types/types.js'
+import { Base, CustomLogger, Item } from '../types/types.js'
+import { DefermentManagerOptions } from '../operations/options.js'
 
 export class DefermentManager {
-  #deferments: DeferredBase[] = []
+  private deferments: Map<string, DeferredBase> = new Map()
+  private timer?: ReturnType<typeof setTimeout>
+  private logger: CustomLogger
+  private currentSize = 0
+  private disposed = false
+
+  constructor(private options: DefermentManagerOptions) {
+    this.resetGlobalTimer()
+    this.logger = options.logger || ((): void => {})
+  }
+
+  private now(): number {
+    return Date.now()
+  }
+
+  isDeferred(id: string): boolean {
+    return this.deferments.has(id)
+  }
+
+  get(id: string): DeferredBase | undefined {
+    if (this.disposed) throw new Error('DefermentManager is disposed')
+    return this.deferments.get(id)
+  }
 
   async defer(params: { id: string }): Promise<Base> {
-    const deferredBase = this.#deferments.find((x) => x.id === params.id)
+    if (this.disposed) throw new Error('DefermentManager is disposed')
+    const now = this.now()
+    const deferredBase = this.deferments.get(params.id)
     if (deferredBase) {
-      return await deferredBase.promise
+      deferredBase.setAccess(now)
+      return deferredBase.getPromise()
     }
-    const d = new DeferredBase(params.id)
-    this.#deferments.push(d)
-    return d.promise
+    const notYetFound = new DeferredBase(
+      this.options.ttlms,
+      params.id,
+      now + this.options.ttlms
+    )
+    this.deferments.set(params.id, notYetFound)
+    return notYetFound.getPromise()
   }
 
   undefer(item: Item): void {
-    const deferredIndex = this.#deferments.findIndex((x) => x.id === item.baseId)
-    if (deferredIndex !== -1) {
-      const deferredBase = this.#deferments[deferredIndex]
-      deferredBase.resolve(item.base)
-      this.#deferments.splice(deferredIndex, 1)
+    if (this.disposed) throw new Error('DefermentManager is disposed')
+    const now = this.now()
+    this.currentSize += item.size || 0
+    //order matters here with found before undefer
+    const deferredBase = this.deferments.get(item.baseId)
+    if (deferredBase) {
+      deferredBase.found(item)
+      deferredBase.setAccess(now)
+    } else {
+      const existing = new DeferredBase(this.options.ttlms, item.baseId, now)
+      existing.found(item)
+      this.deferments.set(item.baseId, existing)
+    }
+  }
+
+  private resetGlobalTimer(): void {
+    const run = (): void => {
+      this.cleanDeferments()
+      this.timer = setTimeout(run, this.options.ttlms)
+    }
+    this.timer = setTimeout(run, this.options.ttlms)
+  }
+
+  dispose(): void {
+    if (this.disposed) return
+    this.disposed = true
+    if (this.timer) {
+      clearTimeout(this.timer)
+      this.timer = undefined
     }
+    this.clearDeferments()
+  }
+
+  private clearDeferments(): void {
+    let waiting = 0
+    for (const deferredBase of this.deferments.values()) {
+      deferredBase.done(0)
+      if (deferredBase.getItem() === undefined) {
+        waiting++
+      }
+    }
+    this.currentSize = 0
+    this.deferments.clear()
+    this.logger('cleared deferments, left', waiting)
+  }
+
+  private cleanDeferments(): void {
+    const maxSizeBytes = this.options.maxSizeInMb * 1024 * 1024
+    if (this.currentSize < maxSizeBytes) {
+      this.logger(
+        'deferments size is ok, no need to clean',
+        this.currentSize,
+        maxSizeBytes
+      )
+      return
+    }
+    const now = this.now()
+    let cleaned = 0
+    const start = performance.now()
+    for (const deferredBase of Array.from(this.deferments.values())
+      .filter((x) => x.isExpired(now))
+      .sort((a, b) => this.compareMaybeBasesBySize(a.getItem(), b.getItem()))) {
+      if (deferredBase.done(now)) {
+        this.currentSize -= deferredBase.getItem()?.size || 0
+        this.deferments.delete(deferredBase.getId())
+        cleaned++
+        if (this.currentSize < maxSizeBytes) {
+          break
+        }
+      }
+    }
+    this.logger(
+      'cleaned deferments, cleaned, left',
+      cleaned,
+      this.deferments.size,
+      performance.now() - start
+    )
+    return
+  }
+
+  compareMaybeBasesBySize(a: Item | undefined, b: Item | undefined): number {
+    if (a === undefined && b === undefined) return 0
+    if (a === undefined) return -1
+    if (b === undefined) return 1
+    return this.compareMaybe(a.size, b.size)
+  }
+
+  compareMaybe(a: number | undefined, b: number | undefined): number {
+    if (a === undefined && b === undefined) return 0
+    if (a === undefined) return -1
+    if (b === undefined) return 1
+    return a - b
   }
 }

package/src/helpers/deferredBase.ts

@@ -1,17 +1,55 @@
-import { Base } from '../types/types.js'
+import { Base, Item } from '../types/types.js'
 
 export class DeferredBase {
-  promise: Promise<Base>
-  resolve!: (value: Base) => void
-  reject!: (reason?: Error) => void
+  private promise: Promise<Base>
+  private resolve!: (value: Base) => void
+  private reject!: (reason?: Error) => void
+  private item?: Item
 
-  readonly id: string
+  private readonly id: string
+  private expiresAt: number // Timestamp in ms
+  private ttl: number // ttl in ms
 
-  constructor(id: string) {
+  constructor(ttl: number, id: string, expiresAt: number) {
+    this.expiresAt = expiresAt
+    this.ttl = ttl
     this.id = id
     this.promise = new Promise<Base>((resolve, reject) => {
       this.resolve = resolve
       this.reject = reject
     })
   }
+
+  getId(): string {
+    return this.id
+  }
+
+  getItem(): Item | undefined {
+    return this.item
+  }
+
+  getPromise(): Promise<Base> {
+    return this.promise
+  }
+
+  isExpired(now: number): boolean {
+    return this.item !== undefined && now > this.expiresAt
+  }
+  setAccess(now: number): void {
+    this.expiresAt = now + this.ttl
+  }
+
+  found(value: Item): void {
+    this.item = value
+    this.resolve(value.base)
+  }
+  done(now: number): boolean {
+    if (this.item) {
+      this.resolve(this.item.base)
+    }
+    if (this.isExpired(now)) {
+      return true
+    }
+    return false
+  }
 }

package/src/helpers/keyedQueue.ts

@@ -0,0 +1,45 @@
+export default class KeyedQueue<K, V> {
+  private _map: Map<K, V>
+  private _order: K[]
+
+  constructor() {
+    this._map = new Map<K, V>()
+    this._order = []
+  }
+
+  enqueue(key: K, value: V): boolean {
+    if (this._map.has(key)) {
+      return false // Key already exists
+    }
+    this._map.set(key, value)
+    this._order.push(key)
+    return true
+  }
+
+  get(key: K): V | undefined {
+    return this._map.get(key)
+  }
+
+  has(key: K): boolean {
+    return this._map.has(key)
+  }
+
+  get size(): number {
+    return this._order.length
+  }
+
+  spliceValues(start: number, deleteCount: number): V[] {
+    const splicedKeys = this._order.splice(start, deleteCount)
+    const result: V[] = []
+
+    for (const key of splicedKeys) {
+      const value = this._map.get(key)
+      if (value !== undefined) {
+        result.push(value)
+        this._map.delete(key)
+      }
+    }
+
+    return result
+  }
+}

package/src/helpers/memoryPump.ts

@@ -0,0 +1,40 @@
+import { Item } from '../types/types.js'
+import { Pump } from './pump.js'
+import Queue from './queue.js'
+
+export class MemoryPump implements Pump {
+  #items: Map<string, Item> = new Map()
+
+  add(item: Item): void {
+    this.#items.set(item.baseId, item)
+  }
+
+  async pumpItems(params: {
+    ids: string[]
+    foundItems: Queue<Item>
+    notFoundItems: Queue<string>
+  }): Promise<void> {
+    const { ids, foundItems, notFoundItems } = params
+    for (const id of ids) {
+      const item = this.#items.get(id)
+      if (item) {
+        foundItems.add(item)
+      } else {
+        notFoundItems.add(id)
+      }
+    }
+    return Promise.resolve()
+  }
+
+  async *gather(ids: string[]): AsyncGenerator<Item> {
+    for (const id of ids) {
+      const item = this.#items.get(id)
+      if (item) {
+        yield item
+      }
+    }
+    return Promise.resolve()
+  }
+
+  async disposeAsync(): Promise<void> {}
+}

package/src/helpers/pump.ts

@@ -0,0 +1,8 @@
+import { Downloader } from '../operations/interfaces.js'
+import { Item } from '../types/types.js'
+import Queue from './queue.js'
+
+export interface Pump extends Queue<Item> {
+  gather(ids: string[], downloader: Downloader): AsyncGenerator<Item>
+  disposeAsync(): Promise<void>
+}

package/src/index.ts

@@ -1,4 +1,3 @@
-import ObjectLoader2 from './operations/objectLoader2.js'
-
-export default ObjectLoader2
-export { MemoryDatabase } from './operations/memoryDatabase.js'
+export { MemoryDatabase } from './operations/databases/memoryDatabase.js'
+export { ObjectLoader2 } from './operations/objectLoader2.js'
+export { ObjectLoader2Factory } from './operations/objectLoader2Factory.js'

package/src/operations/__snapshots__/objectLoader2.spec.ts.snap

@@ -2,35 +2,35 @@
 
 exports[`objectloader2 > add extra header 1`] = `
 {
-  "baseId": "baseId",
+  "base": {
+    "__closure": {
+      "child1Id": 100,
+    },
+    "id": "rootId",
+    "speckle_type": "type",
+  },
+  "baseId": "rootId",
 }
 `;
 
 exports[`objectloader2 > can get a root object from cache 1`] = `
 {
+  "base": {
+    "id": "baseId",
+    "speckle_type": "type",
+  },
   "baseId": "baseId",
 }
 `;
 
 exports[`objectloader2 > can get a root object from downloader 1`] = `
 {
-  "baseId": "baseId",
-}
-`;
-
-exports[`objectloader2 > can get root/child object from cache using iterator 1`] = `
-[
-  {
-    "__closure": {
-      "child1Id": 100,
-    },
-    "id": "rootId",
+  "base": {
+    "id": "baseId",
     "speckle_type": "type",
   },
-  {
-    "id": "child1Id",
-  },
-]
+  "baseId": "baseId",
+}
 `;
 
 exports[`objectloader2 > can get root/child object from memory cache using iterator and getObject 1`] = `

package/src/operations/{__snapshots__ → databases/__snapshots__}/indexedDatabase.spec.ts.snap

@@ -29,24 +29,3 @@ exports[`database cache > write two items to queue use getItem 2`] = `
   "baseId": "id2",
 }
 `;
-
-exports[`database cache > write two items to queue use processItems 1`] = `
-[
-  {
-    "base": {
-      "id": "id",
-      "speckle_type": "type",
-    },
-    "baseId": "id1",
-  },
-  {
-    "base": {
-      "id": "id",
-      "speckle_type": "type",
-    },
-    "baseId": "id2",
-  },
-]
-`;
-
-exports[`database cache > write two items to queue use processItems 2`] = `[]`;

package/src/operations/{indexedDatabase.spec.ts → databases/indexedDatabase.spec.ts}

@@ -1,16 +1,14 @@
 import { describe, expect, test } from 'vitest'
 import IndexedDatabase from './indexedDatabase.js'
 import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
-import { Item } from '../types/types.js'
-import BufferQueue from '../helpers/bufferQueue.js'
+import { Item } from '../../types/types.js'
 
 describe('database cache', () => {
   test('write single item to queue use getItem', async () => {
     const i: Item = { baseId: 'id', base: { id: 'id', speckle_type: 'type' } }
     const database = new IndexedDatabase({
       indexedDB: new IDBFactory(),
-      keyRange: IDBKeyRange,
-      maxCacheBatchWriteWait: 200
+      keyRange: IDBKeyRange
     })
     await database.add(i)
     await database.disposeAsync()
@@ -36,28 +34,4 @@ describe('database cache', () => {
     const x2 = await database.getItem({ id: i2.baseId })
     expect(x2).toMatchSnapshot()
   })
-
-  test('write two items to queue use processItems', async () => {
-    const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } }
-    const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } }
-    const database = new IndexedDatabase({
-      indexedDB: new IDBFactory(),
-      keyRange: IDBKeyRange
-    })
-    await database.add(i1)
-    await database.add(i2)
-    await database.disposeAsync()
-
-    const foundItems = new BufferQueue<Item>()
-    const notFoundItems = new BufferQueue<string>()
-
-    await database.processItems({
-      ids: [i1.baseId, i2.baseId],
-      foundItems,
-      notFoundItems
-    })
-
-    expect(foundItems.values()).toMatchSnapshot()
-    expect(notFoundItems.values()).toMatchSnapshot()
-  })
 })

package/src/operations/databases/indexedDatabase.ts

@@ -0,0 +1,150 @@
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+import BatchingQueue from '../../helpers/batchingQueue.js'
+import { CustomLogger, Item } from '../../types/types.js'
+import { isSafari } from '@speckle/shared'
+import { Dexie, DexieOptions, Table } from 'dexie'
+import { Database } from '../interfaces.js'
+
+class ObjectStore extends Dexie {
+  static #databaseName: string = 'speckle-cache'
+  objects!: Table<Item, string> // Table type: <entity, primaryKey>
+
+  constructor(options: DexieOptions) {
+    super(ObjectStore.#databaseName, options)
+
+    this.version(1).stores({
+      objects: 'baseId, item' // baseId is primary key
+    })
+  }
+}
+
+export interface IndexedDatabaseOptions {
+  logger?: CustomLogger
+  indexedDB?: IDBFactory
+  keyRange?: {
+    bound: Function
+    lowerBound: Function
+    upperBound: Function
+  }
+}
+
+export default class IndexedDatabase implements Database {
+  #options: IndexedDatabaseOptions
+  #logger: CustomLogger
+
+  #cacheDB?: ObjectStore
+
+  #writeQueue: BatchingQueue<Item> | undefined
+
+  // #count: number = 0
+
+  constructor(options: IndexedDatabaseOptions) {
+    this.#options = options
+    this.#logger = options.logger || ((): void => {})
+  }
+
+  async getAll(keys: string[]): Promise<(Item | undefined)[]> {
+    await this.#setupCacheDb()
+    let items: (Item | undefined)[] = []
+    //   this.#count++
+    //  const startTime = performance.now()
+    // this.#logger('Start read ' + x + ' ' + batch.length)
+
+    //faster than BulkGet with dexie
+    await this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => {
+      const gets = keys.map((key) => this.#cacheDB!.objects.get(key))
+      const cachedData = await Promise.all(gets)
+      items = cachedData
+    })
+    // const endTime = performance.now()
+    // const duration = endTime - startTime
+    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
+
+    return items
+  }
+
+  async #openDatabase(): Promise<ObjectStore> {
+    const db = new ObjectStore({
+      indexedDB: this.#options.indexedDB ?? globalThis.indexedDB,
+      IDBKeyRange: this.#options.keyRange ?? IDBKeyRange,
+      chromeTransactionDurability: 'relaxed'
+    })
+    await db.open()
+    return db
+  }
+
+  async #setupCacheDb(): Promise<void> {
+    if (this.#cacheDB !== undefined) {
+      return
+    }
+
+    // Initialize
+    await this.#safariFix()
+    this.#cacheDB = await this.#openDatabase()
+  }
+
+  //this is for testing only - in the real world we will not use this
+  async add(item: Item): Promise<void> {
+    await this.#setupCacheDb()
+    await this.#cacheDB!.transaction('rw', this.#cacheDB!.objects, async () => {
+      return await this.#cacheDB?.objects.add(item)
+    })
+  }
+
+  async getItem(params: { id: string }): Promise<Item | undefined> {
+    const { id } = params
+    await this.#setupCacheDb()
+    //might not be in the real DB yet, so check the write queue first
+    if (this.#writeQueue) {
+      const item = this.#writeQueue.get(id)
+      if (item) {
+        return item
+      }
+    }
+
+    return this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => {
+      return await this.#cacheDB?.objects.get(id)
+    })
+  }
+
+  async cacheSaveBatch(params: { batch: Item[] }): Promise<void> {
+    await this.#setupCacheDb()
+    const { batch } = params
+    //const x = this.#count
+    //this.#count++
+
+    // const startTime = performance.now()
+    //  this.#logger('Start save ' + x + ' ' + batch.length)
+    await this.#cacheDB!.objects.bulkPut(batch)
+    // const endTime = performance.now()
+    // const duration = endTime - startTime
+    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
+  }
+
+  /**
+   * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
+   * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
+   */
+  async #safariFix(): Promise<void> {
+    // No point putting other browsers or older versions of Safari through this mess.
+    if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve()
+
+    let intervalId: ReturnType<typeof setInterval>
+
+    return new Promise<void>((resolve: () => void) => {
+      const tryIdb = (): Promise<IDBDatabaseInfo[]> | undefined =>
+        this.#options.indexedDB?.databases().finally(resolve)
+      intervalId = setInterval(() => {
+        void tryIdb()
+      }, 100)
+      void tryIdb()
+    }).finally(() => clearInterval(intervalId))
+  }
+
+  async disposeAsync(): Promise<void> {
+    this.#cacheDB?.close()
+    this.#cacheDB = undefined
+    await this.#writeQueue?.disposeAsync()
+    this.#writeQueue = undefined
+  }
+}

package/src/operations/databases/memoryDatabase.ts

@@ -0,0 +1,43 @@
+import { Base, Item } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import { MemoryDatabaseOptions } from '../options.js'
+
+export class MemoryDatabase implements Database {
+  private items: Map<string, Base>
+
+  constructor(options?: MemoryDatabaseOptions) {
+    this.items = options?.items || new Map<string, Base>()
+  }
+
+  getAll(keys: string[]): Promise<(Item | undefined)[]> {
+    const found: (Item | undefined)[] = []
+    for (const key of keys) {
+      const item = this.items.get(key)
+      if (item) {
+        found.push({ baseId: key, base: item })
+      } else {
+        found.push(undefined)
+      }
+    }
+    return Promise.resolve(found)
+  }
+
+  cacheSaveBatch({ batch }: { batch: Item[] }): Promise<void> {
+    for (const item of batch) {
+      this.items.set(item.baseId, item.base)
+    }
+    return Promise.resolve()
+  }
+
+  getItem(params: { id: string }): Promise<Item | undefined> {
+    const item = this.items.get(params.id)
+    if (item) {
+      return Promise.resolve({ baseId: params.id, base: item })
+    }
+    return Promise.resolve(undefined)
+  }
+
+  disposeAsync(): Promise<void> {
+    return Promise.resolve()
+  }
+}