@speckle/objectloader2 2.25.4 → 2.25.7

package/src/helpers/defermentManager.ts

@@ -8,6 +8,9 @@ export class DefermentManager {
   private logger: CustomLogger
   private currentSize = 0
   private disposed = false
+  //tracks total deferment requests for each id
+  //this is used to prevent cleaning up deferments that are still being requested
+  private totalDefermentRequests: Map<string, number> = new Map()
 
   constructor(private options: DefermentManagerOptions) {
     this.resetGlobalTimer()
@@ -29,6 +32,7 @@ export class DefermentManager {
 
   async defer(params: { id: string }): Promise<Base> {
     if (this.disposed) throw new Error('DefermentManager is disposed')
+    this.trackDefermentRequest(params.id)
     const now = this.now()
     const deferredBase = this.deferments.get(params.id)
     if (deferredBase) {
@@ -44,6 +48,15 @@ export class DefermentManager {
     return notYetFound.getPromise()
   }
 
+  private trackDefermentRequest(id: string): void {
+    const request = this.totalDefermentRequests.get(id)
+    if (request) {
+      this.totalDefermentRequests.set(id, request + 1)
+    } else {
+      this.totalDefermentRequests.set(id, 1)
+    }
+  }
+
   undefer(item: Item): void {
     if (this.disposed) throw new Error('DefermentManager is disposed')
     const now = this.now()
@@ -108,6 +121,12 @@ export class DefermentManager {
       .filter((x) => x.isExpired(now))
       .sort((a, b) => this.compareMaybeBasesBySize(a.getItem(), b.getItem()))) {
       if (deferredBase.done(now)) {
+        //if the deferment is done but has been requested multiple times,
+        //we do not clean it up to allow the requests to resolve
+        const requestCount = this.totalDefermentRequests.get(deferredBase.getId())
+        if (requestCount && requestCount > 1) {
+          return
+        }
         this.currentSize -= deferredBase.getItem()?.size || 0
         this.deferments.delete(deferredBase.getId())
         cleaned++
@@ -117,7 +136,7 @@ export class DefermentManager {
       }
     }
     this.logger(
-      'cleaned deferments, cleaned, left',
+      'cleaned deferments: cleaned, left, time',
       cleaned,
       this.deferments.size,
       performance.now() - start

package/src/index.ts

@@ -1,3 +1,2 @@
-export { MemoryDatabase } from './operations/databases/memoryDatabase.js'
 export { ObjectLoader2 } from './operations/objectLoader2.js'
 export { ObjectLoader2Factory } from './operations/objectLoader2Factory.js'

package/src/operations/databases/__snapshots__/indexedDatabase.spec.ts.snap

@@ -1,31 +1,18 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
-exports[`database cache > write single item to queue use getItem 1`] = `
-{
-  "base": {
-    "id": "id",
-    "speckle_type": "type",
+exports[`IndexedDatabase > should add and get multiple items 1`] = `
+[
+  {
+    "baseId": "id1",
+    "item": {
+      "foo": "bar",
+    },
   },
-  "baseId": "id",
-}
-`;
-
-exports[`database cache > write two items to queue use getItem 1`] = `
-{
-  "base": {
-    "id": "id",
-    "speckle_type": "type",
-  },
-  "baseId": "id1",
-}
-`;
-
-exports[`database cache > write two items to queue use getItem 2`] = `
-{
-  "base": {
-    "id": "id",
-    "speckle_type": "type",
+  {
+    "baseId": "id2",
+    "item": {
+      "foo": "bar",
+    },
   },
-  "baseId": "id2",
-}
+]
 `;

package/src/operations/databases/indexedDatabase.spec.ts

@@ -1,37 +1,33 @@
-import { describe, expect, test } from 'vitest'
-import IndexedDatabase from './indexedDatabase.js'
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
 import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
+import IndexedDatabase, { IndexedDatabaseOptions } from './indexedDatabase.js'
 import { Item } from '../../types/types.js'
 
-describe('database cache', () => {
-  test('write single item to queue use getItem', async () => {
-    const i: Item = { baseId: 'id', base: { id: 'id', speckle_type: 'type' } }
-    const database = new IndexedDatabase({
-      indexedDB: new IDBFactory(),
-      keyRange: IDBKeyRange
-    })
-    await database.add(i)
-    await database.disposeAsync()
+// Mock Item
+const defaultItem = (id: string): Item => ({ baseId: id, item: { foo: 'bar' } })
 
-    const x = await database.getItem({ id: 'id' })
-    expect(x).toMatchSnapshot()
+describe('IndexedDatabase', () => {
+  let db: IndexedDatabase
+  let options: IndexedDatabaseOptions
+
+  beforeEach(() => {
+    options = { indexedDB: new IDBFactory(), keyRange: IDBKeyRange }
+    db = new IndexedDatabase(options)
   })
 
-  test('write two items to queue use getItem', async () => {
-    const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } }
-    const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } }
-    const database = new IndexedDatabase({
-      indexedDB: new IDBFactory(),
-      keyRange: IDBKeyRange
-    })
-    await database.add(i1)
-    await database.add(i2)
-    await database.disposeAsync()
+  afterEach(async () => {
+    await db.disposeAsync()
+  })
 
-    const x1 = await database.getItem({ id: i1.baseId })
-    expect(x1).toMatchSnapshot()
+  it('should add and get multiple items', async () => {
+    const items = [defaultItem('id1'), defaultItem('id2')]
+    await db.cacheSaveBatch({ batch: items })
+    const result = await db.getAll(['id1', 'id2'])
+    expect(result).toMatchSnapshot()
+    expect(result).toEqual(items)
+  })
 
-    const x2 = await database.getItem({ id: i2.baseId })
-    expect(x2).toMatchSnapshot()
+  it('should dispose without error', async () => {
+    await expect(db.disposeAsync()).resolves.not.toThrow()
   })
 })

package/src/operations/databases/indexedDatabase.ts

@@ -5,7 +5,7 @@ import { isSafari } from '@speckle/shared'
 import { Dexie, DexieOptions, Table } from 'dexie'
 import { Database } from '../interfaces.js'
 
-class ObjectStore extends Dexie {
+export class ObjectStore extends Dexie {
   static #databaseName: string = 'speckle-cache'
   objects!: Table<Item, string> // Table type: <entity, primaryKey>
 
@@ -83,30 +83,6 @@ export default class IndexedDatabase implements Database {
     this.#cacheDB = await this.#openDatabase()
   }
 
-  //this is for testing only - in the real world we will not use this
-  async add(item: Item): Promise<void> {
-    await this.#setupCacheDb()
-    await this.#cacheDB!.transaction('rw', this.#cacheDB!.objects, async () => {
-      return await this.#cacheDB?.objects.add(item)
-    })
-  }
-
-  async getItem(params: { id: string }): Promise<Item | undefined> {
-    const { id } = params
-    await this.#setupCacheDb()
-    //might not be in the real DB yet, so check the write queue first
-    if (this.#writeQueue) {
-      const item = this.#writeQueue.get(id)
-      if (item) {
-        return item
-      }
-    }
-
-    return this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => {
-      return await this.#cacheDB?.objects.get(id)
-    })
-  }
-
   async cacheSaveBatch(params: { batch: Item[] }): Promise<void> {
     await this.#setupCacheDb()
     const { batch } = params

package/src/operations/databases/memoryDatabase.spec.ts

@@ -0,0 +1,48 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { MemoryDatabase } from './memoryDatabase.js'
+import { Base, Item } from '../../types/types.js'
+
+const makeItem = (id: string, foo = 'bar'): Item => ({
+  baseId: id,
+  base: { foo } as unknown as Base
+})
+
+describe('MemoryDatabase', () => {
+  let db: MemoryDatabase
+
+  beforeEach(() => {
+    db = new MemoryDatabase()
+  })
+
+  it('should return undefined for missing keys', async () => {
+    const result = await db.getAll(['missing'])
+    expect(result).toEqual([undefined])
+  })
+
+  it('should add and retrieve a single item', async () => {
+    const item = makeItem('id1')
+    await db.cacheSaveBatch({ batch: [item] })
+    const result = await db.getAll(['id1'])
+    expect(result).toEqual([item])
+  })
+
+  it('should add and retrieve multiple items', async () => {
+    const items = [makeItem('id1'), makeItem('id2', 'baz')]
+    await db.cacheSaveBatch({ batch: items })
+    const result = await db.getAll(['id1', 'id2'])
+    expect(result).toEqual(items)
+  })
+
+  it('should overwrite items with the same key', async () => {
+    const item1 = makeItem('id1', 'foo')
+    const item2 = makeItem('id1', 'bar')
+    await db.cacheSaveBatch({ batch: [item1] })
+    await db.cacheSaveBatch({ batch: [item2] })
+    const result = await db.getAll(['id1'])
+    expect(result).toEqual([item2])
+  })
+
+  it('disposeAsync should resolve', async () => {
+    await expect(db.disposeAsync()).resolves.not.toThrow()
+  })
+})

package/src/operations/databases/memoryDatabase.ts

@@ -29,14 +29,6 @@ export class MemoryDatabase implements Database {
     return Promise.resolve()
   }
 
-  getItem(params: { id: string }): Promise<Item | undefined> {
-    const item = this.items.get(params.id)
-    if (item) {
-      return Promise.resolve({ baseId: params.id, base: item })
-    }
-    return Promise.resolve(undefined)
-  }
-
   disposeAsync(): Promise<void> {
     return Promise.resolve()
   }

package/src/operations/downloaders/__snapshots__/memoryDownloader.spec.ts.snap

@@ -0,0 +1,12 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`MemoryDownloader > should add found item to results queue 1`] = `
+Map {
+  "id1" => {
+    "foo": "foo",
+  },
+  "id2" => {
+    "foo": "bar",
+  },
+}
+`;

package/src/operations/downloaders/memoryDownloader.spec.ts

@@ -0,0 +1,49 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { MemoryDownloader } from './memoryDownloader.js'
+import { Base, Item } from '../../types/types.js'
+import Queue from '../../helpers/queue.js'
+import BufferQueue from '../../helpers/bufferQueue.js'
+
+const makeBase = (foo: string): Base => ({ foo } as unknown as Base)
+
+describe('MemoryDownloader', () => {
+  let items: Map<string, Base>
+  let downloader: MemoryDownloader
+  let results: Queue<Item>
+
+  beforeEach(() => {
+    items = new Map([
+      ['id1', makeBase('foo')],
+      ['id2', makeBase('bar')]
+    ])
+    downloader = new MemoryDownloader('id1', items)
+    results = new BufferQueue<Item>()
+  })
+
+  it('should download the root item', async () => {
+    const item = await downloader.downloadSingle()
+    expect(item).toEqual({ baseId: 'id1', base: { foo: 'foo' } })
+  })
+
+  it('should throw if root item is missing', async () => {
+    const missingDownloader = new MemoryDownloader('missing', items)
+    await expect(missingDownloader.downloadSingle()).rejects.toThrow(
+      'Method not implemented.'
+    )
+  })
+
+  it('should add found item to results queue', () => {
+    downloader.initializePool({ results, total: 2 })
+    downloader.add('id2')
+    expect(items).toMatchSnapshot()
+  })
+
+  it('should throw if added item is missing', () => {
+    downloader.initializePool({ results, total: 2 })
+    expect(() => downloader.add('missing')).toThrow()
+  })
+
+  it('disposeAsync should resolve', async () => {
+    await expect(downloader.disposeAsync()).resolves.not.toThrow()
+  })
+})

package/src/operations/downloaders/memoryDownloader.ts

@@ -23,7 +23,7 @@ export class MemoryDownloader implements Downloader {
     if (root) {
       return Promise.resolve({ baseId: this.#rootId, base: root })
     }
-    throw new Error('Method not implemented.')
+    return Promise.reject(new Error('Method not implemented.'))
   }
   disposeAsync(): Promise<void> {
     return Promise.resolve()

package/src/operations/downloaders/serverDownloader.ts

@@ -53,7 +53,7 @@ export default class ServerDownloader implements Downloader {
       return [total]
     }
 
-    return [10000, 30000, 10000, 1000]
+    return [10000, 25000, 10000, 1000]
   }
 
   initializePool(params: {

package/src/operations/interfaces.ts

@@ -13,7 +13,6 @@ export interface Downloader extends Queue<string> {
 
 export interface Database {
   getAll(keys: string[]): Promise<(Item | undefined)[]>
-  getItem(params: { id: string }): Promise<Item | undefined>
   cacheSaveBatch(params: { batch: Item[] }): Promise<void>
   disposeAsync(): Promise<void>
 }

package/src/operations/objectLoader2.ts

@@ -41,7 +41,7 @@ export class ObjectLoader2 {
     this.#database = options.database
     this.#deferments = new DefermentManager({
       maxSizeInMb: 2_000, // 2 GBs
-      ttlms: 5_000, // 5 seconds
+      ttlms: 15_000, // 15 seconds
       logger: this.#logger
     })
     this.#cache = new CacheReader(this.#database, this.#deferments, cacheOptions)
@@ -55,13 +55,17 @@ export class ObjectLoader2 {
   }
 
   async disposeAsync(): Promise<void> {
-    await Promise.all([this.#downloader.disposeAsync(), this.#cache.disposeAsync()])
+    await Promise.all([
+      this.#downloader.disposeAsync(),
+      this.#cache.disposeAsync(),
+      this.#pump.disposeAsync()
+    ])
     this.#deferments.dispose()
   }
 
   async getRootObject(): Promise<Item | undefined> {
     if (!this.#root) {
-      this.#root = await this.#database.getItem({ id: this.#rootId })
+      this.#root = (await this.#database.getAll([this.#rootId]))[0]
       if (!this.#root) {
         this.#root = await this.#downloader.downloadSingle()
       }
@@ -90,7 +94,11 @@ export class ObjectLoader2 {
     yield rootItem.base
     if (!rootItem.base.__closure) return
 
-    const children = Object.keys(rootItem.base.__closure)
+    //sort the closures by their values descending
+    const sortedClosures = Object.entries(rootItem.base.__closure).sort(
+      (a, b) => b[1] - a[1]
+    )
+    const children = sortedClosures.map((x) => x[0])
     const total = children.length
     this.#downloader.initializePool({
       results: new AggregateQueue(this.#gathered, this.#pump),

package/src/operations/objectLoader2Factory.ts

@@ -5,6 +5,14 @@ import { MemoryDownloader } from './downloaders/memoryDownloader.js'
 import ServerDownloader from './downloaders/serverDownloader.js'
 import { ObjectLoader2 } from './objectLoader2.js'
 
+export interface ObjectLoader2FactoryOptions {
+  useMemoryCache?: boolean
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+  keyRange?: { bound: Function; lowerBound: Function; upperBound: Function }
+  indexedDB?: IDBFactory
+  logger?: CustomLogger
+}
+
 export class ObjectLoader2Factory {
   static createFromObjects(objects: Base[]): ObjectLoader2 {
     const root = objects[0]
@@ -31,26 +39,40 @@ export class ObjectLoader2Factory {
     objectId: string
     token?: string
     headers?: Headers
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
-    keyRange?: { bound: Function; lowerBound: Function; upperBound: Function }
-    indexedDB?: IDBFactory
-    logger?: CustomLogger
+    options?: ObjectLoader2FactoryOptions
   }): ObjectLoader2 {
-    const loader = new ObjectLoader2({
-      rootId: params.objectId,
-      downloader: new ServerDownloader({
-        serverUrl: params.serverUrl,
-        streamId: params.streamId,
-        objectId: params.objectId,
-        token: params.token,
-        headers: params.headers
-      }),
-      database: new IndexedDatabase({
-        logger: params.logger,
-        indexedDB: params.indexedDB,
-        keyRange: params.keyRange
+    let loader: ObjectLoader2
+    if (params.options?.useMemoryCache) {
+      loader = new ObjectLoader2({
+        rootId: params.objectId,
+        downloader: new ServerDownloader({
+          serverUrl: params.serverUrl,
+          streamId: params.streamId,
+          objectId: params.objectId,
+          token: params.token,
+          headers: params.headers
+        }),
+        database: new MemoryDatabase({
+          items: new Map<string, Base>()
+        })
       })
-    })
+    } else {
+      loader = new ObjectLoader2({
+        rootId: params.objectId,
+        downloader: new ServerDownloader({
+          serverUrl: params.serverUrl,
+          streamId: params.streamId,
+          objectId: params.objectId,
+          token: params.token,
+          headers: params.headers
+        }),
+        database: new IndexedDatabase({
+          logger: params.options?.logger,
+          indexedDB: params.options?.indexedDB,
+          keyRange: params.options?.keyRange
+        })
+      })
+    }
     return loader
   }
 }

package/src/test/e2e.spec.ts

@@ -14,8 +14,10 @@ describe('e2e', () => {
         serverUrl: 'https://app.speckle.systems',
         streamId: 'da9e320dad',
         objectId: '31d10c0cea569a1e26809658ed27e281',
-        indexedDB: new IDBFactory(),
-        keyRange: IDBKeyRange
+        options: {
+          indexedDB: new IDBFactory(),
+          keyRange: IDBKeyRange
+        }
       })
 
       const getObjectPromise = loader.getObject({