@speckle/objectloader2 2.24.2 → 2.25.1

package/src/operations/objectLoader2.spec.ts

@@ -1,56 +1,45 @@
 import { describe, expect, test } from 'vitest'
-import ObjectLoader2 from './objectLoader2.js'
+import { ObjectLoader2 } from './objectLoader2.js'
 import { Base, Item } from '../types/types.js'
-import { Cache, Downloader } from './interfaces.js'
-import Queue from '../helpers/queue.js'
-import { MemoryDatabase } from './memoryDatabase.js'
-import { MemoryDownloader } from './memoryDownloader.js'
-import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
+import { MemoryDownloader } from './downloaders/memoryDownloader.js'
+import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
+import { MemoryDatabase } from './databases/memoryDatabase.js'
+import IndexedDatabase from './databases/indexedDatabase.js'
 
 describe('objectloader2', () => {
   test('can get a root object from cache', async () => {
-    const root = { baseId: 'baseId' } as unknown as Item
-    const cache = {
-      getItem(params: { id: string }): Promise<Item> {
-        expect(params.id).toBe(root.baseId)
-        return Promise.resolve(root)
-      }
-    } as Cache
-    const downloader = {} as Downloader
+    const rootId = 'baseId'
+    const rootBase: Base = { id: 'baseId', speckle_type: 'type' }
+    const downloader = new MemoryDownloader(
+      rootId,
+      new Map<string, Base>([[rootId, rootBase]])
+    )
     const loader = new ObjectLoader2({
-      serverUrl: 'a',
-      streamId: 'b',
-      objectId: root.baseId,
-      cache,
-      downloader
+      rootId,
+      downloader,
+      database: new IndexedDatabase({
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
     })
     const x = await loader.getRootObject()
     expect(x).toMatchSnapshot()
   })
 
   test('can get a root object from downloader', async () => {
-    const root = { baseId: 'baseId' } as unknown as Item
-    const cache = {
-      getItem(params: { id: string }): Promise<Item | undefined> {
-        expect(params.id).toBe(root.baseId)
-        return Promise.resolve<Item | undefined>(undefined)
-      },
-      add(item: Item): Promise<void> {
-        expect(item).toBe(root)
-        return Promise.resolve()
-      }
-    } as Cache
-    const downloader = {
-      downloadSingle(): Promise<Item> {
-        return Promise.resolve(root)
-      }
-    } as Downloader
+    const rootId = 'baseId'
+    const rootBase: Base = { id: 'baseId', speckle_type: 'type' }
+    const downloader = new MemoryDownloader(
+      rootId,
+      new Map<string, Base>([[rootId, rootBase]])
+    )
     const loader = new ObjectLoader2({
-      serverUrl: 'a',
-      streamId: 'b',
-      objectId: root.baseId,
-      cache,
-      downloader
+      rootId,
+      downloader,
+      database: new IndexedDatabase({
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
     })
     const x = await loader.getRootObject()
     expect(x).toMatchSnapshot()
@@ -59,83 +48,24 @@ describe('objectloader2', () => {
   test('can get single object from cache using iterator', async () => {
     const rootId = 'baseId'
     const rootBase: Base = { id: 'baseId', speckle_type: 'type' }
-    const root = { baseId: rootId, base: rootBase } as unknown as Item
-    const cache = {
-      getItem(params: { id: string }): Promise<Item | undefined> {
-        expect(params.id).toBe(rootId)
-        return Promise.resolve(root)
-      }
-    } as Cache
-    const downloader = {} as Downloader
-    const loader = new ObjectLoader2({
-      serverUrl: 'a',
-      streamId: 'b',
-      objectId: rootId,
-      cache,
-      downloader
-    })
-    const r = []
-    for await (const x of loader.getObjectIterator()) {
-      r.push(x)
-    }
-
-    expect(r).toMatchSnapshot()
-  })
-
-  test('can get root/child object from cache using iterator', async () => {
-    const child1Base = { id: 'child1Id' }
-    const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
-
-    const rootId = 'rootId'
-    const rootBase: Base = {
-      id: 'rootId',
-      speckle_type: 'type',
-      __closure: { child1Id: 100 }
-    }
-    const root = {
-      baseId: rootId,
-      base: rootBase
-    } as unknown as Item
-
-    const cache = {
-      getItem(params: { id: string }): Promise<Item | undefined> {
-        expect(params.id).toBe(root.baseId)
-        return Promise.resolve(root)
-      },
-      processItems(params: {
-        ids: string[]
-        foundItems: Queue<Item>
 
-        notFoundItems: Queue<string>
-      }): Promise<void> {
-        expect(params.ids.length).toBe(1)
-        expect(params.ids[0]).toBe(child1.baseId)
-        params.foundItems.add(child1)
-        return Promise.resolve()
-      },
-      disposeAsync(): Promise<void> {
-        return Promise.resolve()
-      }
-    } as Cache
-    const downloader = {
-      initializePool(params: { total: number }): void {
-        expect(params.total).toBe(1)
-      },
-      disposeAsync(): Promise<void> {
-        return Promise.resolve()
-      }
-    } as Downloader
+    const downloader = new MemoryDownloader(
+      rootId,
+      new Map<string, Base>([[rootId, rootBase]])
+    )
     const loader = new ObjectLoader2({
-      serverUrl: 'a',
-      streamId: 'b',
-      objectId: root.baseId,
-      cache,
-      downloader
+      rootId,
+      downloader,
+      database: new IndexedDatabase({
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
     })
     const r = []
     for await (const x of loader.getObjectIterator()) {
       r.push(x)
     }
+
     expect(r).toMatchSnapshot()
   })
 
@@ -152,19 +82,18 @@ describe('objectloader2', () => {
     const root = {
       baseId: rootId,
       base: rootBase
-    } as unknown as Item
+    } as Item
 
-    const records: Record<string, Base> = {}
-    records[root.baseId] = rootBase
-    records[child1.baseId] = child1Base
+    const records: Map<string, Base> = new Map<string, Base>()
+    records.set(root.baseId, rootBase)
+    records.set(child1.baseId, child1Base)
 
     const loader = new ObjectLoader2({
-      serverUrl: 'a',
-      streamId: 'b',
-      objectId: root.baseId,
-      cache: new MemoryDatabase({ items: records }),
-      downloader: new MemoryDownloader(rootId, records)
+      rootId: root.baseId,
+      downloader: new MemoryDownloader(rootId, records),
+      database: new MemoryDatabase({ items: records })
     })
+
     const r = []
     const obj = loader.getObject({ id: child1.baseId })
     for await (const x of loader.getObjectIterator()) {
@@ -193,18 +122,17 @@ describe('objectloader2', () => {
       base: rootBase
     } as unknown as Item
 
-    const records: Record<string, Base> = {}
-    records[root.baseId] = rootBase
-    records[child1.baseId] = child1Base
+    const records: Map<string, Base> = new Map<string, Base>()
+    records.set(root.baseId, rootBase)
+    records.set(child1.baseId, child1Base)
 
-    const results: AsyncGeneratorQueue<Item> = new AsyncGeneratorQueue<Item>()
     const loader = new ObjectLoader2({
-      serverUrl: 'a',
-      streamId: 'b',
-      objectId: root.baseId,
-      results,
-      cache: new MemoryDatabase(),
-      downloader: new MemoryDownloader(rootId, records, results)
+      rootId: root.baseId,
+      downloader: new MemoryDownloader(rootId, records),
+      database: new IndexedDatabase({
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
     })
     const r = []
     const obj = loader.getObject({ id: child1.baseId })
@@ -220,26 +148,30 @@ describe('objectloader2', () => {
   })
 
   test('add extra header', async () => {
-    const root = { baseId: 'baseId' } as unknown as Item
-    const cache = {
-      getItem(params: { id: string }): Promise<Item> {
-        expect(params.id).toBe(root.baseId)
-        return Promise.resolve(root)
-      }
-    } as Cache
-    const downloader = {} as Downloader
+    const rootId = 'rootId'
+    const rootBase: Base = {
+      id: 'rootId',
+      speckle_type: 'type',
+      __closure: { child1Id: 100 }
+    }
+    const root = {
+      baseId: rootId,
+      base: rootBase
+    } as Item
+
+    const records: Map<string, Base> = new Map<string, Base>()
+    records.set(root.baseId, rootBase)
     const headers = new Headers()
     headers.set('x-test', 'asdf')
     const loader = new ObjectLoader2({
-      serverUrl: 'a',
-      streamId: 'b',
-      objectId: root.baseId,
-      headers,
-      cache,
-      downloader
+      rootId: root.baseId,
+      downloader: new MemoryDownloader(rootId, records),
+      database: new IndexedDatabase({
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
     })
     const x = await loader.getRootObject()
-    expect(x).toBe(root)
     expect(x).toMatchSnapshot()
   })
 

package/src/operations/objectLoader2.ts

@@ -1,81 +1,79 @@
 import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
-import { Cache, Downloader } from './interfaces.js'
-import IndexedDatabase from './indexedDatabase.js'
-import ServerDownloader from './serverDownloader.js'
+import { Downloader, Database } from './interfaces.js'
 import { CustomLogger, Base, Item } from '../types/types.js'
-import { ObjectLoader2Options } from './options.js'
-import { MemoryDownloader } from './memoryDownloader.js'
-import { MemoryDatabase } from './memoryDatabase.js'
+import { CacheOptions, ObjectLoader2Options } from './options.js'
 import { DefermentManager } from '../helpers/defermentManager.js'
+import { CacheReader } from '../helpers/cacheReader.js'
+import { CachePump } from '../helpers/cachePump.js'
+import AggregateQueue from '../helpers/aggregateQueue.js'
+import { ObjectLoader2Factory } from './objectLoader2Factory.js'
 
-export default class ObjectLoader2 {
-  #objectId: string
+export class ObjectLoader2 {
+  #rootId: string
 
   #logger: CustomLogger
 
-  #database: Cache
+  #database: Database
   #downloader: Downloader
+  #pump: CachePump
+  #cache: CacheReader
 
   #deferments: DefermentManager
 
   #gathered: AsyncGeneratorQueue<Item>
 
-  constructor(options: ObjectLoader2Options) {
-    this.#objectId = options.objectId
+  #root?: Item = undefined
 
+  constructor(options: ObjectLoader2Options) {
+    this.#rootId = options.rootId
     this.#logger = options.logger || console.log
-    this.#gathered = options.results || new AsyncGeneratorQueue()
-    this.#deferments = new DefermentManager()
-    this.#database =
-      options.cache ||
-      new IndexedDatabase({
-        logger: this.#logger,
-        maxCacheReadSize: 10_000,
-        maxCacheWriteSize: 5_000,
-        indexedDB: options.indexedDB,
-        keyRange: options.keyRange
-      })
-    this.#downloader =
-      options.downloader ||
-      new ServerDownloader({
-        database: this.#database,
-        results: this.#gathered,
-        serverUrl: options.serverUrl,
-        streamId: options.streamId,
-        objectId: this.#objectId,
-        token: options.token,
-        headers: options.headers
-      })
+
+    const cacheOptions: CacheOptions = {
+      logger: this.#logger,
+      maxCacheReadSize: 10_000,
+      maxCacheWriteSize: 10_000,
+      maxWriteQueueSize: 40_000,
+      maxCacheBatchWriteWait: 3_000,
+      maxCacheBatchReadWait: 3_000
+    }
+
+    this.#gathered = new AsyncGeneratorQueue()
+    this.#database = options.database
+    this.#deferments = new DefermentManager({
+      maxSizeInMb: 2_000, // 2 GBs
+      ttlms: 5_000, // 5 seconds
+      logger: this.#logger
+    })
+    this.#cache = new CacheReader(this.#database, this.#deferments, cacheOptions)
+    this.#pump = new CachePump(
+      this.#database,
+      this.#gathered,
+      this.#deferments,
+      cacheOptions
+    )
+    this.#downloader = options.downloader
   }
 
   async disposeAsync(): Promise<void> {
-    await Promise.all([
-      this.#database.disposeAsync(),
-      this.#downloader.disposeAsync(),
-      this.#gathered.dispose()
-    ])
+    await Promise.all([this.#downloader.disposeAsync(), this.#cache.disposeAsync()])
+    this.#deferments.dispose()
   }
 
   async getRootObject(): Promise<Item | undefined> {
-    const cachedRootObject = await this.#database.getItem({ id: this.#objectId })
-    if (cachedRootObject) {
-      return cachedRootObject
+    if (!this.#root) {
+      this.#root = await this.#database.getItem({ id: this.#rootId })
+      if (!this.#root) {
+        this.#root = await this.#downloader.downloadSingle()
+      }
     }
-    const rootItem = await this.#downloader.downloadSingle()
-
-    await this.#database.add(rootItem)
-    return rootItem
+    return this.#root
   }
 
   async getObject(params: { id: string }): Promise<Base> {
-    const item = await this.#database.getItem({ id: params.id })
-    if (item) {
-      return item.base
-    }
-    return await this.#deferments.defer({ id: params.id })
+    return await this.#cache.getObject({ id: params.id })
   }
 
-  async getTotalObjectCount() {
+  async getTotalObjectCount(): Promise<number> {
     const rootObj = await this.getRootObject()
     const totalChildrenCount = Object.keys(rootObj?.base.__closure || {}).length
     return totalChildrenCount + 1 //count the root
@@ -87,47 +85,27 @@ export default class ObjectLoader2 {
       this.#logger('No root object found!')
       return
     }
+    //only for root
+    this.#pump.add(rootItem)
     yield rootItem.base
     if (!rootItem.base.__closure) return
 
     const children = Object.keys(rootItem.base.__closure)
     const total = children.length
-    this.#downloader.initializePool({ total })
-    const processPromise = this.#database.processItems({
-      ids: children,
-      foundItems: this.#gathered,
-      notFoundItems: this.#downloader
+    this.#downloader.initializePool({
+      results: new AggregateQueue(this.#gathered, this.#pump),
+      total
     })
-    let count = 0
-    for await (const item of this.#gathered.consume()) {
-      this.#deferments.undefer(item)
+    for await (const item of this.#pump.gather(children, this.#downloader)) {
       yield item.base
-      count++
-      if (count >= total) {
-        await this.disposeAsync()
-      }
     }
-    await processPromise
   }
 
   static createFromObjects(objects: Base[]): ObjectLoader2 {
-    const root = objects[0]
-    const records: Record<string, Base> = {}
-    objects.forEach((element) => {
-      records[element.id] = element
-    })
-    const loader = new ObjectLoader2({
-      serverUrl: 'dummy',
-      streamId: 'dummy',
-      objectId: root.id,
-      cache: new MemoryDatabase({ items: records }),
-      downloader: new MemoryDownloader(root.id, records)
-    })
-    return loader
+    return ObjectLoader2Factory.createFromObjects(objects)
   }
 
   static createFromJSON(json: string): ObjectLoader2 {
-    const jsonObj = JSON.parse(json) as Base[]
-    return this.createFromObjects(jsonObj)
+    return ObjectLoader2Factory.createFromJSON(json)
   }
 }

package/src/operations/objectLoader2Factory.ts

@@ -0,0 +1,56 @@
+import { Base, CustomLogger } from '../types/types.js'
+import IndexedDatabase from './databases/indexedDatabase.js'
+import { MemoryDatabase } from './databases/memoryDatabase.js'
+import { MemoryDownloader } from './downloaders/memoryDownloader.js'
+import ServerDownloader from './downloaders/serverDownloader.js'
+import { ObjectLoader2 } from './objectLoader2.js'
+
+export class ObjectLoader2Factory {
+  static createFromObjects(objects: Base[]): ObjectLoader2 {
+    const root = objects[0]
+    const records: Map<string, Base> = new Map<string, Base>()
+    objects.forEach((element) => {
+      records.set(element.id, element)
+    })
+    const loader = new ObjectLoader2({
+      rootId: root.id,
+      database: new MemoryDatabase({ items: records }),
+      downloader: new MemoryDownloader(root.id, records)
+    })
+    return loader
+  }
+
+  static createFromJSON(json: string): ObjectLoader2 {
+    const jsonObj = JSON.parse(json) as Base[]
+    return this.createFromObjects(jsonObj)
+  }
+
+  static createFromUrl(params: {
+    serverUrl: string
+    streamId: string
+    objectId: string
+    token?: string
+    headers?: Headers
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+    keyRange?: { bound: Function; lowerBound: Function; upperBound: Function }
+    indexedDB?: IDBFactory
+    logger?: CustomLogger
+  }): ObjectLoader2 {
+    const loader = new ObjectLoader2({
+      rootId: params.objectId,
+      downloader: new ServerDownloader({
+        serverUrl: params.serverUrl,
+        streamId: params.streamId,
+        objectId: params.objectId,
+        token: params.token,
+        headers: params.headers
+      }),
+      database: new IndexedDatabase({
+        logger: params.logger,
+        indexedDB: params.indexedDB,
+        keyRange: params.keyRange
+      })
+    })
+    return loader
+  }
+}

package/src/operations/options.ts

@@ -1,48 +1,29 @@
-/* eslint-disable @typescript-eslint/no-unsafe-function-type */
-import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
-import Queue from '../helpers/queue.js'
-import { Base, CustomLogger, Fetcher, Item } from '../types/types.js'
-import { Cache, Downloader } from './interfaces.js'
+import { Base, CustomLogger } from '../types/types.js'
+import { Downloader, Database } from './interfaces.js'
 
 export interface ObjectLoader2Options {
-  keyRange?: { bound: Function; lowerBound: Function; upperBound: Function }
-  indexedDB?: IDBFactory
-  serverUrl: string
-  streamId: string
-  objectId: string
-  token?: string
+  rootId: string
+  downloader: Downloader
+  database: Database
   logger?: CustomLogger
-  headers?: Headers
-  results?: AsyncGeneratorQueue<Item>
-  cache?: Cache
-  downloader?: Downloader
 }
-export interface BaseDatabaseOptions {
+
+export interface CacheOptions {
   logger?: CustomLogger
-  indexedDB?: IDBFactory
-  keyRange?: {
-    bound: Function
-    lowerBound: Function
-    upperBound: Function
-  }
-  maxCacheReadSize?: number
-  maxCacheWriteSize?: number
-  maxCacheBatchWriteWait?: number
+  maxCacheReadSize: number
+  maxCacheWriteSize: number
+  maxCacheBatchWriteWait: number
+  maxCacheBatchReadWait: number
+  maxWriteQueueSize: number
 }
 
-export interface BaseDownloadOptions {
-  serverUrl: string
-  streamId: string
-  objectId: string
-  token?: string
-  headers?: Headers
-
-  fetch?: Fetcher
-  database: Cache
-  results: Queue<Item>
+export interface MemoryDatabaseOptions {
+  logger?: CustomLogger
+  items?: Map<string, Base>
 }
 
-export interface MemoryDatabaseOptions {
+export interface DefermentManagerOptions {
   logger?: CustomLogger
-  items?: Record<string, Base>
+  maxSizeInMb: number
+  ttlms: number
 }

package/src/operations/traverser.spec.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from 'vitest'
 import { Base } from '../types/types.js'
-import ObjectLoader2 from './objectLoader2.js'
+import { ObjectLoader2 } from './objectLoader2.js'
 import Traverser from './traverser.js'
 
 describe('Traverser', () => {

package/src/operations/traverser.ts

@@ -1,5 +1,5 @@
 import { Base, DataChunk, isBase, isReference, isScalar } from '../types/types.js'
-import ObjectLoader2 from './objectLoader2.js'
+import { ObjectLoader2 } from './objectLoader2.js'
 
 export type ProgressStage = 'download' | 'construction'
 export type OnProgress = (e: {

package/src/test/e2e.spec.ts

@@ -1,8 +1,8 @@
 import { describe, test, expect } from 'vitest'
 import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
-import ObjectLoader2 from '../operations/objectLoader2.js'
 import { Base } from '../types/types.js'
-import { TIME } from '@speckle/shared'
+import { TIME_MS } from '@speckle/shared'
+import { ObjectLoader2Factory } from '../operations/objectLoader2Factory.js'
 
 describe('e2e', () => {
   test(
@@ -10,7 +10,7 @@ describe('e2e', () => {
     async () => {
       // Revit sample house (good for bim-like stuff with many display meshes)
       //const resource = 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
-      const loader = new ObjectLoader2({
+      const loader = ObjectLoader2Factory.createFromUrl({
         serverUrl: 'https://app.speckle.systems',
         streamId: 'da9e320dad',
         objectId: '31d10c0cea569a1e26809658ed27e281',
@@ -36,6 +36,6 @@ describe('e2e', () => {
       expect(base2).toBeDefined()
       expect(base2.id).toBe('3841e3cbc45d52c47bc2f1b7b0ad4eb9')
     },
-    10 * TIME.second
+    10 * TIME_MS.second
   )
 })

package/src/types/types.ts

@@ -8,6 +8,7 @@ export type Fetcher = (
 export interface Item {
   baseId: string
   base: Base
+  size?: number
 }
 
 export interface Base {
@@ -58,3 +59,13 @@ export function isScalar(
     type === 'undefined'
   )
 }
+
+export function take<T>(it: Iterator<T>, count: number): T[] {
+  const result: T[] = []
+  for (let i = 0; i < count; i++) {
+    const itr = it.next()
+    if (itr.done) break
+    result.push(itr.value)
+  }
+  return result
+}