@speckle/objectloader2 2.25.4 → 2.25.8

package/src/core/objectLoader2.ts

@@ -0,0 +1,162 @@
+import { MemoryCache } from '../deferment/MemoryCache.js'
+import { DefermentManager } from '../deferment/defermentManager.js'
+import AggregateQueue from '../queues/aggregateQueue.js'
+import AsyncGeneratorQueue from '../queues/asyncGeneratorQueue.js'
+import { CustomLogger, take } from '../types/functions.js'
+import { Item, Base } from '../types/types.js'
+import { Database, Downloader } from './interfaces.js'
+import { ObjectLoader2Factory } from './objectLoader2Factory.js'
+import { ObjectLoader2Options, CacheOptions } from './options.js'
+import { CacheReader } from './stages/cacheReader.js'
+import { CacheWriter } from './stages/cacheWriter.js'
+
+const MAX_CLOSURES_TO_TAKE = 100
+const EXPECTED_CLOSURE_VALUE = 100
+
+export class ObjectLoader2 {
+  #rootId: string
+
+  #logger: CustomLogger
+
+  #database: Database
+  #downloader: Downloader
+  #cacheReader: CacheReader
+  #cacheWriter: CacheWriter
+
+  #deferments: DefermentManager
+  #cache: MemoryCache
+
+  #gathered: AsyncGeneratorQueue<Item>
+
+  #root?: Item = undefined
+  #isRootStored = false
+
+  constructor(options: ObjectLoader2Options) {
+    this.#rootId = options.rootId
+    this.#logger = options.logger || ((): void => {})
+    this.#logger('ObjectLoader2 initialized with rootId:', this.#rootId)
+
+    const cacheOptions: CacheOptions = {
+      logger: this.#logger,
+      maxCacheReadSize: 10_000,
+      maxCacheWriteSize: 10_000,
+      maxWriteQueueSize: 40_000,
+      maxCacheBatchWriteWait: 100, //100 ms, next to nothing!
+      maxCacheBatchReadWait: 100 //100 ms, next to nothing!
+    }
+
+    this.#gathered = new AsyncGeneratorQueue()
+
+    this.#database = options.database
+    this.#cache = new MemoryCache(
+      {
+        maxSizeInMb: 500, // 500 MB
+        ttlms: 5_000 // 5 seconds
+      },
+      this.#logger
+    )
+    this.#deferments = new DefermentManager(this.#cache, this.#logger)
+    this.#downloader = options.downloader
+    this.#cacheReader = new CacheReader(
+      this.#database,
+      this.#deferments,
+      this.#logger,
+      cacheOptions
+    )
+    this.#cacheReader.initializeQueue(this.#gathered, this.#downloader)
+    this.#cacheWriter = new CacheWriter(
+      this.#database,
+      this.#logger,
+      this.#deferments,
+      cacheOptions,
+      (id: string) => {
+        this.#cacheReader.requestItem(id)
+      }
+    )
+  }
+
+  async disposeAsync(): Promise<void> {
+    await Promise.all([
+      this.#gathered.disposeAsync(),
+      this.#downloader.disposeAsync(),
+      this.#cacheWriter.disposeAsync(),
+      this.#cacheReader.disposeAsync()
+    ])
+    this.#deferments.dispose()
+    this.#cache.dispose()
+  }
+
+  async getRootObject(): Promise<Item | undefined> {
+    if (!this.#root) {
+      this.#root = (await this.#database.getAll([this.#rootId]))[0]
+      if (!this.#root) {
+        this.#root = await this.#downloader.downloadSingle()
+      } else {
+        this.#isRootStored = true
+      }
+    }
+    return this.#root
+  }
+
+  async getObject(params: { id: string }): Promise<Base> {
+    return await this.#cacheReader.getObject({ id: params.id })
+  }
+
+  async getTotalObjectCount(): Promise<number> {
+    const rootObj = await this.getRootObject()
+    const totalChildrenCount = Object.keys(rootObj?.base?.__closure || {}).length
+    return totalChildrenCount + 1 //count the root
+  }
+
+  async *getObjectIterator(): AsyncGenerator<Base> {
+    const rootItem = await this.getRootObject()
+    if (rootItem?.base === undefined) {
+      this.#logger('No root object found!')
+      return
+    }
+    if (!rootItem.base.__closure) {
+      yield rootItem.base
+      return
+    }
+
+    //sort the closures by their values descending
+    const sortedClosures = Object.entries(rootItem.base.__closure ?? []).sort(
+      (a, b) => b[1] - a[1]
+    )
+    this.#logger(
+      'calculated closures: ',
+      !take(sortedClosures.values(), MAX_CLOSURES_TO_TAKE).every(
+        (x) => x[1] === EXPECTED_CLOSURE_VALUE
+      )
+    )
+    const children = sortedClosures.map((x) => x[0])
+    const total = children.length + 1 // +1 for the root object
+    this.#downloader.initializePool({
+      results: new AggregateQueue(this.#gathered, this.#cacheWriter),
+      total
+    })
+    //only for root
+    this.#gathered.add(rootItem)
+    this.#cacheReader.requestAll(children)
+    let count = 0
+    for await (const item of this.#gathered.consume()) {
+      yield item.base! //always defined, as we add it to the queue
+      count++
+      if (count >= total) {
+        break
+      }
+    }
+    if (!this.#isRootStored) {
+      await this.#database.saveBatch({ batch: [rootItem] })
+      this.#isRootStored = true
+    }
+  }
+
+  static createFromObjects(objects: Base[]): ObjectLoader2 {
+    return ObjectLoader2Factory.createFromObjects(objects)
+  }
+
+  static createFromJSON(json: string): ObjectLoader2 {
+    return ObjectLoader2Factory.createFromJSON(json)
+  }
+}

package/src/core/objectLoader2Factory.ts

@@ -0,0 +1,87 @@
+import { CustomLogger, getFeatureFlag, ObjectLoader2Flags } from '../types/functions.js'
+import { Base } from '../types/types.js'
+import { ObjectLoader2 } from './objectLoader2.js'
+import IndexedDatabase from './stages/indexedDatabase.js'
+import { MemoryDatabase } from './stages/memory/memoryDatabase.js'
+import { MemoryDownloader } from './stages/memory/memoryDownloader.js'
+import ServerDownloader from './stages/serverDownloader.js'
+
+export interface ObjectLoader2FactoryOptions {
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+  keyRange?: { bound: Function; lowerBound: Function; upperBound: Function }
+  indexedDB?: IDBFactory
+  logger?: CustomLogger
+}
+
+export class ObjectLoader2Factory {
+  static createFromObjects(objects: Base[]): ObjectLoader2 {
+    const root = objects[0]
+    const records: Map<string, Base> = new Map<string, Base>()
+    objects.forEach((element) => {
+      records.set(element.id, element)
+    })
+    const loader = new ObjectLoader2({
+      rootId: root.id,
+      database: new MemoryDatabase({ items: records }),
+      downloader: new MemoryDownloader(root.id, records)
+    })
+    return loader
+  }
+
+  static createFromJSON(json: string): ObjectLoader2 {
+    const jsonObj = JSON.parse(json) as Base[]
+    return this.createFromObjects(jsonObj)
+  }
+
+  static createFromUrl(params: {
+    serverUrl: string
+    streamId: string
+    objectId: string
+    token?: string
+    headers?: Headers
+    options?: ObjectLoader2FactoryOptions
+  }): ObjectLoader2 {
+    const log = ObjectLoader2Factory.getLogger(params.options?.logger)
+    let database
+    if (getFeatureFlag(ObjectLoader2Flags.DEBUG) === 'true') {
+      this.logger('Using DEBUG mode for ObjectLoader2Factory')
+    }
+    if (getFeatureFlag(ObjectLoader2Flags.USE_CACHE) === 'true') {
+      database = new IndexedDatabase({
+        indexedDB: params.options?.indexedDB,
+        keyRange: params.options?.keyRange
+      })
+    } else {
+      database = new MemoryDatabase({
+        items: new Map<string, Base>()
+      })
+      this.logger(
+        'Disabled persistent caching for ObjectLoader2.  Using MemoryDatabase'
+      )
+    }
+    const loader = new ObjectLoader2({
+      rootId: params.objectId,
+      downloader: new ServerDownloader({
+        serverUrl: params.serverUrl,
+        streamId: params.streamId,
+        objectId: params.objectId,
+        token: params.token,
+        headers: params.headers
+      }),
+      database,
+      logger: log
+    })
+    return loader
+  }
+
+  static getLogger(providedLogger?: CustomLogger): CustomLogger | undefined {
+    if (getFeatureFlag(ObjectLoader2Flags.DEBUG) === 'true') {
+      return providedLogger || this.logger
+    }
+    return providedLogger
+  }
+
+  static logger: CustomLogger = (m?: string, ...optionalParams: unknown[]) => {
+    console.log(`[debug] ${m}`, ...optionalParams)
+  }
+}

package/src/{operations → core}/options.ts

@@ -1,4 +1,5 @@
-import { Base, CustomLogger } from '../types/types.js'
+import { CustomLogger } from '../types/functions.js'
+import { Base } from '../types/types.js'
 import { Downloader, Database } from './interfaces.js'
 
 export interface ObjectLoader2Options {

package/src/core/stages/ItemStore.ts

@@ -0,0 +1,196 @@
+/* eslint-disable @typescript-eslint/no-base-to-string */
+/* eslint-disable @typescript-eslint/prefer-promise-reject-errors */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+import { isSafari } from '@speckle/shared'
+import { Item } from '../../types/types.js'
+
+/**
+ * A wrapper class for IndexedDB to simplify common database operations.
+ */
+export interface ItemStoreOptions {
+  indexedDB?: IDBFactory
+  keyRange?: {
+    bound: Function
+    lowerBound: Function
+    upperBound: Function
+  }
+}
+export class ItemStore {
+  #options: ItemStoreOptions
+
+  #db: IDBDatabase | undefined = undefined
+  readonly #dbName: string
+  readonly #storeName: string
+
+  constructor(options: ItemStoreOptions, dbName: string, storeName: string) {
+    this.#options = options
+    this.#dbName = dbName
+    this.#storeName = storeName
+  }
+
+  /**
+   * Initializes the database connection and creates the object store if needed.
+   * This must be called before any other database operations.
+   */
+  async init(): Promise<void> {
+    if (this.#db) return
+    await this.#safariFix()
+    return this.#openDatabase()
+  }
+
+  /**
+   * Opens the database, and if there's an error, deletes the database and tries again.
+   */
+  async #openDatabase(): Promise<void> {
+    const idb = this.#options.indexedDB ?? indexedDB
+
+    return new Promise((resolve, reject) => {
+      const request = idb.open(this.#dbName, 1)
+
+      request.onerror = (): any => {
+        console.warn(
+          `Failed to open database: ${this.#dbName}, deleting and trying again`
+        )
+        // Delete the database and try again
+        const deleteRequest = idb.deleteDatabase(this.#dbName)
+        deleteRequest.onsuccess = (): any => {
+          // Try opening again after deletion
+          void this.#openDatabase().then(resolve).catch(reject)
+        }
+        deleteRequest.onerror = (): any => {
+          reject(`Failed to delete and reopen database: ${this.#dbName}`)
+        }
+      }
+
+      request.onupgradeneeded = (event): any => {
+        const db = (event.target as IDBOpenDBRequest).result
+        if (db.objectStoreNames.contains(this.#storeName)) {
+          db.deleteObjectStore(this.#storeName)
+        }
+        db.createObjectStore(this.#storeName, { keyPath: 'baseId' })
+      }
+
+      request.onsuccess = (event): any => {
+        this.#db = (event.target as IDBOpenDBRequest).result
+        resolve()
+      }
+    })
+  }
+
+  #getDB(): IDBDatabase {
+    if (!this.#db) {
+      throw new Error('Database not initialized. Call init() first.')
+    }
+    return this.#db
+  }
+
+  /**
+   * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
+   * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
+   */
+  async #safariFix(): Promise<void> {
+    // No point putting other browsers or older versions of Safari through this mess.
+    if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve()
+
+    let intervalId: ReturnType<typeof setInterval>
+
+    return new Promise<void>((resolve: () => void) => {
+      const tryIdb = (): Promise<IDBDatabaseInfo[]> | undefined =>
+        this.#options.indexedDB?.databases().finally(resolve)
+      intervalId = setInterval(() => {
+        void tryIdb()
+      }, 100)
+      void tryIdb()
+    }).finally(() => clearInterval(intervalId))
+  }
+
+  /**
+   * Inserts or updates an array of items in a single transaction.
+   * @param data The array of items to insert.
+   */
+  bulkInsert(data: Item[]): Promise<void> {
+    return new Promise((resolve, reject) => {
+      try {
+        const transaction = this.#getDB().transaction(this.#storeName, 'readwrite', {
+          durability: 'relaxed'
+        })
+        const store = transaction.objectStore(this.#storeName)
+
+        transaction.onerror = (): any => {
+          reject(`Transaction error: ${transaction.error}`)
+        }
+        transaction.oncomplete = (): any => {
+          resolve()
+        }
+
+        data.forEach((item) => store.put(item))
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
+
+  /**
+   * Retrieves an array of items from the object store based on their IDs.
+   * @param ids The array of IDs to retrieve.
+   */
+  bulkGet(ids: string[]): Promise<(Item | undefined)[]> {
+    return new Promise((resolve, reject) => {
+      if (ids.length === 0) {
+        return resolve([])
+      }
+      try {
+        const transaction = this.#getDB().transaction(this.#storeName, 'readonly', {
+          durability: 'relaxed'
+        })
+        const store = transaction.objectStore(this.#storeName)
+        const promises: Promise<Item | undefined>[] = []
+
+        for (const id of ids) {
+          promises.push(
+            new Promise((resolveGet, rejectGet) => {
+              const request = store.get(id)
+              request.onerror = (): void =>
+                rejectGet(`Request error for id ${id}: ${request.error}`)
+              // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+              request.onsuccess = (): void => resolveGet(request.result)
+            })
+          )
+        }
+
+        Promise.all(promises)
+          .then((results) => {
+            resolve(results)
+          })
+          .catch(reject)
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
+
+  /**
+   * Retrieves all items from the object store.
+   */
+  getAll(): Promise<Item[]> {
+    return new Promise((resolve, reject) => {
+      try {
+        const transaction = this.#getDB().transaction(this.#storeName, 'readonly')
+        const store = transaction.objectStore(this.#storeName)
+        const request = store.getAll()
+
+        request.onerror = (): any => reject(`Request error: ${request.error}`)
+        request.onsuccess = (): any => resolve(request.result)
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
+
+  close(): void {
+    if (!this.#db) return
+    this.#db.close()
+    this.#db = undefined
+  }
+}

package/src/core/stages/__snapshots__/indexedDatabase.spec.ts.snap

@@ -0,0 +1,18 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`IndexedDatabase > should add and get multiple items 1`] = `
+[
+  {
+    "base": {
+      "foo": "bar",
+    },
+    "baseId": "id1",
+  },
+  {
+    "base": {
+      "foo": "bar",
+    },
+    "baseId": "id2",
+  },
+]
+`;

package/src/{operations/downloaders → core/stages}/__snapshots__/serverDownloader.spec.ts.snap

@@ -56,6 +56,31 @@ exports[`downloader > download batch of three 1`] = `
 ]
 `;
 
+exports[`downloader > download batch of three with Objects.Other.RawEncoding 1`] = `
+[
+  {
+    "base": {
+      "id": "id1",
+      "speckle_type": "type",
+    },
+    "baseId": "id1",
+    "size": 34,
+  },
+  {
+    "base": undefined,
+    "baseId": "id2",
+  },
+  {
+    "base": {
+      "id": "id3",
+      "speckle_type": "type",
+    },
+    "baseId": "id3",
+    "size": 34,
+  },
+]
+`;
+
 exports[`downloader > download batch of two 1`] = `
 [
   {

package/src/{helpers → core/stages}/cacheReader.spec.ts

@@ -1,19 +1,22 @@
-import { describe, expect, test } from 'vitest'
-import { Base, Item } from '../types/types.js'
-import { DefermentManager } from './defermentManager.js'
+import { describe, test, expect } from 'vitest'
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import { Item, Base } from '../../types/types.js'
 import { CacheReader } from './cacheReader.js'
-import { MemoryDatabase } from '../operations/databases/memoryDatabase.js'
+import { MemoryDatabase } from './memory/memoryDatabase.js'
+import { MemoryCache } from '../../deferment/MemoryCache.js'
 
 describe('CacheReader testing', () => {
   test('deferred getObject', async () => {
     const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } }
 
-    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
+    const cache = new MemoryCache({ maxSizeInMb: 1, ttlms: 1 }, () => {})
+    const deferments = new DefermentManager(cache, () => {})
     const cacheReader = new CacheReader(
       new MemoryDatabase({
-        items: new Map<string, Base>([[i1.baseId, i1.base]])
+        items: new Map<string, Base>([[i1.baseId, i1.base!]])
       }),
       deferments,
+      () => {},
       {
         maxCacheReadSize: 1,
         maxCacheWriteSize: 1,
@@ -26,7 +29,9 @@ describe('CacheReader testing', () => {
     const objPromise = cacheReader.getObject({
       id: i1.baseId
     })
-    deferments.undefer(i1)
+    deferments.undefer(i1, (id: string) => {
+      throw new Error(`Requesting item ${id} not implemented`)
+    })
     const base = await objPromise
 
     expect(base).toMatchSnapshot()

package/src/core/stages/cacheReader.ts

@@ -0,0 +1,88 @@
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
+import Queue from '../../queues/queue.js'
+import { CustomLogger } from '../../types/functions.js'
+import { Item, Base } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import { CacheOptions } from '../options.js'
+
+export class CacheReader {
+  #database: Database
+  #defermentManager: DefermentManager
+  #logger: CustomLogger
+  #options: CacheOptions
+  #readQueue: BatchingQueue<string> | undefined
+  #foundQueue: Queue<Item> | undefined
+  #notFoundQueue: Queue<string> | undefined
+
+  constructor(
+    database: Database,
+    defermentManager: DefermentManager,
+    logger: CustomLogger,
+    options: CacheOptions
+  ) {
+    this.#database = database
+    this.#defermentManager = defermentManager
+    this.#logger = logger
+    this.#options = options
+  }
+
+  initializeQueue(foundQueue: Queue<Item>, notFoundQueue: Queue<string>): void {
+    this.#foundQueue = foundQueue
+    this.#notFoundQueue = notFoundQueue
+  }
+
+  getObject(params: { id: string }): Promise<Base> {
+    const [p, b] = this.#defermentManager.defer({ id: params.id })
+    if (!b) {
+      this.requestItem(params.id)
+    }
+    return p
+  }
+
+  #createReadQueue(): void {
+    if (!this.#readQueue) {
+      this.#readQueue = new BatchingQueue({
+        batchSize: this.#options.maxCacheReadSize,
+        maxWaitTime: this.#options.maxCacheBatchReadWait,
+        processFunction: this.#processBatch
+      })
+    }
+  }
+
+  requestItem(id: string): void {
+    this.#createReadQueue()
+    if (!this.#readQueue?.get(id)) {
+      this.#readQueue?.add(id, id)
+    }
+  }
+
+  requestAll(keys: string[]): void {
+    this.#createReadQueue()
+
+    this.#readQueue?.addAll(keys, keys)
+  }
+
+  #processBatch = async (batch: string[]): Promise<void> => {
+    const start = performance.now()
+    const items = await this.#database.getAll(batch)
+    for (let i = 0; i < items.length; i++) {
+      const item = items[i]
+      if (item) {
+        this.#foundQueue?.add(item)
+        this.#defermentManager.undefer(item, (id) => this.requestItem(id))
+      } else {
+        this.#notFoundQueue?.add(batch[i])
+      }
+    }
+    this.#logger(
+      `readBatch: batch ${batch.length}, time ${
+        performance.now() - start
+      } ms left ${this.#readQueue?.count()}`
+    )
+  }
+
+  disposeAsync(): Promise<void> {
+    return this.#readQueue?.disposeAsync() || Promise.resolve()
+  }
+}