@speckle/objectloader2 2.25.7 → 2.25.8

package/src/helpers/defermentManager.defermentTotals.spec.ts

@@ -1,53 +0,0 @@
-import { describe, test, expect, beforeEach } from 'vitest'
-import { DefermentManager } from './defermentManager.js'
-import { DefermentManagerOptions } from '../operations/options.js'
-import { Base, Item } from '../types/types.js'
-
-const makeItem = (id: string, size = 1): Item => ({
-  baseId: id,
-  base: { foo: 'bar' } as unknown as Base,
-  size
-})
-
-describe('DefermentManager totalDefermentRequests', () => {
-  let manager: DefermentManager
-  let options: DefermentManagerOptions
-
-  beforeEach(() => {
-    options = { maxSizeInMb: 1, ttlms: 1000, logger: (): void => {} }
-    manager = new DefermentManager(options)
-  })
-
-  test('tracks deferment requests for each id', () => {
-    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-    manager.defer({ id: 'a' })
-    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-    manager.defer({ id: 'a' })
-    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-    manager.defer({ id: 'b' })
-    // @ts-expect-error: access private for test
-    expect(manager.totalDefermentRequests.get('a')).toBe(2)
-    // @ts-expect-error: access private for test
-    expect(manager.totalDefermentRequests.get('b')).toBe(1)
-  })
-
-  test('increments and does not reset on undefer', () => {
-    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-    manager.defer({ id: 'x' })
-    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-    manager.defer({ id: 'x' })
-    manager.undefer(makeItem('x'))
-    // @ts-expect-error: access private for test
-    expect(manager.totalDefermentRequests.get('x')).toBe(2)
-    // @ts-expect-error: access private for test
-    const deferredBase = manager.deferments.get('x')
-    expect(deferredBase).toBeDefined()
-    expect(deferredBase?.getId()).toBe('x')
-  })
-
-  test('does not increment for undefer only', () => {
-    manager.undefer(makeItem('y'))
-    // @ts-expect-error: access private for test
-    expect(manager.totalDefermentRequests.get('y')).toBeUndefined()
-  })
-})

package/src/helpers/defermentManager.disposal.spec.ts

@@ -1,28 +0,0 @@
-import { describe, it, expect } from 'vitest'
-import { DefermentManager } from './defermentManager.js'
-import { DefermentManagerOptions } from '../operations/options.js'
-import { Item } from '../types/types.js'
-
-describe('DefermentManager disposal', () => {
-  const options: DefermentManagerOptions = { ttlms: 10, maxSizeInMb: 1 }
-  const makeItem = (id: string): Item => ({
-    baseId: id,
-    base: { id, speckle_type: 'test' }
-  })
-
-  it('should throw on get/defer/undefer after dispose', async () => {
-    const manager = new DefermentManager(options)
-    manager.dispose()
-    expect(() => manager.get('a')).toThrow('DefermentManager is disposed')
-    expect(() => manager.undefer(makeItem('a'))).toThrow('DefermentManager is disposed')
-    await expect(manager.defer({ id: 'a' })).rejects.toThrow(
-      'DefermentManager is disposed'
-    )
-  })
-
-  it('dispose is idempotent', () => {
-    const manager = new DefermentManager(options)
-    manager.dispose()
-    expect(() => manager.dispose()).not.toThrow()
-  })
-})

package/src/helpers/defermentManager.spec.ts

@@ -1,37 +0,0 @@
-/* eslint-disable @typescript-eslint/no-unsafe-member-access */
-/* eslint-disable @typescript-eslint/no-explicit-any  */
-import { describe, expect, test } from 'vitest'
-import { DefermentManager } from './defermentManager.js'
-
-describe('deferments', () => {
-  test('defer one', async () => {
-    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
-    const x = deferments.defer({ id: 'id' })
-    expect(x).toBeInstanceOf(Promise)
-    deferments.undefer({ baseId: 'id', base: { id: 'id', speckle_type: 'type' } })
-    const b = await x
-    expect(b).toMatchSnapshot()
-  })
-
-  test('expireAt timeout', async () => {
-    const now = 1
-    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
-    deferments['now'] = (): number => now
-    const x = deferments.defer({ id: 'id' })
-    expect(x).toBeInstanceOf(Promise)
-    const d = deferments.get('id')
-    expect(d).toBeDefined()
-    expect(d?.getId()).toBe('id')
-    expect((d as any).expiresAt).toBe(2)
-    expect((d as any).ttl).toBe(1)
-    expect((d as any).item).toBeUndefined()
-    expect(d?.isExpired(1)).toBe(false)
-    deferments.undefer({ baseId: 'id', base: { id: 'id', speckle_type: 'type' } })
-    await x
-    expect((d as any).expiresAt).toBe(2)
-    expect((d as any).ttl).toBe(1)
-    expect((d as any).item).toBeDefined()
-    expect(d?.isExpired(1)).toBe(false)
-    expect(d?.isExpired(3)).toBe(true)
-  })
-})

package/src/helpers/defermentManager.ts

@@ -1,160 +0,0 @@
-import { DeferredBase } from './deferredBase.js'
-import { Base, CustomLogger, Item } from '../types/types.js'
-import { DefermentManagerOptions } from '../operations/options.js'
-
-export class DefermentManager {
-  private deferments: Map<string, DeferredBase> = new Map()
-  private timer?: ReturnType<typeof setTimeout>
-  private logger: CustomLogger
-  private currentSize = 0
-  private disposed = false
-  //tracks total deferment requests for each id
-  //this is used to prevent cleaning up deferments that are still being requested
-  private totalDefermentRequests: Map<string, number> = new Map()
-
-  constructor(private options: DefermentManagerOptions) {
-    this.resetGlobalTimer()
-    this.logger = options.logger || ((): void => {})
-  }
-
-  private now(): number {
-    return Date.now()
-  }
-
-  isDeferred(id: string): boolean {
-    return this.deferments.has(id)
-  }
-
-  get(id: string): DeferredBase | undefined {
-    if (this.disposed) throw new Error('DefermentManager is disposed')
-    return this.deferments.get(id)
-  }
-
-  async defer(params: { id: string }): Promise<Base> {
-    if (this.disposed) throw new Error('DefermentManager is disposed')
-    this.trackDefermentRequest(params.id)
-    const now = this.now()
-    const deferredBase = this.deferments.get(params.id)
-    if (deferredBase) {
-      deferredBase.setAccess(now)
-      return deferredBase.getPromise()
-    }
-    const notYetFound = new DeferredBase(
-      this.options.ttlms,
-      params.id,
-      now + this.options.ttlms
-    )
-    this.deferments.set(params.id, notYetFound)
-    return notYetFound.getPromise()
-  }
-
-  private trackDefermentRequest(id: string): void {
-    const request = this.totalDefermentRequests.get(id)
-    if (request) {
-      this.totalDefermentRequests.set(id, request + 1)
-    } else {
-      this.totalDefermentRequests.set(id, 1)
-    }
-  }
-
-  undefer(item: Item): void {
-    if (this.disposed) throw new Error('DefermentManager is disposed')
-    const now = this.now()
-    this.currentSize += item.size || 0
-    //order matters here with found before undefer
-    const deferredBase = this.deferments.get(item.baseId)
-    if (deferredBase) {
-      deferredBase.found(item)
-      deferredBase.setAccess(now)
-    } else {
-      const existing = new DeferredBase(this.options.ttlms, item.baseId, now)
-      existing.found(item)
-      this.deferments.set(item.baseId, existing)
-    }
-  }
-
-  private resetGlobalTimer(): void {
-    const run = (): void => {
-      this.cleanDeferments()
-      this.timer = setTimeout(run, this.options.ttlms)
-    }
-    this.timer = setTimeout(run, this.options.ttlms)
-  }
-
-  dispose(): void {
-    if (this.disposed) return
-    this.disposed = true
-    if (this.timer) {
-      clearTimeout(this.timer)
-      this.timer = undefined
-    }
-    this.clearDeferments()
-  }
-
-  private clearDeferments(): void {
-    let waiting = 0
-    for (const deferredBase of this.deferments.values()) {
-      deferredBase.done(0)
-      if (deferredBase.getItem() === undefined) {
-        waiting++
-      }
-    }
-    this.currentSize = 0
-    this.deferments.clear()
-    this.logger('cleared deferments, left', waiting)
-  }
-
-  private cleanDeferments(): void {
-    const maxSizeBytes = this.options.maxSizeInMb * 1024 * 1024
-    if (this.currentSize < maxSizeBytes) {
-      this.logger(
-        'deferments size is ok, no need to clean',
-        this.currentSize,
-        maxSizeBytes
-      )
-      return
-    }
-    const now = this.now()
-    let cleaned = 0
-    const start = performance.now()
-    for (const deferredBase of Array.from(this.deferments.values())
-      .filter((x) => x.isExpired(now))
-      .sort((a, b) => this.compareMaybeBasesBySize(a.getItem(), b.getItem()))) {
-      if (deferredBase.done(now)) {
-        //if the deferment is done but has been requested multiple times,
-        //we do not clean it up to allow the requests to resolve
-        const requestCount = this.totalDefermentRequests.get(deferredBase.getId())
-        if (requestCount && requestCount > 1) {
-          return
-        }
-        this.currentSize -= deferredBase.getItem()?.size || 0
-        this.deferments.delete(deferredBase.getId())
-        cleaned++
-        if (this.currentSize < maxSizeBytes) {
-          break
-        }
-      }
-    }
-    this.logger(
-      'cleaned deferments: cleaned, left, time',
-      cleaned,
-      this.deferments.size,
-      performance.now() - start
-    )
-    return
-  }
-
-  compareMaybeBasesBySize(a: Item | undefined, b: Item | undefined): number {
-    if (a === undefined && b === undefined) return 0
-    if (a === undefined) return -1
-    if (b === undefined) return 1
-    return this.compareMaybe(a.size, b.size)
-  }
-
-  compareMaybe(a: number | undefined, b: number | undefined): number {
-    if (a === undefined && b === undefined) return 0
-    if (a === undefined) return -1
-    if (b === undefined) return 1
-    return a - b
-  }
-}

package/src/helpers/deferredBase.ts

@@ -1,55 +0,0 @@
-import { Base, Item } from '../types/types.js'
-
-export class DeferredBase {
-  private promise: Promise<Base>
-  private resolve!: (value: Base) => void
-  private reject!: (reason?: Error) => void
-  private item?: Item
-
-  private readonly id: string
-  private expiresAt: number // Timestamp in ms
-  private ttl: number // ttl in ms
-
-  constructor(ttl: number, id: string, expiresAt: number) {
-    this.expiresAt = expiresAt
-    this.ttl = ttl
-    this.id = id
-    this.promise = new Promise<Base>((resolve, reject) => {
-      this.resolve = resolve
-      this.reject = reject
-    })
-  }
-
-  getId(): string {
-    return this.id
-  }
-
-  getItem(): Item | undefined {
-    return this.item
-  }
-
-  getPromise(): Promise<Base> {
-    return this.promise
-  }
-
-  isExpired(now: number): boolean {
-    return this.item !== undefined && now > this.expiresAt
-  }
-  setAccess(now: number): void {
-    this.expiresAt = now + this.ttl
-  }
-
-  found(value: Item): void {
-    this.item = value
-    this.resolve(value.base)
-  }
-  done(now: number): boolean {
-    if (this.item) {
-      this.resolve(this.item.base)
-    }
-    if (this.isExpired(now)) {
-      return true
-    }
-    return false
-  }
-}

package/src/helpers/memoryPump.ts

@@ -1,40 +0,0 @@
-import { Item } from '../types/types.js'
-import { Pump } from './pump.js'
-import Queue from './queue.js'
-
-export class MemoryPump implements Pump {
-  #items: Map<string, Item> = new Map()
-
-  add(item: Item): void {
-    this.#items.set(item.baseId, item)
-  }
-
-  async pumpItems(params: {
-    ids: string[]
-    foundItems: Queue<Item>
-    notFoundItems: Queue<string>
-  }): Promise<void> {
-    const { ids, foundItems, notFoundItems } = params
-    for (const id of ids) {
-      const item = this.#items.get(id)
-      if (item) {
-        foundItems.add(item)
-      } else {
-        notFoundItems.add(id)
-      }
-    }
-    return Promise.resolve()
-  }
-
-  async *gather(ids: string[]): AsyncGenerator<Item> {
-    for (const id of ids) {
-      const item = this.#items.get(id)
-      if (item) {
-        yield item
-      }
-    }
-    return Promise.resolve()
-  }
-
-  async disposeAsync(): Promise<void> {}
-}

package/src/helpers/pump.ts

@@ -1,8 +0,0 @@
-import { Downloader } from '../operations/interfaces.js'
-import { Item } from '../types/types.js'
-import Queue from './queue.js'
-
-export interface Pump extends Queue<Item> {
-  gather(ids: string[], downloader: Downloader): AsyncGenerator<Item>
-  disposeAsync(): Promise<void>
-}

package/src/operations/databases/indexedDatabase.ts

@@ -1,126 +0,0 @@
-/* eslint-disable @typescript-eslint/no-unsafe-function-type */
-import BatchingQueue from '../../helpers/batchingQueue.js'
-import { CustomLogger, Item } from '../../types/types.js'
-import { isSafari } from '@speckle/shared'
-import { Dexie, DexieOptions, Table } from 'dexie'
-import { Database } from '../interfaces.js'
-
-export class ObjectStore extends Dexie {
-  static #databaseName: string = 'speckle-cache'
-  objects!: Table<Item, string> // Table type: <entity, primaryKey>
-
-  constructor(options: DexieOptions) {
-    super(ObjectStore.#databaseName, options)
-
-    this.version(1).stores({
-      objects: 'baseId, item' // baseId is primary key
-    })
-  }
-}
-
-export interface IndexedDatabaseOptions {
-  logger?: CustomLogger
-  indexedDB?: IDBFactory
-  keyRange?: {
-    bound: Function
-    lowerBound: Function
-    upperBound: Function
-  }
-}
-
-export default class IndexedDatabase implements Database {
-  #options: IndexedDatabaseOptions
-  #logger: CustomLogger
-
-  #cacheDB?: ObjectStore
-
-  #writeQueue: BatchingQueue<Item> | undefined
-
-  // #count: number = 0
-
-  constructor(options: IndexedDatabaseOptions) {
-    this.#options = options
-    this.#logger = options.logger || ((): void => {})
-  }
-
-  async getAll(keys: string[]): Promise<(Item | undefined)[]> {
-    await this.#setupCacheDb()
-    let items: (Item | undefined)[] = []
-    //   this.#count++
-    //  const startTime = performance.now()
-    // this.#logger('Start read ' + x + ' ' + batch.length)
-
-    //faster than BulkGet with dexie
-    await this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => {
-      const gets = keys.map((key) => this.#cacheDB!.objects.get(key))
-      const cachedData = await Promise.all(gets)
-      items = cachedData
-    })
-    // const endTime = performance.now()
-    // const duration = endTime - startTime
-    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
-
-    return items
-  }
-
-  async #openDatabase(): Promise<ObjectStore> {
-    const db = new ObjectStore({
-      indexedDB: this.#options.indexedDB ?? globalThis.indexedDB,
-      IDBKeyRange: this.#options.keyRange ?? IDBKeyRange,
-      chromeTransactionDurability: 'relaxed'
-    })
-    await db.open()
-    return db
-  }
-
-  async #setupCacheDb(): Promise<void> {
-    if (this.#cacheDB !== undefined) {
-      return
-    }
-
-    // Initialize
-    await this.#safariFix()
-    this.#cacheDB = await this.#openDatabase()
-  }
-
-  async cacheSaveBatch(params: { batch: Item[] }): Promise<void> {
-    await this.#setupCacheDb()
-    const { batch } = params
-    //const x = this.#count
-    //this.#count++
-
-    // const startTime = performance.now()
-    //  this.#logger('Start save ' + x + ' ' + batch.length)
-    await this.#cacheDB!.objects.bulkPut(batch)
-    // const endTime = performance.now()
-    // const duration = endTime - startTime
-    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
-  }
-
-  /**
-   * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
-   * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
-   */
-  async #safariFix(): Promise<void> {
-    // No point putting other browsers or older versions of Safari through this mess.
-    if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve()
-
-    let intervalId: ReturnType<typeof setInterval>
-
-    return new Promise<void>((resolve: () => void) => {
-      const tryIdb = (): Promise<IDBDatabaseInfo[]> | undefined =>
-        this.#options.indexedDB?.databases().finally(resolve)
-      intervalId = setInterval(() => {
-        void tryIdb()
-      }, 100)
-      void tryIdb()
-    }).finally(() => clearInterval(intervalId))
-  }
-
-  async disposeAsync(): Promise<void> {
-    this.#cacheDB?.close()
-    this.#cacheDB = undefined
-    await this.#writeQueue?.disposeAsync()
-    this.#writeQueue = undefined
-  }
-}

package/src/operations/objectLoader2.ts

@@ -1,119 +0,0 @@
-import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
-import { Downloader, Database } from './interfaces.js'
-import { CustomLogger, Base, Item } from '../types/types.js'
-import { CacheOptions, ObjectLoader2Options } from './options.js'
-import { DefermentManager } from '../helpers/defermentManager.js'
-import { CacheReader } from '../helpers/cacheReader.js'
-import { CachePump } from '../helpers/cachePump.js'
-import AggregateQueue from '../helpers/aggregateQueue.js'
-import { ObjectLoader2Factory } from './objectLoader2Factory.js'
-
-export class ObjectLoader2 {
-  #rootId: string
-
-  #logger: CustomLogger
-
-  #database: Database
-  #downloader: Downloader
-  #pump: CachePump
-  #cache: CacheReader
-
-  #deferments: DefermentManager
-
-  #gathered: AsyncGeneratorQueue<Item>
-
-  #root?: Item = undefined
-
-  constructor(options: ObjectLoader2Options) {
-    this.#rootId = options.rootId
-    this.#logger = options.logger || console.log
-
-    const cacheOptions: CacheOptions = {
-      logger: this.#logger,
-      maxCacheReadSize: 10_000,
-      maxCacheWriteSize: 10_000,
-      maxWriteQueueSize: 40_000,
-      maxCacheBatchWriteWait: 3_000,
-      maxCacheBatchReadWait: 3_000
-    }
-
-    this.#gathered = new AsyncGeneratorQueue()
-    this.#database = options.database
-    this.#deferments = new DefermentManager({
-      maxSizeInMb: 2_000, // 2 GBs
-      ttlms: 15_000, // 15 seconds
-      logger: this.#logger
-    })
-    this.#cache = new CacheReader(this.#database, this.#deferments, cacheOptions)
-    this.#pump = new CachePump(
-      this.#database,
-      this.#gathered,
-      this.#deferments,
-      cacheOptions
-    )
-    this.#downloader = options.downloader
-  }
-
-  async disposeAsync(): Promise<void> {
-    await Promise.all([
-      this.#downloader.disposeAsync(),
-      this.#cache.disposeAsync(),
-      this.#pump.disposeAsync()
-    ])
-    this.#deferments.dispose()
-  }
-
-  async getRootObject(): Promise<Item | undefined> {
-    if (!this.#root) {
-      this.#root = (await this.#database.getAll([this.#rootId]))[0]
-      if (!this.#root) {
-        this.#root = await this.#downloader.downloadSingle()
-      }
-    }
-    return this.#root
-  }
-
-  async getObject(params: { id: string }): Promise<Base> {
-    return await this.#cache.getObject({ id: params.id })
-  }
-
-  async getTotalObjectCount(): Promise<number> {
-    const rootObj = await this.getRootObject()
-    const totalChildrenCount = Object.keys(rootObj?.base.__closure || {}).length
-    return totalChildrenCount + 1 //count the root
-  }
-
-  async *getObjectIterator(): AsyncGenerator<Base> {
-    const rootItem = await this.getRootObject()
-    if (rootItem === undefined) {
-      this.#logger('No root object found!')
-      return
-    }
-    //only for root
-    this.#pump.add(rootItem)
-    yield rootItem.base
-    if (!rootItem.base.__closure) return
-
-    //sort the closures by their values descending
-    const sortedClosures = Object.entries(rootItem.base.__closure).sort(
-      (a, b) => b[1] - a[1]
-    )
-    const children = sortedClosures.map((x) => x[0])
-    const total = children.length
-    this.#downloader.initializePool({
-      results: new AggregateQueue(this.#gathered, this.#pump),
-      total
-    })
-    for await (const item of this.#pump.gather(children, this.#downloader)) {
-      yield item.base
-    }
-  }
-
-  static createFromObjects(objects: Base[]): ObjectLoader2 {
-    return ObjectLoader2Factory.createFromObjects(objects)
-  }
-
-  static createFromJSON(json: string): ObjectLoader2 {
-    return ObjectLoader2Factory.createFromJSON(json)
-  }
-}