@speckle/objectloader2 2.25.4 → 2.25.8

package/src/helpers/cacheReader.ts

@@ -1,64 +0,0 @@
-import { Database } from '../operations/interfaces.js'
-import { CacheOptions } from '../operations/options.js'
-import { Base, CustomLogger, Item } from '../types/types.js'
-import BatchingQueue from './batchingQueue.js'
-import { DefermentManager } from './defermentManager.js'
-
-export class CacheReader {
-  #database: Database
-  #defermentManager: DefermentManager
-  #logger: CustomLogger
-  #options: CacheOptions
-  #readQueue: BatchingQueue<string> | undefined
-
-  constructor(
-    database: Database,
-    defermentManager: DefermentManager,
-    options: CacheOptions
-  ) {
-    this.#database = database
-    this.#defermentManager = defermentManager
-    this.#options = options
-    this.#logger = options.logger || ((): void => {})
-  }
-
-  async getObject(params: { id: string }): Promise<Base> {
-    if (!this.#defermentManager.isDeferred(params.id)) {
-      this.#getItem(params.id)
-    }
-    return await this.#defermentManager.defer({ id: params.id })
-  }
-
-  #getItem(id: string): void {
-    if (!this.#readQueue) {
-      this.#readQueue = new BatchingQueue({
-        batchSize: this.#options.maxCacheReadSize,
-        maxWaitTime: this.#options.maxCacheBatchReadWait,
-        processFunction: this.#processBatch
-      })
-    }
-    if (!this.#readQueue.get(id)) {
-      this.#readQueue.add(id, id)
-    }
-  }
-
-  async getAll(keys: string[]): Promise<(Item | undefined)[]> {
-    return this.#database.getAll(keys)
-  }
-
-  #processBatch = async (batch: string[]): Promise<void> => {
-    const items = await this.#database.getAll(batch)
-    for (let i = 0; i < items.length; i++) {
-      if (items[i]) {
-        this.#defermentManager.undefer(items[i]!)
-      } else {
-        //this is okay!
-        //this.#logger(`Item ${batch[i]} not found in cache`)
-      }
-    }
-  }
-
-  async disposeAsync(): Promise<void> {
-    await this.#readQueue?.disposeAsync()
-  }
-}

package/src/helpers/defermentManager.disposal.spec.ts

@@ -1,28 +0,0 @@
-import { describe, it, expect } from 'vitest'
-import { DefermentManager } from './defermentManager.js'
-import { DefermentManagerOptions } from '../operations/options.js'
-import { Item } from '../types/types.js'
-
-describe('DefermentManager disposal', () => {
-  const options: DefermentManagerOptions = { ttlms: 10, maxSizeInMb: 1 }
-  const makeItem = (id: string): Item => ({
-    baseId: id,
-    base: { id, speckle_type: 'test' }
-  })
-
-  it('should throw on get/defer/undefer after dispose', async () => {
-    const manager = new DefermentManager(options)
-    manager.dispose()
-    expect(() => manager.get('a')).toThrow('DefermentManager is disposed')
-    expect(() => manager.undefer(makeItem('a'))).toThrow('DefermentManager is disposed')
-    await expect(manager.defer({ id: 'a' })).rejects.toThrow(
-      'DefermentManager is disposed'
-    )
-  })
-
-  it('dispose is idempotent', () => {
-    const manager = new DefermentManager(options)
-    manager.dispose()
-    expect(() => manager.dispose()).not.toThrow()
-  })
-})

package/src/helpers/defermentManager.spec.ts

@@ -1,37 +0,0 @@
-/* eslint-disable @typescript-eslint/no-unsafe-member-access */
-/* eslint-disable @typescript-eslint/no-explicit-any  */
-import { describe, expect, test } from 'vitest'
-import { DefermentManager } from './defermentManager.js'
-
-describe('deferments', () => {
-  test('defer one', async () => {
-    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
-    const x = deferments.defer({ id: 'id' })
-    expect(x).toBeInstanceOf(Promise)
-    deferments.undefer({ baseId: 'id', base: { id: 'id', speckle_type: 'type' } })
-    const b = await x
-    expect(b).toMatchSnapshot()
-  })
-
-  test('expireAt timeout', async () => {
-    const now = 1
-    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
-    deferments['now'] = (): number => now
-    const x = deferments.defer({ id: 'id' })
-    expect(x).toBeInstanceOf(Promise)
-    const d = deferments.get('id')
-    expect(d).toBeDefined()
-    expect(d?.getId()).toBe('id')
-    expect((d as any).expiresAt).toBe(2)
-    expect((d as any).ttl).toBe(1)
-    expect((d as any).item).toBeUndefined()
-    expect(d?.isExpired(1)).toBe(false)
-    deferments.undefer({ baseId: 'id', base: { id: 'id', speckle_type: 'type' } })
-    await x
-    expect((d as any).expiresAt).toBe(2)
-    expect((d as any).ttl).toBe(1)
-    expect((d as any).item).toBeDefined()
-    expect(d?.isExpired(1)).toBe(false)
-    expect(d?.isExpired(3)).toBe(true)
-  })
-})

package/src/helpers/defermentManager.ts

@@ -1,141 +0,0 @@
-import { DeferredBase } from './deferredBase.js'
-import { Base, CustomLogger, Item } from '../types/types.js'
-import { DefermentManagerOptions } from '../operations/options.js'
-
-export class DefermentManager {
-  private deferments: Map<string, DeferredBase> = new Map()
-  private timer?: ReturnType<typeof setTimeout>
-  private logger: CustomLogger
-  private currentSize = 0
-  private disposed = false
-
-  constructor(private options: DefermentManagerOptions) {
-    this.resetGlobalTimer()
-    this.logger = options.logger || ((): void => {})
-  }
-
-  private now(): number {
-    return Date.now()
-  }
-
-  isDeferred(id: string): boolean {
-    return this.deferments.has(id)
-  }
-
-  get(id: string): DeferredBase | undefined {
-    if (this.disposed) throw new Error('DefermentManager is disposed')
-    return this.deferments.get(id)
-  }
-
-  async defer(params: { id: string }): Promise<Base> {
-    if (this.disposed) throw new Error('DefermentManager is disposed')
-    const now = this.now()
-    const deferredBase = this.deferments.get(params.id)
-    if (deferredBase) {
-      deferredBase.setAccess(now)
-      return deferredBase.getPromise()
-    }
-    const notYetFound = new DeferredBase(
-      this.options.ttlms,
-      params.id,
-      now + this.options.ttlms
-    )
-    this.deferments.set(params.id, notYetFound)
-    return notYetFound.getPromise()
-  }
-
-  undefer(item: Item): void {
-    if (this.disposed) throw new Error('DefermentManager is disposed')
-    const now = this.now()
-    this.currentSize += item.size || 0
-    //order matters here with found before undefer
-    const deferredBase = this.deferments.get(item.baseId)
-    if (deferredBase) {
-      deferredBase.found(item)
-      deferredBase.setAccess(now)
-    } else {
-      const existing = new DeferredBase(this.options.ttlms, item.baseId, now)
-      existing.found(item)
-      this.deferments.set(item.baseId, existing)
-    }
-  }
-
-  private resetGlobalTimer(): void {
-    const run = (): void => {
-      this.cleanDeferments()
-      this.timer = setTimeout(run, this.options.ttlms)
-    }
-    this.timer = setTimeout(run, this.options.ttlms)
-  }
-
-  dispose(): void {
-    if (this.disposed) return
-    this.disposed = true
-    if (this.timer) {
-      clearTimeout(this.timer)
-      this.timer = undefined
-    }
-    this.clearDeferments()
-  }
-
-  private clearDeferments(): void {
-    let waiting = 0
-    for (const deferredBase of this.deferments.values()) {
-      deferredBase.done(0)
-      if (deferredBase.getItem() === undefined) {
-        waiting++
-      }
-    }
-    this.currentSize = 0
-    this.deferments.clear()
-    this.logger('cleared deferments, left', waiting)
-  }
-
-  private cleanDeferments(): void {
-    const maxSizeBytes = this.options.maxSizeInMb * 1024 * 1024
-    if (this.currentSize < maxSizeBytes) {
-      this.logger(
-        'deferments size is ok, no need to clean',
-        this.currentSize,
-        maxSizeBytes
-      )
-      return
-    }
-    const now = this.now()
-    let cleaned = 0
-    const start = performance.now()
-    for (const deferredBase of Array.from(this.deferments.values())
-      .filter((x) => x.isExpired(now))
-      .sort((a, b) => this.compareMaybeBasesBySize(a.getItem(), b.getItem()))) {
-      if (deferredBase.done(now)) {
-        this.currentSize -= deferredBase.getItem()?.size || 0
-        this.deferments.delete(deferredBase.getId())
-        cleaned++
-        if (this.currentSize < maxSizeBytes) {
-          break
-        }
-      }
-    }
-    this.logger(
-      'cleaned deferments, cleaned, left',
-      cleaned,
-      this.deferments.size,
-      performance.now() - start
-    )
-    return
-  }
-
-  compareMaybeBasesBySize(a: Item | undefined, b: Item | undefined): number {
-    if (a === undefined && b === undefined) return 0
-    if (a === undefined) return -1
-    if (b === undefined) return 1
-    return this.compareMaybe(a.size, b.size)
-  }
-
-  compareMaybe(a: number | undefined, b: number | undefined): number {
-    if (a === undefined && b === undefined) return 0
-    if (a === undefined) return -1
-    if (b === undefined) return 1
-    return a - b
-  }
-}

package/src/helpers/deferredBase.ts

@@ -1,55 +0,0 @@
-import { Base, Item } from '../types/types.js'
-
-export class DeferredBase {
-  private promise: Promise<Base>
-  private resolve!: (value: Base) => void
-  private reject!: (reason?: Error) => void
-  private item?: Item
-
-  private readonly id: string
-  private expiresAt: number // Timestamp in ms
-  private ttl: number // ttl in ms
-
-  constructor(ttl: number, id: string, expiresAt: number) {
-    this.expiresAt = expiresAt
-    this.ttl = ttl
-    this.id = id
-    this.promise = new Promise<Base>((resolve, reject) => {
-      this.resolve = resolve
-      this.reject = reject
-    })
-  }
-
-  getId(): string {
-    return this.id
-  }
-
-  getItem(): Item | undefined {
-    return this.item
-  }
-
-  getPromise(): Promise<Base> {
-    return this.promise
-  }
-
-  isExpired(now: number): boolean {
-    return this.item !== undefined && now > this.expiresAt
-  }
-  setAccess(now: number): void {
-    this.expiresAt = now + this.ttl
-  }
-
-  found(value: Item): void {
-    this.item = value
-    this.resolve(value.base)
-  }
-  done(now: number): boolean {
-    if (this.item) {
-      this.resolve(this.item.base)
-    }
-    if (this.isExpired(now)) {
-      return true
-    }
-    return false
-  }
-}

package/src/helpers/memoryPump.ts

@@ -1,40 +0,0 @@
-import { Item } from '../types/types.js'
-import { Pump } from './pump.js'
-import Queue from './queue.js'
-
-export class MemoryPump implements Pump {
-  #items: Map<string, Item> = new Map()
-
-  add(item: Item): void {
-    this.#items.set(item.baseId, item)
-  }
-
-  async pumpItems(params: {
-    ids: string[]
-    foundItems: Queue<Item>
-    notFoundItems: Queue<string>
-  }): Promise<void> {
-    const { ids, foundItems, notFoundItems } = params
-    for (const id of ids) {
-      const item = this.#items.get(id)
-      if (item) {
-        foundItems.add(item)
-      } else {
-        notFoundItems.add(id)
-      }
-    }
-    return Promise.resolve()
-  }
-
-  async *gather(ids: string[]): AsyncGenerator<Item> {
-    for (const id of ids) {
-      const item = this.#items.get(id)
-      if (item) {
-        yield item
-      }
-    }
-    return Promise.resolve()
-  }
-
-  async disposeAsync(): Promise<void> {}
-}

package/src/helpers/pump.ts

@@ -1,8 +0,0 @@
-import { Downloader } from '../operations/interfaces.js'
-import { Item } from '../types/types.js'
-import Queue from './queue.js'
-
-export interface Pump extends Queue<Item> {
-  gather(ids: string[], downloader: Downloader): AsyncGenerator<Item>
-  disposeAsync(): Promise<void>
-}

package/src/operations/databases/__snapshots__/indexedDatabase.spec.ts.snap

@@ -1,31 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`database cache > write single item to queue use getItem 1`] = `
-{
-  "base": {
-    "id": "id",
-    "speckle_type": "type",
-  },
-  "baseId": "id",
-}
-`;
-
-exports[`database cache > write two items to queue use getItem 1`] = `
-{
-  "base": {
-    "id": "id",
-    "speckle_type": "type",
-  },
-  "baseId": "id1",
-}
-`;
-
-exports[`database cache > write two items to queue use getItem 2`] = `
-{
-  "base": {
-    "id": "id",
-    "speckle_type": "type",
-  },
-  "baseId": "id2",
-}
-`;

package/src/operations/databases/indexedDatabase.spec.ts

@@ -1,37 +0,0 @@
-import { describe, expect, test } from 'vitest'
-import IndexedDatabase from './indexedDatabase.js'
-import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
-import { Item } from '../../types/types.js'
-
-describe('database cache', () => {
-  test('write single item to queue use getItem', async () => {
-    const i: Item = { baseId: 'id', base: { id: 'id', speckle_type: 'type' } }
-    const database = new IndexedDatabase({
-      indexedDB: new IDBFactory(),
-      keyRange: IDBKeyRange
-    })
-    await database.add(i)
-    await database.disposeAsync()
-
-    const x = await database.getItem({ id: 'id' })
-    expect(x).toMatchSnapshot()
-  })
-
-  test('write two items to queue use getItem', async () => {
-    const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } }
-    const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } }
-    const database = new IndexedDatabase({
-      indexedDB: new IDBFactory(),
-      keyRange: IDBKeyRange
-    })
-    await database.add(i1)
-    await database.add(i2)
-    await database.disposeAsync()
-
-    const x1 = await database.getItem({ id: i1.baseId })
-    expect(x1).toMatchSnapshot()
-
-    const x2 = await database.getItem({ id: i2.baseId })
-    expect(x2).toMatchSnapshot()
-  })
-})

package/src/operations/databases/indexedDatabase.ts

@@ -1,150 +0,0 @@
-/* eslint-disable @typescript-eslint/no-unsafe-function-type */
-import BatchingQueue from '../../helpers/batchingQueue.js'
-import { CustomLogger, Item } from '../../types/types.js'
-import { isSafari } from '@speckle/shared'
-import { Dexie, DexieOptions, Table } from 'dexie'
-import { Database } from '../interfaces.js'
-
-class ObjectStore extends Dexie {
-  static #databaseName: string = 'speckle-cache'
-  objects!: Table<Item, string> // Table type: <entity, primaryKey>
-
-  constructor(options: DexieOptions) {
-    super(ObjectStore.#databaseName, options)
-
-    this.version(1).stores({
-      objects: 'baseId, item' // baseId is primary key
-    })
-  }
-}
-
-export interface IndexedDatabaseOptions {
-  logger?: CustomLogger
-  indexedDB?: IDBFactory
-  keyRange?: {
-    bound: Function
-    lowerBound: Function
-    upperBound: Function
-  }
-}
-
-export default class IndexedDatabase implements Database {
-  #options: IndexedDatabaseOptions
-  #logger: CustomLogger
-
-  #cacheDB?: ObjectStore
-
-  #writeQueue: BatchingQueue<Item> | undefined
-
-  // #count: number = 0
-
-  constructor(options: IndexedDatabaseOptions) {
-    this.#options = options
-    this.#logger = options.logger || ((): void => {})
-  }
-
-  async getAll(keys: string[]): Promise<(Item | undefined)[]> {
-    await this.#setupCacheDb()
-    let items: (Item | undefined)[] = []
-    //   this.#count++
-    //  const startTime = performance.now()
-    // this.#logger('Start read ' + x + ' ' + batch.length)
-
-    //faster than BulkGet with dexie
-    await this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => {
-      const gets = keys.map((key) => this.#cacheDB!.objects.get(key))
-      const cachedData = await Promise.all(gets)
-      items = cachedData
-    })
-    // const endTime = performance.now()
-    // const duration = endTime - startTime
-    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
-
-    return items
-  }
-
-  async #openDatabase(): Promise<ObjectStore> {
-    const db = new ObjectStore({
-      indexedDB: this.#options.indexedDB ?? globalThis.indexedDB,
-      IDBKeyRange: this.#options.keyRange ?? IDBKeyRange,
-      chromeTransactionDurability: 'relaxed'
-    })
-    await db.open()
-    return db
-  }
-
-  async #setupCacheDb(): Promise<void> {
-    if (this.#cacheDB !== undefined) {
-      return
-    }
-
-    // Initialize
-    await this.#safariFix()
-    this.#cacheDB = await this.#openDatabase()
-  }
-
-  //this is for testing only - in the real world we will not use this
-  async add(item: Item): Promise<void> {
-    await this.#setupCacheDb()
-    await this.#cacheDB!.transaction('rw', this.#cacheDB!.objects, async () => {
-      return await this.#cacheDB?.objects.add(item)
-    })
-  }
-
-  async getItem(params: { id: string }): Promise<Item | undefined> {
-    const { id } = params
-    await this.#setupCacheDb()
-    //might not be in the real DB yet, so check the write queue first
-    if (this.#writeQueue) {
-      const item = this.#writeQueue.get(id)
-      if (item) {
-        return item
-      }
-    }
-
-    return this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => {
-      return await this.#cacheDB?.objects.get(id)
-    })
-  }
-
-  async cacheSaveBatch(params: { batch: Item[] }): Promise<void> {
-    await this.#setupCacheDb()
-    const { batch } = params
-    //const x = this.#count
-    //this.#count++
-
-    // const startTime = performance.now()
-    //  this.#logger('Start save ' + x + ' ' + batch.length)
-    await this.#cacheDB!.objects.bulkPut(batch)
-    // const endTime = performance.now()
-    // const duration = endTime - startTime
-    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
-  }
-
-  /**
-   * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
-   * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
-   */
-  async #safariFix(): Promise<void> {
-    // No point putting other browsers or older versions of Safari through this mess.
-    if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve()
-
-    let intervalId: ReturnType<typeof setInterval>
-
-    return new Promise<void>((resolve: () => void) => {
-      const tryIdb = (): Promise<IDBDatabaseInfo[]> | undefined =>
-        this.#options.indexedDB?.databases().finally(resolve)
-      intervalId = setInterval(() => {
-        void tryIdb()
-      }, 100)
-      void tryIdb()
-    }).finally(() => clearInterval(intervalId))
-  }
-
-  async disposeAsync(): Promise<void> {
-    this.#cacheDB?.close()
-    this.#cacheDB = undefined
-    await this.#writeQueue?.disposeAsync()
-    this.#writeQueue = undefined
-  }
-}