@speckle/objectloader2 2.24.2 → 2.25.0

package/dist/commonjs/index.js

@@ -1,11 +1,10 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MemoryDatabase = void 0;
-const objectLoader2_js_1 = __importDefault(require("./operations/objectLoader2.js"));
-exports.default = objectLoader2_js_1.default;
-var memoryDatabase_js_1 = require("./operations/memoryDatabase.js");
+exports.ObjectLoader2Factory = exports.ObjectLoader2 = exports.MemoryDatabase = void 0;
+var memoryDatabase_js_1 = require("./operations/databases/memoryDatabase.js");
 Object.defineProperty(exports, "MemoryDatabase", { enumerable: true, get: function () { return memoryDatabase_js_1.MemoryDatabase; } });
+var objectLoader2_js_1 = require("./operations/objectLoader2.js");
+Object.defineProperty(exports, "ObjectLoader2", { enumerable: true, get: function () { return objectLoader2_js_1.ObjectLoader2; } });
+var objectLoader2Factory_js_1 = require("./operations/objectLoader2Factory.js");
+Object.defineProperty(exports, "ObjectLoader2Factory", { enumerable: true, get: function () { return objectLoader2Factory_js_1.ObjectLoader2Factory; } });
 //# sourceMappingURL=index.js.map

package/dist/esm/index.js

@@ -1,4 +1,4 @@
-import ObjectLoader2 from './operations/objectLoader2.js';
-export default ObjectLoader2;
-export { MemoryDatabase } from './operations/memoryDatabase.js';
+export { MemoryDatabase } from './operations/databases/memoryDatabase.js';
+export { ObjectLoader2 } from './operations/objectLoader2.js';
+export { ObjectLoader2Factory } from './operations/objectLoader2Factory.js';
 //# sourceMappingURL=index.js.map

package/eslint.config.mjs

@@ -29,7 +29,9 @@ const configs = [
       }
     },
     rules: {
-      '@typescript-eslint/restrict-template-expressions': 'off'
+      '@typescript-eslint/restrict-template-expressions': 'off',
+      '@typescript-eslint/await-thenable': 'error',
+      '@typescript-eslint/explicit-function-return-type': 'error'
     }
   },
   {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@speckle/objectloader2",
-  "version": "2.24.2",
+  "version": "2.25.0",
   "description": "This is an updated objectloader for the Speckle viewer written in typescript",
   "main": "./dist/commonjs/index.js",
   "module": "./dist/esm/index.js",
@@ -33,7 +33,7 @@
   "author": "AEC Systems",
   "license": "Apache-2.0",
   "dependencies": {
-    "@speckle/shared": "^2.24.2",
+    "@speckle/shared": "^2.25.0",
     "dexie": "^4.0.11"
   },
   "devDependencies": {

package/src/helpers/__snapshots__/cachePump.spec.ts.snap

@@ -0,0 +1,31 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`CachePump testing > write two items to queue use pumpItems that are NOT found 1`] = `[]`;
+
+exports[`CachePump testing > write two items to queue use pumpItems that are NOT found 2`] = `
+[
+  "id1",
+  "id2",
+]
+`;
+
+exports[`CachePump testing > write two items to queue use pumpItems that are found 1`] = `
+[
+  {
+    "base": {
+      "id": "id",
+      "speckle_type": "type",
+    },
+    "baseId": "id1",
+  },
+  {
+    "base": {
+      "id": "id",
+      "speckle_type": "type",
+    },
+    "baseId": "id2",
+  },
+]
+`;
+
+exports[`CachePump testing > write two items to queue use pumpItems that are found 2`] = `[]`;

package/src/helpers/__snapshots__/cacheReader.spec.ts.snap

@@ -0,0 +1,8 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`CacheReader testing > deferred getObject 1`] = `
+{
+  "id": "id",
+  "speckle_type": "type",
+}
+`;

package/src/helpers/aggregateQueue.ts

@@ -0,0 +1,20 @@
+import Queue from './queue.js'
+
+export default class AggregateQueue<T> implements Queue<T> {
+  #queue1: Queue<T>
+  #queue2: Queue<T>
+
+  constructor(queue1: Queue<T>, queue2: Queue<T>) {
+    this.#queue1 = queue1
+    this.#queue2 = queue2
+  }
+
+  add(value: T): void {
+    this.#queue1.add(value)
+    this.#queue2.add(value)
+  }
+
+  values(): T[] {
+    throw new Error('Not implemented')
+  }
+}

package/src/helpers/batchedPool.ts

@@ -6,7 +6,7 @@ export default class BatchedPool<T> {
   #baseInterval: number
 
   #processingLoop: Promise<void>
-  #finished = false
+  #disposed = false
 
   constructor(params: {
     concurrencyAndSizes: number[]
@@ -27,22 +27,18 @@ export default class BatchedPool<T> {
     return this.#queue.splice(0, Math.min(batchSize, this.#queue.length))
   }
 
-  async #runWorker(batchSize: number) {
-    while (!this.#finished || this.#queue.length > 0) {
+  async #runWorker(batchSize: number): Promise<void> {
+    while (!this.#disposed || this.#queue.length > 0) {
       if (this.#queue.length > 0) {
         const batch = this.getBatch(batchSize)
-        try {
-          await this.#processFunction(batch)
-        } catch (e) {
-          console.error(e)
-        }
+        await this.#processFunction(batch)
       }
       await this.#delay(this.#baseInterval)
     }
   }
 
   async disposeAsync(): Promise<void> {
-    this.#finished = true
+    this.#disposed = true
     await this.#processingLoop
   }
 

package/src/helpers/batchingQueue.ts

@@ -1,7 +1,7 @@
-import Queue from './queue.js'
+import KeyedQueue from './keyedQueue.js'
 
-export default class BatchingQueue<T> implements Queue<T> {
-  #queue: T[] = []
+export default class BatchingQueue<T> {
+  #queue: KeyedQueue<string, T> = new KeyedQueue<string, T>()
   #batchSize: number
   #processFunction: (batch: T[]) => Promise<void>
 
@@ -10,7 +10,7 @@ export default class BatchingQueue<T> implements Queue<T> {
   #maxInterval: number
 
   #processingLoop: Promise<void>
-  #finished = false
+  #disposed = false
 
   constructor(params: {
     batchSize: number
@@ -26,32 +26,40 @@ export default class BatchingQueue<T> implements Queue<T> {
   }
 
   async disposeAsync(): Promise<void> {
-    this.#finished = true
+    this.#disposed = true
     await this.#processingLoop
   }
 
-  add(item: T): void {
-    this.#queue.push(item)
+  add(key: string, item: T): void {
+    this.#queue.enqueue(key, item)
+  }
+
+  get(id: string): T | undefined {
+    return this.#queue.get(id)
   }
 
   count(): number {
-    return this.#queue.length
+    return this.#queue.size
+  }
+
+  isDisposed(): boolean {
+    return this.#disposed
   }
 
   #getBatch(batchSize: number): T[] {
-    return this.#queue.splice(0, Math.min(batchSize, this.#queue.length))
+    return this.#queue.spliceValues(0, Math.min(batchSize, this.#queue.size))
   }
 
   async #loop(): Promise<void> {
     let interval = this.#baseInterval
-    while (!this.#finished || this.#queue.length > 0) {
+    while (!this.#disposed || this.#queue.size > 0) {
       const startTime = performance.now()
-      if (this.#queue.length > 0) {
+      if (this.#queue.size > 0) {
         const batch = this.#getBatch(this.#batchSize)
         //console.log('running with queue size of ' + this.#queue.length)
         await this.#processFunction(batch)
       }
-      if (this.#queue.length < this.#batchSize / 2) {
+      if (this.#queue.size < this.#batchSize / 2) {
         //refigure interval
         const endTime = performance.now()
         const duration = endTime - startTime
@@ -62,7 +70,7 @@ export default class BatchingQueue<T> implements Queue<T> {
         }
         /*console.log(
           'queue is waiting ' +
-            interval / TIME_MS.second +
+            interval / 1000 +
             ' with queue size of ' +
             this.#queue.length
         )*/

package/src/helpers/cachePump.disposal.spec.ts

@@ -0,0 +1,49 @@
+import { describe, expect, test } from 'vitest'
+import { CachePump } from './cachePump.js'
+import { Database } from '../operations/interfaces.js'
+import AsyncGeneratorQueue from './asyncGeneratorQueue.js'
+import { Item } from '../types/types.js'
+import { DefermentManager } from './defermentManager.js'
+
+const makeDatabase = (): Database =>
+  ({
+    cacheSaveBatch: async (): Promise<void> => {},
+    getAll: async (): Promise<(Item | undefined)[]> => Promise.resolve([])
+  } as unknown as Database)
+const makeGathered = (): AsyncGeneratorQueue<Item> =>
+  ({
+    add: () => {},
+    async *consume() {}
+  } as unknown as AsyncGeneratorQueue<Item>)
+const makeDeferments = (): DefermentManager =>
+  ({
+    undefer: () => {}
+  } as unknown as DefermentManager)
+describe('CachePump disposal', () => {
+  test('disposeAsync is idempotent and always resolves', async () => {
+    const pump = new CachePump(makeDatabase(), makeGathered(), makeDeferments(), {
+      maxCacheWriteSize: 2,
+      maxCacheBatchWriteWait: 100,
+      maxCacheBatchReadWait: 1,
+      maxWriteQueueSize: 2,
+      maxCacheReadSize: 2
+    })
+    await pump.disposeAsync()
+    await expect(pump.disposeAsync()).resolves.toBeUndefined()
+  })
+
+  test('should not throw on add after dispose if writeQueue was never created', async () => {
+    const pump = new CachePump(makeDatabase(), makeGathered(), makeDeferments(), {
+      maxCacheWriteSize: 2,
+      maxCacheBatchWriteWait: 100,
+      maxCacheBatchReadWait: 1,
+      maxWriteQueueSize: 2,
+      maxCacheReadSize: 2
+    })
+    await pump.disposeAsync()
+    // Should not throw, but will not add anything
+    expect(() =>
+      pump.add({ baseId: 'a', base: { id: 'b', speckle_type: 'type' } })
+    ).not.toThrow()
+  })
+})

package/src/helpers/cachePump.spec.ts

@@ -0,0 +1,103 @@
+import { describe, expect, test } from 'vitest'
+import { CachePump } from './cachePump.js'
+import { Base, Item } from '../types/types.js'
+import BufferQueue from './bufferQueue.js'
+import AsyncGeneratorQueue from './asyncGeneratorQueue.js'
+import { DefermentManager } from './defermentManager.js'
+import { MemoryDatabase } from '../operations/databases/memoryDatabase.js'
+import { Database } from '../operations/interfaces.js'
+
+describe('CachePump testing', () => {
+  test('write two items to queue use pumpItems that are NOT found', async () => {
+    const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } }
+
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
+    const cachePump = new CachePump(new MemoryDatabase({}), gathered, deferments, {
+      maxCacheReadSize: 1,
+      maxCacheWriteSize: 1,
+      maxCacheBatchWriteWait: 1,
+      maxCacheBatchReadWait: 1,
+      maxWriteQueueSize: 1
+    })
+
+    const foundItems = new BufferQueue<Item>()
+    const notFoundItems = new BufferQueue<string>()
+
+    await cachePump.pumpItems({
+      ids: [i1.baseId, i2.baseId],
+      foundItems,
+      notFoundItems
+    })
+
+    expect(foundItems.values()).toMatchSnapshot()
+    expect(notFoundItems.values()).toMatchSnapshot()
+    await cachePump.disposeAsync()
+  })
+
+  test('write two items to queue use pumpItems that are found', async () => {
+    const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } }
+
+    const db = new Map<string, Base>()
+    db.set(i1.baseId, i1.base)
+    db.set(i2.baseId, i2.base)
+
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
+    const cachePump = new CachePump(
+      new MemoryDatabase({ items: db }),
+      gathered,
+      deferments,
+      {
+        maxCacheReadSize: 1,
+        maxCacheWriteSize: 1,
+        maxCacheBatchWriteWait: 1,
+        maxCacheBatchReadWait: 1,
+        maxWriteQueueSize: 1
+      }
+    )
+
+    const foundItems = new BufferQueue<Item>()
+    const notFoundItems = new BufferQueue<string>()
+
+    await cachePump.pumpItems({
+      ids: [i1.baseId, i2.baseId],
+      foundItems,
+      notFoundItems
+    })
+
+    expect(foundItems.values()).toMatchSnapshot()
+    expect(notFoundItems.values()).toMatchSnapshot()
+    await cachePump.disposeAsync()
+  })
+
+  test('can dispose while waiting and not wait', async () => {
+    const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } }
+
+    const db: Database = {
+      getAll: async () => Promise.resolve([])
+    } as unknown as Database
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
+    const cachePump = new CachePump(db, gathered, deferments, {
+      maxCacheReadSize: 1,
+      maxCacheWriteSize: 1,
+      maxCacheBatchWriteWait: 1,
+      maxCacheBatchReadWait: 1,
+      maxWriteQueueSize: 1
+    })
+
+    const foundItems = new BufferQueue<Item>()
+    const notFoundItems = new BufferQueue<string>()
+
+    await cachePump.disposeAsync()
+    await cachePump.pumpItems({
+      ids: [i1.baseId, i2.baseId],
+      foundItems,
+      notFoundItems
+    })
+  })
+})

package/src/helpers/cachePump.ts

@@ -0,0 +1,99 @@
+import { TIME } from '@speckle/shared'
+import { Database } from '../operations/interfaces.js'
+import { CacheOptions } from '../operations/options.js'
+import { CustomLogger, Item } from '../types/types.js'
+import BatchingQueue from './batchingQueue.js'
+import Queue from './queue.js'
+import { Downloader } from '../operations/interfaces.js'
+import { DefermentManager } from './defermentManager.js'
+import AsyncGeneratorQueue from './asyncGeneratorQueue.js'
+import { Pump } from './pump.js'
+
+export class CachePump implements Pump {
+  #writeQueue: BatchingQueue<Item> | undefined
+  #database: Database
+  #logger: CustomLogger
+  #deferments: DefermentManager
+
+  #gathered: AsyncGeneratorQueue<Item>
+
+  #options: CacheOptions
+
+  constructor(
+    database: Database,
+    gathered: AsyncGeneratorQueue<Item>,
+    deferments: DefermentManager,
+    options: CacheOptions
+  ) {
+    this.#database = database
+    this.#gathered = gathered
+    this.#deferments = deferments
+    this.#options = options
+    this.#logger = options.logger || ((): void => {})
+  }
+
+  add(item: Item): void {
+    if (!this.#writeQueue) {
+      this.#writeQueue = new BatchingQueue({
+        batchSize: this.#options.maxCacheWriteSize,
+        maxWaitTime: this.#options.maxCacheBatchWriteWait,
+        processFunction: (batch: Item[]): Promise<void> =>
+          this.#database.cacheSaveBatch({ batch })
+      })
+    }
+    this.#writeQueue.add(item.baseId, item)
+  }
+
+  async disposeAsync(): Promise<void> {
+    await this.#writeQueue?.disposeAsync()
+  }
+
+  async pumpItems(params: {
+    ids: string[]
+    foundItems: Queue<Item>
+    notFoundItems: Queue<string>
+  }): Promise<void> {
+    const { ids, foundItems, notFoundItems } = params
+    const maxCacheReadSize = this.#options.maxCacheReadSize
+
+    for (let i = 0; i < ids.length; ) {
+      if (this.#writeQueue?.isDisposed()) break
+      if ((this.#writeQueue?.count() ?? 0) > this.#options.maxWriteQueueSize) {
+        this.#logger(
+          'pausing reads (# in write queue: ' + this.#writeQueue?.count() + ')'
+        )
+        await new Promise((resolve) => setTimeout(resolve, TIME.second)) // Pause for 1 second, protects against out of memory
+        continue
+      }
+      const batch = ids.slice(i, i + maxCacheReadSize)
+      const cachedData = await this.#database.getAll(batch)
+      for (let i = 0; i < cachedData.length; i++) {
+        if (cachedData[i]) {
+          foundItems.add(cachedData[i]!)
+        } else {
+          notFoundItems.add(batch[i])
+        }
+      }
+      i += maxCacheReadSize
+    }
+  }
+
+  async *gather(ids: string[], downloader: Downloader): AsyncGenerator<Item> {
+    const total = ids.length
+    const pumpPromise = this.pumpItems({
+      ids,
+      foundItems: this.#gathered,
+      notFoundItems: downloader
+    })
+    let count = 0
+    for await (const item of this.#gathered.consume()) {
+      this.#deferments.undefer(item)
+      yield item
+      count++
+      if (count >= total) {
+        this.#gathered.dispose()
+      }
+    }
+    await pumpPromise
+  }
+}

package/src/helpers/cacheReader.spec.ts

@@ -0,0 +1,35 @@
+import { describe, expect, test } from 'vitest'
+import { Base, Item } from '../types/types.js'
+import { DefermentManager } from './defermentManager.js'
+import { CacheReader } from './cacheReader.js'
+import { MemoryDatabase } from '../operations/databases/memoryDatabase.js'
+
+describe('CacheReader testing', () => {
+  test('deferred getObject', async () => {
+    const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } }
+
+    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
+    const cacheReader = new CacheReader(
+      new MemoryDatabase({
+        items: new Map<string, Base>([[i1.baseId, i1.base]])
+      }),
+      deferments,
+      {
+        maxCacheReadSize: 1,
+        maxCacheWriteSize: 1,
+        maxCacheBatchWriteWait: 1,
+        maxCacheBatchReadWait: 1,
+        maxWriteQueueSize: 1
+      }
+    )
+
+    const objPromise = cacheReader.getObject({
+      id: i1.baseId
+    })
+    deferments.undefer(i1)
+    const base = await objPromise
+
+    expect(base).toMatchSnapshot()
+    await cacheReader.disposeAsync()
+  })
+})

package/src/helpers/cacheReader.ts

@@ -0,0 +1,64 @@
+import { Database } from '../operations/interfaces.js'
+import { CacheOptions } from '../operations/options.js'
+import { Base, CustomLogger, Item } from '../types/types.js'
+import BatchingQueue from './batchingQueue.js'
+import { DefermentManager } from './defermentManager.js'
+
+export class CacheReader {
+  #database: Database
+  #defermentManager: DefermentManager
+  #logger: CustomLogger
+  #options: CacheOptions
+  #readQueue: BatchingQueue<string> | undefined
+
+  constructor(
+    database: Database,
+    defermentManager: DefermentManager,
+    options: CacheOptions
+  ) {
+    this.#database = database
+    this.#defermentManager = defermentManager
+    this.#options = options
+    this.#logger = options.logger || ((): void => {})
+  }
+
+  async getObject(params: { id: string }): Promise<Base> {
+    if (!this.#defermentManager.isDeferred(params.id)) {
+      this.#getItem(params.id)
+    }
+    return await this.#defermentManager.defer({ id: params.id })
+  }
+
+  #getItem(id: string): void {
+    if (!this.#readQueue) {
+      this.#readQueue = new BatchingQueue({
+        batchSize: this.#options.maxCacheReadSize,
+        maxWaitTime: this.#options.maxCacheBatchReadWait,
+        processFunction: this.#processBatch
+      })
+    }
+    if (!this.#readQueue.get(id)) {
+      this.#readQueue.add(id, id)
+    }
+  }
+
+  async getAll(keys: string[]): Promise<(Item | undefined)[]> {
+    return this.#database.getAll(keys)
+  }
+
+  #processBatch = async (batch: string[]): Promise<void> => {
+    const items = await this.#database.getAll(batch)
+    for (let i = 0; i < items.length; i++) {
+      if (items[i]) {
+        this.#defermentManager.undefer(items[i]!)
+      } else {
+        //this is okay!
+        //this.#logger(`Item ${batch[i]} not found in cache`)
+      }
+    }
+  }
+
+  async disposeAsync(): Promise<void> {
+    await this.#readQueue?.disposeAsync()
+  }
+}

package/src/helpers/defermentManager.disposal.spec.ts

@@ -0,0 +1,28 @@
+import { describe, it, expect } from 'vitest'
+import { DefermentManager } from './defermentManager.js'
+import { DefermentManagerOptions } from '../operations/options.js'
+import { Item } from '../types/types.js'
+
+describe('DefermentManager disposal', () => {
+  const options: DefermentManagerOptions = { ttlms: 10, maxSizeInMb: 1 }
+  const makeItem = (id: string): Item => ({
+    baseId: id,
+    base: { id, speckle_type: 'test' }
+  })
+
+  it('should throw on get/defer/undefer after dispose', async () => {
+    const manager = new DefermentManager(options)
+    manager.dispose()
+    expect(() => manager.get('a')).toThrow('DefermentManager is disposed')
+    expect(() => manager.undefer(makeItem('a'))).toThrow('DefermentManager is disposed')
+    await expect(manager.defer({ id: 'a' })).rejects.toThrow(
+      'DefermentManager is disposed'
+    )
+  })
+
+  it('dispose is idempotent', () => {
+    const manager = new DefermentManager(options)
+    manager.dispose()
+    expect(() => manager.dispose()).not.toThrow()
+  })
+})

package/src/helpers/defermentManager.spec.ts

@@ -1,13 +1,37 @@
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable @typescript-eslint/no-explicit-any  */
 import { describe, expect, test } from 'vitest'
 import { DefermentManager } from './defermentManager.js'
 
 describe('deferments', () => {
   test('defer one', async () => {
-    const deferments = new DefermentManager()
+    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
     const x = deferments.defer({ id: 'id' })
     expect(x).toBeInstanceOf(Promise)
     deferments.undefer({ baseId: 'id', base: { id: 'id', speckle_type: 'type' } })
     const b = await x
     expect(b).toMatchSnapshot()
   })
+
+  test('expireAt timeout', async () => {
+    const now = 1
+    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
+    deferments['now'] = (): number => now
+    const x = deferments.defer({ id: 'id' })
+    expect(x).toBeInstanceOf(Promise)
+    const d = deferments.get('id')
+    expect(d).toBeDefined()
+    expect(d?.getId()).toBe('id')
+    expect((d as any).expiresAt).toBe(2)
+    expect((d as any).ttl).toBe(1)
+    expect((d as any).item).toBeUndefined()
+    expect(d?.isExpired(1)).toBe(false)
+    deferments.undefer({ baseId: 'id', base: { id: 'id', speckle_type: 'type' } })
+    await x
+    expect((d as any).expiresAt).toBe(2)
+    expect((d as any).ttl).toBe(1)
+    expect((d as any).item).toBeDefined()
+    expect(d?.isExpired(1)).toBe(false)
+    expect(d?.isExpired(3)).toBe(true)
+  })
 })