@speckle/objectloader2 2.25.4 → 2.25.8

package/src/core/stages/cacheWriter.spec.ts

@@ -0,0 +1,213 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { Item } from '../../types/types.js'
+import { CacheWriter } from './cacheWriter.js'
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import { Database } from '../interfaces.js'
+import { MemoryCache, MemoryCacheOptions } from '../../deferment/MemoryCache.js'
+import { CacheOptions } from '../options.js'
+import { CustomLogger } from '../../types/functions.js'
+
+// Mock implementations
+class MockDatabase implements Database {
+  savedItems: Item[] = []
+
+  getAll(/* keys */): Promise<(Item | undefined)[]> {
+    return Promise.resolve([])
+  }
+
+  saveBatch({ batch }: { batch: Item[] }): Promise<void> {
+    this.savedItems.push(...batch)
+    return Promise.resolve()
+  }
+
+  disposeAsync(): Promise<void> {
+    return Promise.resolve()
+  }
+}
+
+describe('CacheWriter', () => {
+  let database: MockDatabase
+  let defermentManager: DefermentManager
+  let cacheWriter: CacheWriter
+  let logger: CustomLogger
+  let requestItemMock: (id: string) => void
+  let options: CacheOptions
+  let memoryCache: MemoryCache
+
+  beforeEach(() => {
+    database = new MockDatabase()
+    logger = vi.fn() as CustomLogger
+    const memoryCacheOptions: MemoryCacheOptions = {
+      maxSizeInMb: 100,
+      ttlms: 60000
+    }
+    memoryCache = new MemoryCache(memoryCacheOptions, logger)
+    defermentManager = new DefermentManager(memoryCache, logger)
+    requestItemMock = vi.fn()
+
+    options = {
+      maxCacheWriteSize: 10,
+      maxCacheBatchWriteWait: 50,
+      maxCacheReadSize: 10,
+      maxCacheBatchReadWait: 50,
+      maxWriteQueueSize: 100
+    }
+
+    cacheWriter = new CacheWriter(
+      database,
+      logger,
+      defermentManager,
+      options,
+      requestItemMock
+    )
+  })
+
+  afterEach(async () => {
+    await cacheWriter.disposeAsync()
+  })
+
+  it('should write items to the database', async () => {
+    const item: Item = {
+      baseId: 'test-id',
+      base: { id: 'test-id', speckle_type: 'test-type' }
+    }
+
+    cacheWriter.add(item)
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, options.maxCacheBatchWriteWait + 10)
+    )
+
+    expect(database.savedItems).toHaveLength(1)
+    expect(database.savedItems[0].baseId).toBe('test-id')
+    expect(database.savedItems[0].base).toEqual({
+      id: 'test-id',
+      speckle_type: 'test-type'
+    })
+  })
+
+  it('should write multiple items in a batch', async () => {
+    const items = [
+      {
+        baseId: 'item1',
+        base: { id: 'item1', speckle_type: 'test-type' }
+      },
+      {
+        baseId: 'item2',
+        base: { id: 'item2', speckle_type: 'test-type' }
+      },
+      {
+        baseId: 'item3',
+        base: { id: 'item3', speckle_type: 'test-type' }
+      }
+    ]
+
+    items.forEach((item) => cacheWriter.add(item))
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, options.maxCacheBatchWriteWait + 10)
+    )
+
+    expect(database.savedItems).toHaveLength(3)
+    expect(database.savedItems.map((item) => item.baseId)).toEqual([
+      'item1',
+      'item2',
+      'item3'
+    ])
+  })
+
+  it('should call undefer on the defermentManager', () => {
+    const spy = vi.spyOn(defermentManager, 'undefer')
+    const item: Item = {
+      baseId: 'test-id',
+      base: { id: 'test-id', speckle_type: 'test-type' }
+    }
+
+    cacheWriter.add(item)
+
+    expect(spy).toHaveBeenCalledWith(item, requestItemMock)
+  })
+
+  it('should handle items with id but no base', async () => {
+    const item: Item = {
+      baseId: 'test-id-no-base'
+      // No base property
+    }
+
+    // Adding an item with no base should not throw an error
+    expect(() => cacheWriter.add(item)).not.toThrow()
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, options.maxCacheBatchWriteWait + 10)
+    )
+
+    expect(database.savedItems).toHaveLength(1)
+    expect(database.savedItems[0].baseId).toBe('test-id-no-base')
+    expect(database.savedItems[0].base).toBeUndefined()
+  })
+
+  it('should process items in batches according to maxCacheWriteSize', async () => {
+    const spy = vi.spyOn(database, 'saveBatch')
+    const smallBatchOptions: CacheOptions = {
+      ...options,
+      maxCacheWriteSize: 2, // Set small batch size
+      maxCacheBatchWriteWait: 100
+    }
+
+    const smallBatchCacheWriter = new CacheWriter(
+      database,
+      logger,
+      defermentManager,
+      smallBatchOptions,
+      requestItemMock
+    )
+
+    // Add 5 items
+    const items = Array.from({ length: 5 }, (_, i) => ({
+      baseId: `batch-item-${i}`,
+      base: { id: `batch-item-${i}`, speckle_type: 'test-type' }
+    }))
+
+    items.forEach((item) => smallBatchCacheWriter.add(item))
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, smallBatchOptions.maxCacheBatchWriteWait * 2 + 50)
+    )
+
+    // With batch size 2, we expect 3 calls to saveBatch: 2 items + 2 items + 1 item
+    expect(spy).toHaveBeenCalledTimes(3)
+
+    await smallBatchCacheWriter.disposeAsync()
+  })
+
+  it('should be disposed correctly', async () => {
+    expect(cacheWriter.isDisposed).toBe(false)
+    await cacheWriter.disposeAsync()
+    expect(cacheWriter.isDisposed).toBe(true)
+  })
+
+  it('should write directly with writeAll method', async () => {
+    const items = [
+      {
+        baseId: 'direct1',
+        base: { id: 'direct1', speckle_type: 'test-type' }
+      },
+      {
+        baseId: 'direct2',
+        base: { id: 'direct2', speckle_type: 'test-type' }
+      }
+    ]
+
+    await cacheWriter.writeAll(items)
+
+    expect(database.savedItems).toHaveLength(2)
+    expect(database.savedItems.map((item) => item.baseId)).toEqual([
+      'direct1',
+      'direct2'
+    ])
+  })
+})

package/src/core/stages/cacheWriter.ts

@@ -0,0 +1,64 @@
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
+import Queue from '../../queues/queue.js'
+import { CustomLogger } from '../../types/functions.js'
+import { Item } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import { CacheOptions } from '../options.js'
+
+export class CacheWriter implements Queue<Item> {
+  #writeQueue: BatchingQueue<Item> | undefined
+  #database: Database
+  #defermentManager: DefermentManager
+  #requestItem: (id: string) => void
+  #logger: CustomLogger
+  #options: CacheOptions
+  #disposed = false
+
+  constructor(
+    database: Database,
+    logger: CustomLogger,
+    defermentManager: DefermentManager,
+    options: CacheOptions,
+    requestItem: (id: string) => void
+  ) {
+    this.#database = database
+    this.#options = options
+    this.#logger = logger
+    this.#defermentManager = defermentManager
+    this.#requestItem = requestItem
+  }
+
+  add(item: Item): void {
+    if (!this.#writeQueue) {
+      this.#writeQueue = new BatchingQueue({
+        batchSize: this.#options.maxCacheWriteSize,
+        maxWaitTime: this.#options.maxCacheBatchWriteWait,
+        processFunction: async (batch: Item[]): Promise<void> => {
+          await this.writeAll(batch)
+        }
+      })
+    }
+    this.#writeQueue.add(item.baseId, item)
+    this.#defermentManager.undefer(item, this.#requestItem)
+  }
+
+  async writeAll(items: Item[]): Promise<void> {
+    const start = performance.now()
+    await this.#database.saveBatch({ batch: items })
+    this.#logger(
+      `writeBatch: wrote ${items.length}, time ${
+        performance.now() - start
+      } ms left ${this.#writeQueue?.count()}`
+    )
+  }
+
+  async disposeAsync(): Promise<void> {
+    this.#disposed = true
+    await this.#writeQueue?.disposeAsync()
+  }
+
+  get isDisposed(): boolean {
+    return this.#disposed
+  }
+}

package/src/core/stages/indexedDatabase.spec.ts

@@ -0,0 +1,37 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
+
+import { Base, Item } from '../../types/types.js'
+import IndexedDatabase, { IndexedDatabaseOptions } from './indexedDatabase.js'
+
+// Mock Item
+const defaultItem = (id: string): Item => ({
+  baseId: id,
+  base: { foo: 'bar' } as unknown as Base
+})
+
+describe('IndexedDatabase', () => {
+  let db: IndexedDatabase
+  let options: IndexedDatabaseOptions
+
+  beforeEach(() => {
+    options = { indexedDB: new IDBFactory(), keyRange: IDBKeyRange }
+    db = new IndexedDatabase(options)
+  })
+
+  afterEach(async () => {
+    await db.disposeAsync()
+  })
+
+  it('should add and get multiple items', async () => {
+    const items = [defaultItem('id1'), defaultItem('id2')]
+    await db.saveBatch({ batch: items })
+    const result = await db.getAll(['id1', 'id2'])
+    expect(result).toMatchSnapshot()
+    expect(result).toEqual(items)
+  })
+
+  it('should dispose without error', async () => {
+    await expect(db.disposeAsync()).resolves.not.toThrow()
+  })
+})

package/src/core/stages/indexedDatabase.ts

@@ -0,0 +1,49 @@
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+import { Item } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
+import { ItemStore } from './ItemStore.js'
+
+export interface IndexedDatabaseOptions {
+  indexedDB?: IDBFactory
+  keyRange?: {
+    bound: Function
+    lowerBound: Function
+    upperBound: Function
+  }
+}
+
+export default class IndexedDatabase implements Database {
+  #options: IndexedDatabaseOptions
+  #cacheDB: ItemStore
+  #writeQueue: BatchingQueue<Item> | undefined
+
+  constructor(options: IndexedDatabaseOptions) {
+    this.#options = options
+    this.#cacheDB = new ItemStore(
+      {
+        indexedDB: this.#options.indexedDB,
+        keyRange: this.#options.keyRange
+      },
+      'speckle-cache',
+      'cache'
+    )
+  }
+
+  async getAll(keys: string[]): Promise<(Item | undefined)[]> {
+    await this.#cacheDB.init()
+    return await this.#cacheDB.bulkGet(keys)
+  }
+
+  async saveBatch(params: { batch: Item[] }): Promise<void> {
+    await this.#cacheDB.init()
+    const { batch } = params
+    await this.#cacheDB.bulkInsert(batch)
+  }
+
+  async disposeAsync(): Promise<void> {
+    await this.#writeQueue?.disposeAsync()
+    this.#writeQueue = undefined
+    this.#cacheDB.close()
+  }
+}

package/src/core/stages/memory/__snapshots__/memoryDownloader.spec.ts.snap

@@ -0,0 +1,12 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`MemoryDownloader > should add found item to results queue 1`] = `
+Map {
+  "id1" => {
+    "foo": "foo",
+  },
+  "id2" => {
+    "foo": "bar",
+  },
+}
+`;

package/src/core/stages/memory/memoryDatabase.spec.ts

@@ -0,0 +1,48 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { Item, Base } from '../../../types/types.js'
+import { MemoryDatabase } from './memoryDatabase.js'
+
+const makeItem = (id: string, foo = 'bar'): Item => ({
+  baseId: id,
+  base: { foo } as unknown as Base
+})
+
+describe('MemoryDatabase', () => {
+  let db: MemoryDatabase
+
+  beforeEach(() => {
+    db = new MemoryDatabase()
+  })
+
+  it('should return undefined for missing keys', async () => {
+    const result = await db.getAll(['missing'])
+    expect(result).toEqual([undefined])
+  })
+
+  it('should add and retrieve a single item', async () => {
+    const item = makeItem('id1')
+    await db.saveBatch({ batch: [item] })
+    const result = await db.getAll(['id1'])
+    expect(result).toEqual([item])
+  })
+
+  it('should add and retrieve multiple items', async () => {
+    const items = [makeItem('id1'), makeItem('id2', 'baz')]
+    await db.saveBatch({ batch: items })
+    const result = await db.getAll(['id1', 'id2'])
+    expect(result).toEqual(items)
+  })
+
+  it('should overwrite items with the same key', async () => {
+    const item1 = makeItem('id1', 'foo')
+    const item2 = makeItem('id1', 'bar')
+    await db.saveBatch({ batch: [item1] })
+    await db.saveBatch({ batch: [item2] })
+    const result = await db.getAll(['id1'])
+    expect(result).toEqual([item2])
+  })
+
+  it('disposeAsync should resolve', async () => {
+    await expect(db.disposeAsync()).resolves.not.toThrow()
+  })
+})

package/src/{operations/databases → core/stages/memory}/memoryDatabase.ts

@@ -1,6 +1,6 @@
-import { Base, Item } from '../../types/types.js'
-import { Database } from '../interfaces.js'
-import { MemoryDatabaseOptions } from '../options.js'
+import { Base, Item } from '../../../types/types.js'
+import { Database } from '../../interfaces.js'
+import { MemoryDatabaseOptions } from '../../options.js'
 
 export class MemoryDatabase implements Database {
   private items: Map<string, Base>
@@ -22,21 +22,16 @@ export class MemoryDatabase implements Database {
     return Promise.resolve(found)
   }
 
-  cacheSaveBatch({ batch }: { batch: Item[] }): Promise<void> {
+  saveBatch({ batch }: { batch: Item[] }): Promise<void> {
     for (const item of batch) {
+      if (!item.baseId || !item.base) {
+        throw new Error('Item must have a baseId and base')
+      }
       this.items.set(item.baseId, item.base)
     }
     return Promise.resolve()
   }
 
-  getItem(params: { id: string }): Promise<Item | undefined> {
-    const item = this.items.get(params.id)
-    if (item) {
-      return Promise.resolve({ baseId: params.id, base: item })
-    }
-    return Promise.resolve(undefined)
-  }
-
   disposeAsync(): Promise<void> {
     return Promise.resolve()
   }

package/src/core/stages/memory/memoryDownloader.spec.ts

@@ -0,0 +1,49 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { MemoryDownloader } from './memoryDownloader.js'
+import BufferQueue from '../../../queues/bufferQueue.js'
+import Queue from '../../../queues/queue.js'
+import { Base, Item } from '../../../types/types.js'
+
+const makeBase = (foo: string): Base => ({ foo } as unknown as Base)
+
+describe('MemoryDownloader', () => {
+  let items: Map<string, Base>
+  let downloader: MemoryDownloader
+  let results: Queue<Item>
+
+  beforeEach(() => {
+    items = new Map([
+      ['id1', makeBase('foo')],
+      ['id2', makeBase('bar')]
+    ])
+    downloader = new MemoryDownloader('id1', items)
+    results = new BufferQueue<Item>()
+  })
+
+  it('should download the root item', async () => {
+    const item = await downloader.downloadSingle()
+    expect(item).toEqual({ baseId: 'id1', base: { foo: 'foo' } })
+  })
+
+  it('should throw if root item is missing', async () => {
+    const missingDownloader = new MemoryDownloader('missing', items)
+    await expect(missingDownloader.downloadSingle()).rejects.toThrow(
+      'Method not implemented.'
+    )
+  })
+
+  it('should add found item to results queue', () => {
+    downloader.initializePool({ results, total: 2 })
+    downloader.add('id2')
+    expect(items).toMatchSnapshot()
+  })
+
+  it('should throw if added item is missing', () => {
+    downloader.initializePool({ results, total: 2 })
+    expect(() => downloader.add('missing')).toThrow()
+  })
+
+  it('disposeAsync should resolve', async () => {
+    await expect(downloader.disposeAsync()).resolves.not.toThrow()
+  })
+})

package/src/{operations/downloaders → core/stages/memory}/memoryDownloader.ts

@@ -1,6 +1,6 @@
-import Queue from '../../helpers/queue.js'
-import { Base, Item } from '../../types/types.js'
-import { Downloader } from '../interfaces.js'
+import Queue from '../../../queues/queue.js'
+import { Base, Item } from '../../../types/types.js'
+import { Downloader } from '../../interfaces.js'
 
 export class MemoryDownloader implements Downloader {
   #items: Map<string, Base>
@@ -23,7 +23,7 @@ export class MemoryDownloader implements Downloader {
     if (root) {
       return Promise.resolve({ baseId: this.#rootId, base: root })
     }
-    throw new Error('Method not implemented.')
+    return Promise.reject(new Error('Method not implemented.'))
   }
   disposeAsync(): Promise<void> {
     return Promise.resolve()