@speckle/objectloader2 2.25.7 → 2.25.9

package/src/core/stages/cacheWriter.spec.ts

@@ -0,0 +1,213 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { Item } from '../../types/types.js'
+import { CacheWriter } from './cacheWriter.js'
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import { Database } from '../interfaces.js'
+import { MemoryCache, MemoryCacheOptions } from '../../deferment/MemoryCache.js'
+import { CacheOptions } from '../options.js'
+import { CustomLogger } from '../../types/functions.js'
+
+// Mock implementations
+class MockDatabase implements Database {
+  savedItems: Item[] = []
+
+  getAll(/* keys */): Promise<(Item | undefined)[]> {
+    return Promise.resolve([])
+  }
+
+  saveBatch({ batch }: { batch: Item[] }): Promise<void> {
+    this.savedItems.push(...batch)
+    return Promise.resolve()
+  }
+
+  disposeAsync(): Promise<void> {
+    return Promise.resolve()
+  }
+}
+
+describe('CacheWriter', () => {
+  let database: MockDatabase
+  let defermentManager: DefermentManager
+  let cacheWriter: CacheWriter
+  let logger: CustomLogger
+  let requestItemMock: (id: string) => void
+  let options: CacheOptions
+  let memoryCache: MemoryCache
+
+  beforeEach(() => {
+    database = new MockDatabase()
+    logger = vi.fn() as CustomLogger
+    const memoryCacheOptions: MemoryCacheOptions = {
+      maxSizeInMb: 100,
+      ttlms: 60000
+    }
+    memoryCache = new MemoryCache(memoryCacheOptions, logger)
+    defermentManager = new DefermentManager(memoryCache, logger)
+    requestItemMock = vi.fn()
+
+    options = {
+      maxCacheWriteSize: 10,
+      maxCacheBatchWriteWait: 50,
+      maxCacheReadSize: 10,
+      maxCacheBatchReadWait: 50,
+      maxWriteQueueSize: 100
+    }
+
+    cacheWriter = new CacheWriter(
+      database,
+      logger,
+      defermentManager,
+      options,
+      requestItemMock
+    )
+  })
+
+  afterEach(async () => {
+    await cacheWriter.disposeAsync()
+  })
+
+  it('should write items to the database', async () => {
+    const item: Item = {
+      baseId: 'test-id',
+      base: { id: 'test-id', speckle_type: 'test-type' }
+    }
+
+    cacheWriter.add(item)
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, options.maxCacheBatchWriteWait + 10)
+    )
+
+    expect(database.savedItems).toHaveLength(1)
+    expect(database.savedItems[0].baseId).toBe('test-id')
+    expect(database.savedItems[0].base).toEqual({
+      id: 'test-id',
+      speckle_type: 'test-type'
+    })
+  })
+
+  it('should write multiple items in a batch', async () => {
+    const items = [
+      {
+        baseId: 'item1',
+        base: { id: 'item1', speckle_type: 'test-type' }
+      },
+      {
+        baseId: 'item2',
+        base: { id: 'item2', speckle_type: 'test-type' }
+      },
+      {
+        baseId: 'item3',
+        base: { id: 'item3', speckle_type: 'test-type' }
+      }
+    ]
+
+    items.forEach((item) => cacheWriter.add(item))
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, options.maxCacheBatchWriteWait + 10)
+    )
+
+    expect(database.savedItems).toHaveLength(3)
+    expect(database.savedItems.map((item) => item.baseId)).toEqual([
+      'item1',
+      'item2',
+      'item3'
+    ])
+  })
+
+  it('should call undefer on the defermentManager', () => {
+    const spy = vi.spyOn(defermentManager, 'undefer')
+    const item: Item = {
+      baseId: 'test-id',
+      base: { id: 'test-id', speckle_type: 'test-type' }
+    }
+
+    cacheWriter.add(item)
+
+    expect(spy).toHaveBeenCalledWith(item, requestItemMock)
+  })
+
+  it('should handle items with id but no base', async () => {
+    const item: Item = {
+      baseId: 'test-id-no-base'
+      // No base property
+    }
+
+    // Adding an item with no base should not throw an error
+    expect(() => cacheWriter.add(item)).not.toThrow()
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, options.maxCacheBatchWriteWait + 10)
+    )
+
+    expect(database.savedItems).toHaveLength(1)
+    expect(database.savedItems[0].baseId).toBe('test-id-no-base')
+    expect(database.savedItems[0].base).toBeUndefined()
+  })
+
+  it('should process items in batches according to maxCacheWriteSize', async () => {
+    const spy = vi.spyOn(database, 'saveBatch')
+    const smallBatchOptions: CacheOptions = {
+      ...options,
+      maxCacheWriteSize: 2, // Set small batch size
+      maxCacheBatchWriteWait: 100
+    }
+
+    const smallBatchCacheWriter = new CacheWriter(
+      database,
+      logger,
+      defermentManager,
+      smallBatchOptions,
+      requestItemMock
+    )
+
+    // Add 5 items
+    const items = Array.from({ length: 5 }, (_, i) => ({
+      baseId: `batch-item-${i}`,
+      base: { id: `batch-item-${i}`, speckle_type: 'test-type' }
+    }))
+
+    items.forEach((item) => smallBatchCacheWriter.add(item))
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, smallBatchOptions.maxCacheBatchWriteWait * 2 + 50)
+    )
+
+    // With batch size 2, we expect 3 calls to saveBatch: 2 items + 2 items + 1 item
+    expect(spy).toHaveBeenCalledTimes(3)
+
+    await smallBatchCacheWriter.disposeAsync()
+  })
+
+  it('should be disposed correctly', async () => {
+    expect(cacheWriter.isDisposed).toBe(false)
+    await cacheWriter.disposeAsync()
+    expect(cacheWriter.isDisposed).toBe(true)
+  })
+
+  it('should write directly with writeAll method', async () => {
+    const items = [
+      {
+        baseId: 'direct1',
+        base: { id: 'direct1', speckle_type: 'test-type' }
+      },
+      {
+        baseId: 'direct2',
+        base: { id: 'direct2', speckle_type: 'test-type' }
+      }
+    ]
+
+    await cacheWriter.writeAll(items)
+
+    expect(database.savedItems).toHaveLength(2)
+    expect(database.savedItems.map((item) => item.baseId)).toEqual([
+      'direct1',
+      'direct2'
+    ])
+  })
+})

package/src/core/stages/cacheWriter.ts

@@ -0,0 +1,64 @@
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
+import Queue from '../../queues/queue.js'
+import { CustomLogger } from '../../types/functions.js'
+import { Item } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import { CacheOptions } from '../options.js'
+
+export class CacheWriter implements Queue<Item> {
+  #writeQueue: BatchingQueue<Item> | undefined
+  #database: Database
+  #defermentManager: DefermentManager
+  #requestItem: (id: string) => void
+  #logger: CustomLogger
+  #options: CacheOptions
+  #disposed = false
+
+  constructor(
+    database: Database,
+    logger: CustomLogger,
+    defermentManager: DefermentManager,
+    options: CacheOptions,
+    requestItem: (id: string) => void
+  ) {
+    this.#database = database
+    this.#options = options
+    this.#logger = logger
+    this.#defermentManager = defermentManager
+    this.#requestItem = requestItem
+  }
+
+  add(item: Item): void {
+    if (!this.#writeQueue) {
+      this.#writeQueue = new BatchingQueue({
+        batchSize: this.#options.maxCacheWriteSize,
+        maxWaitTime: this.#options.maxCacheBatchWriteWait,
+        processFunction: async (batch: Item[]): Promise<void> => {
+          await this.writeAll(batch)
+        }
+      })
+    }
+    this.#writeQueue.add(item.baseId, item)
+    this.#defermentManager.undefer(item, this.#requestItem)
+  }
+
+  async writeAll(items: Item[]): Promise<void> {
+    const start = performance.now()
+    await this.#database.saveBatch({ batch: items })
+    this.#logger(
+      `writeBatch: wrote ${items.length}, time ${
+        performance.now() - start
+      } ms left ${this.#writeQueue?.count()}`
+    )
+  }
+
+  async disposeAsync(): Promise<void> {
+    this.#disposed = true
+    await this.#writeQueue?.disposeAsync()
+  }
+
+  get isDisposed(): boolean {
+    return this.#disposed
+  }
+}

package/src/{operations/databases → core/stages}/indexedDatabase.spec.ts

@@ -1,10 +1,14 @@
 import { describe, it, expect, beforeEach, afterEach } from 'vitest'
 import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
+
+import { Base, Item } from '../../types/types.js'
 import IndexedDatabase, { IndexedDatabaseOptions } from './indexedDatabase.js'
-import { Item } from '../../types/types.js'
 
 // Mock Item
-const defaultItem = (id: string): Item => ({ baseId: id, item: { foo: 'bar' } })
+const defaultItem = (id: string): Item => ({
+  baseId: id,
+  base: { foo: 'bar' } as unknown as Base
+})
 
 describe('IndexedDatabase', () => {
   let db: IndexedDatabase
@@ -21,7 +25,7 @@ describe('IndexedDatabase', () => {
 
   it('should add and get multiple items', async () => {
     const items = [defaultItem('id1'), defaultItem('id2')]
-    await db.cacheSaveBatch({ batch: items })
+    await db.saveBatch({ batch: items })
     const result = await db.getAll(['id1', 'id2'])
     expect(result).toMatchSnapshot()
     expect(result).toEqual(items)

package/src/core/stages/indexedDatabase.ts

@@ -0,0 +1,49 @@
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+import { Item } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
+import { ItemStore } from './ItemStore.js'
+
+export interface IndexedDatabaseOptions {
+  indexedDB?: IDBFactory
+  keyRange?: {
+    bound: Function
+    lowerBound: Function
+    upperBound: Function
+  }
+}
+
+export default class IndexedDatabase implements Database {
+  #options: IndexedDatabaseOptions
+  #cacheDB: ItemStore
+  #writeQueue: BatchingQueue<Item> | undefined
+
+  constructor(options: IndexedDatabaseOptions) {
+    this.#options = options
+    this.#cacheDB = new ItemStore(
+      {
+        indexedDB: this.#options.indexedDB,
+        keyRange: this.#options.keyRange
+      },
+      'speckle-cache',
+      'cache'
+    )
+  }
+
+  async getAll(keys: string[]): Promise<(Item | undefined)[]> {
+    await this.#cacheDB.init()
+    return await this.#cacheDB.bulkGet(keys)
+  }
+
+  async saveBatch(params: { batch: Item[] }): Promise<void> {
+    await this.#cacheDB.init()
+    const { batch } = params
+    await this.#cacheDB.bulkInsert(batch)
+  }
+
+  async disposeAsync(): Promise<void> {
+    await this.#writeQueue?.disposeAsync()
+    this.#writeQueue = undefined
+    this.#cacheDB.close()
+  }
+}

package/src/{operations/databases → core/stages/memory}/memoryDatabase.spec.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect, beforeEach } from 'vitest'
+import { Item, Base } from '../../../types/types.js'
 import { MemoryDatabase } from './memoryDatabase.js'
-import { Base, Item } from '../../types/types.js'
 
 const makeItem = (id: string, foo = 'bar'): Item => ({
   baseId: id,
@@ -21,14 +21,14 @@ describe('MemoryDatabase', () => {
 
   it('should add and retrieve a single item', async () => {
     const item = makeItem('id1')
-    await db.cacheSaveBatch({ batch: [item] })
+    await db.saveBatch({ batch: [item] })
     const result = await db.getAll(['id1'])
     expect(result).toEqual([item])
   })
 
   it('should add and retrieve multiple items', async () => {
     const items = [makeItem('id1'), makeItem('id2', 'baz')]
-    await db.cacheSaveBatch({ batch: items })
+    await db.saveBatch({ batch: items })
     const result = await db.getAll(['id1', 'id2'])
     expect(result).toEqual(items)
   })
@@ -36,8 +36,8 @@ describe('MemoryDatabase', () => {
   it('should overwrite items with the same key', async () => {
     const item1 = makeItem('id1', 'foo')
     const item2 = makeItem('id1', 'bar')
-    await db.cacheSaveBatch({ batch: [item1] })
-    await db.cacheSaveBatch({ batch: [item2] })
+    await db.saveBatch({ batch: [item1] })
+    await db.saveBatch({ batch: [item2] })
     const result = await db.getAll(['id1'])
     expect(result).toEqual([item2])
   })

package/src/{operations/databases → core/stages/memory}/memoryDatabase.ts

@@ -1,6 +1,6 @@
-import { Base, Item } from '../../types/types.js'
-import { Database } from '../interfaces.js'
-import { MemoryDatabaseOptions } from '../options.js'
+import { Base, Item } from '../../../types/types.js'
+import { Database } from '../../interfaces.js'
+import { MemoryDatabaseOptions } from '../../options.js'
 
 export class MemoryDatabase implements Database {
   private items: Map<string, Base>
@@ -22,8 +22,11 @@ export class MemoryDatabase implements Database {
     return Promise.resolve(found)
   }
 
-  cacheSaveBatch({ batch }: { batch: Item[] }): Promise<void> {
+  saveBatch({ batch }: { batch: Item[] }): Promise<void> {
     for (const item of batch) {
+      if (!item.baseId || !item.base) {
+        throw new Error('Item must have a baseId and base')
+      }
       this.items.set(item.baseId, item.base)
     }
     return Promise.resolve()

package/src/{operations/downloaders → core/stages/memory}/memoryDownloader.spec.ts

@@ -1,8 +1,8 @@
 import { describe, it, expect, beforeEach } from 'vitest'
 import { MemoryDownloader } from './memoryDownloader.js'
-import { Base, Item } from '../../types/types.js'
-import Queue from '../../helpers/queue.js'
-import BufferQueue from '../../helpers/bufferQueue.js'
+import BufferQueue from '../../../queues/bufferQueue.js'
+import Queue from '../../../queues/queue.js'
+import { Base, Item } from '../../../types/types.js'
 
 const makeBase = (foo: string): Base => ({ foo } as unknown as Base)
 

package/src/{operations/downloaders → core/stages/memory}/memoryDownloader.ts

@@ -1,6 +1,6 @@
-import Queue from '../../helpers/queue.js'
-import { Base, Item } from '../../types/types.js'
-import { Downloader } from '../interfaces.js'
+import Queue from '../../../queues/queue.js'
+import { Base, Item } from '../../../types/types.js'
+import { Downloader } from '../../interfaces.js'
 
 export class MemoryDownloader implements Downloader {
   #items: Map<string, Base>

package/src/{operations/downloaders → core/stages}/serverDownloader.spec.ts

@@ -3,14 +3,14 @@ import createFetchMock from 'vitest-fetch-mock'
 import { vi } from 'vitest'
 import { Item } from '../../types/types.js'
 import ServerDownloader from './serverDownloader.js'
-import { MemoryPump } from '../../helpers/memoryPump.js'
+import AsyncGeneratorQueue from '../../queues/asyncGeneratorQueue.js'
 
 describe('downloader', () => {
   test('download batch of one', async () => {
     const fetchMocker = createFetchMock(vi)
     const i: Item = { baseId: 'id', base: { id: 'id', speckle_type: 'type' } }
     fetchMocker.mockResponseOnce('id\t' + JSON.stringify(i.base) + '\n')
-    const pump = new MemoryPump()
+    const gathered = new AsyncGeneratorQueue<Item>()
     const downloader = new ServerDownloader({
       serverUrl: 'http://speckle.test',
       streamId: 'streamId',
@@ -18,12 +18,18 @@ describe('downloader', () => {
       token: 'token',
       fetch: fetchMocker
     })
-    downloader.initializePool({ results: pump, total: 1, maxDownloadBatchWait: 200 })
+    downloader.initializePool({
+      results: gathered,
+      total: 1,
+      maxDownloadBatchWait: 200
+    })
     downloader.add('id')
-    await downloader.disposeAsync()
     const r = []
-    for await (const x of pump.gather([i.baseId])) {
+    for await (const x of gathered.consume()) {
       r.push(x)
+      if (r.length >= 1) {
+        break
+      }
     }
 
     expect(r).toMatchSnapshot()
@@ -38,7 +44,7 @@ describe('downloader', () => {
       'id1\t' + JSON.stringify(i1.base) + '\nid2\t' + JSON.stringify(i2.base) + '\n'
     )
 
-    const pump = new MemoryPump()
+    const gathered = new AsyncGeneratorQueue<Item>()
     const downloader = new ServerDownloader({
       serverUrl: 'http://speckle.test',
       streamId: 'streamId',
@@ -47,13 +53,20 @@ describe('downloader', () => {
 
       fetch: fetchMocker
     })
-    downloader.initializePool({ results: pump, total: 2, maxDownloadBatchWait: 200 })
+    downloader.initializePool({
+      results: gathered,
+      total: 2,
+      maxDownloadBatchWait: 200
+    })
     downloader.add('id1')
     downloader.add('id2')
     await downloader.disposeAsync()
     const r = []
-    for await (const x of pump.gather([i1.baseId, i2.baseId])) {
+    for await (const x of gathered.consume()) {
       r.push(x)
+      if (r.length >= 2) {
+        break
+      }
     }
 
     expect(r).toMatchSnapshot()
@@ -75,7 +88,7 @@ describe('downloader', () => {
         '\n'
     )
 
-    const pump = new MemoryPump()
+    const gathered = new AsyncGeneratorQueue<Item>()
     const downloader = new ServerDownloader({
       serverUrl: 'http://speckle.test',
       streamId: 'streamId',
@@ -84,20 +97,98 @@ describe('downloader', () => {
 
       fetch: fetchMocker
     })
-    downloader.initializePool({ results: pump, total: 3, maxDownloadBatchWait: 200 })
+    downloader.initializePool({
+      results: gathered,
+      total: 3,
+      maxDownloadBatchWait: 200
+    })
     downloader.add('id1')
     downloader.add('id2')
     downloader.add('id3')
     await downloader.disposeAsync()
     const r = []
-    for await (const x of pump.gather([i1.baseId, i2.baseId, i3.baseId])) {
+    for await (const x of gathered.consume()) {
       r.push(x)
+      if (r.length >= 3) {
+        break
+      }
     }
 
     expect(r).toMatchSnapshot()
     await downloader.disposeAsync()
   })
 
+  test('download batch of three with Objects.Other.RawEncoding', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i1: Item = { baseId: 'id1', base: { id: 'id1', speckle_type: 'type' } }
+    const i2: Item = {
+      baseId: 'id2',
+      base: { id: 'id2', speckle_type: 'Objects.Other.RawEncoding' }
+    }
+    const i3: Item = { baseId: 'id3', base: { id: 'id3', speckle_type: 'type' } }
+    fetchMocker.mockResponseOnce(
+      'id1\t' +
+        JSON.stringify(i1.base) +
+        '\nid2\t' +
+        JSON.stringify(i2.base) +
+        '\nid3\t' +
+        JSON.stringify(i3.base) +
+        '\n'
+    )
+
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://speckle.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    downloader.initializePool({
+      results: gathered,
+      total: 3,
+      maxDownloadBatchWait: 200
+    })
+    downloader.add('id1')
+    downloader.add('id2')
+    downloader.add('id3')
+    await downloader.disposeAsync()
+    const r = []
+    for await (const x of gathered.consume()) {
+      r.push(x)
+      if (r.length >= 3) {
+        break
+      }
+    }
+
+    expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test("download Objects.Other.RawEncoding doesn't exist", async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = {
+      baseId: 'id',
+      base: {
+        id: 'id',
+        speckle_type: 'Objects.Other.RawEncoding',
+        __closure: { childIds: 1 }
+      }
+    }
+    fetchMocker.mockResponseOnce(JSON.stringify(i.base))
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://speckle.test',
+      streamId: 'streamId',
+      objectId: i.baseId,
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    await expect(downloader.downloadSingle()).rejects.toThrow()
+    await downloader.disposeAsync()
+  })
+
   test('download single exists', async () => {
     const fetchMocker = createFetchMock(vi)
     const i: Item = {