@speckle/objectloader2 2.25.7 → 2.25.9

package/src/deferment/defermentManager.test.ts

@@ -0,0 +1,220 @@
+import { vi, describe, it, expect } from 'vitest'
+import { DefermentManager } from './defermentManager.js'
+import { MemoryCache } from './MemoryCache.js'
+import { CustomLogger } from '../types/functions.js'
+import { Item } from '../types/types.js'
+
+describe('DefermentManager', () => {
+  it('should be created', () => {
+    const mockLogger: CustomLogger = vi.fn()
+    const mockCache = {
+      get: vi.fn(),
+      add: vi.fn()
+    } as unknown as MemoryCache
+    const defermentManager = new DefermentManager(mockCache, mockLogger)
+    expect(defermentManager).toBeDefined()
+  })
+
+  describe('defer', () => {
+    it('should return a resolved promise if item is in cache', async () => {
+      const mockLogger: CustomLogger = vi.fn()
+      const get = vi.fn()
+      const add = vi.fn()
+      const mockCache = {
+        get,
+        add
+      } as unknown as MemoryCache
+      const defermentManager = new DefermentManager(mockCache, mockLogger)
+
+      const item: Item = {
+        // eslint-disable-next-line camelcase
+        base: { id: 'testId', speckle_type: 'Base' },
+        baseId: 'testId'
+      }
+      get.mockReturnValue(item)
+
+      const [promise, wasInCache] = defermentManager.defer({ id: 'testId' })
+      const result = await promise
+
+      expect(wasInCache).toBe(true)
+      expect(result).toEqual(item.base)
+      expect(get).toHaveBeenCalledWith('testId')
+    })
+
+    it('should return the same promise if item is already outstanding', () => {
+      const mockLogger: CustomLogger = vi.fn()
+      const get = vi.fn()
+      const add = vi.fn()
+      const mockCache = {
+        get,
+        add
+      } as unknown as MemoryCache
+      const defermentManager = new DefermentManager(mockCache, mockLogger)
+
+      const [promise1, wasInCache1] = defermentManager.defer({ id: 'testId' })
+      const [promise2, wasInCache2] = defermentManager.defer({ id: 'testId' })
+
+      expect(wasInCache1).toBe(false)
+      expect(wasInCache2).toBe(true)
+      expect(promise1).toBe(promise2)
+    })
+
+    it('should create a new deferred base if not in cache and not outstanding', () => {
+      const mockLogger: CustomLogger = vi.fn()
+      const get = vi.fn()
+      const add = vi.fn()
+      const mockCache = {
+        get,
+        add
+      } as unknown as MemoryCache
+      const defermentManager = new DefermentManager(mockCache, mockLogger)
+
+      const [promise, wasInCache] = defermentManager.defer({ id: 'testId' })
+
+      expect(wasInCache).toBe(false)
+      expect(promise).toBeInstanceOf(Promise)
+    })
+
+    it('should throw if disposed', () => {
+      const mockLogger: CustomLogger = vi.fn()
+      const get = vi.fn()
+      const add = vi.fn()
+      const mockCache = {
+        get,
+        add
+      } as unknown as MemoryCache
+      const defermentManager = new DefermentManager(mockCache, mockLogger)
+
+      defermentManager.dispose()
+      expect(() => defermentManager.defer({ id: 'testId' })).toThrow(
+        'DefermentManager is disposed'
+      )
+    })
+  })
+
+  describe('undefer', () => {
+    it('should resolve the promise when an outstanding item is found', async () => {
+      const mockLogger: CustomLogger = vi.fn()
+      const get = vi.fn()
+      const add = vi.fn()
+      const mockCache = {
+        get,
+        add
+      } as unknown as MemoryCache
+      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const requestItem = vi.fn()
+
+      const [promise] = defermentManager.defer({ id: 'testId' })
+
+      const item: Item = {
+        // eslint-disable-next-line camelcase
+        base: { id: 'testId', speckle_type: 'Base' },
+        baseId: 'testId'
+      }
+      defermentManager.undefer(item, requestItem)
+
+      const result = await promise
+      expect(result).toEqual(item.base)
+    })
+
+    it('should log an error if item has no base', () => {
+      const mockLogger: CustomLogger = vi.fn()
+      const get = vi.fn()
+      const add = vi.fn()
+      const mockCache = {
+        get,
+        add
+      } as unknown as MemoryCache
+      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const requestItem = vi.fn()
+
+      const item: Item = { baseId: 'testId' }
+      defermentManager.undefer(item, requestItem)
+      expect(mockLogger).toHaveBeenCalledWith('undefer called with no base', item)
+    })
+
+    it('should add to cache and request item if not outstanding', () => {
+      const mockLogger: CustomLogger = vi.fn()
+      const get = vi.fn()
+      const add = vi.fn()
+      const mockCache = {
+        get,
+        add
+      } as unknown as MemoryCache
+      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const requestItem = vi.fn()
+
+      const item: Item = {
+        // eslint-disable-next-line camelcase
+        base: { id: 'testId', speckle_type: 'Base' },
+        baseId: 'testId'
+      }
+      add.mockImplementation((_item: Item, getDependencies?: (id: string) => void) => {
+        if (getDependencies) getDependencies('testId')
+      })
+
+      defermentManager.undefer(item, requestItem)
+
+      expect(add).toHaveBeenCalledWith(item, expect.any(Function))
+      expect(requestItem).toHaveBeenCalledWith('testId')
+    })
+
+    it('should throw if disposed', () => {
+      const mockLogger: CustomLogger = vi.fn()
+      const get = vi.fn()
+      const add = vi.fn()
+      const mockCache = {
+        get,
+        add
+      } as unknown as MemoryCache
+      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const requestItem = vi.fn()
+
+      defermentManager.dispose()
+      const item: Item = {
+        // eslint-disable-next-line camelcase
+        base: { id: 'testId', speckle_type: 'Base' },
+        baseId: 'testId'
+      }
+      expect(() => defermentManager.undefer(item, requestItem)).toThrow(
+        'DefermentManager is disposed'
+      )
+    })
+  })
+
+  describe('dispose', () => {
+    it('should clear all outstanding deferments', () => {
+      const mockLogger: CustomLogger = vi.fn()
+      const get = vi.fn()
+      const add = vi.fn()
+      const mockCache = {
+        get,
+        add
+      } as unknown as MemoryCache
+      const defermentManager = new DefermentManager(mockCache, mockLogger)
+
+      void defermentManager.defer({ id: 'testId' })
+      defermentManager.dispose()
+      // @ts-expect-error - accessing private property for testing
+      expect(defermentManager.outstanding.size).toBe(0)
+    })
+
+    it('should not do anything if already disposed', () => {
+      const mockLogger: CustomLogger = vi.fn()
+      const get = vi.fn()
+      const add = vi.fn()
+      const mockCache = {
+        get,
+        add
+      } as unknown as MemoryCache
+      const defermentManager = new DefermentManager(mockCache, mockLogger)
+
+      defermentManager.dispose()
+      // @ts-expect-error - accessing private property for testing
+      const outstanding = defermentManager.outstanding
+      const clearSpy = vi.spyOn(outstanding, 'clear')
+      defermentManager.dispose()
+      expect(clearSpy).not.toHaveBeenCalled()
+    })
+  })
+})

package/src/deferment/defermentManager.ts

@@ -0,0 +1,59 @@
+import { DeferredBase } from './deferredBase.js'
+import { CustomLogger } from '../types/functions.js'
+import { Item, Base } from '../types/types.js'
+import { MemoryCache } from './MemoryCache.js'
+
+export class DefermentManager {
+  private outstanding: Map<string, DeferredBase> = new Map()
+  private logger: CustomLogger
+  private disposed = false
+  private cache: MemoryCache
+
+  constructor(cache: MemoryCache, logger: CustomLogger) {
+    this.cache = cache
+    this.logger = logger
+  }
+
+  defer(params: { id: string }): [Promise<Base>, boolean] {
+    if (this.disposed) throw new Error('DefermentManager is disposed')
+    const item = this.cache.get(params.id)
+    if (item) {
+      return [Promise.resolve(item.base!), true]
+    }
+    const deferredBase = this.outstanding.get(params.id)
+    if (deferredBase) {
+      return [deferredBase.getPromise(), true]
+    }
+    const notYetFound = new DeferredBase(params.id)
+    this.outstanding.set(params.id, notYetFound)
+    return [notYetFound.getPromise(), false]
+  }
+
+  undefer(item: Item, requestItem: (id: string) => void): void {
+    if (this.disposed) throw new Error('DefermentManager is disposed')
+    const base = item.base
+    if (!base) {
+      this.logger('undefer called with no base', item)
+      return
+    }
+    this.cache.add(item, (id) => {
+      if (!this.outstanding.has(id)) {
+        requestItem(id)
+      }
+    })
+
+    //order matters here with found before undefer
+    const deferredBase = this.outstanding.get(item.baseId)
+    if (deferredBase) {
+      deferredBase.found(base)
+      this.outstanding.delete(item.baseId)
+    }
+  }
+
+  dispose(): void {
+    if (this.disposed) return
+    this.disposed = true
+    this.logger('cleared deferments, left', this.outstanding.size)
+    this.outstanding.clear()
+  }
+}

package/src/deferment/deferredBase.ts

@@ -0,0 +1,29 @@
+import { Base } from '../types/types.js'
+
+export class DeferredBase {
+  private promise: Promise<Base>
+  private resolve!: (value: Base) => void
+  private reject!: (reason?: Error) => void
+
+  private readonly id: string
+
+  constructor(id: string) {
+    this.id = id
+    this.promise = new Promise<Base>((resolve, reject) => {
+      this.resolve = resolve
+      this.reject = reject
+    })
+  }
+
+  getId(): string {
+    return this.id
+  }
+
+  getPromise(): Promise<Base> {
+    return this.promise
+  }
+
+  found(value: Base): void {
+    this.resolve(value)
+  }
+}

package/src/index.ts

@@ -1,2 +1,3 @@
-export { ObjectLoader2 } from './operations/objectLoader2.js'
-export { ObjectLoader2Factory } from './operations/objectLoader2Factory.js'
+export { ObjectLoader2 } from './core/objectLoader2.js'
+export { ObjectLoader2Factory } from './core/objectLoader2Factory.js'
+export { getFeatureFlag, ObjectLoader2Flags } from './types/functions.js'

package/src/{helpers → queues}/aggregateQueue.ts

@@ -8,13 +8,17 @@ export default class AggregateQueue<T> implements Queue<T> {
     this.#queue1 = queue1
     this.#queue2 = queue2
   }
+  async disposeAsync(): Promise<void> {
+    await this.#queue1.disposeAsync()
+    await this.#queue2.disposeAsync()
+  }
 
   add(value: T): void {
     this.#queue1.add(value)
     this.#queue2.add(value)
   }
 
-  values(): T[] {
+  values(): never {
     throw new Error('Not implemented')
   }
 }

package/src/{helpers → queues}/asyncGeneratorQueue.ts

@@ -29,7 +29,8 @@ export default class AsyncGeneratorQueue<T> implements Queue<T> {
       }
     }
   }
-  dispose(): void {
+  disposeAsync(): Promise<void> {
     this.#finished = true
+    return Promise.resolve()
   }
 }

package/src/{helpers → queues}/batchedPool.ts

@@ -1,4 +1,6 @@
-export default class BatchedPool<T> {
+import Queue from './queue.js'
+
+export default class BatchedPool<T> implements Queue<T> {
   #queue: T[] = []
   #concurrencyAndSizes: number[]
   #processFunction: (batch: T[]) => Promise<void>

package/src/queues/batchingQueue.dispose.test.ts

@@ -0,0 +1,74 @@
+import { describe, test, expect, vi } from 'vitest'
+import BatchingQueue from './batchingQueue.js'
+
+describe('BatchingQueue disposal', () => {
+  test('should drain the queue on dispose', async () => {
+    const processFunction = vi.fn().mockResolvedValue(undefined)
+    const queue = new BatchingQueue<{ id: string }>({
+      batchSize: 5,
+      maxWaitTime: 1000,
+      processFunction
+    })
+
+    const items = Array.from({ length: 3 }, (_, i) => ({ id: `item-${i}` }))
+    items.forEach((item) => queue.add(item.id, item))
+
+    expect(queue.count()).toBe(3)
+
+    await queue.disposeAsync()
+
+    expect(processFunction).toHaveBeenCalledWith(items)
+    expect(queue.count()).toBe(0)
+    expect(queue.isDisposed()).toBe(true)
+  })
+
+  test('should wait for processing to finish before disposing', async () => {
+    let resolveProcess: (value: void | PromiseLike<void>) => void = () => {}
+    const processPromise = new Promise<void>((resolve) => {
+      resolveProcess = resolve
+    })
+
+    const processFunction = vi.fn().mockImplementation(() => processPromise)
+
+    const queue = new BatchingQueue<{ id: string }>({
+      batchSize: 2,
+      maxWaitTime: 100,
+      processFunction
+    })
+
+    const items1 = [{ id: 'item-1' }, { id: 'item-2' }]
+    items1.forEach((item) => queue.add(item.id, item))
+
+    // First batch is processing
+    expect(processFunction).toHaveBeenCalledWith(items1)
+
+    const items2 = [{ id: 'item-3' }]
+    items2.forEach((item) => queue.add(item.id, item))
+
+    const disposePromise = queue.disposeAsync()
+
+    // Queue should be disposed now, but processing is still ongoing
+    expect(queue.isDisposed()).toBe(true)
+    resolveProcess()
+    await disposePromise
+
+    expect(processFunction).toHaveBeenCalledTimes(2)
+    expect(processFunction).toHaveBeenCalledWith(items2)
+    expect(queue.count()).toBe(0)
+    expect(queue.isDisposed()).toBe(true)
+  })
+
+  test('adding items after dispose should do nothing', async () => {
+    const processFunction = vi.fn().mockResolvedValue(undefined)
+    const queue = new BatchingQueue<string>({
+      batchSize: 5,
+      maxWaitTime: 1000,
+      processFunction
+    })
+
+    await queue.disposeAsync()
+    queue.add('key1', 'item1')
+    expect(queue.count()).toBe(0)
+    expect(processFunction).not.toHaveBeenCalled()
+  })
+})

package/src/queues/batchingQueue.test.ts

@@ -0,0 +1,149 @@
+import { describe, test, expect, vi } from 'vitest'
+import BatchingQueue from './batchingQueue.js'
+
+describe('BatchingQueue', () => {
+  test('should add items and process them in batches', async () => {
+    const processSpy = vi.fn()
+    const queue = new BatchingQueue({
+      batchSize: 2,
+      maxWaitTime: 100,
+      processFunction: async (batch: string[]): Promise<void> => {
+        await new Promise((resolve) => setTimeout(resolve, 0))
+        processSpy(batch)
+      }
+    })
+
+    try {
+      queue.add('key1', 'item1')
+      queue.add('key2', 'item2')
+
+      await new Promise((resolve) => setTimeout(resolve, 200))
+
+      expect(processSpy).toHaveBeenCalledTimes(1)
+      expect(processSpy).toHaveBeenCalledWith(['item1', 'item2'])
+    } finally {
+      await queue.disposeAsync()
+    }
+  })
+
+  test('should process items after timeout if batch size is not reached', async () => {
+    const processSpy = vi.fn()
+    const queue = new BatchingQueue({
+      batchSize: 5,
+      maxWaitTime: 100,
+      processFunction: async (batch: string[]): Promise<void> => {
+        await new Promise((resolve) => setTimeout(resolve, 0))
+        processSpy(batch)
+      }
+    })
+
+    try {
+      queue.add('key1', 'item1')
+      queue.add('key2', 'item2')
+
+      await new Promise((resolve) => setTimeout(resolve, 200))
+
+      expect(processSpy).toHaveBeenCalledTimes(1)
+      expect(processSpy).toHaveBeenCalledWith(['item1', 'item2'])
+    } finally {
+      await queue.disposeAsync()
+    }
+  })
+
+  test('should handle multiple batches correctly', async () => {
+    const processSpy = vi.fn()
+    const queue = new BatchingQueue({
+      batchSize: 2,
+      maxWaitTime: 100,
+      processFunction: async (batch: string[]): Promise<void> => {
+        await new Promise((resolve) => setTimeout(resolve, 0))
+        processSpy(batch)
+      }
+    })
+
+    try {
+      queue.add('key1', 'item1')
+      queue.add('key2', 'item2')
+      queue.add('key3', 'item3')
+      queue.add('key4', 'item4')
+
+      await new Promise((resolve) => setTimeout(resolve, 200))
+
+      expect(processSpy).toHaveBeenCalledTimes(2)
+      expect(processSpy).toHaveBeenCalledWith(['item1', 'item2'])
+      expect(processSpy).toHaveBeenCalledWith(['item3', 'item4'])
+    } finally {
+      await queue.disposeAsync()
+    }
+  })
+
+  test('should retrieve items by key', async () => {
+    const queue = new BatchingQueue<string>({
+      batchSize: 3,
+      maxWaitTime: 100,
+      processFunction: async (): Promise<void> => {
+        await new Promise((resolve) => setTimeout(resolve, 0))
+      }
+    })
+    try {
+      queue.add('key1', 'item1')
+      queue.add('key2', 'item2')
+
+      expect(queue.get('key1')).toBe('item1')
+      expect(queue.get('key2')).toBe('item2')
+      expect(queue.get('key3')).toBeUndefined()
+    } finally {
+      await queue.disposeAsync()
+    }
+  })
+
+  test('should return correct count of items', async () => {
+    const queue = new BatchingQueue<string>({
+      batchSize: 3,
+      maxWaitTime: 100,
+      processFunction: async (): Promise<void> => {
+        await new Promise((resolve) => setTimeout(resolve, 0))
+      }
+    })
+    try {
+      expect(queue.count()).toBe(0)
+
+      queue.add('key1', 'item1')
+      queue.add('key2', 'item2')
+
+      expect(queue.count()).toBe(2)
+    } finally {
+      await queue.disposeAsync()
+    }
+  })
+
+  test('should not process items if already processing', async () => {
+    const processSpy = vi.fn()
+    const queue = new BatchingQueue({
+      batchSize: 2,
+      maxWaitTime: 100,
+      processFunction: async (batch: string[]): Promise<void> => {
+        processSpy(batch)
+        await new Promise((resolve) => setTimeout(resolve, 300))
+      }
+    })
+
+    try {
+      queue.add('key1', 'item1')
+      queue.add('key2', 'item2')
+      queue.add('key3', 'item3')
+
+      await new Promise((resolve) => setTimeout(resolve, 200))
+
+      expect(processSpy).toHaveBeenCalledTimes(1)
+      expect(processSpy).toHaveBeenCalledWith(['item1', 'item2'])
+
+      await new Promise((resolve) => setTimeout(resolve, 500))
+
+      expect(processSpy).toHaveBeenCalledTimes(2)
+      expect(processSpy).toHaveBeenCalledWith(['item3'])
+    } finally {
+      await queue.disposeAsync()
+    }
+  })
+})