@speckle/objectloader2 2.26.0 → 2.26.2

package/src/core/objectLoader2Factory.test.ts

@@ -0,0 +1,135 @@
+/* eslint-disable camelcase */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { ObjectLoader2Factory } from './objectLoader2Factory.js'
+import { Base } from '../types/types.js'
+
+describe('ObjectLoader2Factory', () => {
+  let testObjects: Base[]
+
+  beforeEach(() => {
+    testObjects = [
+      {
+        id: 'root-id',
+        speckle_type: 'Base',
+        __closure: {
+          'child-1': 1,
+          'child-2': 2
+        }
+      },
+      {
+        id: 'child-1',
+        speckle_type: 'Base'
+      },
+      {
+        id: 'child-2',
+        speckle_type: 'Base'
+      }
+    ]
+  })
+
+  describe('createFromObjects', () => {
+    it('should create ObjectLoader2 from array of objects', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      expect(loader).toBeDefined()
+
+      // Test that we can get the root object
+      const rootObject = await loader.getRootObject()
+      expect(rootObject?.baseId).toBe('root-id')
+      expect(rootObject?.base?.speckle_type).toBe('Base')
+
+      await loader.disposeAsync()
+    })
+
+    it('should use first object as root', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      const rootObject = await loader.getRootObject()
+      expect(rootObject?.baseId).toBe('root-id')
+      expect(rootObject?.base?.__closure).toEqual({
+        'child-1': 1,
+        'child-2': 2
+      })
+
+      await loader.disposeAsync()
+    })
+
+    it('should allow iteration over all objects', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      const objects: Base[] = []
+      for await (const obj of loader.getObjectIterator()) {
+        objects.push(obj)
+      }
+
+      expect(objects).toHaveLength(3)
+      expect(objects[0].id).toBe('root-id')
+
+      await loader.disposeAsync()
+    })
+
+    it('should get total object count correctly', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      const totalCount = await loader.getTotalObjectCount()
+      expect(totalCount).toBe(3) // root + 2 children
+
+      await loader.disposeAsync()
+    })
+
+    it('should handle empty objects array', () => {
+      expect(() => {
+        ObjectLoader2Factory.createFromObjects([])
+      }).toThrow()
+    })
+
+    it('should get individual objects by id', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      const rootObj = await loader.getObject({ id: 'root-id' })
+      expect(rootObj.id).toBe('root-id')
+      expect(rootObj.speckle_type).toBe('Base')
+
+      const child1 = await loader.getObject({ id: 'child-1' })
+      expect(child1.id).toBe('child-1')
+
+      const child2 = await loader.getObject({ id: 'child-2' })
+      expect(child2.id).toBe('child-2')
+
+      await loader.disposeAsync()
+    })
+
+    it('should get individual objects by id that does not exist', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      const rootObj = await loader.getObject({ id: 'root-id' })
+      expect(rootObj.id).toBe('root-id')
+      expect(rootObj.speckle_type).toBe('Base')
+
+      const getObjectPromise = loader.getObject({ id: 'child-11111' })
+      await expect(getObjectPromise).rejects.toThrow()
+
+      await loader.disposeAsync()
+    })
+  })
+
+  describe('createFromJSON', () => {
+    it('should create ObjectLoader2 from JSON string', async () => {
+      const json = JSON.stringify(testObjects)
+      const loader = ObjectLoader2Factory.createFromJSON(json)
+
+      expect(loader).toBeDefined()
+
+      const rootObject = await loader.getRootObject()
+      expect(rootObject?.baseId).toBe('root-id')
+
+      await loader.disposeAsync()
+    })
+
+    it('should handle invalid JSON', () => {
+      expect(() => {
+        ObjectLoader2Factory.createFromJSON('invalid json')
+      }).toThrow()
+    })
+  })
+})

package/src/core/objectLoader2Factory.ts

@@ -1,5 +1,11 @@
-import { CustomLogger, getFeatureFlag, ObjectLoader2Flags } from '../types/functions.js'
-import { Base } from '../types/types.js'
+import { DefermentManager, MemoryOnlyDeferment } from '../deferment/defermentManager.js'
+import {
+  CustomLogger,
+  Fetcher,
+  getFeatureFlag,
+  ObjectLoader2Flags
+} from '../types/functions.js'
+import { Base, ObjectAttributeMask } from '../types/types.js'
 import { ObjectLoader2 } from './objectLoader2.js'
 import { IndexedDatabase } from './stages/indexedDatabase.js'
 import { MemoryDatabase } from './stages/memory/memoryDatabase.js'
@@ -10,6 +16,10 @@ export interface ObjectLoader2FactoryOptions {
   // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
   keyRange?: { bound: Function; lowerBound: Function; upperBound: Function }
   indexedDB?: IDBFactory
+  fetch?: Fetcher
+  attributeMask?: ObjectAttributeMask
+  useCache?: boolean
+  debug?: boolean
   logger?: CustomLogger
 }
 
@@ -22,6 +32,7 @@ export class ObjectLoader2Factory {
     })
     const loader = new ObjectLoader2({
       rootId: root.id,
+      deferments: new MemoryOnlyDeferment(records),
       database: new MemoryDatabase({ items: records }),
       downloader: new MemoryDownloader(root.id, records)
     })
@@ -40,13 +51,21 @@ export class ObjectLoader2Factory {
     token?: string
     headers?: Headers
     options?: ObjectLoader2FactoryOptions
+    attributeMask?: ObjectAttributeMask
   }): ObjectLoader2 {
     const log = ObjectLoader2Factory.getLogger(params.options?.logger)
     let database
-    if (getFeatureFlag(ObjectLoader2Flags.DEBUG) === 'true') {
+    if (
+      params.options?.debug === true ||
+      getFeatureFlag(ObjectLoader2Flags.DEBUG) === 'true'
+    ) {
       this.logger('Using DEBUG mode for ObjectLoader2Factory')
     }
-    if (getFeatureFlag(ObjectLoader2Flags.USE_CACHE) === 'true') {
+    const useCache = params.options?.useCache ?? true
+    const flag = getFeatureFlag(ObjectLoader2Flags.USE_CACHE)
+    const flagAllowsCache = flag !== 'false'
+
+    if (useCache && flagAllowsCache) {
       database = new IndexedDatabase({
         indexedDB: params.options?.indexedDB,
         keyRange: params.options?.keyRange
@@ -59,18 +78,22 @@ export class ObjectLoader2Factory {
         'Disabled persistent caching for ObjectLoader2.  Using MemoryDatabase'
       )
     }
+    const logger = log || (((): void => {}) as CustomLogger)
     const loader = new ObjectLoader2({
       rootId: params.objectId,
+      deferments: new DefermentManager(logger),
       downloader: new ServerDownloader({
         serverUrl: params.serverUrl,
         streamId: params.streamId,
         objectId: params.objectId,
         token: params.token,
         headers: params.headers,
-        logger: log || ((): void => {})
+        fetch: params.options?.fetch,
+        attributeMask: params.attributeMask,
+        logger
       }),
       database,
-      logger: log
+      logger
     })
     return loader
   }

package/src/core/options.ts

@@ -1,9 +1,11 @@
+import { Deferment } from '../deferment/defermentManager.js'
 import { CustomLogger } from '../types/functions.js'
 import { Base } from '../types/types.js'
 import { Downloader, Database } from './interfaces.js'
 
 export interface ObjectLoader2Options {
   rootId: string
+  deferments: Deferment
   downloader: Downloader
   database: Database
   logger?: CustomLogger

package/src/core/stages/cacheReader.ts

@@ -1,4 +1,4 @@
-import { DefermentManager } from '../../deferment/defermentManager.js'
+import { Deferment } from '../../deferment/defermentManager.js'
 import BatchingQueue from '../../queues/batchingQueue.js'
 import Queue from '../../queues/queue.js'
 import { CustomLogger } from '../../types/functions.js'
@@ -8,7 +8,7 @@ import { CacheOptions } from '../options.js'
 
 export class CacheReader {
   #database: Database
-  #defermentManager: DefermentManager
+  #defermentManager: Deferment
   #logger: CustomLogger
   #options: CacheOptions
   #readQueue: BatchingQueue<string> | undefined
@@ -17,7 +17,7 @@ export class CacheReader {
 
   constructor(
     database: Database,
-    defermentManager: DefermentManager,
+    defermentManager: Deferment,
     logger: CustomLogger,
     options: CacheOptions
   ) {

package/src/core/stages/cacheWriter.ts

@@ -1,4 +1,4 @@
-import { DefermentManager } from '../../deferment/defermentManager.js'
+import { Deferment } from '../../deferment/defermentManager.js'
 import BatchingQueue from '../../queues/batchingQueue.js'
 import Queue from '../../queues/queue.js'
 import { CustomLogger } from '../../types/functions.js'
@@ -9,7 +9,7 @@ import { CacheOptions } from '../options.js'
 export class CacheWriter implements Queue<Item> {
   #writeQueue: BatchingQueue<Item> | undefined
   #database: Database
-  #defermentManager: DefermentManager
+  #deferment: Deferment
   #requestItem: (id: string) => void
   #logger: CustomLogger
   #options: CacheOptions
@@ -18,14 +18,14 @@ export class CacheWriter implements Queue<Item> {
   constructor(
     database: Database,
     logger: CustomLogger,
-    defermentManager: DefermentManager,
+    deferment: Deferment,
     options: CacheOptions,
     requestItem: (id: string) => void
   ) {
     this.#database = database
     this.#options = options
     this.#logger = logger
-    this.#defermentManager = defermentManager
+    this.#deferment = deferment
     this.#requestItem = requestItem
   }
 
@@ -40,7 +40,7 @@ export class CacheWriter implements Queue<Item> {
       })
     }
     this.#writeQueue.add(item.baseId, item)
-    this.#defermentManager.undefer(item, this.#requestItem)
+    this.#deferment.undefer(item, this.#requestItem)
   }
 
   async writeAll(items: Item[]): Promise<void> {

package/src/core/stages/serverDownloader.ts

@@ -2,7 +2,7 @@ import BatchingQueue from '../../queues/batchingQueue.js'
 import Queue from '../../queues/queue.js'
 import { ObjectLoaderRuntimeError } from '../../types/errors.js'
 import { CustomLogger, Fetcher, indexOf, isBase, take } from '../../types/functions.js'
-import { Item } from '../../types/types.js'
+import { Item, ObjectAttributeMask } from '../../types/types.js'
 import { Downloader } from '../interfaces.js'
 
 export interface ServerDownloaderOptions {
@@ -13,6 +13,7 @@ export interface ServerDownloaderOptions {
   headers?: Headers
   logger: CustomLogger
   fetch?: Fetcher
+  attributeMask?: ObjectAttributeMask
 }
 
 const MAX_SAFARI_DECODE_BYTES = 2 * 1024 * 1024 * 1024 - 1024 * 1024 // 2GB minus a margin
@@ -51,9 +52,10 @@ export default class ServerDownloader implements Downloader {
     if (this.#options.token) {
       this.#headers['Authorization'] = `Bearer ${this.#options.token}`
     }
-    this.#requestUrlChildren = `${this.#options.serverUrl}/api/getobjects/${
+    this.#requestUrlChildren = `${this.#options.serverUrl}/api/v2/projects/${
       this.#options.streamId
-    }`
+    }/object-stream/`
+
     this.#requestUrlRootObj = `${this.#options.serverUrl}/objects/${
       this.#options.streamId
     }/${this.#options.objectId}/single`
@@ -117,11 +119,12 @@ Chrome's behavior: Chrome generally handles larger data sizes without this speci
 
     const start = performance.now()
     this.#logger(`Downloading batch of ${batch.length} items...`)
+    const attributeMask = this.#options.attributeMask
     const keys = new Set<string>(batch)
     const response = await this.#fetch(url, {
       method: 'POST',
       headers: { ...headers, 'Content-Type': 'application/json' },
-      body: JSON.stringify({ objects: JSON.stringify(batch) })
+      body: JSON.stringify({ objectIds: batch, attributeMask })
     })
 
     this.#validateResponse(response)

package/src/deferment/defermentManager.test.ts

@@ -11,7 +11,7 @@ describe('DefermentManager', () => {
       get: vi.fn(),
       add: vi.fn()
     } as unknown as MemoryCache
-    const defermentManager = new DefermentManager(mockCache, mockLogger)
+    const defermentManager = new DefermentManager(mockLogger, mockCache)
     expect(defermentManager).toBeDefined()
   })
 
@@ -24,7 +24,7 @@ describe('DefermentManager', () => {
         get,
         add
       } as unknown as MemoryCache
-      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const defermentManager = new DefermentManager(mockLogger, mockCache)
 
       const item: Item = {
         // eslint-disable-next-line camelcase
@@ -49,7 +49,7 @@ describe('DefermentManager', () => {
         get,
         add
       } as unknown as MemoryCache
-      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const defermentManager = new DefermentManager(mockLogger, mockCache)
 
       const [promise1, wasInCache1] = defermentManager.defer({ id: 'testId' })
       const [promise2, wasInCache2] = defermentManager.defer({ id: 'testId' })
@@ -67,7 +67,7 @@ describe('DefermentManager', () => {
         get,
         add
       } as unknown as MemoryCache
-      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const defermentManager = new DefermentManager(mockLogger, mockCache)
 
       const [promise, wasInCache] = defermentManager.defer({ id: 'testId' })
 
@@ -83,7 +83,7 @@ describe('DefermentManager', () => {
         get,
         add
       } as unknown as MemoryCache
-      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const defermentManager = new DefermentManager(mockLogger, mockCache)
 
       defermentManager.dispose()
       expect(() => defermentManager.defer({ id: 'testId' })).toThrow(
@@ -101,7 +101,7 @@ describe('DefermentManager', () => {
         get,
         add
       } as unknown as MemoryCache
-      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const defermentManager = new DefermentManager(mockLogger, mockCache)
       const requestItem = vi.fn()
 
       const [promise] = defermentManager.defer({ id: 'testId' })
@@ -125,7 +125,7 @@ describe('DefermentManager', () => {
         get,
         add
       } as unknown as MemoryCache
-      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const defermentManager = new DefermentManager(mockLogger, mockCache)
       const requestItem = vi.fn()
 
       const item: Item = { baseId: 'testId' }
@@ -141,7 +141,7 @@ describe('DefermentManager', () => {
         get,
         add
       } as unknown as MemoryCache
-      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const defermentManager = new DefermentManager(mockLogger, mockCache)
       const requestItem = vi.fn()
 
       const item: Item = {
@@ -167,7 +167,7 @@ describe('DefermentManager', () => {
         get,
         add
       } as unknown as MemoryCache
-      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const defermentManager = new DefermentManager(mockLogger, mockCache)
       const requestItem = vi.fn()
 
       defermentManager.dispose()
@@ -191,7 +191,7 @@ describe('DefermentManager', () => {
         get,
         add
       } as unknown as MemoryCache
-      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const defermentManager = new DefermentManager(mockLogger, mockCache)
 
       void defermentManager.defer({ id: 'testId' })
       defermentManager.dispose()
@@ -207,7 +207,7 @@ describe('DefermentManager', () => {
         get,
         add
       } as unknown as MemoryCache
-      const defermentManager = new DefermentManager(mockCache, mockLogger)
+      const defermentManager = new DefermentManager(mockLogger, mockCache)
 
       defermentManager.dispose()
       // @ts-expect-error - accessing private property for testing

package/src/deferment/defermentManager.ts

@@ -3,15 +3,50 @@ import { CustomLogger } from '../types/functions.js'
 import { Item, Base } from '../types/types.js'
 import { MemoryCache } from './MemoryCache.js'
 
-export class DefermentManager {
+export interface Deferment {
+  defer(params: { id: string }): [Promise<Base>, boolean]
+  undefer(item: Item, requestItem: (id: string) => void): void
+  dispose(): void
+}
+
+export class MemoryOnlyDeferment implements Deferment {
+  private items: Map<string, Base>
+
+  constructor(items: Map<string, Base>) {
+    this.items = items
+  }
+  defer(params: { id: string }): [Promise<Base>, boolean] {
+    const item = this.items.get(params.id)
+    if (item) {
+      return [Promise.resolve(item), true]
+    }
+    return [Promise.reject(new Error('Not found in cache: ' + params.id)), false]
+  }
+  undefer(): void {
+    //no-op
+  }
+  dispose(): void {
+    //no-op
+  }
+}
+
+export class DefermentManager implements Deferment {
   private outstanding: Map<string, DeferredBase> = new Map()
   private logger: CustomLogger
   private disposed = false
   private cache: MemoryCache
 
-  constructor(cache: MemoryCache, logger: CustomLogger) {
-    this.cache = cache
+  constructor(logger: CustomLogger, cache?: MemoryCache) {
     this.logger = logger
+    this.cache =
+      cache ||
+      new MemoryCache(
+        {
+          maxSizeInMb: 500, // 500 MB
+          ttlms: 5_000 // 5 seconds
+        },
+        logger
+      )
   }
 
   defer(params: { id: string }): [Promise<Base>, boolean] {
@@ -55,5 +90,6 @@ export class DefermentManager {
     this.disposed = true
     this.logger('cleared deferments, left', this.outstanding.size)
     this.outstanding.clear()
+    this.cache.dispose()
   }
 }

package/src/queues/batchingQueue.dispose.test.ts

@@ -17,7 +17,7 @@ describe('BatchingQueue disposal', () => {
 
     await queue.disposeAsync()
 
-    expect(processFunction).toHaveBeenCalledWith(items)
+    expect(processFunction).not.toHaveBeenCalled()
     expect(queue.count()).toBe(0)
     expect(queue.isDisposed()).toBe(true)
   })
@@ -52,8 +52,6 @@ describe('BatchingQueue disposal', () => {
     resolveProcess()
     await disposePromise
 
-    expect(processFunction).toHaveBeenCalledTimes(2)
-    expect(processFunction).toHaveBeenCalledWith(items2)
     expect(queue.count()).toBe(0)
     expect(queue.isDisposed()).toBe(true)
   })

package/src/queues/batchingQueue.ts

@@ -1,4 +1,3 @@
-import { CustomLogger } from '../types/functions.js'
 import KeyedQueue from './keyedQueue.js'
 
 /**
@@ -13,7 +12,6 @@ export default class BatchingQueue<T> {
   #processFunction: (batch: T[]) => Promise<void>
   #timeoutId: ReturnType<typeof setTimeout> | null = null
   #isProcessing = false
-  #logger: CustomLogger
 
   #disposed = false
   #batchTimeout: number
@@ -41,12 +39,10 @@ export default class BatchingQueue<T> {
     batchSize: number
     maxWaitTime: number
     processFunction: (batch: T[]) => Promise<void>
-    logger?: CustomLogger
   }) {
     this.#batchSize = params.batchSize
     this.#processFunction = params.processFunction
     this.#batchTimeout = params.maxWaitTime
-    this.#logger = params.logger || ((): void => {})
   }
 
   async disposeAsync(): Promise<void> {
@@ -109,11 +105,13 @@ export default class BatchingQueue<T> {
     this.#isProcessing = true
 
     const batchToProcess = this.#getBatch(this.#batchSize)
+    if (this.#disposed) return
 
     try {
       await this.#processFunction(batchToProcess)
     } catch (error) {
-      this.#logger('Batch processing failed:', error)
+      console.error('Batch processing failed:', error)
+      this.#disposed = true
     } finally {
       this.#isProcessing = false
     }

package/src/types/types.ts

@@ -19,3 +19,8 @@ export interface Reference {
 export interface DataChunk extends Base {
   data?: Base[]
 }
+
+export type ObjectAttributeMask =
+  | { include: string[] }
+  | { exclude: string[] }
+  | undefined