@speckle/objectloader2 2.23.22 → 2.24.0

package/dist/commonjs/index.js

@@ -3,6 +3,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryDatabase = void 0;
 const objectLoader2_js_1 = __importDefault(require("./operations/objectLoader2.js"));
 exports.default = objectLoader2_js_1.default;
+var memoryDatabase_js_1 = require("./operations/memoryDatabase.js");
+Object.defineProperty(exports, "MemoryDatabase", { enumerable: true, get: function () { return memoryDatabase_js_1.MemoryDatabase; } });
 //# sourceMappingURL=index.js.map

package/dist/esm/index.js

@@ -1,3 +1,4 @@
 import ObjectLoader2 from './operations/objectLoader2.js';
 export default ObjectLoader2;
+export { MemoryDatabase } from './operations/memoryDatabase.js';
 //# sourceMappingURL=index.js.map

package/eslint.config.mjs

@@ -49,6 +49,7 @@ const configs = [
   {
     files: ['**/*.spec.ts'],
     rules: {
+      camelcase: 'off',
       '@typescript-eslint/no-unused-expressions': 'off'
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@speckle/objectloader2",
-  "version": "2.23.22",
+  "version": "2.24.0",
   "description": "This is an updated objectloader for the Speckle viewer written in typescript",
   "main": "./dist/commonjs/index.js",
   "module": "./dist/esm/index.js",
@@ -33,7 +33,7 @@
   "author": "AEC Systems",
   "license": "Apache-2.0",
   "dependencies": {
-    "@speckle/shared": "^2.23.22",
+    "@speckle/shared": "^2.24.0",
     "dexie": "^4.0.11"
   },
   "devDependencies": {

package/src/helpers/__snapshots__/defermentManager.spec.ts.snap

@@ -0,0 +1,8 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`deferments > defer one 1`] = `
+{
+  "id": "id",
+  "speckle_type": "type",
+}
+`;

package/src/helpers/batchingQueue.ts

@@ -62,7 +62,7 @@ export default class BatchingQueue<T> implements Queue<T> {
         }
         /*console.log(
           'queue is waiting ' +
-            interval / 1000 +
+            interval / TIME_MS.second +
             ' with queue size of ' +
             this.#queue.length
         )*/

package/src/helpers/defermentManager.spec.ts

@@ -0,0 +1,13 @@
+import { describe, expect, test } from 'vitest'
+import { DefermentManager } from './defermentManager.js'
+
+describe('deferments', () => {
+  test('defer one', async () => {
+    const deferments = new DefermentManager()
+    const x = deferments.defer({ id: 'id' })
+    expect(x).toBeInstanceOf(Promise)
+    deferments.undefer({ baseId: 'id', base: { id: 'id', speckle_type: 'type' } })
+    const b = await x
+    expect(b).toMatchSnapshot()
+  })
+})

package/src/helpers/defermentManager.ts

@@ -0,0 +1,25 @@
+import { DeferredBase } from './deferredBase.js'
+import { Base, Item } from '../types/types.js'
+
+export class DefermentManager {
+  #deferments: DeferredBase[] = []
+
+  async defer(params: { id: string }): Promise<Base> {
+    const deferredBase = this.#deferments.find((x) => x.id === params.id)
+    if (deferredBase) {
+      return await deferredBase.promise
+    }
+    const d = new DeferredBase(params.id)
+    this.#deferments.push(d)
+    return d.promise
+  }
+
+  undefer(item: Item): void {
+    const deferredIndex = this.#deferments.findIndex((x) => x.id === item.baseId)
+    if (deferredIndex !== -1) {
+      const deferredBase = this.#deferments[deferredIndex]
+      deferredBase.resolve(item.base)
+      this.#deferments.splice(deferredIndex, 1)
+    }
+  }
+}

package/src/index.ts

@@ -1,3 +1,4 @@
 import ObjectLoader2 from './operations/objectLoader2.js'
 
 export default ObjectLoader2
+export { MemoryDatabase } from './operations/memoryDatabase.js'

package/src/operations/__snapshots__/indexedDatabase.spec.ts.snap

@@ -0,0 +1,52 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`database cache > write single item to queue use getItem 1`] = `
+{
+  "base": {
+    "id": "id",
+    "speckle_type": "type",
+  },
+  "baseId": "id",
+}
+`;
+
+exports[`database cache > write two items to queue use getItem 1`] = `
+{
+  "base": {
+    "id": "id",
+    "speckle_type": "type",
+  },
+  "baseId": "id1",
+}
+`;
+
+exports[`database cache > write two items to queue use getItem 2`] = `
+{
+  "base": {
+    "id": "id",
+    "speckle_type": "type",
+  },
+  "baseId": "id2",
+}
+`;
+
+exports[`database cache > write two items to queue use processItems 1`] = `
+[
+  {
+    "base": {
+      "id": "id",
+      "speckle_type": "type",
+    },
+    "baseId": "id1",
+  },
+  {
+    "base": {
+      "id": "id",
+      "speckle_type": "type",
+    },
+    "baseId": "id2",
+  },
+]
+`;
+
+exports[`database cache > write two items to queue use processItems 2`] = `[]`;

package/src/operations/__snapshots__/objectLoader2.spec.ts.snap

@@ -0,0 +1,149 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`objectloader2 > add extra header 1`] = `
+{
+  "baseId": "baseId",
+}
+`;
+
+exports[`objectloader2 > can get a root object from cache 1`] = `
+{
+  "baseId": "baseId",
+}
+`;
+
+exports[`objectloader2 > can get a root object from downloader 1`] = `
+{
+  "baseId": "baseId",
+}
+`;
+
+exports[`objectloader2 > can get root/child object from cache using iterator 1`] = `
+[
+  {
+    "__closure": {
+      "child1Id": 100,
+    },
+    "id": "rootId",
+    "speckle_type": "type",
+  },
+  {
+    "id": "child1Id",
+  },
+]
+`;
+
+exports[`objectloader2 > can get root/child object from memory cache using iterator and getObject 1`] = `
+[
+  {
+    "__closure": {
+      "child1Id": 100,
+    },
+    "id": "rootId",
+    "speckle_type": "type",
+  },
+  {
+    "id": "child1Id",
+    "speckle_type": "type",
+  },
+]
+`;
+
+exports[`objectloader2 > can get root/child object from memory cache using iterator and getObject 2`] = `
+{
+  "id": "child1Id",
+  "speckle_type": "type",
+}
+`;
+
+exports[`objectloader2 > can get root/child object from memory downloader using iterator and getObject 1`] = `
+[
+  {
+    "__closure": {
+      "child1Id": 100,
+    },
+    "id": "rootId",
+    "speckle_type": "type",
+  },
+  {
+    "id": "child1Id",
+    "speckle_type": "type",
+  },
+]
+`;
+
+exports[`objectloader2 > can get root/child object from memory downloader using iterator and getObject 2`] = `
+{
+  "id": "child1Id",
+  "speckle_type": "type",
+}
+`;
+
+exports[`objectloader2 > can get single object from cache using iterator 1`] = `
+[
+  {
+    "id": "baseId",
+    "speckle_type": "type",
+  },
+]
+`;
+
+exports[`objectloader2 > createFromJSON test 1`] = `
+[
+  {
+    "__closure": {
+      "0e61e61edee00404ec6e0f9f594bce24": 100,
+      "f70738e3e3e593ac11099a6ed6b71154": 100,
+    },
+    "applicationId": "1",
+    "arr": null,
+    "attachedProp": null,
+    "crazyProp": null,
+    "detachedProp": null,
+    "detachedProp2": null,
+    "dynamicProp": 123,
+    "id": "efeadaca70a85ae6d3acfc93a8b380db",
+    "list": [
+      {
+        "__closure": null,
+        "referencedId": "0e61e61edee00404ec6e0f9f594bce24",
+        "speckle_type": "reference",
+      },
+    ],
+    "list2": [
+      {
+        "__closure": null,
+        "referencedId": "f70738e3e3e593ac11099a6ed6b71154",
+        "speckle_type": "reference",
+      },
+    ],
+    "speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SampleObjectBase2",
+  },
+  {
+    "applicationId": null,
+    "data": [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+    ],
+    "id": "0e61e61edee00404ec6e0f9f594bce24",
+    "speckle_type": "Speckle.Core.Models.DataChunk",
+  },
+  {
+    "applicationId": null,
+    "data": [
+      1,
+      10,
+    ],
+    "id": "f70738e3e3e593ac11099a6ed6b71154",
+    "speckle_type": "Speckle.Core.Models.DataChunk",
+  },
+]
+`;

package/src/operations/__snapshots__/serverDownloader.spec.ts.snap

@@ -0,0 +1,58 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`downloader > add extra header 1`] = `
+{
+  "base": {
+    "__closure": {
+      "childIds": 1,
+    },
+    "id": "id",
+    "speckle_type": "type",
+  },
+  "baseId": "id",
+}
+`;
+
+exports[`downloader > download batch of one 1`] = `
+[
+  {
+    "base": {
+      "id": "id",
+      "speckle_type": "type",
+    },
+    "baseId": "id",
+  },
+]
+`;
+
+exports[`downloader > download batch of two 1`] = `
+[
+  {
+    "base": {
+      "id": "id1",
+      "speckle_type": "type",
+    },
+    "baseId": "id1",
+  },
+  {
+    "base": {
+      "id": "id2",
+      "speckle_type": "type",
+    },
+    "baseId": "id2",
+  },
+]
+`;
+
+exports[`downloader > download single exists 1`] = `
+{
+  "base": {
+    "__closure": {
+      "childIds": 1,
+    },
+    "id": "id",
+    "speckle_type": "type",
+  },
+  "baseId": "id",
+}
+`;

package/src/operations/__snapshots__/traverser.spec.ts.snap

@@ -0,0 +1,45 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Traverser > root and two children with referenceId 1`] = `
+{
+  "applicationId": "1",
+  "arr": null,
+  "attachedProp": null,
+  "crazyProp": null,
+  "detachedProp": null,
+  "detachedProp2": null,
+  "dynamicProp": 123,
+  "id": "efeadaca70a85ae6d3acfc93a8b380db",
+  "list": [
+    {
+      "applicationId": null,
+      "data": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8,
+        9,
+        10,
+      ],
+      "id": "0e61e61edee00404ec6e0f9f594bce24",
+      "speckle_type": "Speckle.Core.Models.DataChunk",
+    },
+  ],
+  "list2": [
+    {
+      "applicationId": null,
+      "data": [
+        1,
+        10,
+      ],
+      "id": "f70738e3e3e593ac11099a6ed6b71154",
+      "speckle_type": "Speckle.Core.Models.DataChunk",
+    },
+  ],
+  "speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SampleObjectBase2",
+}
+`;

package/src/operations/indexedDatabase.spec.ts

@@ -6,7 +6,7 @@ import BufferQueue from '../helpers/bufferQueue.js'
 
 describe('database cache', () => {
   test('write single item to queue use getItem', async () => {
-    const i: Item = { baseId: 'id', base: { id: 'id' } }
+    const i: Item = { baseId: 'id', base: { id: 'id', speckle_type: 'type' } }
     const database = new IndexedDatabase({
       indexedDB: new IDBFactory(),
       keyRange: IDBKeyRange,
@@ -16,13 +16,12 @@ describe('database cache', () => {
     await database.disposeAsync()
 
     const x = await database.getItem({ id: 'id' })
-    expect(x).toBeDefined()
-    expect(JSON.stringify(x)).toBe(JSON.stringify(i))
+    expect(x).toMatchSnapshot()
   })
 
   test('write two items to queue use getItem', async () => {
-    const i1: Item = { baseId: 'id1', base: { id: 'id' } }
-    const i2: Item = { baseId: 'id2', base: { id: 'id' } }
+    const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } }
     const database = new IndexedDatabase({
       indexedDB: new IDBFactory(),
       keyRange: IDBKeyRange
@@ -32,17 +31,15 @@ describe('database cache', () => {
     await database.disposeAsync()
 
     const x1 = await database.getItem({ id: i1.baseId })
-    expect(x1).toBeDefined()
-    expect(JSON.stringify(x1)).toBe(JSON.stringify(i1))
+    expect(x1).toMatchSnapshot()
 
     const x2 = await database.getItem({ id: i2.baseId })
-    expect(x2).toBeDefined()
-    expect(JSON.stringify(x2)).toBe(JSON.stringify(i2))
+    expect(x2).toMatchSnapshot()
   })
 
-  test('write two items to queue use getItem', async () => {
-    const i1: Item = { baseId: 'id1', base: { id: 'id' } }
-    const i2: Item = { baseId: 'id2', base: { id: 'id' } }
+  test('write two items to queue use processItems', async () => {
+    const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } }
     const database = new IndexedDatabase({
       indexedDB: new IDBFactory(),
       keyRange: IDBKeyRange
@@ -60,10 +57,7 @@ describe('database cache', () => {
       notFoundItems
     })
 
-    expect(foundItems.values().length).toBe(2)
-    expect(JSON.stringify(foundItems.values()[0])).toBe(JSON.stringify(i1))
-    expect(JSON.stringify(foundItems.values()[1])).toBe(JSON.stringify(i2))
-
-    expect(notFoundItems.values().length).toBe(0)
+    expect(foundItems.values()).toMatchSnapshot()
+    expect(notFoundItems.values()).toMatchSnapshot()
   })
 })

package/src/operations/indexedDatabase.ts

@@ -1,7 +1,7 @@
 import BatchingQueue from '../helpers/batchingQueue.js'
 import Queue from '../helpers/queue.js'
 import { CustomLogger, Item } from '../types/types.js'
-import { isSafari } from '@speckle/shared'
+import { isSafari, TIME } from '@speckle/shared'
 import { BaseDatabaseOptions } from './options.js'
 import { Cache } from './interfaces.js'
 import { Dexie, DexieOptions, Table } from 'dexie'
@@ -91,7 +91,7 @@ export default class IndexedDatabase implements Cache {
         this.#logger(
           'pausing reads (# in write queue: ' + this.#writeQueue?.count() + ')'
         )
-        await new Promise((resolve) => setTimeout(resolve, 1000)) // Pause for 1 second, protects against out of memory
+        await new Promise((resolve) => setTimeout(resolve, TIME.second)) // Pause for 1 second, protects against out of memory
         continue
       }
       const batch = ids.slice(i, i + maxCacheReadSize)
@@ -114,7 +114,7 @@ export default class IndexedDatabase implements Cache {
       })
       // const endTime = performance.now()
       // const duration = endTime - startTime
-      // this.#logger('Read batch ' + x + ' ' + batch.length + ' ' + duration / 1000)
+      // this.#logger('Read batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
 
       // interate down here to help with pausing
       i += maxCacheReadSize
@@ -143,7 +143,7 @@ export default class IndexedDatabase implements Cache {
     await cacheDB.objects.bulkPut(batch)
     // const endTime = performance.now()
     // const duration = endTime - startTime
-    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / 1000)
+    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
   }
 
   /**

package/src/operations/memoryDatabase.ts

@@ -0,0 +1,42 @@
+import Queue from '../helpers/queue.js'
+import { Base, Item } from '../types/types.js'
+import { Cache } from './interfaces.js'
+import { MemoryDatabaseOptions } from './options.js'
+
+export class MemoryDatabase implements Cache {
+  #items: Record<string, Base>
+  constructor(options?: MemoryDatabaseOptions) {
+    this.#items = options?.items || {}
+  }
+
+  getItem(params: { id: string }): Promise<Item | undefined> {
+    const item = this.#items[params.id]
+    if (item) {
+      return Promise.resolve({ baseId: params.id, base: item })
+    }
+    return Promise.resolve(undefined)
+  }
+  processItems(params: {
+    ids: string[]
+    foundItems: Queue<Item>
+    notFoundItems: Queue<string>
+  }): Promise<void> {
+    const { ids, foundItems, notFoundItems } = params
+    for (const id of ids) {
+      const item = this.#items[id]
+      if (item) {
+        foundItems.add({ baseId: id, base: item })
+      } else {
+        notFoundItems.add(id)
+      }
+    }
+    return Promise.resolve()
+  }
+  add(item: Item): Promise<void> {
+    this.#items[item.baseId] = item.base
+    return Promise.resolve()
+  }
+  disposeAsync(): Promise<void> {
+    return Promise.resolve()
+  }
+}

package/src/operations/memoryDownloader.ts

@@ -0,0 +1,38 @@
+import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
+import { Base, Item } from '../types/types.js'
+import { Downloader } from './interfaces.js'
+
+export class MemoryDownloader implements Downloader {
+  #items: Record<string, Base>
+  #rootId: string
+  #results?: AsyncGeneratorQueue<Item>
+
+  constructor(
+    rootId: string,
+    items: Record<string, Base>,
+    results?: AsyncGeneratorQueue<Item>
+  ) {
+    this.#rootId = rootId
+    this.#items = items
+    this.#results = results
+  }
+  initializePool(): void {}
+  downloadSingle(): Promise<Item> {
+    const root = this.#items[this.#rootId]
+    if (root) {
+      return Promise.resolve({ baseId: this.#rootId, base: root })
+    }
+    throw new Error('Method not implemented.')
+  }
+  disposeAsync(): Promise<void> {
+    return Promise.resolve()
+  }
+  add(id: string): void {
+    const base = this.#items[id]
+    if (base) {
+      this.#results?.add({ baseId: id, base })
+      return
+    }
+    throw new Error('Method not implemented.')
+  }
+}

package/src/operations/objectLoader2.spec.ts

@@ -3,6 +3,9 @@ import ObjectLoader2 from './objectLoader2.js'
 import { Base, Item } from '../types/types.js'
 import { Cache, Downloader } from './interfaces.js'
 import Queue from '../helpers/queue.js'
+import { MemoryDatabase } from './memoryDatabase.js'
+import { MemoryDownloader } from './memoryDownloader.js'
+import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
 
 describe('objectloader2', () => {
   test('can get a root object from cache', async () => {
@@ -22,7 +25,7 @@ describe('objectloader2', () => {
       downloader
     })
     const x = await loader.getRootObject()
-    expect(x).toBe(root)
+    expect(x).toMatchSnapshot()
   })
 
   test('can get a root object from downloader', async () => {
@@ -50,12 +53,12 @@ describe('objectloader2', () => {
       downloader
     })
     const x = await loader.getRootObject()
-    expect(x).toBe(root)
+    expect(x).toMatchSnapshot()
   })
 
   test('can get single object from cache using iterator', async () => {
     const rootId = 'baseId'
-    const rootBase: Base = { id: 'baseId' }
+    const rootBase: Base = { id: 'baseId', speckle_type: 'type' }
     const root = { baseId: rootId, base: rootBase } as unknown as Item
     const cache = {
       getItem(params: { id: string }): Promise<Item | undefined> {
@@ -76,8 +79,7 @@ describe('objectloader2', () => {
       r.push(x)
     }
 
-    expect(r.length).toBe(1)
-    expect(r[0]).toBe(rootBase)
+    expect(r).toMatchSnapshot()
   })
 
   test('can get root/child object from cache using iterator', async () => {
@@ -85,7 +87,11 @@ describe('objectloader2', () => {
     const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
 
     const rootId = 'rootId'
-    const rootBase: Base = { id: 'rootId', __closure: { child1Id: 100 } }
+    const rootBase: Base = {
+      id: 'rootId',
+      speckle_type: 'type',
+      __closure: { child1Id: 100 }
+    }
     const root = {
       baseId: rootId,
       base: rootBase
@@ -130,10 +136,87 @@ describe('objectloader2', () => {
     for await (const x of loader.getObjectIterator()) {
       r.push(x)
     }
+    expect(r).toMatchSnapshot()
+  })
+
+  test('can get root/child object from memory cache using iterator and getObject', async () => {
+    const child1Base = { id: 'child1Id', speckle_type: 'type' } as Base
+    const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
+
+    const rootId = 'rootId'
+    const rootBase: Base = {
+      id: 'rootId',
+      speckle_type: 'type',
+      __closure: { child1Id: 100 }
+    }
+    const root = {
+      baseId: rootId,
+      base: rootBase
+    } as unknown as Item
+
+    const records: Record<string, Base> = {}
+    records[root.baseId] = rootBase
+    records[child1.baseId] = child1Base
+
+    const loader = new ObjectLoader2({
+      serverUrl: 'a',
+      streamId: 'b',
+      objectId: root.baseId,
+      cache: new MemoryDatabase({ items: records }),
+      downloader: new MemoryDownloader(rootId, records)
+    })
+    const r = []
+    const obj = loader.getObject({ id: child1.baseId })
+    for await (const x of loader.getObjectIterator()) {
+      r.push(x)
+    }
 
-    expect(r.length).toBe(2)
-    expect(r[0]).toBe(rootBase)
-    expect(r[1]).toBe(child1Base)
+    expect(obj).toBeDefined()
+    expect(r).toMatchSnapshot()
+    const obj2 = await obj
+    expect(obj2).toBe(child1Base)
+    expect(obj2).toMatchSnapshot()
+  })
+
+  test('can get root/child object from memory downloader using iterator and getObject', async () => {
+    const child1Base = { id: 'child1Id', speckle_type: 'type' } as Base
+    const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
+
+    const rootId = 'rootId'
+    const rootBase: Base = {
+      id: 'rootId',
+      speckle_type: 'type',
+      __closure: { child1Id: 100 }
+    }
+    const root = {
+      baseId: rootId,
+      base: rootBase
+    } as unknown as Item
+
+    const records: Record<string, Base> = {}
+    records[root.baseId] = rootBase
+    records[child1.baseId] = child1Base
+
+    const results: AsyncGeneratorQueue<Item> = new AsyncGeneratorQueue<Item>()
+    const loader = new ObjectLoader2({
+      serverUrl: 'a',
+      streamId: 'b',
+      objectId: root.baseId,
+      results,
+      cache: new MemoryDatabase(),
+      downloader: new MemoryDownloader(rootId, records, results)
+    })
+    const r = []
+    const obj = loader.getObject({ id: child1.baseId })
+    for await (const x of loader.getObjectIterator()) {
+      r.push(x)
+    }
+
+    expect(obj).toBeDefined()
+    expect(r).toMatchSnapshot()
+    const obj2 = await obj
+    expect(obj2).toBe(child1Base)
+    expect(obj2).toMatchSnapshot()
   })
 
   test('add extra header', async () => {
@@ -157,5 +240,58 @@ describe('objectloader2', () => {
     })
     const x = await loader.getRootObject()
     expect(x).toBe(root)
+    expect(x).toMatchSnapshot()
+  })
+
+  test('createFromJSON test', async () => {
+    const root = `{
+  "list": [{
+    "speckle_type": "reference",
+    "referencedId": "0e61e61edee00404ec6e0f9f594bce24",
+    "__closure": null
+  }],
+  "list2": [{
+    "speckle_type": "reference",
+    "referencedId": "f70738e3e3e593ac11099a6ed6b71154",
+    "__closure": null
+  }],
+  "arr": null,
+  "detachedProp": null,
+  "detachedProp2": null,
+  "attachedProp": null,
+  "crazyProp": null,
+  "applicationId": "1",
+  "speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SampleObjectBase2",
+  "dynamicProp": 123,
+  "id": "efeadaca70a85ae6d3acfc93a8b380db",
+  "__closure": {
+    "0e61e61edee00404ec6e0f9f594bce24": 100,
+    "f70738e3e3e593ac11099a6ed6b71154": 100
+  }
+}`
+
+    const list1 = `{
+  "data": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0],
+  "applicationId": null,
+  "speckle_type": "Speckle.Core.Models.DataChunk",
+  "id": "0e61e61edee00404ec6e0f9f594bce24"
+}`
+
+    const list2 = `{
+  "data": [1.0, 10.0],
+  "applicationId": null,
+  "speckle_type": "Speckle.Core.Models.DataChunk",
+  "id": "f70738e3e3e593ac11099a6ed6b71154"
+}`
+    const rootObj = JSON.parse(root) as Base
+    const list1Obj = JSON.parse(list1) as Base
+    const list2Obj = JSON.parse(list2) as Base
+
+    const loader = ObjectLoader2.createFromObjects([rootObj, list1Obj, list2Obj])
+    const r = []
+    for await (const x of loader.getObjectIterator()) {
+      r.push(x)
+    }
+    expect(r).toMatchSnapshot()
   })
 })

package/src/operations/objectLoader2.ts

@@ -4,7 +4,9 @@ import IndexedDatabase from './indexedDatabase.js'
 import ServerDownloader from './serverDownloader.js'
 import { CustomLogger, Base, Item } from '../types/types.js'
 import { ObjectLoader2Options } from './options.js'
-import { DeferredBase } from '../helpers/deferredBase.js'
+import { MemoryDownloader } from './memoryDownloader.js'
+import { MemoryDatabase } from './memoryDatabase.js'
+import { DefermentManager } from '../helpers/defermentManager.js'
 
 export default class ObjectLoader2 {
   #objectId: string
@@ -14,21 +16,22 @@ export default class ObjectLoader2 {
   #database: Cache
   #downloader: Downloader
 
-  #gathered: AsyncGeneratorQueue<Item>
+  #deferments: DefermentManager
 
-  #buffer: DeferredBase[] = []
+  #gathered: AsyncGeneratorQueue<Item>
 
   constructor(options: ObjectLoader2Options) {
     this.#objectId = options.objectId
 
     this.#logger = options.logger || console.log
-    this.#gathered = new AsyncGeneratorQueue()
+    this.#gathered = options.results || new AsyncGeneratorQueue()
+    this.#deferments = new DefermentManager()
     this.#database =
       options.cache ||
       new IndexedDatabase({
         logger: this.#logger,
-        maxCacheReadSize: 10000,
-        maxCacheWriteSize: 5000,
+        maxCacheReadSize: 10_000,
+        maxCacheWriteSize: 5_000,
         indexedDB: options.indexedDB,
         keyRange: options.keyRange
       })
@@ -69,13 +72,7 @@ export default class ObjectLoader2 {
     if (item) {
       return item.base
     }
-    const deferredBase = this.#buffer.find((x) => x.id === params.id)
-    if (deferredBase) {
-      return await deferredBase.promise
-    }
-    const d = new DeferredBase(params.id)
-    this.#buffer.push(d)
-    return d
+    return await this.#deferments.defer({ id: params.id })
   }
 
   async getTotalObjectCount() {
@@ -103,12 +100,7 @@ export default class ObjectLoader2 {
     })
     let count = 0
     for await (const item of this.#gathered.consume()) {
-      const deferredIndex = this.#buffer.findIndex((x) => x.id === item.baseId)
-      if (deferredIndex !== -1) {
-        const deferredBase = this.#buffer[deferredIndex]
-        deferredBase.resolve(item.base)
-        this.#buffer.splice(deferredIndex, 1)
-      }
+      this.#deferments.undefer(item)
       yield item.base
       count++
       if (count >= total) {
@@ -117,4 +109,25 @@ export default class ObjectLoader2 {
     }
     await processPromise
   }
+
+  static createFromObjects(objects: Base[]): ObjectLoader2 {
+    const root = objects[0]
+    const records: Record<string, Base> = {}
+    objects.forEach((element) => {
+      records[element.id] = element
+    })
+    const loader = new ObjectLoader2({
+      serverUrl: 'dummy',
+      streamId: 'dummy',
+      objectId: root.id,
+      cache: new MemoryDatabase({ items: records }),
+      downloader: new MemoryDownloader(root.id, records)
+    })
+    return loader
+  }
+
+  static createFromJSON(json: string): ObjectLoader2 {
+    const jsonObj = JSON.parse(json) as Base[]
+    return this.createFromObjects(jsonObj)
+  }
 }

package/src/operations/options.ts

@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-function-type */
+import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
 import Queue from '../helpers/queue.js'
-import { CustomLogger, Fetcher, Item } from '../types/types.js'
+import { Base, CustomLogger, Fetcher, Item } from '../types/types.js'
 import { Cache, Downloader } from './interfaces.js'
 
 export interface ObjectLoader2Options {
@@ -12,6 +13,7 @@ export interface ObjectLoader2Options {
   token?: string
   logger?: CustomLogger
   headers?: Headers
+  results?: AsyncGeneratorQueue<Item>
   cache?: Cache
   downloader?: Downloader
 }
@@ -39,3 +41,8 @@ export interface BaseDownloadOptions {
   database: Cache
   results: Queue<Item>
 }
+
+export interface MemoryDatabaseOptions {
+  logger?: CustomLogger
+  items?: Record<string, Base>
+}

package/src/operations/serverDownloader.spec.ts

@@ -9,9 +9,9 @@ import ServerDownloader from './serverDownloader.js'
 describe('downloader', () => {
   test('download batch of one', async () => {
     const fetchMocker = createFetchMock(vi)
-    const i: Item = { baseId: 'id', base: { id: 'id' } }
+    const i: Item = { baseId: 'id', base: { id: 'id', speckle_type: 'type' } }
     fetchMocker.mockResponseOnce('id\t' + JSON.stringify(i.base) + '\n')
-    const results = new AsyncGeneratorQueue<Item>()
+    const results = new AsyncGeneratorQueue()
     const db = {
       async add(): Promise<void> {
         return Promise.resolve()
@@ -36,18 +36,17 @@ describe('downloader', () => {
       r.push(x)
     }
 
-    expect(r.length).toBe(1)
-    expect(JSON.stringify(r[0])).toBe(JSON.stringify(i))
+    expect(r).toMatchSnapshot()
   })
 
   test('download batch of two', async () => {
     const fetchMocker = createFetchMock(vi)
-    const i1: Item = { baseId: 'id1', base: { id: 'id1' } }
-    const i2: Item = { baseId: 'id2', base: { id: 'id2' } }
+    const i1: Item = { baseId: 'id1', base: { id: 'id1', speckle_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id2', speckle_type: 'type' } }
     fetchMocker.mockResponseOnce(
       'id1\t' + JSON.stringify(i1.base) + '\nid2\t' + JSON.stringify(i2.base) + '\n'
     )
-    const results = new AsyncGeneratorQueue<Item>()
+    const results = new AsyncGeneratorQueue()
     const db = {
       async add(): Promise<void> {
         return Promise.resolve()
@@ -72,16 +71,17 @@ describe('downloader', () => {
       r.push(x)
     }
 
-    expect(r.length).toBe(2)
-    expect(JSON.stringify(r[0])).toBe(JSON.stringify(i1))
-    expect(JSON.stringify(r[1])).toBe(JSON.stringify(i2))
+    expect(r).toMatchSnapshot()
   })
 
   test('download single exists', async () => {
     const fetchMocker = createFetchMock(vi)
-    const i: Item = { baseId: 'id', base: { id: 'id', __closure: { childIds: 1 } } }
+    const i: Item = {
+      baseId: 'id',
+      base: { id: 'id', speckle_type: 'type', __closure: { childIds: 1 } }
+    }
     fetchMocker.mockResponseOnce(JSON.stringify(i.base))
-    const results = new AsyncGeneratorQueue<Item>()
+    const results = new AsyncGeneratorQueue()
     const db = {
       async add(): Promise<void> {
         return Promise.resolve()
@@ -98,17 +98,20 @@ describe('downloader', () => {
       fetch: fetchMocker
     })
     const x = await downloader.downloadSingle()
-    expect(JSON.stringify(x)).toBe(JSON.stringify(i))
+    expect(x).toMatchSnapshot()
   })
 
   test('add extra header', async () => {
     const fetchMocker = createFetchMock(vi)
-    const i: Item = { baseId: 'id', base: { id: 'id', __closure: { childIds: 1 } } }
+    const i: Item = {
+      baseId: 'id',
+      base: { id: 'id', speckle_type: 'type', __closure: { childIds: 1 } }
+    }
     fetchMocker.mockResponseIf(
       (req) => req.headers.get('x-test') === 'asdf',
       JSON.stringify(i.base)
     )
-    const results = new AsyncGeneratorQueue<Item>()
+    const results = new AsyncGeneratorQueue()
     const db = {
       async add(): Promise<void> {
         return Promise.resolve()
@@ -128,6 +131,6 @@ describe('downloader', () => {
       fetch: fetchMocker
     })
     const x = await downloader.downloadSingle()
-    expect(JSON.stringify(x)).toBe(JSON.stringify(i))
+    expect(x).toMatchSnapshot()
   })
 })

package/src/operations/traverser.spec.ts

@@ -0,0 +1,58 @@
+import { describe, expect, test } from 'vitest'
+import { Base } from '../types/types.js'
+import ObjectLoader2 from './objectLoader2.js'
+import Traverser from './traverser.js'
+
+describe('Traverser', () => {
+  test('root and two children with referenceId', async () => {
+    const root = `{
+  "list": [{
+    "speckle_type": "reference",
+    "referencedId": "0e61e61edee00404ec6e0f9f594bce24",
+    "__closure": null
+  }],
+  "list2": [{
+    "speckle_type": "reference",
+    "referencedId": "f70738e3e3e593ac11099a6ed6b71154",
+    "__closure": null
+  }],
+  "arr": null,
+  "detachedProp": null,
+  "detachedProp2": null,
+  "attachedProp": null,
+  "crazyProp": null,
+  "applicationId": "1",
+  "speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SampleObjectBase2",
+  "dynamicProp": 123,
+  "id": "efeadaca70a85ae6d3acfc93a8b380db",
+  "__closure": {
+    "0e61e61edee00404ec6e0f9f594bce24": 100,
+    "f70738e3e3e593ac11099a6ed6b71154": 100
+  }
+}`
+
+    const list1 = `{
+  "data": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0],
+  "applicationId": null,
+  "speckle_type": "Speckle.Core.Models.DataChunk",
+  "id": "0e61e61edee00404ec6e0f9f594bce24"
+}`
+
+    const list2 = `{
+  "data": [1.0, 10.0],
+  "applicationId": null,
+  "speckle_type": "Speckle.Core.Models.DataChunk",
+  "id": "f70738e3e3e593ac11099a6ed6b71154"
+}`
+
+    const rootObj = JSON.parse(root) as Base
+    const list1Obj = JSON.parse(list1) as Base
+    const list2Obj = JSON.parse(list2) as Base
+
+    const loader = ObjectLoader2.createFromObjects([rootObj, list1Obj, list2Obj])
+
+    const traverser = new Traverser(loader)
+    const r = await traverser.traverse()
+    expect(r).toMatchSnapshot()
+  })
+})

package/src/operations/traverser.ts

@@ -0,0 +1,112 @@
+import { Base, DataChunk, isBase, isReference, isScalar } from '../types/types.js'
+import ObjectLoader2 from './objectLoader2.js'
+
+export type ProgressStage = 'download' | 'construction'
+export type OnProgress = (e: {
+  stage: ProgressStage
+  current: number
+  total: number
+}) => void
+
+export interface TraverserOptions {
+  excludeProps?: string[]
+}
+
+export default class Traverser {
+  #loader: ObjectLoader2
+  #options: TraverserOptions
+
+  #totalChildrenCount = 0
+  #traversedReferencesCount = 0
+
+  constructor(loader: ObjectLoader2, options?: TraverserOptions) {
+    this.#options = options || {}
+    this.#loader = loader
+  }
+
+  async traverse(onProgress?: OnProgress): Promise<Base> {
+    let firstObjectPromise: Promise<Base> | undefined = undefined
+    for await (const obj of this.#loader.getObjectIterator()) {
+      if (!firstObjectPromise) {
+        firstObjectPromise = this.traverseBase(obj, onProgress)
+      }
+    }
+
+    if (firstObjectPromise) {
+      return await firstObjectPromise
+    } else {
+      throw new Error('No objects found')
+    }
+  }
+
+  async traverseArray(array: Array<unknown>, onProgress?: OnProgress): Promise<void> {
+    for (let i = 0; i < 10; i++) {
+      const prop = array[i]
+      if (isScalar(prop)) continue
+      if (isBase(prop)) {
+        array[i] = await this.traverseBase(prop, onProgress)
+      } else if (isReference(prop)) {
+        array[i] = await this.traverseBase(
+          await this.#loader.getObject({ id: prop.referencedId }),
+          onProgress
+        )
+      }
+    }
+  }
+
+  async traverseBase(base: Base, onProgress?: OnProgress): Promise<Base> {
+    for (const ignoredProp of this.#options.excludeProps || []) {
+      delete (base as never)[ignoredProp]
+    }
+    if (base.__closure) {
+      const ids = Object.keys(base.__closure)
+      const promises: Promise<Base>[] = []
+      for (const id of ids) {
+        promises.push(
+          this.traverseBase(await this.#loader.getObject({ id }), onProgress)
+        )
+      }
+      await Promise.all(promises)
+    }
+    delete (base as never)['__closure']
+
+    // De-chunk
+    if (base.speckle_type?.includes('DataChunk')) {
+      const chunk = base as DataChunk
+      if (chunk.data) {
+        await this.traverseArray(chunk.data, onProgress)
+      }
+    }
+
+    //other props
+    for (const prop in base) {
+      if (prop === '__closure') continue
+      if (prop === 'referenceId') continue
+      if (prop === 'speckle_type') continue
+      if (prop === 'data') continue
+      const baseProp = (base as unknown as Record<string, unknown>)[prop]
+      if (isScalar(baseProp)) continue
+      if (isBase(baseProp)) {
+        await this.traverseBase(baseProp, onProgress)
+      } else if (isReference(baseProp)) {
+        await this.traverseBase(
+          await this.#loader.getObject({ id: baseProp.referencedId }),
+          onProgress
+        )
+      } else if (Array.isArray(baseProp)) {
+        await this.traverseArray(baseProp, onProgress)
+      }
+    }
+    if (onProgress) {
+      onProgress({
+        stage: 'construction',
+        current:
+          ++this.#traversedReferencesCount > this.#totalChildrenCount
+            ? this.#totalChildrenCount
+            : this.#traversedReferencesCount,
+        total: this.#totalChildrenCount
+      })
+    }
+    return base
+  }
+}

package/src/test/e2e.spec.ts

@@ -2,35 +2,40 @@ import { describe, test, expect } from 'vitest'
 import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
 import ObjectLoader2 from '../operations/objectLoader2.js'
 import { Base } from '../types/types.js'
+import { TIME } from '@speckle/shared'
 
 describe('e2e', () => {
-  test('download small model', async () => {
-    // Revit sample house (good for bim-like stuff with many display meshes)
-    //const resource = 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
-    const loader = new ObjectLoader2({
-      serverUrl: 'https://app.speckle.systems',
-      streamId: 'da9e320dad',
-      objectId: '31d10c0cea569a1e26809658ed27e281',
-      indexedDB: new IDBFactory(),
-      keyRange: IDBKeyRange
-    })
+  test(
+    'download small model',
+    async () => {
+      // Revit sample house (good for bim-like stuff with many display meshes)
+      //const resource = 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
+      const loader = new ObjectLoader2({
+        serverUrl: 'https://app.speckle.systems',
+        streamId: 'da9e320dad',
+        objectId: '31d10c0cea569a1e26809658ed27e281',
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
 
-    const getObjectPromise = loader.getObject({
-      id: '1708a78e057e8115f924c620ba686db6'
-    })
+      const getObjectPromise = loader.getObject({
+        id: '1708a78e057e8115f924c620ba686db6'
+      })
 
-    const bases: Base[] = []
-    for await (const obj of loader.getObjectIterator()) {
-      bases.push(obj)
-    }
+      const bases: Base[] = []
+      for await (const obj of loader.getObjectIterator()) {
+        bases.push(obj)
+      }
 
-    expect(await loader.getTotalObjectCount()).toBe(1328)
-    expect(bases.length).toBe(1328)
-    const base = await getObjectPromise
-    expect(base).toBeDefined()
-    expect(base.id).toBe('1708a78e057e8115f924c620ba686db6')
-    const base2 = await loader.getObject({ id: '3841e3cbc45d52c47bc2f1b7b0ad4eb9' })
-    expect(base2).toBeDefined()
-    expect(base2.id).toBe('3841e3cbc45d52c47bc2f1b7b0ad4eb9')
-  }, 10000)
+      expect(await loader.getTotalObjectCount()).toBe(1328)
+      expect(bases.length).toBe(1328)
+      const base = await getObjectPromise
+      expect(base).toBeDefined()
+      expect(base.id).toBe('1708a78e057e8115f924c620ba686db6')
+      const base2 = await loader.getObject({ id: '3841e3cbc45d52c47bc2f1b7b0ad4eb9' })
+      expect(base2).toBeDefined()
+      expect(base2.id).toBe('3841e3cbc45d52c47bc2f1b7b0ad4eb9')
+    },
+    10 * TIME.second
+  )
 })

package/src/types/types.ts

@@ -12,9 +12,20 @@ export interface Item {
 
 export interface Base {
   id: string
+  speckle_type: string
   __closure?: Record<string, number>
 }
 
+export interface Reference {
+  speckle_type: string
+  referencedId: string
+  __closure?: Record<string, number>
+}
+
+export interface DataChunk extends Base {
+  data?: Base[]
+}
+
 export function isBase(maybeBase?: unknown): maybeBase is Base {
   return (
     maybeBase !== null &&
@@ -23,3 +34,27 @@ export function isBase(maybeBase?: unknown): maybeBase is Base {
     typeof maybeBase.id === 'string'
   )
 }
+
+export function isReference(maybeRef?: unknown): maybeRef is Reference {
+  return (
+    maybeRef !== null &&
+    typeof maybeRef === 'object' &&
+    'referencedId' in maybeRef &&
+    typeof maybeRef.referencedId === 'string'
+  )
+}
+
+export function isScalar(
+  value: unknown
+): value is string | number | boolean | bigint | symbol | undefined {
+  const type = typeof value
+  return (
+    value === null ||
+    type === 'string' ||
+    type === 'number' ||
+    type === 'boolean' ||
+    type === 'bigint' ||
+    type === 'symbol' ||
+    type === 'undefined'
+  )
+}