@kingsy/objectloader2 2.25.7

package/src/operations/objectLoader2.spec.ts

@@ -0,0 +1,229 @@
+import { describe, expect, test } from 'vitest'
+import { ObjectLoader2 } from './objectLoader2.js'
+import { Base, Item } from '../types/types.js'
+import { MemoryDownloader } from './downloaders/memoryDownloader.js'
+import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
+import { MemoryDatabase } from './databases/memoryDatabase.js'
+import IndexedDatabase from './databases/indexedDatabase.js'
+
+describe('objectloader2', () => {
+  test('can get a root object from cache', async () => {
+    const rootId = 'baseId'
+    const rootBase: Base = { id: 'baseId', kingsy_type: 'type' }
+    const downloader = new MemoryDownloader(
+      rootId,
+      new Map<string, Base>([[rootId, rootBase]])
+    )
+    const loader = new ObjectLoader2({
+      rootId,
+      downloader,
+      database: new IndexedDatabase({
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
+    })
+    const x = await loader.getRootObject()
+    expect(x).toMatchSnapshot()
+  })
+
+  test('can get a root object from downloader', async () => {
+    const rootId = 'baseId'
+    const rootBase: Base = { id: 'baseId', kingsy_type: 'type' }
+    const downloader = new MemoryDownloader(
+      rootId,
+      new Map<string, Base>([[rootId, rootBase]])
+    )
+    const loader = new ObjectLoader2({
+      rootId,
+      downloader,
+      database: new IndexedDatabase({
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
+    })
+    const x = await loader.getRootObject()
+    expect(x).toMatchSnapshot()
+  })
+
+  test('can get single object from cache using iterator', async () => {
+    const rootId = 'baseId'
+    const rootBase: Base = { id: 'baseId', kingsy_type: 'type' }
+
+    const downloader = new MemoryDownloader(
+      rootId,
+      new Map<string, Base>([[rootId, rootBase]])
+    )
+    const loader = new ObjectLoader2({
+      rootId,
+      downloader,
+      database: new IndexedDatabase({
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
+    })
+    const r = []
+    for await (const x of loader.getObjectIterator()) {
+      r.push(x)
+    }
+
+    expect(r).toMatchSnapshot()
+  })
+
+  test('can get root/child object from memory cache using iterator and getObject', async () => {
+    const child1Base = { id: 'child1Id', kingsy_type: 'type' } as Base
+    const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
+
+    const rootId = 'rootId'
+    const rootBase: Base = {
+      id: 'rootId',
+      kingsy_type: 'type',
+      __closure: { child1Id: 100 }
+    }
+    const root = {
+      baseId: rootId,
+      base: rootBase
+    } as Item
+
+    const records: Map<string, Base> = new Map<string, Base>()
+    records.set(root.baseId, rootBase)
+    records.set(child1.baseId, child1Base)
+
+    const loader = new ObjectLoader2({
+      rootId: root.baseId,
+      downloader: new MemoryDownloader(rootId, records),
+      database: new MemoryDatabase({ items: records })
+    })
+
+    const r = []
+    const obj = loader.getObject({ id: child1.baseId })
+    for await (const x of loader.getObjectIterator()) {
+      r.push(x)
+    }
+
+    expect(obj).toBeDefined()
+    expect(r).toMatchSnapshot()
+    const obj2 = await obj
+    expect(obj2).toBe(child1Base)
+    expect(obj2).toMatchSnapshot()
+  })
+
+  test('can get root/child object from memory downloader using iterator and getObject', async () => {
+    const child1Base = { id: 'child1Id', kingsy_type: 'type' } as Base
+    const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
+
+    const rootId = 'rootId'
+    const rootBase: Base = {
+      id: 'rootId',
+      kingsy_type: 'type',
+      __closure: { child1Id: 100 }
+    }
+    const root = {
+      baseId: rootId,
+      base: rootBase
+    } as unknown as Item
+
+    const records: Map<string, Base> = new Map<string, Base>()
+    records.set(root.baseId, rootBase)
+    records.set(child1.baseId, child1Base)
+
+    const loader = new ObjectLoader2({
+      rootId: root.baseId,
+      downloader: new MemoryDownloader(rootId, records),
+      database: new IndexedDatabase({
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
+    })
+    const r = []
+    const obj = loader.getObject({ id: child1.baseId })
+    for await (const x of loader.getObjectIterator()) {
+      r.push(x)
+    }
+
+    expect(obj).toBeDefined()
+    expect(r).toMatchSnapshot()
+    const obj2 = await obj
+    expect(obj2).toBe(child1Base)
+    expect(obj2).toMatchSnapshot()
+  })
+
+  test('add extra header', async () => {
+    const rootId = 'rootId'
+    const rootBase: Base = {
+      id: 'rootId',
+      kingsy_type: 'type',
+      __closure: { child1Id: 100 }
+    }
+    const root = {
+      baseId: rootId,
+      base: rootBase
+    } as Item
+
+    const records: Map<string, Base> = new Map<string, Base>()
+    records.set(root.baseId, rootBase)
+    const headers = new Headers()
+    headers.set('x-test', 'asdf')
+    const loader = new ObjectLoader2({
+      rootId: root.baseId,
+      downloader: new MemoryDownloader(rootId, records),
+      database: new IndexedDatabase({
+        indexedDB: new IDBFactory(),
+        keyRange: IDBKeyRange
+      })
+    })
+    const x = await loader.getRootObject()
+    expect(x).toMatchSnapshot()
+  })
+
+  test('createFromJSON test', async () => {
+    const root = `{
+  "list": [{
+    "kingsy_type": "reference",
+    "referencedId": "0e61e61edee00404ec6e0f9f594bce24",
+    "__closure": null
+  }],
+  "list2": [{
+    "kingsy_type": "reference",
+    "referencedId": "f70738e3e3e593ac11099a6ed6b71154",
+    "__closure": null
+  }],
+  "arr": null,
+  "detachedProp": null,
+  "detachedProp2": null,
+  "attachedProp": null,
+  "crazyProp": null,
+  "applicationId": "1",
+  "kingsy_type": "KingSy.Core.Tests.Unit.Models.BaseTests+SampleObjectBase2",
+  "dynamicProp": 123,
+  "id": "efeadaca70a85ae6d3acfc93a8b380db",
+  "__closure": {
+    "0e61e61edee00404ec6e0f9f594bce24": 100,
+    "f70738e3e3e593ac11099a6ed6b71154": 100
+  }
+}`
+
+    const list1 = `{
+  "data": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0],
+  "applicationId": null,
+  "kingsy_type": "KingSy.Core.Models.DataChunk",
+  "id": "0e61e61edee00404ec6e0f9f594bce24"
+}`
+
+    const list2 = `{
+  "data": [1.0, 10.0],
+  "applicationId": null,
+  "kingsy_type": "KingSy.Core.Models.DataChunk",
+  "id": "f70738e3e3e593ac11099a6ed6b71154"
+}`
+    const rootObj = JSON.parse(root) as Base
+    const list1Obj = JSON.parse(list1) as Base
+    const list2Obj = JSON.parse(list2) as Base
+
+    const loader = ObjectLoader2.createFromObjects([rootObj, list1Obj, list2Obj])
+    const r = []
+    for await (const x of loader.getObjectIterator()) {
+      r.push(x)
+    }
+    expect(r).toMatchSnapshot()
+  })
+})

package/src/operations/objectLoader2.ts

@@ -0,0 +1,119 @@
+import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
+import { Downloader, Database } from './interfaces.js'
+import { CustomLogger, Base, Item } from '../types/types.js'
+import { CacheOptions, ObjectLoader2Options } from './options.js'
+import { DefermentManager } from '../helpers/defermentManager.js'
+import { CacheReader } from '../helpers/cacheReader.js'
+import { CachePump } from '../helpers/cachePump.js'
+import AggregateQueue from '../helpers/aggregateQueue.js'
+import { ObjectLoader2Factory } from './objectLoader2Factory.js'
+
+export class ObjectLoader2 {
+  #rootId: string
+
+  #logger: CustomLogger
+
+  #database: Database
+  #downloader: Downloader
+  #pump: CachePump
+  #cache: CacheReader
+
+  #deferments: DefermentManager
+
+  #gathered: AsyncGeneratorQueue<Item>
+
+  #root?: Item = undefined
+
+  constructor(options: ObjectLoader2Options) {
+    this.#rootId = options.rootId
+    this.#logger = options.logger || console.log
+
+    const cacheOptions: CacheOptions = {
+      logger: this.#logger,
+      maxCacheReadSize: 10_000,
+      maxCacheWriteSize: 10_000,
+      maxWriteQueueSize: 40_000,
+      maxCacheBatchWriteWait: 3_000,
+      maxCacheBatchReadWait: 3_000
+    }
+
+    this.#gathered = new AsyncGeneratorQueue()
+    this.#database = options.database
+    this.#deferments = new DefermentManager({
+      maxSizeInMb: 2_000, // 2 GBs
+      ttlms: 15_000, // 15 seconds
+      logger: this.#logger
+    })
+    this.#cache = new CacheReader(this.#database, this.#deferments, cacheOptions)
+    this.#pump = new CachePump(
+      this.#database,
+      this.#gathered,
+      this.#deferments,
+      cacheOptions
+    )
+    this.#downloader = options.downloader
+  }
+
+  async disposeAsync(): Promise<void> {
+    await Promise.all([
+      this.#downloader.disposeAsync(),
+      this.#cache.disposeAsync(),
+      this.#pump.disposeAsync()
+    ])
+    this.#deferments.dispose()
+  }
+
+  async getRootObject(): Promise<Item | undefined> {
+    if (!this.#root) {
+      this.#root = (await this.#database.getAll([this.#rootId]))[0]
+      if (!this.#root) {
+        this.#root = await this.#downloader.downloadSingle()
+      }
+    }
+    return this.#root
+  }
+
+  async getObject(params: { id: string }): Promise<Base> {
+    return await this.#cache.getObject({ id: params.id })
+  }
+
+  async getTotalObjectCount(): Promise<number> {
+    const rootObj = await this.getRootObject()
+    const totalChildrenCount = Object.keys(rootObj?.base.__closure || {}).length
+    return totalChildrenCount + 1 //count the root
+  }
+
+  async *getObjectIterator(): AsyncGenerator<Base> {
+    const rootItem = await this.getRootObject()
+    if (rootItem === undefined) {
+      this.#logger('No root object found!')
+      return
+    }
+    //only for root
+    this.#pump.add(rootItem)
+    yield rootItem.base
+    if (!rootItem.base.__closure) return
+
+    //sort the closures by their values descending
+    const sortedClosures = Object.entries(rootItem.base.__closure).sort(
+      (a, b) => b[1] - a[1]
+    )
+    const children = sortedClosures.map((x) => x[0])
+    const total = children.length
+    this.#downloader.initializePool({
+      results: new AggregateQueue(this.#gathered, this.#pump),
+      total
+    })
+    for await (const item of this.#pump.gather(children, this.#downloader)) {
+      yield item.base
+    }
+  }
+
+  static createFromObjects(objects: Base[]): ObjectLoader2 {
+    return ObjectLoader2Factory.createFromObjects(objects)
+  }
+
+  static createFromJSON(json: string): ObjectLoader2 {
+    return ObjectLoader2Factory.createFromJSON(json)
+  }
+}

package/src/operations/objectLoader2Factory.ts

@@ -0,0 +1,78 @@
+import { Base, CustomLogger } from '../types/types.js'
+import IndexedDatabase from './databases/indexedDatabase.js'
+import { MemoryDatabase } from './databases/memoryDatabase.js'
+import { MemoryDownloader } from './downloaders/memoryDownloader.js'
+import ServerDownloader from './downloaders/serverDownloader.js'
+import { ObjectLoader2 } from './objectLoader2.js'
+
+export interface ObjectLoader2FactoryOptions {
+  useMemoryCache?: boolean
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+  keyRange?: { bound: Function; lowerBound: Function; upperBound: Function }
+  indexedDB?: IDBFactory
+  logger?: CustomLogger
+}
+
+export class ObjectLoader2Factory {
+  static createFromObjects(objects: Base[]): ObjectLoader2 {
+    const root = objects[0]
+    const records: Map<string, Base> = new Map<string, Base>()
+    objects.forEach((element) => {
+      records.set(element.id, element)
+    })
+    const loader = new ObjectLoader2({
+      rootId: root.id,
+      database: new MemoryDatabase({ items: records }),
+      downloader: new MemoryDownloader(root.id, records)
+    })
+    return loader
+  }
+
+  static createFromJSON(json: string): ObjectLoader2 {
+    const jsonObj = JSON.parse(json) as Base[]
+    return this.createFromObjects(jsonObj)
+  }
+
+  static createFromUrl(params: {
+    serverUrl: string
+    streamId: string
+    objectId: string
+    token?: string
+    headers?: Headers
+    options?: ObjectLoader2FactoryOptions
+  }): ObjectLoader2 {
+    let loader: ObjectLoader2
+    if (params.options?.useMemoryCache) {
+      loader = new ObjectLoader2({
+        rootId: params.objectId,
+        downloader: new ServerDownloader({
+          serverUrl: params.serverUrl,
+          streamId: params.streamId,
+          objectId: params.objectId,
+          token: params.token,
+          headers: params.headers
+        }),
+        database: new MemoryDatabase({
+          items: new Map<string, Base>()
+        })
+      })
+    } else {
+      loader = new ObjectLoader2({
+        rootId: params.objectId,
+        downloader: new ServerDownloader({
+          serverUrl: params.serverUrl,
+          streamId: params.streamId,
+          objectId: params.objectId,
+          token: params.token,
+          headers: params.headers
+        }),
+        database: new IndexedDatabase({
+          logger: params.options?.logger,
+          indexedDB: params.options?.indexedDB,
+          keyRange: params.options?.keyRange
+        })
+      })
+    }
+    return loader
+  }
+}

package/src/operations/options.ts

@@ -0,0 +1,29 @@
+import { Base, CustomLogger } from '../types/types.js'
+import { Downloader, Database } from './interfaces.js'
+
+export interface ObjectLoader2Options {
+  rootId: string
+  downloader: Downloader
+  database: Database
+  logger?: CustomLogger
+}
+
+export interface CacheOptions {
+  logger?: CustomLogger
+  maxCacheReadSize: number
+  maxCacheWriteSize: number
+  maxCacheBatchWriteWait: number
+  maxCacheBatchReadWait: number
+  maxWriteQueueSize: number
+}
+
+export interface MemoryDatabaseOptions {
+  logger?: CustomLogger
+  items?: Map<string, Base>
+}
+
+export interface DefermentManagerOptions {
+  logger?: CustomLogger
+  maxSizeInMb: number
+  ttlms: number
+}

package/src/operations/traverser.spec.ts

@@ -0,0 +1,58 @@
+import { describe, expect, test } from 'vitest'
+import { Base } from '../types/types.js'
+import { ObjectLoader2 } from './objectLoader2.js'
+import Traverser from './traverser.js'
+
+describe('Traverser', () => {
+  test('root and two children with referenceId', async () => {
+    const root = `{
+  "list": [{
+    "kingsy_type": "reference",
+    "referencedId": "0e61e61edee00404ec6e0f9f594bce24",
+    "__closure": null
+  }],
+  "list2": [{
+    "kingsy_type": "reference",
+    "referencedId": "f70738e3e3e593ac11099a6ed6b71154",
+    "__closure": null
+  }],
+  "arr": null,
+  "detachedProp": null,
+  "detachedProp2": null,
+  "attachedProp": null,
+  "crazyProp": null,
+  "applicationId": "1",
+  "kingsy_type": "KingSy.Core.Tests.Unit.Models.BaseTests+SampleObjectBase2",
+  "dynamicProp": 123,
+  "id": "efeadaca70a85ae6d3acfc93a8b380db",
+  "__closure": {
+    "0e61e61edee00404ec6e0f9f594bce24": 100,
+    "f70738e3e3e593ac11099a6ed6b71154": 100
+  }
+}`
+
+    const list1 = `{
+  "data": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0],
+  "applicationId": null,
+  "kingsy_type": "KingSy.Core.Models.DataChunk",
+  "id": "0e61e61edee00404ec6e0f9f594bce24"
+}`
+
+    const list2 = `{
+  "data": [1.0, 10.0],
+  "applicationId": null,
+  "kingsy_type": "KingSy.Core.Models.DataChunk",
+  "id": "f70738e3e3e593ac11099a6ed6b71154"
+}`
+
+    const rootObj = JSON.parse(root) as Base
+    const list1Obj = JSON.parse(list1) as Base
+    const list2Obj = JSON.parse(list2) as Base
+
+    const loader = ObjectLoader2.createFromObjects([rootObj, list1Obj, list2Obj])
+
+    const traverser = new Traverser(loader)
+    const r = await traverser.traverse()
+    expect(r).toMatchSnapshot()
+  })
+})

package/src/operations/traverser.ts

@@ -0,0 +1,112 @@
+import { Base, DataChunk, isBase, isReference, isScalar } from '../types/types.js'
+import { ObjectLoader2 } from './objectLoader2.js'
+
+export type ProgressStage = 'download' | 'construction'
+export type OnProgress = (e: {
+  stage: ProgressStage
+  current: number
+  total: number
+}) => void
+
+export interface TraverserOptions {
+  excludeProps?: string[]
+}
+
+export default class Traverser {
+  #loader: ObjectLoader2
+  #options: TraverserOptions
+
+  #totalChildrenCount = 0
+  #traversedReferencesCount = 0
+
+  constructor(loader: ObjectLoader2, options?: TraverserOptions) {
+    this.#options = options || {}
+    this.#loader = loader
+  }
+
+  async traverse(onProgress?: OnProgress): Promise<Base> {
+    let firstObjectPromise: Promise<Base> | undefined = undefined
+    for await (const obj of this.#loader.getObjectIterator()) {
+      if (!firstObjectPromise) {
+        firstObjectPromise = this.traverseBase(obj, onProgress)
+      }
+    }
+
+    if (firstObjectPromise) {
+      return await firstObjectPromise
+    } else {
+      throw new Error('No objects found')
+    }
+  }
+
+  async traverseArray(array: Array<unknown>, onProgress?: OnProgress): Promise<void> {
+    for (let i = 0; i < 10; i++) {
+      const prop = array[i]
+      if (isScalar(prop)) continue
+      if (isBase(prop)) {
+        array[i] = await this.traverseBase(prop, onProgress)
+      } else if (isReference(prop)) {
+        array[i] = await this.traverseBase(
+          await this.#loader.getObject({ id: prop.referencedId }),
+          onProgress
+        )
+      }
+    }
+  }
+
+  async traverseBase(base: Base, onProgress?: OnProgress): Promise<Base> {
+    for (const ignoredProp of this.#options.excludeProps || []) {
+      delete (base as never)[ignoredProp]
+    }
+    if (base.__closure) {
+      const ids = Object.keys(base.__closure)
+      const promises: Promise<Base>[] = []
+      for (const id of ids) {
+        promises.push(
+          this.traverseBase(await this.#loader.getObject({ id }), onProgress)
+        )
+      }
+      await Promise.all(promises)
+    }
+    delete (base as never)['__closure']
+
+    // De-chunk
+    if (base.kingsy_type?.includes('DataChunk')) {
+      const chunk = base as DataChunk
+      if (chunk.data) {
+        await this.traverseArray(chunk.data, onProgress)
+      }
+    }
+
+    //other props
+    for (const prop in base) {
+      if (prop === '__closure') continue
+      if (prop === 'referenceId') continue
+      if (prop === 'kingsy_type') continue
+      if (prop === 'data') continue
+      const baseProp = (base as unknown as Record<string, unknown>)[prop]
+      if (isScalar(baseProp)) continue
+      if (isBase(baseProp)) {
+        await this.traverseBase(baseProp, onProgress)
+      } else if (isReference(baseProp)) {
+        await this.traverseBase(
+          await this.#loader.getObject({ id: baseProp.referencedId }),
+          onProgress
+        )
+      } else if (Array.isArray(baseProp)) {
+        await this.traverseArray(baseProp, onProgress)
+      }
+    }
+    if (onProgress) {
+      onProgress({
+        stage: 'construction',
+        current:
+          ++this.#traversedReferencesCount > this.#totalChildrenCount
+            ? this.#totalChildrenCount
+            : this.#traversedReferencesCount,
+        total: this.#totalChildrenCount
+      })
+    }
+    return base
+  }
+}

package/src/test/e2e.spec.ts

@@ -0,0 +1,43 @@
+import { describe, test, expect } from 'vitest'
+import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
+import { Base } from '../types/types.js'
+import { TIME_MS } from '@kingsy/shared'
+import { ObjectLoader2Factory } from '../operations/objectLoader2Factory.js'
+
+describe('e2e', () => {
+  test(
+    'download small model',
+    async () => {
+      // Revit sample house (good for bim-like stuff with many display meshes)
+      //const resource = 'https://app.kingsy.systems/streams/da9e320dad/commits/5388ef24b8'
+      const loader = ObjectLoader2Factory.createFromUrl({
+        serverUrl: 'https://app.kingsy.systems',
+        streamId: 'da9e320dad',
+        objectId: '31d10c0cea569a1e26809658ed27e281',
+        options: {
+          indexedDB: new IDBFactory(),
+          keyRange: IDBKeyRange
+        }
+      })
+
+      const getObjectPromise = loader.getObject({
+        id: '1708a78e057e8115f924c620ba686db6'
+      })
+
+      const bases: Base[] = []
+      for await (const obj of loader.getObjectIterator()) {
+        bases.push(obj)
+      }
+
+      expect(await loader.getTotalObjectCount()).toBe(1328)
+      expect(bases.length).toBe(1328)
+      const base = await getObjectPromise
+      expect(base).toBeDefined()
+      expect(base.id).toBe('1708a78e057e8115f924c620ba686db6')
+      const base2 = await loader.getObject({ id: '3841e3cbc45d52c47bc2f1b7b0ad4eb9' })
+      expect(base2).toBeDefined()
+      expect(base2.id).toBe('3841e3cbc45d52c47bc2f1b7b0ad4eb9')
+    },
+    10 * TIME_MS.second
+  )
+})

package/src/types/errors.ts

@@ -0,0 +1,25 @@
+/**
+ * Base ObjectLoader error
+ */
+class BaseError extends Error {
+  /**
+   * Default message if none is passed
+   */
+  static defaultMessage = 'Unexpected error occurred'
+
+  /**
+   * @param {string} [message]
+   */
+  constructor(message: string) {
+    message ||= new.target.defaultMessage
+    super(message)
+  }
+}
+
+export class ObjectLoaderConfigurationError extends BaseError {
+  static defaultMessage = 'Object loader configured incorrectly!'
+}
+
+export class ObjectLoaderRuntimeError extends BaseError {
+  static defaultMessage = 'Object loader encountered a runtime problem!'
+}