@synchronized-console/objectloader2 2.25.7 → 2.31.14

package/src/core/objectLoader2.ts

@@ -0,0 +1,152 @@
+import { Deferment } from '../deferment/defermentManager.js'
+import AggregateQueue from '../queues/aggregateQueue.js'
+import AsyncGeneratorQueue from '../queues/asyncGeneratorQueue.js'
+import { CustomLogger, take } from '../types/functions.js'
+import { Item, Base } from '../types/types.js'
+import { Database, Downloader } from './interfaces.js'
+import { ObjectLoader2Factory } from './objectLoader2Factory.js'
+import { ObjectLoader2Options, CacheOptions } from './options.js'
+import { CacheReader } from './stages/cacheReader.js'
+import { CacheWriter } from './stages/cacheWriter.js'
+
+const MAX_CLOSURES_TO_TAKE = 100
+const EXPECTED_CLOSURE_VALUE = 100
+
+export class ObjectLoader2 {
+  #rootId: string
+
+  #logger: CustomLogger
+
+  #database: Database
+  #downloader: Downloader
+  #cacheReader: CacheReader
+  #cacheWriter: CacheWriter
+
+  #deferments: Deferment
+
+  #gathered: AsyncGeneratorQueue<Item>
+
+  #root?: Item = undefined
+  #isRootStored = false
+
+  constructor(options: ObjectLoader2Options) {
+    this.#rootId = options.rootId
+    this.#logger = options.logger || ((): void => {})
+    this.#logger('ObjectLoader2 initialized with rootId:', this.#rootId)
+
+    const cacheOptions: CacheOptions = {
+      logger: this.#logger,
+      maxCacheReadSize: 10_000,
+      maxCacheWriteSize: 10_000,
+      maxWriteQueueSize: 40_000,
+      maxCacheBatchWriteWait: 100, //100 ms, next to nothing!
+      maxCacheBatchReadWait: 100 //100 ms, next to nothing!
+    }
+
+    this.#gathered = new AsyncGeneratorQueue()
+
+    this.#database = options.database
+    this.#deferments = options.deferments
+    this.#downloader = options.downloader
+    this.#cacheReader = new CacheReader(
+      this.#database,
+      this.#deferments,
+      this.#logger,
+      cacheOptions
+    )
+    this.#cacheReader.initializeQueue(this.#gathered, this.#downloader)
+    this.#cacheWriter = new CacheWriter(
+      this.#database,
+      this.#logger,
+      this.#deferments,
+      cacheOptions,
+      (id: string) => {
+        this.#cacheReader.requestItem(id)
+      }
+    )
+  }
+
+  async disposeAsync(): Promise<void> {
+    await Promise.all([
+      this.#gathered.disposeAsync(),
+      this.#downloader.disposeAsync(),
+      this.#cacheWriter.disposeAsync(),
+      this.#cacheReader.disposeAsync()
+    ])
+    this.#deferments.dispose()
+  }
+
+  async getRootObject(): Promise<Item | undefined> {
+    if (!this.#root) {
+      this.#root = (await this.#database.getAll([this.#rootId]))[0]
+      if (!this.#root) {
+        this.#root = await this.#downloader.downloadSingle()
+      } else {
+        this.#isRootStored = true
+      }
+    }
+    return this.#root
+  }
+
+  async getObject(params: { id: string }): Promise<Base> {
+    return await this.#cacheReader.getObject({ id: params.id })
+  }
+
+  async getTotalObjectCount(): Promise<number> {
+    const rootObj = await this.getRootObject()
+    const totalChildrenCount = Object.keys(rootObj?.base?.__closure || {}).length
+    return totalChildrenCount + 1 //count the root
+  }
+
+  async *getObjectIterator(): AsyncGenerator<Base> {
+    const rootItem = await this.getRootObject()
+    if (rootItem?.base === undefined) {
+      this.#logger('No root object found!')
+      return
+    }
+    if (!rootItem.base.__closure) {
+      yield rootItem.base
+      return
+    }
+
+    //sort the closures by their values descending
+    const sortedClosures = Object.entries(rootItem.base.__closure ?? []).sort(
+      (a, b) => b[1] - a[1]
+    )
+    this.#logger(
+      'calculated closures: ',
+      !take(sortedClosures.values(), MAX_CLOSURES_TO_TAKE).every(
+        (x) => x[1] === EXPECTED_CLOSURE_VALUE
+      )
+    )
+    const children = sortedClosures.map((x) => x[0])
+    const total = children.length + 1 // +1 for the root object
+    this.#downloader.initialize({
+      results: new AggregateQueue(this.#gathered, this.#cacheWriter),
+      total
+    })
+    //only for root
+    this.#gathered.add(rootItem)
+    this.#cacheReader.requestAll(children)
+    let count = 0
+    for await (const item of this.#gathered.consume()) {
+      yield item.base! //always defined, as we add it to the queue
+      count++
+      if (count >= total) {
+        break
+      }
+    }
+    if (!this.#isRootStored) {
+      await this.#database.putAll([rootItem])
+      this.#isRootStored = true
+    }
+  }
+
+  static createFromObjects(objects: Base[]): ObjectLoader2 {
+    return ObjectLoader2Factory.createFromObjects(objects)
+  }
+
+  static createFromJSON(json: string): ObjectLoader2 {
+    return ObjectLoader2Factory.createFromJSON(json)
+  }
+}

package/src/core/objectLoader2Factory.test.ts

@@ -0,0 +1,135 @@
+/* eslint-disable camelcase */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { ObjectLoader2Factory } from './objectLoader2Factory.js'
+import { Base } from '../types/types.js'
+
+describe('ObjectLoader2Factory', () => {
+  let testObjects: Base[]
+
+  beforeEach(() => {
+    testObjects = [
+      {
+        id: 'root-id',
+        synchronized_type: 'Base',
+        __closure: {
+          'child-1': 1,
+          'child-2': 2
+        }
+      },
+      {
+        id: 'child-1',
+        synchronized_type: 'Base'
+      },
+      {
+        id: 'child-2',
+        synchronized_type: 'Base'
+      }
+    ]
+  })
+
+  describe('createFromObjects', () => {
+    it('should create ObjectLoader2 from array of objects', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      expect(loader).toBeDefined()
+
+      // Test that we can get the root object
+      const rootObject = await loader.getRootObject()
+      expect(rootObject?.baseId).toBe('root-id')
+      expect(rootObject?.base?.synchronized_type).toBe('Base')
+
+      await loader.disposeAsync()
+    })
+
+    it('should use first object as root', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      const rootObject = await loader.getRootObject()
+      expect(rootObject?.baseId).toBe('root-id')
+      expect(rootObject?.base?.__closure).toEqual({
+        'child-1': 1,
+        'child-2': 2
+      })
+
+      await loader.disposeAsync()
+    })
+
+    it('should allow iteration over all objects', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      const objects: Base[] = []
+      for await (const obj of loader.getObjectIterator()) {
+        objects.push(obj)
+      }
+
+      expect(objects).toHaveLength(3)
+      expect(objects[0].id).toBe('root-id')
+
+      await loader.disposeAsync()
+    })
+
+    it('should get total object count correctly', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      const totalCount = await loader.getTotalObjectCount()
+      expect(totalCount).toBe(3) // root + 2 children
+
+      await loader.disposeAsync()
+    })
+
+    it('should handle empty objects array', () => {
+      expect(() => {
+        ObjectLoader2Factory.createFromObjects([])
+      }).toThrow()
+    })
+
+    it('should get individual objects by id', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      const rootObj = await loader.getObject({ id: 'root-id' })
+      expect(rootObj.id).toBe('root-id')
+      expect(rootObj.synchronized_type).toBe('Base')
+
+      const child1 = await loader.getObject({ id: 'child-1' })
+      expect(child1.id).toBe('child-1')
+
+      const child2 = await loader.getObject({ id: 'child-2' })
+      expect(child2.id).toBe('child-2')
+
+      await loader.disposeAsync()
+    })
+
+    it('should get individual objects by id that does not exist', async () => {
+      const loader = ObjectLoader2Factory.createFromObjects(testObjects)
+
+      const rootObj = await loader.getObject({ id: 'root-id' })
+      expect(rootObj.id).toBe('root-id')
+      expect(rootObj.synchronized_type).toBe('Base')
+
+      const getObjectPromise = loader.getObject({ id: 'child-11111' })
+      await expect(getObjectPromise).rejects.toThrow()
+
+      await loader.disposeAsync()
+    })
+  })
+
+  describe('createFromJSON', () => {
+    it('should create ObjectLoader2 from JSON string', async () => {
+      const json = JSON.stringify(testObjects)
+      const loader = ObjectLoader2Factory.createFromJSON(json)
+
+      expect(loader).toBeDefined()
+
+      const rootObject = await loader.getRootObject()
+      expect(rootObject?.baseId).toBe('root-id')
+
+      await loader.disposeAsync()
+    })
+
+    it('should handle invalid JSON', () => {
+      expect(() => {
+        ObjectLoader2Factory.createFromJSON('invalid json')
+      }).toThrow()
+    })
+  })
+})

package/src/core/objectLoader2Factory.ts

@@ -0,0 +1,113 @@
+import { DefermentManager, MemoryOnlyDeferment } from '../deferment/defermentManager.js'
+import {
+  CustomLogger,
+  Fetcher,
+  getFeatureFlag,
+  ObjectLoader2Flags
+} from '../types/functions.js'
+import { Base, ObjectAttributeMask } from '../types/types.js'
+import { ObjectLoader2 } from './objectLoader2.js'
+import { IndexedDatabase } from './stages/indexedDatabase.js'
+import { MemoryDatabase } from './stages/memory/memoryDatabase.js'
+import { MemoryDownloader } from './stages/memory/memoryDownloader.js'
+import ServerDownloader from './stages/serverDownloader.js'
+
+export interface ObjectLoader2FactoryOptions {
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+  keyRange?: { bound: Function; lowerBound: Function; upperBound: Function }
+  indexedDB?: IDBFactory
+  fetch?: Fetcher
+  attributeMask?: ObjectAttributeMask
+  useCache?: boolean
+  debug?: boolean
+  logger?: CustomLogger
+}
+
+export class ObjectLoader2Factory {
+  static createFromObjects(objects: Base[]): ObjectLoader2 {
+    const root = objects[0]
+    const records: Map<string, Base> = new Map<string, Base>()
+    objects.forEach((element) => {
+      records.set(element.id, element)
+    })
+    const loader = new ObjectLoader2({
+      rootId: root.id,
+      deferments: new MemoryOnlyDeferment(records),
+      database: new MemoryDatabase({ items: records }),
+      downloader: new MemoryDownloader(root.id, records)
+    })
+    return loader
+  }
+
+  static createFromJSON(json: string): ObjectLoader2 {
+    const jsonObj = JSON.parse(json) as Base[]
+    return this.createFromObjects(jsonObj)
+  }
+
+  static createFromUrl(params: {
+    serverUrl: string
+    streamId: string
+    objectId: string
+    token?: string
+    headers?: Headers
+    options?: ObjectLoader2FactoryOptions
+    attributeMask?: ObjectAttributeMask
+    objectTypeMask?: string[]
+  }): ObjectLoader2 {
+    const log = ObjectLoader2Factory.getLogger(params.options?.logger)
+    let database
+    if (
+      params.options?.debug === true ||
+      getFeatureFlag(ObjectLoader2Flags.DEBUG) === 'true'
+    ) {
+      this.logger('Using DEBUG mode for ObjectLoader2Factory')
+    }
+    const useCache = params.options?.useCache ?? true
+    const flag = getFeatureFlag(ObjectLoader2Flags.USE_CACHE)
+    const flagAllowsCache = flag !== 'false'
+
+    if (useCache && flagAllowsCache) {
+      database = new IndexedDatabase({
+        indexedDB: params.options?.indexedDB,
+        keyRange: params.options?.keyRange
+      })
+    } else {
+      database = new MemoryDatabase({
+        items: new Map<string, Base>()
+      })
+      this.logger(
+        'Disabled persistent caching for ObjectLoader2.  Using MemoryDatabase'
+      )
+    }
+    const logger = log || (((): void => {}) as CustomLogger)
+    const loader = new ObjectLoader2({
+      rootId: params.objectId,
+      deferments: new DefermentManager(logger),
+      downloader: new ServerDownloader({
+        serverUrl: params.serverUrl,
+        streamId: params.streamId,
+        objectId: params.objectId,
+        token: params.token,
+        headers: params.headers,
+        fetch: params.options?.fetch,
+        attributeMask: params.attributeMask,
+        objectTypeMask: params.objectTypeMask,
+        logger
+      }),
+      database,
+      logger
+    })
+    return loader
+  }
+
+  static getLogger(providedLogger?: CustomLogger): CustomLogger | undefined {
+    if (getFeatureFlag(ObjectLoader2Flags.DEBUG) === 'true') {
+      return providedLogger || this.logger
+    }
+    return providedLogger
+  }
+
+  static logger: CustomLogger = (m?: string, ...optionalParams: unknown[]) => {
+    console.log(`[debug] ${m}`, ...optionalParams)
+  }
+}

package/src/{operations → core}/options.ts

@@ -1,8 +1,11 @@
-import { Base, CustomLogger } from '../types/types.js'
+import { Deferment } from '../deferment/defermentManager.js'
+import { CustomLogger } from '../types/functions.js'
+import { Base } from '../types/types.js'
 import { Downloader, Database } from './interfaces.js'
 
 export interface ObjectLoader2Options {
   rootId: string
+  deferments: Deferment
   downloader: Downloader
   database: Database
   logger?: CustomLogger

package/src/{operations/databases → core/stages}/__snapshots__/indexedDatabase.spec.ts.snap

@@ -3,16 +3,16 @@
 exports[`IndexedDatabase > should add and get multiple items 1`] = `
 [
   {
-    "baseId": "id1",
-    "item": {
+    "base": {
       "foo": "bar",
     },
+    "baseId": "id1",
   },
   {
-    "baseId": "id2",
-    "item": {
+    "base": {
       "foo": "bar",
     },
+    "baseId": "id2",
   },
 ]
 `;

package/src/{operations/downloaders → core/stages}/__snapshots__/serverDownloader.spec.ts.snap

@@ -56,6 +56,31 @@ exports[`downloader > download batch of three 1`] = `
 ]
 `;
 
+exports[`downloader > download batch of three with Objects.Other.RawEncoding 1`] = `
+[
+  {
+    "base": {
+      "id": "id1",
+      "synchronized_type": "type",
+    },
+    "baseId": "id1",
+    "size": 34,
+  },
+  {
+    "base": undefined,
+    "baseId": "id2",
+  },
+  {
+    "base": {
+      "id": "id3",
+      "synchronized_type": "type",
+    },
+    "baseId": "id3",
+    "size": 34,
+  },
+]
+`;
+
 exports[`downloader > download batch of two 1`] = `
 [
   {

package/src/{helpers → core/stages}/cacheReader.spec.ts

@@ -1,19 +1,22 @@
-import { describe, expect, test } from 'vitest'
-import { Base, Item } from '../types/types.js'
-import { DefermentManager } from './defermentManager.js'
+import { describe, test, expect } from 'vitest'
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import { Item, Base } from '../../types/types.js'
 import { CacheReader } from './cacheReader.js'
-import { MemoryDatabase } from '../operations/databases/memoryDatabase.js'
+import { MemoryDatabase } from './memory/memoryDatabase.js'
+import { MemoryCache } from '../../deferment/MemoryCache.js'
 
 describe('CacheReader testing', () => {
   test('deferred getObject', async () => {
     const i1: Item = { baseId: 'id1', base: { id: 'id', synchronized_type: 'type' } }
 
-    const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 })
+    const cache = new MemoryCache({ maxSizeInMb: 1, ttlms: 1 }, () => {})
+    const deferments = new DefermentManager(() => {}, cache)
     const cacheReader = new CacheReader(
       new MemoryDatabase({
-        items: new Map<string, Base>([[i1.baseId, i1.base]])
+        items: new Map<string, Base>([[i1.baseId, i1.base!]])
       }),
       deferments,
+      () => {},
       {
         maxCacheReadSize: 1,
         maxCacheWriteSize: 1,
@@ -26,7 +29,9 @@ describe('CacheReader testing', () => {
     const objPromise = cacheReader.getObject({
       id: i1.baseId
     })
-    deferments.undefer(i1)
+    deferments.undefer(i1, (id: string) => {
+      throw new Error(`Requesting item ${id} not implemented`)
+    })
     const base = await objPromise
 
     expect(base).toMatchSnapshot()

package/src/core/stages/cacheReader.ts

@@ -0,0 +1,88 @@
+import { Deferment } from '../../deferment/defermentManager.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
+import Queue from '../../queues/queue.js'
+import { CustomLogger } from '../../types/functions.js'
+import { Item, Base } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import { CacheOptions } from '../options.js'
+
+export class CacheReader {
+  #database: Database
+  #defermentManager: Deferment
+  #logger: CustomLogger
+  #options: CacheOptions
+  #readQueue: BatchingQueue<string> | undefined
+  #foundQueue: Queue<Item> | undefined
+  #notFoundQueue: Queue<string> | undefined
+
+  constructor(
+    database: Database,
+    defermentManager: Deferment,
+    logger: CustomLogger,
+    options: CacheOptions
+  ) {
+    this.#database = database
+    this.#defermentManager = defermentManager
+    this.#logger = logger
+    this.#options = options
+  }
+
+  initializeQueue(foundQueue: Queue<Item>, notFoundQueue: Queue<string>): void {
+    this.#foundQueue = foundQueue
+    this.#notFoundQueue = notFoundQueue
+  }
+
+  getObject(params: { id: string }): Promise<Base> {
+    const [p, b] = this.#defermentManager.defer({ id: params.id })
+    if (!b) {
+      this.requestItem(params.id)
+    }
+    return p
+  }
+
+  #createReadQueue(): void {
+    if (!this.#readQueue) {
+      this.#readQueue = new BatchingQueue({
+        batchSize: this.#options.maxCacheReadSize,
+        maxWaitTime: this.#options.maxCacheBatchReadWait,
+        processFunction: this.#processBatch
+      })
+    }
+  }
+
+  requestItem(id: string): void {
+    this.#createReadQueue()
+    if (!this.#readQueue?.get(id)) {
+      this.#readQueue?.add(id, id)
+    }
+  }
+
+  requestAll(keys: string[]): void {
+    this.#createReadQueue()
+
+    this.#readQueue?.addAll(keys, keys)
+  }
+
+  #processBatch = async (batch: string[]): Promise<void> => {
+    const start = performance.now()
+    const items = await this.#database.getAll(batch)
+    for (let i = 0; i < items.length; i++) {
+      const item = items[i]
+      if (item) {
+        this.#foundQueue?.add(item)
+        this.#defermentManager.undefer(item, (id) => this.requestItem(id))
+      } else {
+        this.#notFoundQueue?.add(batch[i])
+      }
+    }
+    this.#logger(
+      `readBatch: batch ${batch.length}, time ${
+        performance.now() - start
+      } ms left ${this.#readQueue?.count()}`
+    )
+  }
+
+  disposeAsync(): Promise<void> {
+    return this.#readQueue?.disposeAsync() || Promise.resolve()
+  }
+}