@kingsy/objectloader2 2.25.7

package/src/helpers/defermentManager.ts

@@ -0,0 +1,160 @@
+import { DeferredBase } from './deferredBase.js'
+import { Base, CustomLogger, Item } from '../types/types.js'
+import { DefermentManagerOptions } from '../operations/options.js'
+
+export class DefermentManager {
+  private deferments: Map<string, DeferredBase> = new Map()
+  private timer?: ReturnType<typeof setTimeout>
+  private logger: CustomLogger
+  private currentSize = 0
+  private disposed = false
+  //tracks total deferment requests for each id
+  //this is used to prevent cleaning up deferments that are still being requested
+  private totalDefermentRequests: Map<string, number> = new Map()
+
+  constructor(private options: DefermentManagerOptions) {
+    this.resetGlobalTimer()
+    this.logger = options.logger || ((): void => {})
+  }
+
+  private now(): number {
+    return Date.now()
+  }
+
+  isDeferred(id: string): boolean {
+    return this.deferments.has(id)
+  }
+
+  get(id: string): DeferredBase | undefined {
+    if (this.disposed) throw new Error('DefermentManager is disposed')
+    return this.deferments.get(id)
+  }
+
+  async defer(params: { id: string }): Promise<Base> {
+    if (this.disposed) throw new Error('DefermentManager is disposed')
+    this.trackDefermentRequest(params.id)
+    const now = this.now()
+    const deferredBase = this.deferments.get(params.id)
+    if (deferredBase) {
+      deferredBase.setAccess(now)
+      return deferredBase.getPromise()
+    }
+    const notYetFound = new DeferredBase(
+      this.options.ttlms,
+      params.id,
+      now + this.options.ttlms
+    )
+    this.deferments.set(params.id, notYetFound)
+    return notYetFound.getPromise()
+  }
+
+  private trackDefermentRequest(id: string): void {
+    const request = this.totalDefermentRequests.get(id)
+    if (request) {
+      this.totalDefermentRequests.set(id, request + 1)
+    } else {
+      this.totalDefermentRequests.set(id, 1)
+    }
+  }
+
+  undefer(item: Item): void {
+    if (this.disposed) throw new Error('DefermentManager is disposed')
+    const now = this.now()
+    this.currentSize += item.size || 0
+    //order matters here with found before undefer
+    const deferredBase = this.deferments.get(item.baseId)
+    if (deferredBase) {
+      deferredBase.found(item)
+      deferredBase.setAccess(now)
+    } else {
+      const existing = new DeferredBase(this.options.ttlms, item.baseId, now)
+      existing.found(item)
+      this.deferments.set(item.baseId, existing)
+    }
+  }
+
+  private resetGlobalTimer(): void {
+    const run = (): void => {
+      this.cleanDeferments()
+      this.timer = setTimeout(run, this.options.ttlms)
+    }
+    this.timer = setTimeout(run, this.options.ttlms)
+  }
+
+  dispose(): void {
+    if (this.disposed) return
+    this.disposed = true
+    if (this.timer) {
+      clearTimeout(this.timer)
+      this.timer = undefined
+    }
+    this.clearDeferments()
+  }
+
+  private clearDeferments(): void {
+    let waiting = 0
+    for (const deferredBase of this.deferments.values()) {
+      deferredBase.done(0)
+      if (deferredBase.getItem() === undefined) {
+        waiting++
+      }
+    }
+    this.currentSize = 0
+    this.deferments.clear()
+    this.logger('cleared deferments, left', waiting)
+  }
+
+  private cleanDeferments(): void {
+    const maxSizeBytes = this.options.maxSizeInMb * 1024 * 1024
+    if (this.currentSize < maxSizeBytes) {
+      this.logger(
+        'deferments size is ok, no need to clean',
+        this.currentSize,
+        maxSizeBytes
+      )
+      return
+    }
+    const now = this.now()
+    let cleaned = 0
+    const start = performance.now()
+    for (const deferredBase of Array.from(this.deferments.values())
+      .filter((x) => x.isExpired(now))
+      .sort((a, b) => this.compareMaybeBasesBySize(a.getItem(), b.getItem()))) {
+      if (deferredBase.done(now)) {
+        //if the deferment is done but has been requested multiple times,
+        //we do not clean it up to allow the requests to resolve
+        const requestCount = this.totalDefermentRequests.get(deferredBase.getId())
+        if (requestCount && requestCount > 1) {
+          return
+        }
+        this.currentSize -= deferredBase.getItem()?.size || 0
+        this.deferments.delete(deferredBase.getId())
+        cleaned++
+        if (this.currentSize < maxSizeBytes) {
+          break
+        }
+      }
+    }
+    this.logger(
+      'cleaned deferments: cleaned, left, time',
+      cleaned,
+      this.deferments.size,
+      performance.now() - start
+    )
+    return
+  }
+
+  compareMaybeBasesBySize(a: Item | undefined, b: Item | undefined): number {
+    if (a === undefined && b === undefined) return 0
+    if (a === undefined) return -1
+    if (b === undefined) return 1
+    return this.compareMaybe(a.size, b.size)
+  }
+
+  compareMaybe(a: number | undefined, b: number | undefined): number {
+    if (a === undefined && b === undefined) return 0
+    if (a === undefined) return -1
+    if (b === undefined) return 1
+    return a - b
+  }
+}

package/src/helpers/deferredBase.ts

@@ -0,0 +1,55 @@
+import { Base, Item } from '../types/types.js'
+
+export class DeferredBase {
+  private promise: Promise<Base>
+  private resolve!: (value: Base) => void
+  private reject!: (reason?: Error) => void
+  private item?: Item
+
+  private readonly id: string
+  private expiresAt: number // Timestamp in ms
+  private ttl: number // ttl in ms
+
+  constructor(ttl: number, id: string, expiresAt: number) {
+    this.expiresAt = expiresAt
+    this.ttl = ttl
+    this.id = id
+    this.promise = new Promise<Base>((resolve, reject) => {
+      this.resolve = resolve
+      this.reject = reject
+    })
+  }
+
+  getId(): string {
+    return this.id
+  }
+
+  getItem(): Item | undefined {
+    return this.item
+  }
+
+  getPromise(): Promise<Base> {
+    return this.promise
+  }
+
+  isExpired(now: number): boolean {
+    return this.item !== undefined && now > this.expiresAt
+  }
+  setAccess(now: number): void {
+    this.expiresAt = now + this.ttl
+  }
+
+  found(value: Item): void {
+    this.item = value
+    this.resolve(value.base)
+  }
+  done(now: number): boolean {
+    if (this.item) {
+      this.resolve(this.item.base)
+    }
+    if (this.isExpired(now)) {
+      return true
+    }
+    return false
+  }
+}

package/src/helpers/keyedQueue.ts

@@ -0,0 +1,45 @@
+export default class KeyedQueue<K, V> {
+  private _map: Map<K, V>
+  private _order: K[]
+
+  constructor() {
+    this._map = new Map<K, V>()
+    this._order = []
+  }
+
+  enqueue(key: K, value: V): boolean {
+    if (this._map.has(key)) {
+      return false // Key already exists
+    }
+    this._map.set(key, value)
+    this._order.push(key)
+    return true
+  }
+
+  get(key: K): V | undefined {
+    return this._map.get(key)
+  }
+
+  has(key: K): boolean {
+    return this._map.has(key)
+  }
+
+  get size(): number {
+    return this._order.length
+  }
+
+  spliceValues(start: number, deleteCount: number): V[] {
+    const splicedKeys = this._order.splice(start, deleteCount)
+    const result: V[] = []
+
+    for (const key of splicedKeys) {
+      const value = this._map.get(key)
+      if (value !== undefined) {
+        result.push(value)
+        this._map.delete(key)
+      }
+    }
+
+    return result
+  }
+}

package/src/helpers/memoryPump.ts

@@ -0,0 +1,40 @@
+import { Item } from '../types/types.js'
+import { Pump } from './pump.js'
+import Queue from './queue.js'
+
+export class MemoryPump implements Pump {
+  #items: Map<string, Item> = new Map()
+
+  add(item: Item): void {
+    this.#items.set(item.baseId, item)
+  }
+
+  async pumpItems(params: {
+    ids: string[]
+    foundItems: Queue<Item>
+    notFoundItems: Queue<string>
+  }): Promise<void> {
+    const { ids, foundItems, notFoundItems } = params
+    for (const id of ids) {
+      const item = this.#items.get(id)
+      if (item) {
+        foundItems.add(item)
+      } else {
+        notFoundItems.add(id)
+      }
+    }
+    return Promise.resolve()
+  }
+
+  async *gather(ids: string[]): AsyncGenerator<Item> {
+    for (const id of ids) {
+      const item = this.#items.get(id)
+      if (item) {
+        yield item
+      }
+    }
+    return Promise.resolve()
+  }
+
+  async disposeAsync(): Promise<void> {}
+}

package/src/helpers/pump.ts

@@ -0,0 +1,8 @@
+import { Downloader } from '../operations/interfaces.js'
+import { Item } from '../types/types.js'
+import Queue from './queue.js'
+
+export interface Pump extends Queue<Item> {
+  gather(ids: string[], downloader: Downloader): AsyncGenerator<Item>
+  disposeAsync(): Promise<void>
+}

package/src/helpers/queue.ts

@@ -0,0 +1,3 @@
+export default interface Queue<T> {
+  add(value: T): void
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export { ObjectLoader2 } from './operations/objectLoader2.js'
+export { ObjectLoader2Factory } from './operations/objectLoader2Factory.js'

package/src/operations/__snapshots__/objectLoader2.spec.ts.snap

@@ -0,0 +1,149 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`objectloader2 > add extra header 1`] = `
+{
+  "base": {
+    "__closure": {
+      "child1Id": 100,
+    },
+    "id": "rootId",
+    "kingsy_type": "type",
+  },
+  "baseId": "rootId",
+}
+`;
+
+exports[`objectloader2 > can get a root object from cache 1`] = `
+{
+  "base": {
+    "id": "baseId",
+    "kingsy_type": "type",
+  },
+  "baseId": "baseId",
+}
+`;
+
+exports[`objectloader2 > can get a root object from downloader 1`] = `
+{
+  "base": {
+    "id": "baseId",
+    "kingsy_type": "type",
+  },
+  "baseId": "baseId",
+}
+`;
+
+exports[`objectloader2 > can get root/child object from memory cache using iterator and getObject 1`] = `
+[
+  {
+    "__closure": {
+      "child1Id": 100,
+    },
+    "id": "rootId",
+    "kingsy_type": "type",
+  },
+  {
+    "id": "child1Id",
+    "kingsy_type": "type",
+  },
+]
+`;
+
+exports[`objectloader2 > can get root/child object from memory cache using iterator and getObject 2`] = `
+{
+  "id": "child1Id",
+  "kingsy_type": "type",
+}
+`;
+
+exports[`objectloader2 > can get root/child object from memory downloader using iterator and getObject 1`] = `
+[
+  {
+    "__closure": {
+      "child1Id": 100,
+    },
+    "id": "rootId",
+    "kingsy_type": "type",
+  },
+  {
+    "id": "child1Id",
+    "kingsy_type": "type",
+  },
+]
+`;
+
+exports[`objectloader2 > can get root/child object from memory downloader using iterator and getObject 2`] = `
+{
+  "id": "child1Id",
+  "kingsy_type": "type",
+}
+`;
+
+exports[`objectloader2 > can get single object from cache using iterator 1`] = `
+[
+  {
+    "id": "baseId",
+    "kingsy_type": "type",
+  },
+]
+`;
+
+exports[`objectloader2 > createFromJSON test 1`] = `
+[
+  {
+    "__closure": {
+      "0e61e61edee00404ec6e0f9f594bce24": 100,
+      "f70738e3e3e593ac11099a6ed6b71154": 100,
+    },
+    "applicationId": "1",
+    "arr": null,
+    "attachedProp": null,
+    "crazyProp": null,
+    "detachedProp": null,
+    "detachedProp2": null,
+    "dynamicProp": 123,
+    "id": "efeadaca70a85ae6d3acfc93a8b380db",
+    "list": [
+      {
+        "__closure": null,
+        "referencedId": "0e61e61edee00404ec6e0f9f594bce24",
+        "kingsy_type": "reference",
+      },
+    ],
+    "list2": [
+      {
+        "__closure": null,
+        "referencedId": "f70738e3e3e593ac11099a6ed6b71154",
+        "kingsy_type": "reference",
+      },
+    ],
+    "kingsy_type": "KingSy.Core.Tests.Unit.Models.BaseTests+SampleObjectBase2",
+  },
+  {
+    "applicationId": null,
+    "data": [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+    ],
+    "id": "0e61e61edee00404ec6e0f9f594bce24",
+    "kingsy_type": "KingSy.Core.Models.DataChunk",
+  },
+  {
+    "applicationId": null,
+    "data": [
+      1,
+      10,
+    ],
+    "id": "f70738e3e3e593ac11099a6ed6b71154",
+    "kingsy_type": "KingSy.Core.Models.DataChunk",
+  },
+]
+`;

package/src/operations/__snapshots__/traverser.spec.ts.snap

@@ -0,0 +1,45 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Traverser > root and two children with referenceId 1`] = `
+{
+  "applicationId": "1",
+  "arr": null,
+  "attachedProp": null,
+  "crazyProp": null,
+  "detachedProp": null,
+  "detachedProp2": null,
+  "dynamicProp": 123,
+  "id": "efeadaca70a85ae6d3acfc93a8b380db",
+  "list": [
+    {
+      "applicationId": null,
+      "data": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8,
+        9,
+        10,
+      ],
+      "id": "0e61e61edee00404ec6e0f9f594bce24",
+      "kingsy_type": "KingSy.Core.Models.DataChunk",
+    },
+  ],
+  "list2": [
+    {
+      "applicationId": null,
+      "data": [
+        1,
+        10,
+      ],
+      "id": "f70738e3e3e593ac11099a6ed6b71154",
+      "kingsy_type": "KingSy.Core.Models.DataChunk",
+    },
+  ],
+  "kingsy_type": "KingSy.Core.Tests.Unit.Models.BaseTests+SampleObjectBase2",
+}
+`;

package/src/operations/databases/__snapshots__/indexedDatabase.spec.ts.snap

@@ -0,0 +1,18 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`IndexedDatabase > should add and get multiple items 1`] = `
+[
+  {
+    "baseId": "id1",
+    "item": {
+      "foo": "bar",
+    },
+  },
+  {
+    "baseId": "id2",
+    "item": {
+      "foo": "bar",
+    },
+  },
+]
+`;

package/src/operations/databases/indexedDatabase.spec.ts

@@ -0,0 +1,33 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
+import IndexedDatabase, { IndexedDatabaseOptions } from './indexedDatabase.js'
+import { Item } from '../../types/types.js'
+
+// Mock Item
+const defaultItem = (id: string): Item => ({ baseId: id, item: { foo: 'bar' } })
+
+describe('IndexedDatabase', () => {
+  let db: IndexedDatabase
+  let options: IndexedDatabaseOptions
+
+  beforeEach(() => {
+    options = { indexedDB: new IDBFactory(), keyRange: IDBKeyRange }
+    db = new IndexedDatabase(options)
+  })
+
+  afterEach(async () => {
+    await db.disposeAsync()
+  })
+
+  it('should add and get multiple items', async () => {
+    const items = [defaultItem('id1'), defaultItem('id2')]
+    await db.cacheSaveBatch({ batch: items })
+    const result = await db.getAll(['id1', 'id2'])
+    expect(result).toMatchSnapshot()
+    expect(result).toEqual(items)
+  })
+
+  it('should dispose without error', async () => {
+    await expect(db.disposeAsync()).resolves.not.toThrow()
+  })
+})

package/src/operations/databases/indexedDatabase.ts

@@ -0,0 +1,126 @@
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+import BatchingQueue from '../../helpers/batchingQueue.js'
+import { CustomLogger, Item } from '../../types/types.js'
+import { isSafari } from '@kingsy/shared'
+import { Dexie, DexieOptions, Table } from 'dexie'
+import { Database } from '../interfaces.js'
+
+export class ObjectStore extends Dexie {
+  static #databaseName: string = 'kingsy-cache'
+  objects!: Table<Item, string> // Table type: <entity, primaryKey>
+
+  constructor(options: DexieOptions) {
+    super(ObjectStore.#databaseName, options)
+
+    this.version(1).stores({
+      objects: 'baseId, item' // baseId is primary key
+    })
+  }
+}
+
+export interface IndexedDatabaseOptions {
+  logger?: CustomLogger
+  indexedDB?: IDBFactory
+  keyRange?: {
+    bound: Function
+    lowerBound: Function
+    upperBound: Function
+  }
+}
+
+export default class IndexedDatabase implements Database {
+  #options: IndexedDatabaseOptions
+  #logger: CustomLogger
+
+  #cacheDB?: ObjectStore
+
+  #writeQueue: BatchingQueue<Item> | undefined
+
+  // #count: number = 0
+
+  constructor(options: IndexedDatabaseOptions) {
+    this.#options = options
+    this.#logger = options.logger || ((): void => {})
+  }
+
+  async getAll(keys: string[]): Promise<(Item | undefined)[]> {
+    await this.#setupCacheDb()
+    let items: (Item | undefined)[] = []
+    //   this.#count++
+    //  const startTime = performance.now()
+    // this.#logger('Start read ' + x + ' ' + batch.length)
+
+    //faster than BulkGet with dexie
+    await this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => {
+      const gets = keys.map((key) => this.#cacheDB!.objects.get(key))
+      const cachedData = await Promise.all(gets)
+      items = cachedData
+    })
+    // const endTime = performance.now()
+    // const duration = endTime - startTime
+    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
+
+    return items
+  }
+
+  async #openDatabase(): Promise<ObjectStore> {
+    const db = new ObjectStore({
+      indexedDB: this.#options.indexedDB ?? globalThis.indexedDB,
+      IDBKeyRange: this.#options.keyRange ?? IDBKeyRange,
+      chromeTransactionDurability: 'relaxed'
+    })
+    await db.open()
+    return db
+  }
+
+  async #setupCacheDb(): Promise<void> {
+    if (this.#cacheDB !== undefined) {
+      return
+    }
+
+    // Initialize
+    await this.#safariFix()
+    this.#cacheDB = await this.#openDatabase()
+  }
+
+  async cacheSaveBatch(params: { batch: Item[] }): Promise<void> {
+    await this.#setupCacheDb()
+    const { batch } = params
+    //const x = this.#count
+    //this.#count++
+
+    // const startTime = performance.now()
+    //  this.#logger('Start save ' + x + ' ' + batch.length)
+    await this.#cacheDB!.objects.bulkPut(batch)
+    // const endTime = performance.now()
+    // const duration = endTime - startTime
+    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
+  }
+
+  /**
+   * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
+   * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
+   */
+  async #safariFix(): Promise<void> {
+    // No point putting other browsers or older versions of Safari through this mess.
+    if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve()
+
+    let intervalId: ReturnType<typeof setInterval>
+
+    return new Promise<void>((resolve: () => void) => {
+      const tryIdb = (): Promise<IDBDatabaseInfo[]> | undefined =>
+        this.#options.indexedDB?.databases().finally(resolve)
+      intervalId = setInterval(() => {
+        void tryIdb()
+      }, 100)
+      void tryIdb()
+    }).finally(() => clearInterval(intervalId))
+  }
+
+  async disposeAsync(): Promise<void> {
+    this.#cacheDB?.close()
+    this.#cacheDB = undefined
+    await this.#writeQueue?.disposeAsync()
+    this.#writeQueue = undefined
+  }
+}