@speckle/objectloader2 2.25.7 → 2.25.9

package/dist/commonjs/{operations → core}/interfaces.d.ts

@@ -1,4 +1,4 @@
-import Queue from '../helpers/queue.js';
+import Queue from '../queues/queue.js';
 import { Item } from '../types/types.js';
 export interface Downloader extends Queue<string> {
     initializePool(params: {
@@ -11,7 +11,7 @@ export interface Downloader extends Queue<string> {
 }
 export interface Database {
     getAll(keys: string[]): Promise<(Item | undefined)[]>;
-    cacheSaveBatch(params: {
+    saveBatch(params: {
         batch: Item[];
     }): Promise<void>;
     disposeAsync(): Promise<void>;

package/dist/commonjs/core/interfaces.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"interfaces.d.ts","sourceRoot":"","sources":["../../../src/core/interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,oBAAoB,CAAA;AACtC,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAA;AAExC,MAAM,WAAW,UAAW,SAAQ,KAAK,CAAC,MAAM,CAAC;IAC/C,cAAc,CAAC,MAAM,EAAE;QACrB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;QACpB,KAAK,EAAE,MAAM,CAAA;QACb,oBAAoB,CAAC,EAAE,MAAM,CAAA;KAC9B,GAAG,IAAI,CAAA;IACR,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/B,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9B;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,CAAA;IACrD,SAAS,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,IAAI,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9B"}

package/dist/commonjs/core/interfaces.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../../src/core/interfaces.ts"],"names":[],"mappings":""}

package/dist/commonjs/{operations → core}/objectLoader2.d.ts

@@ -1,4 +1,4 @@
-import { Base, Item } from '../types/types.js';
+import { Item, Base } from '../types/types.js';
 import { ObjectLoader2Options } from './options.js';
 export declare class ObjectLoader2 {
     #private;

package/dist/commonjs/core/objectLoader2.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"objectLoader2.d.ts","sourceRoot":"","sources":["../../../src/core/objectLoader2.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAA;AAG9C,OAAO,EAAE,oBAAoB,EAAgB,MAAM,cAAc,CAAA;AAOjE,qBAAa,aAAa;;gBAkBZ,OAAO,EAAE,oBAAoB;IA4CnC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAW7B,aAAa,IAAI,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAY1C,SAAS,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhD,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC,iBAAiB,IAAI,cAAc,CAAC,IAAI,CAAC;IA4ChD,MAAM,CAAC,iBAAiB,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,aAAa;IAIxD,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa;CAGnD"}

package/dist/commonjs/core/objectLoader2.js

@@ -0,0 +1,128 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ObjectLoader2 = void 0;
+const MemoryCache_js_1 = require("../deferment/MemoryCache.js");
+const defermentManager_js_1 = require("../deferment/defermentManager.js");
+const aggregateQueue_js_1 = __importDefault(require("../queues/aggregateQueue.js"));
+const asyncGeneratorQueue_js_1 = __importDefault(require("../queues/asyncGeneratorQueue.js"));
+const functions_js_1 = require("../types/functions.js");
+const objectLoader2Factory_js_1 = require("./objectLoader2Factory.js");
+const cacheReader_js_1 = require("./stages/cacheReader.js");
+const cacheWriter_js_1 = require("./stages/cacheWriter.js");
+const MAX_CLOSURES_TO_TAKE = 100;
+const EXPECTED_CLOSURE_VALUE = 100;
+class ObjectLoader2 {
+    #rootId;
+    #logger;
+    #database;
+    #downloader;
+    #cacheReader;
+    #cacheWriter;
+    #deferments;
+    #cache;
+    #gathered;
+    #root = undefined;
+    #isRootStored = false;
+    constructor(options) {
+        this.#rootId = options.rootId;
+        this.#logger = options.logger || (() => { });
+        this.#logger('ObjectLoader2 initialized with rootId:', this.#rootId);
+        const cacheOptions = {
+            logger: this.#logger,
+            maxCacheReadSize: 10_000,
+            maxCacheWriteSize: 10_000,
+            maxWriteQueueSize: 40_000,
+            maxCacheBatchWriteWait: 100, //100 ms, next to nothing!
+            maxCacheBatchReadWait: 100 //100 ms, next to nothing!
+        };
+        this.#gathered = new asyncGeneratorQueue_js_1.default();
+        this.#database = options.database;
+        this.#cache = new MemoryCache_js_1.MemoryCache({
+            maxSizeInMb: 500, // 500 MB
+            ttlms: 5_000 // 5 seconds
+        }, this.#logger);
+        this.#deferments = new defermentManager_js_1.DefermentManager(this.#cache, this.#logger);
+        this.#downloader = options.downloader;
+        this.#cacheReader = new cacheReader_js_1.CacheReader(this.#database, this.#deferments, this.#logger, cacheOptions);
+        this.#cacheReader.initializeQueue(this.#gathered, this.#downloader);
+        this.#cacheWriter = new cacheWriter_js_1.CacheWriter(this.#database, this.#logger, this.#deferments, cacheOptions, (id) => {
+            this.#cacheReader.requestItem(id);
+        });
+    }
+    async disposeAsync() {
+        await Promise.all([
+            this.#gathered.disposeAsync(),
+            this.#downloader.disposeAsync(),
+            this.#cacheWriter.disposeAsync(),
+            this.#cacheReader.disposeAsync()
+        ]);
+        this.#deferments.dispose();
+        this.#cache.dispose();
+    }
+    async getRootObject() {
+        if (!this.#root) {
+            this.#root = (await this.#database.getAll([this.#rootId]))[0];
+            if (!this.#root) {
+                this.#root = await this.#downloader.downloadSingle();
+            }
+            else {
+                this.#isRootStored = true;
+            }
+        }
+        return this.#root;
+    }
+    async getObject(params) {
+        return await this.#cacheReader.getObject({ id: params.id });
+    }
+    async getTotalObjectCount() {
+        const rootObj = await this.getRootObject();
+        const totalChildrenCount = Object.keys(rootObj?.base?.__closure || {}).length;
+        return totalChildrenCount + 1; //count the root
+    }
+    async *getObjectIterator() {
+        const rootItem = await this.getRootObject();
+        if (rootItem?.base === undefined) {
+            this.#logger('No root object found!');
+            return;
+        }
+        if (!rootItem.base.__closure) {
+            yield rootItem.base;
+            return;
+        }
+        //sort the closures by their values descending
+        const sortedClosures = Object.entries(rootItem.base.__closure ?? []).sort((a, b) => b[1] - a[1]);
+        this.#logger('calculated closures: ', !(0, functions_js_1.take)(sortedClosures.values(), MAX_CLOSURES_TO_TAKE).every((x) => x[1] === EXPECTED_CLOSURE_VALUE));
+        const children = sortedClosures.map((x) => x[0]);
+        const total = children.length + 1; // +1 for the root object
+        this.#downloader.initializePool({
+            results: new aggregateQueue_js_1.default(this.#gathered, this.#cacheWriter),
+            total
+        });
+        //only for root
+        this.#gathered.add(rootItem);
+        this.#cacheReader.requestAll(children);
+        let count = 0;
+        for await (const item of this.#gathered.consume()) {
+            yield item.base; //always defined, as we add it to the queue
+            count++;
+            if (count >= total) {
+                break;
+            }
+        }
+        if (!this.#isRootStored) {
+            await this.#database.saveBatch({ batch: [rootItem] });
+            this.#isRootStored = true;
+        }
+    }
+    static createFromObjects(objects) {
+        return objectLoader2Factory_js_1.ObjectLoader2Factory.createFromObjects(objects);
+    }
+    static createFromJSON(json) {
+        return objectLoader2Factory_js_1.ObjectLoader2Factory.createFromJSON(json);
+    }
+}
+exports.ObjectLoader2 = ObjectLoader2;
+//# sourceMappingURL=objectLoader2.js.map

package/dist/commonjs/core/objectLoader2.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"objectLoader2.js","sourceRoot":"","sources":["../../../src/core/objectLoader2.ts"],"names":[],"mappings":";;;;;;AAAA,gEAAyD;AACzD,0EAAmE;AACnE,oFAAwD;AACxD,8FAAkE;AAClE,wDAA0D;AAG1D,uEAAgE;AAEhE,4DAAqD;AACrD,4DAAqD;AAErD,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAChC,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC,MAAa,aAAa;IACxB,OAAO,CAAQ;IAEf,OAAO,CAAc;IAErB,SAAS,CAAU;IACnB,WAAW,CAAY;IACvB,YAAY,CAAa;IACzB,YAAY,CAAa;IAEzB,WAAW,CAAkB;IAC7B,MAAM,CAAa;IAEnB,SAAS,CAA2B;IAEpC,KAAK,GAAU,SAAS,CAAA;IACxB,aAAa,GAAG,KAAK,CAAA;IAErB,YAAY,OAA6B;QACvC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAA;QAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,GAAS,EAAE,GAAE,CAAC,CAAC,CAAA;QACjD,IAAI,CAAC,OAAO,CAAC,wCAAwC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;QAEpE,MAAM,YAAY,GAAiB;YACjC,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,gBAAgB,EAAE,MAAM;YACxB,iBAAiB,EAAE,MAAM;YACzB,iBAAiB,EAAE,MAAM;YACzB,sBAAsB,EAAE,GAAG,EAAE,0BAA0B;YACvD,qBAAqB,EAAE,GAAG,CAAC,0BAA0B;SACtD,CAAA;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,gCAAmB,EAAE,CAAA;QAE1C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAA;QACjC,IAAI,CAAC,MAAM,GAAG,IAAI,4BAAW,CAC3B;YACE,WAAW,EAAE,GAAG,EAAE,SAAS;YAC3B,KAAK,EAAE,KAAK,CAAC,YAAY;SAC1B,EACD,IAAI,CAAC,OAAO,CACb,CAAA;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,sCAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;QAClE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,UAAU,CAAA;QACrC,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAW,CACjC,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,OAAO,EACZ,YAAY,CACb,CAAA;QACD,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;QACnE,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAW,CACjC,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,WAAW,EAChB,YAAY,EACZ,CAAC,EAAU,EAAE,EAAE;YACb,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACnC,CAAC,CACF,CAAA;IACH,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE;YAC7B,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE;YAC/B,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE;YAChC,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE;SACjC,CAAC,CAAA;QACF,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAA;QAC1B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAC7D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAA;YACtD,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAA;IACnB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAsB;QACpC,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAA;IAC7D,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;QAC1C,MAAM,kBAAkB,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;QAC7E,OAAO,kBAAkB,GAAG,CAAC,CAAA,CAAC,gBAAgB;IAChD,CAAC;IAED,KAAK,CAAC,CAAC,iBAAiB;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;QAC3C,IAAI,QAAQ,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAA;YACrC,OAAM;QACR,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAC7B,MAAM,QAAQ,CAAC,IAAI,CAAA;YACnB,OAAM;QACR,CAAC;QAED,8CAA8C;QAC9C,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,CACvE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CACtB,CAAA;QACD,IAAI,CAAC,OAAO,CACV,uBAAuB,EACvB,CAAC,IAAA,mBAAI,EAAC,cAAc,CAAC,MAAM,EAAE,EAAE,oBAAoB,CAAC,CAAC,KAAK,CACxD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,sBAAsB,CACvC,CACF,CAAA;QACD,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAChD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA,CAAC,yBAAyB;QAC3D,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC;YAC9B,OAAO,EAAE,IAAI,2BAAc,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC;YAC9D,KAAK;SACN,CAAC,CAAA;QACF,eAAe;QACf,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC5B,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAA;QACtC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,MAAM,IAAI,CAAC,IAAK,CAAA,CAAC,2CAA2C;YAC5D,KAAK,EAAE,CAAA;YACP,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;gBACnB,MAAK;YACP,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;YACrD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;QAC3B,CAAC;IACH,CAAC;IAED,MAAM,CAAC,iBAAiB,CAAC,OAAe;QACtC,OAAO,8CAAoB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAA;IACxD,CAAC;IAED,MAAM,CAAC,cAAc,CAAC,IAAY;QAChC,OAAO,8CAAoB,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;IAClD,CAAC;CACF;AAlJD,sCAkJC"}

package/dist/commonjs/{operations → core}/objectLoader2Factory.d.ts

@@ -1,7 +1,7 @@
-import { Base, CustomLogger } from '../types/types.js';
+import { CustomLogger } from '../types/functions.js';
+import { Base } from '../types/types.js';
 import { ObjectLoader2 } from './objectLoader2.js';
 export interface ObjectLoader2FactoryOptions {
-    useMemoryCache?: boolean;
     keyRange?: {
         bound: Function;
         lowerBound: Function;
@@ -21,5 +21,7 @@ export declare class ObjectLoader2Factory {
         headers?: Headers;
         options?: ObjectLoader2FactoryOptions;
     }): ObjectLoader2;
+    static getLogger(providedLogger?: CustomLogger): CustomLogger | undefined;
+    static logger: CustomLogger;
 }
 //# sourceMappingURL=objectLoader2Factory.d.ts.map

package/dist/commonjs/core/objectLoader2Factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"objectLoader2Factory.d.ts","sourceRoot":"","sources":["../../../src/core/objectLoader2Factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAsC,MAAM,uBAAuB,CAAA;AACxF,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAMlD,MAAM,WAAW,2BAA2B;IAE1C,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,QAAQ,CAAC;QAAC,UAAU,EAAE,QAAQ,CAAC;QAAC,UAAU,EAAE,QAAQ,CAAA;KAAE,CAAA;IAC1E,SAAS,CAAC,EAAE,UAAU,CAAA;IACtB,MAAM,CAAC,EAAE,YAAY,CAAA;CACtB;AAED,qBAAa,oBAAoB;IAC/B,MAAM,CAAC,iBAAiB,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,aAAa;IAcxD,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa;IAKlD,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE;QAC3B,SAAS,EAAE,MAAM,CAAA;QACjB,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,OAAO,CAAC,EAAE,OAAO,CAAA;QACjB,OAAO,CAAC,EAAE,2BAA2B,CAAA;KACtC,GAAG,aAAa;IAkCjB,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,SAAS;IAOzE,MAAM,CAAC,MAAM,EAAE,YAAY,CAE1B;CACF"}

package/dist/commonjs/core/objectLoader2Factory.js

@@ -0,0 +1,74 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ObjectLoader2Factory = void 0;
+const functions_js_1 = require("../types/functions.js");
+const objectLoader2_js_1 = require("./objectLoader2.js");
+const indexedDatabase_js_1 = __importDefault(require("./stages/indexedDatabase.js"));
+const memoryDatabase_js_1 = require("./stages/memory/memoryDatabase.js");
+const memoryDownloader_js_1 = require("./stages/memory/memoryDownloader.js");
+const serverDownloader_js_1 = __importDefault(require("./stages/serverDownloader.js"));
+class ObjectLoader2Factory {
+    static createFromObjects(objects) {
+        const root = objects[0];
+        const records = new Map();
+        objects.forEach((element) => {
+            records.set(element.id, element);
+        });
+        const loader = new objectLoader2_js_1.ObjectLoader2({
+            rootId: root.id,
+            database: new memoryDatabase_js_1.MemoryDatabase({ items: records }),
+            downloader: new memoryDownloader_js_1.MemoryDownloader(root.id, records)
+        });
+        return loader;
+    }
+    static createFromJSON(json) {
+        const jsonObj = JSON.parse(json);
+        return this.createFromObjects(jsonObj);
+    }
+    static createFromUrl(params) {
+        const log = ObjectLoader2Factory.getLogger(params.options?.logger);
+        let database;
+        if ((0, functions_js_1.getFeatureFlag)(functions_js_1.ObjectLoader2Flags.DEBUG) === 'true') {
+            this.logger('Using DEBUG mode for ObjectLoader2Factory');
+        }
+        if ((0, functions_js_1.getFeatureFlag)(functions_js_1.ObjectLoader2Flags.USE_CACHE) === 'true') {
+            database = new indexedDatabase_js_1.default({
+                indexedDB: params.options?.indexedDB,
+                keyRange: params.options?.keyRange
+            });
+        }
+        else {
+            database = new memoryDatabase_js_1.MemoryDatabase({
+                items: new Map()
+            });
+            this.logger('Disabled persistent caching for ObjectLoader2.  Using MemoryDatabase');
+        }
+        const loader = new objectLoader2_js_1.ObjectLoader2({
+            rootId: params.objectId,
+            downloader: new serverDownloader_js_1.default({
+                serverUrl: params.serverUrl,
+                streamId: params.streamId,
+                objectId: params.objectId,
+                token: params.token,
+                headers: params.headers
+            }),
+            database,
+            logger: log
+        });
+        return loader;
+    }
+    static getLogger(providedLogger) {
+        if ((0, functions_js_1.getFeatureFlag)(functions_js_1.ObjectLoader2Flags.DEBUG) === 'true') {
+            return providedLogger || this.logger;
+        }
+        return providedLogger;
+    }
+    static logger = (m, ...optionalParams) => {
+        console.log(`[debug] ${m}`, ...optionalParams);
+    };
+}
+exports.ObjectLoader2Factory = ObjectLoader2Factory;
+//# sourceMappingURL=objectLoader2Factory.js.map

package/dist/commonjs/core/objectLoader2Factory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"objectLoader2Factory.js","sourceRoot":"","sources":["../../../src/core/objectLoader2Factory.ts"],"names":[],"mappings":";;;;;;AAAA,wDAAwF;AAExF,yDAAkD;AAClD,qFAAyD;AACzD,yEAAkE;AAClE,6EAAsE;AACtE,uFAA2D;AAS3D,MAAa,oBAAoB;IAC/B,MAAM,CAAC,iBAAiB,CAAC,OAAe;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,OAAO,GAAsB,IAAI,GAAG,EAAgB,CAAA;QAC1D,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC1B,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;QAClC,CAAC,CAAC,CAAA;QACF,MAAM,MAAM,GAAG,IAAI,gCAAa,CAAC;YAC/B,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,QAAQ,EAAE,IAAI,kCAAc,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;YAChD,UAAU,EAAE,IAAI,sCAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC;SACnD,CAAC,CAAA;QACF,OAAO,MAAM,CAAA;IACf,CAAC;IAED,MAAM,CAAC,cAAc,CAAC,IAAY;QAChC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAW,CAAA;QAC1C,OAAO,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAA;IACxC,CAAC;IAED,MAAM,CAAC,aAAa,CAAC,MAOpB;QACC,MAAM,GAAG,GAAG,oBAAoB,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAClE,IAAI,QAAQ,CAAA;QACZ,IAAI,IAAA,6BAAc,EAAC,iCAAkB,CAAC,KAAK,CAAC,KAAK,MAAM,EAAE,CAAC;YACxD,IAAI,CAAC,MAAM,CAAC,2CAA2C,CAAC,CAAA;QAC1D,CAAC;QACD,IAAI,IAAA,6BAAc,EAAC,iCAAkB,CAAC,SAAS,CAAC,KAAK,MAAM,EAAE,CAAC;YAC5D,QAAQ,GAAG,IAAI,4BAAe,CAAC;gBAC7B,SAAS,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS;gBACpC,QAAQ,EAAE,MAAM,CAAC,OAAO,EAAE,QAAQ;aACnC,CAAC,CAAA;QACJ,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,IAAI,kCAAc,CAAC;gBAC5B,KAAK,EAAE,IAAI,GAAG,EAAgB;aAC/B,CAAC,CAAA;YACF,IAAI,CAAC,MAAM,CACT,sEAAsE,CACvE,CAAA;QACH,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,gCAAa,CAAC;YAC/B,MAAM,EAAE,MAAM,CAAC,QAAQ;YACvB,UAAU,EAAE,IAAI,6BAAgB,CAAC;gBAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC;YACF,QAAQ;YACR,MAAM,EAAE,GAAG;SACZ,CAAC,CAAA;QACF,OAAO,MAAM,CAAA;IACf,CAAC;IAED,MAAM,CAAC,SAAS,CAAC,cAA6B;QAC5C,IAAI,IAAA,6BAAc,EAAC,iCAAkB,CAAC,KAAK,CAAC,KAAK,MAAM,EAAE,CAAC;YACxD,OAAO,cAAc,IAAI,IAAI,CAAC,MAAM,CAAA;QACtC,CAAC;QACD,OAAO,cAAc,CAAA;IACvB,CAAC;IAED,MAAM,CAAC,MAAM,GAAiB,CAAC,CAAU,EAAE,GAAG,cAAyB,EAAE,EAAE;QACzE,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EAAE,GAAG,cAAc,CAAC,CAAA;IAChD,CAAC,CAAA;;AAtEH,oDAuEC"}

package/dist/{esm/operations → commonjs/core}/options.d.ts

@@ -1,4 +1,5 @@
-import { Base, CustomLogger } from '../types/types.js';
+import { CustomLogger } from '../types/functions.js';
+import { Base } from '../types/types.js';
 import { Downloader, Database } from './interfaces.js';
 export interface ObjectLoader2Options {
     rootId: string;

package/dist/commonjs/core/options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../../src/core/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAEtD,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,UAAU,CAAA;IACtB,QAAQ,EAAE,QAAQ,CAAA;IAClB,MAAM,CAAC,EAAE,YAAY,CAAA;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,YAAY,CAAA;IACrB,gBAAgB,EAAE,MAAM,CAAA;IACxB,iBAAiB,EAAE,MAAM,CAAA;IACzB,sBAAsB,EAAE,MAAM,CAAA;IAC9B,qBAAqB,EAAE,MAAM,CAAA;IAC7B,iBAAiB,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,YAAY,CAAA;IACrB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;CAC1B;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,CAAC,EAAE,YAAY,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,MAAM,CAAA;CACd"}

package/dist/commonjs/{operations → core}/options.js.map

@@ -1 +1 @@
-{"version":3,"file":"options.js","sourceRoot":"","sources":["../../../src/operations/options.ts"],"names":[],"mappings":""}
+{"version":3,"file":"options.js","sourceRoot":"","sources":["../../../src/core/options.ts"],"names":[],"mappings":""}

package/dist/commonjs/core/stages/ItemStore.d.ts

@@ -0,0 +1,37 @@
+import { Item } from '../../types/types.js';
+/**
+ * A wrapper class for IndexedDB to simplify common database operations.
+ */
+export interface ItemStoreOptions {
+    indexedDB?: IDBFactory;
+    keyRange?: {
+        bound: Function;
+        lowerBound: Function;
+        upperBound: Function;
+    };
+}
+export declare class ItemStore {
+    #private;
+    constructor(options: ItemStoreOptions, dbName: string, storeName: string);
+    /**
+     * Initializes the database connection and creates the object store if needed.
+     * This must be called before any other database operations.
+     */
+    init(): Promise<void>;
+    /**
+     * Inserts or updates an array of items in a single transaction.
+     * @param data The array of items to insert.
+     */
+    bulkInsert(data: Item[]): Promise<void>;
+    /**
+     * Retrieves an array of items from the object store based on their IDs.
+     * @param ids The array of IDs to retrieve.
+     */
+    bulkGet(ids: string[]): Promise<(Item | undefined)[]>;
+    /**
+     * Retrieves all items from the object store.
+     */
+    getAll(): Promise<Item[]>;
+    close(): void;
+}
+//# sourceMappingURL=ItemStore.d.ts.map

package/dist/commonjs/core/stages/ItemStore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ItemStore.d.ts","sourceRoot":"","sources":["../../../../src/core/stages/ItemStore.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAA;AAE3C;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,CAAC,EAAE,UAAU,CAAA;IACtB,QAAQ,CAAC,EAAE;QACT,KAAK,EAAE,QAAQ,CAAA;QACf,UAAU,EAAE,QAAQ,CAAA;QACpB,UAAU,EAAE,QAAQ,CAAA;KACrB,CAAA;CACF;AACD,qBAAa,SAAS;;gBAOR,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAMxE;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwE3B;;;OAGG;IACH,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBvC;;;OAGG;IACH,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC;IAmCrD;;OAEG;IACH,MAAM,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;IAezB,KAAK,IAAI,IAAI;CAKd"}

package/dist/commonjs/core/stages/ItemStore.js

@@ -0,0 +1,167 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ItemStore = void 0;
+/* eslint-disable @typescript-eslint/no-base-to-string */
+/* eslint-disable @typescript-eslint/prefer-promise-reject-errors */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+const shared_1 = require("@speckle/shared");
+class ItemStore {
+    #options;
+    #db = undefined;
+    #dbName;
+    #storeName;
+    constructor(options, dbName, storeName) {
+        this.#options = options;
+        this.#dbName = dbName;
+        this.#storeName = storeName;
+    }
+    /**
+     * Initializes the database connection and creates the object store if needed.
+     * This must be called before any other database operations.
+     */
+    async init() {
+        if (this.#db)
+            return;
+        await this.#safariFix();
+        return this.#openDatabase();
+    }
+    /**
+     * Opens the database, and if there's an error, deletes the database and tries again.
+     */
+    async #openDatabase() {
+        const idb = this.#options.indexedDB ?? indexedDB;
+        return new Promise((resolve, reject) => {
+            const request = idb.open(this.#dbName, 1);
+            request.onerror = () => {
+                console.warn(`Failed to open database: ${this.#dbName}, deleting and trying again`);
+                // Delete the database and try again
+                const deleteRequest = idb.deleteDatabase(this.#dbName);
+                deleteRequest.onsuccess = () => {
+                    // Try opening again after deletion
+                    void this.#openDatabase().then(resolve).catch(reject);
+                };
+                deleteRequest.onerror = () => {
+                    reject(`Failed to delete and reopen database: ${this.#dbName}`);
+                };
+            };
+            request.onupgradeneeded = (event) => {
+                const db = event.target.result;
+                if (db.objectStoreNames.contains(this.#storeName)) {
+                    db.deleteObjectStore(this.#storeName);
+                }
+                db.createObjectStore(this.#storeName, { keyPath: 'baseId' });
+            };
+            request.onsuccess = (event) => {
+                this.#db = event.target.result;
+                resolve();
+            };
+        });
+    }
+    #getDB() {
+        if (!this.#db) {
+            throw new Error('Database not initialized. Call init() first.');
+        }
+        return this.#db;
+    }
+    /**
+     * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
+     * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
+     */
+    async #safariFix() {
+        // No point putting other browsers or older versions of Safari through this mess.
+        if (!(0, shared_1.isSafari)() || !this.#options.indexedDB?.databases)
+            return Promise.resolve();
+        let intervalId;
+        return new Promise((resolve) => {
+            const tryIdb = () => this.#options.indexedDB?.databases().finally(resolve);
+            intervalId = setInterval(() => {
+                void tryIdb();
+            }, 100);
+            void tryIdb();
+        }).finally(() => clearInterval(intervalId));
+    }
+    /**
+     * Inserts or updates an array of items in a single transaction.
+     * @param data The array of items to insert.
+     */
+    bulkInsert(data) {
+        return new Promise((resolve, reject) => {
+            try {
+                const transaction = this.#getDB().transaction(this.#storeName, 'readwrite', {
+                    durability: 'relaxed'
+                });
+                const store = transaction.objectStore(this.#storeName);
+                transaction.onerror = () => {
+                    reject(`Transaction error: ${transaction.error}`);
+                };
+                transaction.oncomplete = () => {
+                    resolve();
+                };
+                data.forEach((item) => store.put(item));
+            }
+            catch (error) {
+                reject(error);
+            }
+        });
+    }
+    /**
+     * Retrieves an array of items from the object store based on their IDs.
+     * @param ids The array of IDs to retrieve.
+     */
+    bulkGet(ids) {
+        return new Promise((resolve, reject) => {
+            if (ids.length === 0) {
+                return resolve([]);
+            }
+            try {
+                const transaction = this.#getDB().transaction(this.#storeName, 'readonly', {
+                    durability: 'relaxed'
+                });
+                const store = transaction.objectStore(this.#storeName);
+                const promises = [];
+                for (const id of ids) {
+                    promises.push(new Promise((resolveGet, rejectGet) => {
+                        const request = store.get(id);
+                        request.onerror = () => rejectGet(`Request error for id ${id}: ${request.error}`);
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+                        request.onsuccess = () => resolveGet(request.result);
+                    }));
+                }
+                Promise.all(promises)
+                    .then((results) => {
+                    resolve(results);
+                })
+                    .catch(reject);
+            }
+            catch (error) {
+                reject(error);
+            }
+        });
+    }
+    /**
+     * Retrieves all items from the object store.
+     */
+    getAll() {
+        return new Promise((resolve, reject) => {
+            try {
+                const transaction = this.#getDB().transaction(this.#storeName, 'readonly');
+                const store = transaction.objectStore(this.#storeName);
+                const request = store.getAll();
+                request.onerror = () => reject(`Request error: ${request.error}`);
+                request.onsuccess = () => resolve(request.result);
+            }
+            catch (error) {
+                reject(error);
+            }
+        });
+    }
+    close() {
+        if (!this.#db)
+            return;
+        this.#db.close();
+        this.#db = undefined;
+    }
+}
+exports.ItemStore = ItemStore;
+//# sourceMappingURL=ItemStore.js.map

package/dist/commonjs/core/stages/ItemStore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ItemStore.js","sourceRoot":"","sources":["../../../../src/core/stages/ItemStore.ts"],"names":[],"mappings":";;;AAAA,yDAAyD;AACzD,oEAAoE;AACpE,uDAAuD;AACvD,+DAA+D;AAC/D,4CAA0C;AAc1C,MAAa,SAAS;IACpB,QAAQ,CAAkB;IAE1B,GAAG,GAA4B,SAAS,CAAA;IAC/B,OAAO,CAAQ;IACf,UAAU,CAAQ;IAE3B,YAAY,OAAyB,EAAE,MAAc,EAAE,SAAiB;QACtE,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;QACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC7B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,GAAG;YAAE,OAAM;QACpB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QACvB,OAAO,IAAI,CAAC,aAAa,EAAE,CAAA;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACjB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,SAAS,CAAA;QAEhD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;YAEzC,OAAO,CAAC,OAAO,GAAG,GAAQ,EAAE;gBAC1B,OAAO,CAAC,IAAI,CACV,4BAA4B,IAAI,CAAC,OAAO,6BAA6B,CACtE,CAAA;gBACD,oCAAoC;gBACpC,MAAM,aAAa,GAAG,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;gBACtD,aAAa,CAAC,SAAS,GAAG,GAAQ,EAAE;oBAClC,mCAAmC;oBACnC,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;gBACvD,CAAC,CAAA;gBACD,aAAa,CAAC,OAAO,GAAG,GAAQ,EAAE;oBAChC,MAAM,CAAC,yCAAyC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;gBACjE,CAAC,CAAA;YACH,CAAC,CAAA;YAED,OAAO,CAAC,eAAe,GAAG,CAAC,KAAK,EAAO,EAAE;gBACvC,MAAM,EAAE,GAAI,KAAK,CAAC,MAA2B,CAAC,MAAM,CAAA;gBACpD,IAAI,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBAClD,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBACvC,CAAC;gBACD,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAA;YAC9D,CAAC,CAAA;YAED,OAAO,CAAC,SAAS,GAAG,CAAC,KAAK,EAAO,EAAE;gBACjC,IAAI,CAAC,GAAG,GAAI,KAAK,CAAC,MAA2B,CAAC,MAAM,CAAA;gBACpD,OAAO,EAAE,CAAA;YACX,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;QACjE,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAA;IACjB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,iFAAiF;QACjF,IAAI,CAAC,IAAA,iBAAQ,GAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QAEhF,IAAI,UAA0C,CAAA;QAE9C,OAAO,IAAI,OAAO,CAAO,CAAC,OAAmB,EAAE,EAAE;YAC/C,MAAM,MAAM,GAAG,GAA2C,EAAE,CAC1D,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YACvD,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC5B,KAAK,MAAM,EAAE,CAAA;YACf,CAAC,EAAE,GAAG,CAAC,CAAA;YACP,KAAK,MAAM,EAAE,CAAA;QACf,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAA;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,IAAY;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,EAAE;oBAC1E,UAAU,EAAE,SAAS;iBACtB,CAAC,CAAA;gBACF,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBAEtD,WAAW,CAAC,OAAO,GAAG,GAAQ,EAAE;oBAC9B,MAAM,CAAC,sBAAsB,WAAW,CAAC,KAAK,EAAE,CAAC,CAAA;gBACnD,CAAC,CAAA;gBACD,WAAW,CAAC,UAAU,GAAG,GAAQ,EAAE;oBACjC,OAAO,EAAE,CAAA;gBACX,CAAC,CAAA;gBAED,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;YACzC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAA;YACf,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,GAAa;QACnB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,OAAO,OAAO,CAAC,EAAE,CAAC,CAAA;YACpB,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE;oBACzE,UAAU,EAAE,SAAS;iBACtB,CAAC,CAAA;gBACF,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBACtD,MAAM,QAAQ,GAAgC,EAAE,CAAA;gBAEhD,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;oBACrB,QAAQ,CAAC,IAAI,CACX,IAAI,OAAO,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE;wBACpC,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;wBAC7B,OAAO,CAAC,OAAO,GAAG,GAAS,EAAE,CAC3B,SAAS,CAAC,wBAAwB,EAAE,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;wBAC3D,iEAAiE;wBACjE,OAAO,CAAC,SAAS,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;oBAC5D,CAAC,CAAC,CACH,CAAA;gBACH,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;qBAClB,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;oBAChB,OAAO,CAAC,OAAO,CAAC,CAAA;gBAClB,CAAC,CAAC;qBACD,KAAK,CAAC,MAAM,CAAC,CAAA;YAClB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAA;YACf,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;gBAC1E,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBACtD,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,EAAE,CAAA;gBAE9B,OAAO,CAAC,OAAO,GAAG,GAAQ,EAAE,CAAC,MAAM,CAAC,kBAAkB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;gBACtE,OAAO,CAAC,SAAS,GAAG,GAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACxD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAA;YACf,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAM;QACrB,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAA;QAChB,IAAI,CAAC,GAAG,GAAG,SAAS,CAAA;IACtB,CAAC;CACF;AAjLD,8BAiLC"}

package/dist/commonjs/core/stages/cacheReader.d.ts

@@ -0,0 +1,18 @@
+import { DefermentManager } from '../../deferment/defermentManager.js';
+import Queue from '../../queues/queue.js';
+import { CustomLogger } from '../../types/functions.js';
+import { Item, Base } from '../../types/types.js';
+import { Database } from '../interfaces.js';
+import { CacheOptions } from '../options.js';
+export declare class CacheReader {
+    #private;
+    constructor(database: Database, defermentManager: DefermentManager, logger: CustomLogger, options: CacheOptions);
+    initializeQueue(foundQueue: Queue<Item>, notFoundQueue: Queue<string>): void;
+    getObject(params: {
+        id: string;
+    }): Promise<Base>;
+    requestItem(id: string): void;
+    requestAll(keys: string[]): void;
+    disposeAsync(): Promise<void>;
+}
+//# sourceMappingURL=cacheReader.d.ts.map

package/dist/commonjs/core/stages/cacheReader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cacheReader.d.ts","sourceRoot":"","sources":["../../../../src/core/stages/cacheReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAA;AAEtE,OAAO,KAAK,MAAM,uBAAuB,CAAA;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AACvD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAE5C,qBAAa,WAAW;;gBAUpB,QAAQ,EAAE,QAAQ,EAClB,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,YAAY;IAQvB,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI;IAK5E,SAAS,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBhD,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAO7B,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI;IAyBhC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;CAG9B"}

package/dist/commonjs/core/stages/cacheReader.js

@@ -0,0 +1,72 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CacheReader = void 0;
+const batchingQueue_js_1 = __importDefault(require("../../queues/batchingQueue.js"));
+class CacheReader {
+    #database;
+    #defermentManager;
+    #logger;
+    #options;
+    #readQueue;
+    #foundQueue;
+    #notFoundQueue;
+    constructor(database, defermentManager, logger, options) {
+        this.#database = database;
+        this.#defermentManager = defermentManager;
+        this.#logger = logger;
+        this.#options = options;
+    }
+    initializeQueue(foundQueue, notFoundQueue) {
+        this.#foundQueue = foundQueue;
+        this.#notFoundQueue = notFoundQueue;
+    }
+    getObject(params) {
+        const [p, b] = this.#defermentManager.defer({ id: params.id });
+        if (!b) {
+            this.requestItem(params.id);
+        }
+        return p;
+    }
+    #createReadQueue() {
+        if (!this.#readQueue) {
+            this.#readQueue = new batchingQueue_js_1.default({
+                batchSize: this.#options.maxCacheReadSize,
+                maxWaitTime: this.#options.maxCacheBatchReadWait,
+                processFunction: this.#processBatch
+            });
+        }
+    }
+    requestItem(id) {
+        this.#createReadQueue();
+        if (!this.#readQueue?.get(id)) {
+            this.#readQueue?.add(id, id);
+        }
+    }
+    requestAll(keys) {
+        this.#createReadQueue();
+        this.#readQueue?.addAll(keys, keys);
+    }
+    #processBatch = async (batch) => {
+        const start = performance.now();
+        const items = await this.#database.getAll(batch);
+        for (let i = 0; i < items.length; i++) {
+            const item = items[i];
+            if (item) {
+                this.#foundQueue?.add(item);
+                this.#defermentManager.undefer(item, (id) => this.requestItem(id));
+            }
+            else {
+                this.#notFoundQueue?.add(batch[i]);
+            }
+        }
+        this.#logger(`readBatch: batch ${batch.length}, time ${performance.now() - start} ms left ${this.#readQueue?.count()}`);
+    };
+    disposeAsync() {
+        return this.#readQueue?.disposeAsync() || Promise.resolve();
+    }
+}
+exports.CacheReader = CacheReader;
+//# sourceMappingURL=cacheReader.js.map

package/dist/commonjs/core/stages/cacheReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cacheReader.js","sourceRoot":"","sources":["../../../../src/core/stages/cacheReader.ts"],"names":[],"mappings":";;;;;;AACA,qFAAyD;AAOzD,MAAa,WAAW;IACtB,SAAS,CAAU;IACnB,iBAAiB,CAAkB;IACnC,OAAO,CAAc;IACrB,QAAQ,CAAc;IACtB,UAAU,CAAmC;IAC7C,WAAW,CAAyB;IACpC,cAAc,CAA2B;IAEzC,YACE,QAAkB,EAClB,gBAAkC,EAClC,MAAoB,EACpB,OAAqB;QAErB,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAA;QACzB,IAAI,CAAC,iBAAiB,GAAG,gBAAgB,CAAA;QACzC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;IACzB,CAAC;IAED,eAAe,CAAC,UAAuB,EAAE,aAA4B;QACnE,IAAI,CAAC,WAAW,GAAG,UAAU,CAAA;QAC7B,IAAI,CAAC,cAAc,GAAG,aAAa,CAAA;IACrC,CAAC;IAED,SAAS,CAAC,MAAsB;QAC9B,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAA;QAC9D,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC7B,CAAC;QACD,OAAO,CAAC,CAAA;IACV,CAAC;IAED,gBAAgB;QACd,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,IAAI,0BAAa,CAAC;gBAClC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,gBAAgB;gBACzC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,qBAAqB;gBAChD,eAAe,EAAE,IAAI,CAAC,aAAa;aACpC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,WAAW,CAAC,EAAU;QACpB,IAAI,CAAC,gBAAgB,EAAE,CAAA;QACvB,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;QAC9B,CAAC;IACH,CAAC;IAED,UAAU,CAAC,IAAc;QACvB,IAAI,CAAC,gBAAgB,EAAE,CAAA;QAEvB,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IACrC,CAAC;IAED,aAAa,GAAG,KAAK,EAAE,KAAe,EAAiB,EAAE;QACvD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QAC/B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YACrB,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,IAAI,CAAC,CAAA;gBAC3B,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAA;YACpE,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,cAAc,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;YACpC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,CACV,oBAAoB,KAAK,CAAC,MAAM,UAC9B,WAAW,CAAC,GAAG,EAAE,GAAG,KACtB,YAAY,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,CACvC,CAAA;IACH,CAAC,CAAA;IAED,YAAY;QACV,OAAO,IAAI,CAAC,UAAU,EAAE,YAAY,EAAE,IAAI,OAAO,CAAC,OAAO,EAAE,CAAA;IAC7D,CAAC;CACF;AA/ED,kCA+EC"}

package/dist/commonjs/core/stages/cacheWriter.d.ts

@@ -0,0 +1,15 @@
+import { DefermentManager } from '../../deferment/defermentManager.js';
+import Queue from '../../queues/queue.js';
+import { CustomLogger } from '../../types/functions.js';
+import { Item } from '../../types/types.js';
+import { Database } from '../interfaces.js';
+import { CacheOptions } from '../options.js';
+export declare class CacheWriter implements Queue<Item> {
+    #private;
+    constructor(database: Database, logger: CustomLogger, defermentManager: DefermentManager, options: CacheOptions, requestItem: (id: string) => void);
+    add(item: Item): void;
+    writeAll(items: Item[]): Promise<void>;
+    disposeAsync(): Promise<void>;
+    get isDisposed(): boolean;
+}
+//# sourceMappingURL=cacheWriter.d.ts.map

package/dist/commonjs/core/stages/cacheWriter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cacheWriter.d.ts","sourceRoot":"","sources":["../../../../src/core/stages/cacheWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAA;AAEtE,OAAO,KAAK,MAAM,uBAAuB,CAAA;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AACvD,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAE5C,qBAAa,WAAY,YAAW,KAAK,CAAC,IAAI,CAAC;;gBAU3C,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,YAAY,EACpB,gBAAgB,EAAE,gBAAgB,EAClC,OAAO,EAAE,YAAY,EACrB,WAAW,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI;IASnC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAcf,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAUtC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAKnC,IAAI,UAAU,IAAI,OAAO,CAExB;CACF"}