@speckle/objectloader2 2.25.0 → 2.25.4

package/dist/commonjs/operations/traverser.js

@@ -0,0 +1,96 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const types_js_1 = require("../types/types.js");
+class Traverser {
+    #loader;
+    #options;
+    #totalChildrenCount = 0;
+    #traversedReferencesCount = 0;
+    constructor(loader, options) {
+        this.#options = options || {};
+        this.#loader = loader;
+    }
+    async traverse(onProgress) {
+        let firstObjectPromise = undefined;
+        for await (const obj of this.#loader.getObjectIterator()) {
+            if (!firstObjectPromise) {
+                firstObjectPromise = this.traverseBase(obj, onProgress);
+            }
+        }
+        if (firstObjectPromise) {
+            return await firstObjectPromise;
+        }
+        else {
+            throw new Error('No objects found');
+        }
+    }
+    async traverseArray(array, onProgress) {
+        for (let i = 0; i < 10; i++) {
+            const prop = array[i];
+            if ((0, types_js_1.isScalar)(prop))
+                continue;
+            if ((0, types_js_1.isBase)(prop)) {
+                array[i] = await this.traverseBase(prop, onProgress);
+            }
+            else if ((0, types_js_1.isReference)(prop)) {
+                array[i] = await this.traverseBase(await this.#loader.getObject({ id: prop.referencedId }), onProgress);
+            }
+        }
+    }
+    async traverseBase(base, onProgress) {
+        for (const ignoredProp of this.#options.excludeProps || []) {
+            delete base[ignoredProp];
+        }
+        if (base.__closure) {
+            const ids = Object.keys(base.__closure);
+            const promises = [];
+            for (const id of ids) {
+                promises.push(this.traverseBase(await this.#loader.getObject({ id }), onProgress));
+            }
+            await Promise.all(promises);
+        }
+        delete base['__closure'];
+        // De-chunk
+        if (base.speckle_type?.includes('DataChunk')) {
+            const chunk = base;
+            if (chunk.data) {
+                await this.traverseArray(chunk.data, onProgress);
+            }
+        }
+        //other props
+        for (const prop in base) {
+            if (prop === '__closure')
+                continue;
+            if (prop === 'referenceId')
+                continue;
+            if (prop === 'speckle_type')
+                continue;
+            if (prop === 'data')
+                continue;
+            const baseProp = base[prop];
+            if ((0, types_js_1.isScalar)(baseProp))
+                continue;
+            if ((0, types_js_1.isBase)(baseProp)) {
+                await this.traverseBase(baseProp, onProgress);
+            }
+            else if ((0, types_js_1.isReference)(baseProp)) {
+                await this.traverseBase(await this.#loader.getObject({ id: baseProp.referencedId }), onProgress);
+            }
+            else if (Array.isArray(baseProp)) {
+                await this.traverseArray(baseProp, onProgress);
+            }
+        }
+        if (onProgress) {
+            onProgress({
+                stage: 'construction',
+                current: ++this.#traversedReferencesCount > this.#totalChildrenCount
+                    ? this.#totalChildrenCount
+                    : this.#traversedReferencesCount,
+                total: this.#totalChildrenCount
+            });
+        }
+        return base;
+    }
+}
+exports.default = Traverser;
+//# sourceMappingURL=traverser.js.map

package/dist/commonjs/operations/traverser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"traverser.js","sourceRoot":"","sources":["../../../src/operations/traverser.ts"],"names":[],"mappings":";;AAAA,gDAAkF;AAclF,MAAqB,SAAS;IAC5B,OAAO,CAAe;IACtB,QAAQ,CAAkB;IAE1B,mBAAmB,GAAG,CAAC,CAAA;IACvB,yBAAyB,GAAG,CAAC,CAAA;IAE7B,YAAY,MAAqB,EAAE,OAA0B;QAC3D,IAAI,CAAC,QAAQ,GAAG,OAAO,IAAI,EAAE,CAAA;QAC7B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,UAAuB;QACpC,IAAI,kBAAkB,GAA8B,SAAS,CAAA;QAC7D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACxB,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAA;YACzD,CAAC;QACH,CAAC;QAED,IAAI,kBAAkB,EAAE,CAAC;YACvB,OAAO,MAAM,kBAAkB,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAA;QACrC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAqB,EAAE,UAAuB;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YACrB,IAAI,IAAA,mBAAQ,EAAC,IAAI,CAAC;gBAAE,SAAQ;YAC5B,IAAI,IAAA,iBAAM,EAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;YACtD,CAAC;iBAAM,IAAI,IAAA,sBAAW,EAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,YAAY,CAChC,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC,EACvD,UAAU,CACX,CAAA;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,IAAU,EAAE,UAAuB;QACpD,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;YAC3D,OAAQ,IAAc,CAAC,WAAW,CAAC,CAAA;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACvC,MAAM,QAAQ,GAAoB,EAAE,CAAA;YACpC,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBACrB,QAAQ,CAAC,IAAI,CACX,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,CACpE,CAAA;YACH,CAAC;YACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC7B,CAAC;QACD,OAAQ,IAAc,CAAC,WAAW,CAAC,CAAA;QAEnC,WAAW;QACX,IAAI,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,IAAiB,CAAA;YAC/B,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACf,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;YAClD,CAAC;QACH,CAAC;QAED,aAAa;QACb,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,KAAK,WAAW;gBAAE,SAAQ;YAClC,IAAI,IAAI,KAAK,aAAa;gBAAE,SAAQ;YACpC,IAAI,IAAI,KAAK,cAAc;gBAAE,SAAQ;YACrC,IAAI,IAAI,KAAK,MAAM;gBAAE,SAAQ;YAC7B,MAAM,QAAQ,GAAI,IAA2C,CAAC,IAAI,CAAC,CAAA;YACnE,IAAI,IAAA,mBAAQ,EAAC,QAAQ,CAAC;gBAAE,SAAQ;YAChC,IAAI,IAAA,iBAAM,EAAC,QAAQ,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;YAC/C,CAAC;iBAAM,IAAI,IAAA,sBAAW,EAAC,QAAQ,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,CAAC,YAAY,CACrB,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC,EAC3D,UAAU,CACX,CAAA;YACH,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnC,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;YAChD,CAAC;QACH,CAAC;QACD,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC;gBACT,KAAK,EAAE,cAAc;gBACrB,OAAO,EACL,EAAE,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,mBAAmB;oBACzD,CAAC,CAAC,IAAI,CAAC,mBAAmB;oBAC1B,CAAC,CAAC,IAAI,CAAC,yBAAyB;gBACpC,KAAK,EAAE,IAAI,CAAC,mBAAmB;aAChC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;CACF;AAjGD,4BAiGC"}

package/dist/commonjs/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "commonjs"
+}

package/dist/commonjs/types/errors.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Base ObjectLoader error
+ */
+declare class BaseError extends Error {
+    /**
+     * Default message if none is passed
+     */
+    static defaultMessage: string;
+    /**
+     * @param {string} [message]
+     */
+    constructor(message: string);
+}
+export declare class ObjectLoaderConfigurationError extends BaseError {
+    static defaultMessage: string;
+}
+export declare class ObjectLoaderRuntimeError extends BaseError {
+    static defaultMessage: string;
+}
+export {};
+//# sourceMappingURL=errors.d.ts.map

package/dist/commonjs/types/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../src/types/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,cAAM,SAAU,SAAQ,KAAK;IAC3B;;OAEG;IACH,MAAM,CAAC,cAAc,SAA8B;IAEnD;;OAEG;gBACS,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,8BAA+B,SAAQ,SAAS;IAC3D,MAAM,CAAC,cAAc,SAA0C;CAChE;AAED,qBAAa,wBAAyB,SAAQ,SAAS;IACrD,MAAM,CAAC,cAAc,SAAiD;CACvE"}

package/dist/commonjs/types/errors.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ObjectLoaderRuntimeError = exports.ObjectLoaderConfigurationError = void 0;
+/**
+ * Base ObjectLoader error
+ */
+class BaseError extends Error {
+    /**
+     * Default message if none is passed
+     */
+    static defaultMessage = 'Unexpected error occurred';
+    /**
+     * @param {string} [message]
+     */
+    constructor(message) {
+        message ||= new.target.defaultMessage;
+        super(message);
+    }
+}
+class ObjectLoaderConfigurationError extends BaseError {
+    static defaultMessage = 'Object loader configured incorrectly!';
+}
+exports.ObjectLoaderConfigurationError = ObjectLoaderConfigurationError;
+class ObjectLoaderRuntimeError extends BaseError {
+    static defaultMessage = 'Object loader encountered a runtime problem!';
+}
+exports.ObjectLoaderRuntimeError = ObjectLoaderRuntimeError;
+//# sourceMappingURL=errors.js.map

package/dist/commonjs/types/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../../src/types/errors.ts"],"names":[],"mappings":";;;AAAA;;GAEG;AACH,MAAM,SAAU,SAAQ,KAAK;IAC3B;;OAEG;IACH,MAAM,CAAC,cAAc,GAAG,2BAA2B,CAAA;IAEnD;;OAEG;IACH,YAAY,OAAe;QACzB,OAAO,KAAK,GAAG,CAAC,MAAM,CAAC,cAAc,CAAA;QACrC,KAAK,CAAC,OAAO,CAAC,CAAA;IAChB,CAAC;;AAGH,MAAa,8BAA+B,SAAQ,SAAS;IAC3D,MAAM,CAAC,cAAc,GAAG,uCAAuC,CAAA;;AADjE,wEAEC;AAED,MAAa,wBAAyB,SAAQ,SAAS;IACrD,MAAM,CAAC,cAAc,GAAG,8CAA8C,CAAA;;AADxE,4DAEC"}

package/dist/commonjs/types/types.d.ts

@@ -0,0 +1,25 @@
+export type CustomLogger = (message?: string, ...optionalParams: unknown[]) => void;
+export type Fetcher = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+export interface Item {
+    baseId: string;
+    base: Base;
+    size?: number;
+}
+export interface Base {
+    id: string;
+    speckle_type: string;
+    __closure?: Record<string, number>;
+}
+export interface Reference {
+    speckle_type: string;
+    referencedId: string;
+    __closure?: Record<string, number>;
+}
+export interface DataChunk extends Base {
+    data?: Base[];
+}
+export declare function isBase(maybeBase?: unknown): maybeBase is Base;
+export declare function isReference(maybeRef?: unknown): maybeRef is Reference;
+export declare function isScalar(value: unknown): value is string | number | boolean | bigint | symbol | undefined;
+export declare function take<T>(it: Iterator<T>, count: number): T[];
+//# sourceMappingURL=types.d.ts.map

package/dist/commonjs/types/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/types/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,OAAO,EAAE,KAAK,IAAI,CAAA;AAEnF,MAAM,MAAM,OAAO,GAAG,CACpB,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAC7B,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAA;AAEtB,MAAM,WAAW,IAAI;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,IAAI,CAAA;IACV,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAA;IACV,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,SAAS;IACxB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,SAAU,SAAQ,IAAI;IACrC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAA;CACd;AAED,wBAAgB,MAAM,CAAC,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,IAAI,IAAI,CAO7D;AAED,wBAAgB,WAAW,CAAC,QAAQ,CAAC,EAAE,OAAO,GAAG,QAAQ,IAAI,SAAS,CAOrE;AAED,wBAAgB,QAAQ,CACtB,KAAK,EAAE,OAAO,GACb,KAAK,IAAI,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAWlE;AAED,wBAAgB,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,CAAC,EAAE,CAQ3D"}

package/dist/commonjs/types/types.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isBase = isBase;
+exports.isReference = isReference;
+exports.isScalar = isScalar;
+exports.take = take;
+function isBase(maybeBase) {
+    return (maybeBase !== null &&
+        typeof maybeBase === 'object' &&
+        'id' in maybeBase &&
+        typeof maybeBase.id === 'string');
+}
+function isReference(maybeRef) {
+    return (maybeRef !== null &&
+        typeof maybeRef === 'object' &&
+        'referencedId' in maybeRef &&
+        typeof maybeRef.referencedId === 'string');
+}
+function isScalar(value) {
+    const type = typeof value;
+    return (value === null ||
+        type === 'string' ||
+        type === 'number' ||
+        type === 'boolean' ||
+        type === 'bigint' ||
+        type === 'symbol' ||
+        type === 'undefined');
+}
+function take(it, count) {
+    const result = [];
+    for (let i = 0; i < count; i++) {
+        const itr = it.next();
+        if (itr.done)
+            break;
+        result.push(itr.value);
+    }
+    return result;
+}
+//# sourceMappingURL=types.js.map

package/dist/commonjs/types/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/types/types.ts"],"names":[],"mappings":";;AA6BA,wBAOC;AAED,kCAOC;AAED,4BAaC;AAED,oBAQC;AAzCD,SAAgB,MAAM,CAAC,SAAmB;IACxC,OAAO,CACL,SAAS,KAAK,IAAI;QAClB,OAAO,SAAS,KAAK,QAAQ;QAC7B,IAAI,IAAI,SAAS;QACjB,OAAO,SAAS,CAAC,EAAE,KAAK,QAAQ,CACjC,CAAA;AACH,CAAC;AAED,SAAgB,WAAW,CAAC,QAAkB;IAC5C,OAAO,CACL,QAAQ,KAAK,IAAI;QACjB,OAAO,QAAQ,KAAK,QAAQ;QAC5B,cAAc,IAAI,QAAQ;QAC1B,OAAO,QAAQ,CAAC,YAAY,KAAK,QAAQ,CAC1C,CAAA;AACH,CAAC;AAED,SAAgB,QAAQ,CACtB,KAAc;IAEd,MAAM,IAAI,GAAG,OAAO,KAAK,CAAA;IACzB,OAAO,CACL,KAAK,KAAK,IAAI;QACd,IAAI,KAAK,QAAQ;QACjB,IAAI,KAAK,QAAQ;QACjB,IAAI,KAAK,SAAS;QAClB,IAAI,KAAK,QAAQ;QACjB,IAAI,KAAK,QAAQ;QACjB,IAAI,KAAK,WAAW,CACrB,CAAA;AACH,CAAC;AAED,SAAgB,IAAI,CAAI,EAAe,EAAE,KAAa;IACpD,MAAM,MAAM,GAAQ,EAAE,CAAA;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,EAAE,CAAA;QACrB,IAAI,GAAG,CAAC,IAAI;YAAE,MAAK;QACnB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACxB,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC"}

package/dist/esm/helpers/aggregateQueue.d.ts

@@ -0,0 +1,8 @@
+import Queue from './queue.js';
+export default class AggregateQueue<T> implements Queue<T> {
+    #private;
+    constructor(queue1: Queue<T>, queue2: Queue<T>);
+    add(value: T): void;
+    values(): T[];
+}
+//# sourceMappingURL=aggregateQueue.d.ts.map

package/dist/esm/helpers/aggregateQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aggregateQueue.d.ts","sourceRoot":"","sources":["../../../src/helpers/aggregateQueue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAA;AAE9B,MAAM,CAAC,OAAO,OAAO,cAAc,CAAC,CAAC,CAAE,YAAW,KAAK,CAAC,CAAC,CAAC;;gBAI5C,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;IAK9C,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI;IAKnB,MAAM,IAAI,CAAC,EAAE;CAGd"}

package/dist/esm/helpers/aggregateQueue.js

@@ -0,0 +1,16 @@
+export default class AggregateQueue {
+    #queue1;
+    #queue2;
+    constructor(queue1, queue2) {
+        this.#queue1 = queue1;
+        this.#queue2 = queue2;
+    }
+    add(value) {
+        this.#queue1.add(value);
+        this.#queue2.add(value);
+    }
+    values() {
+        throw new Error('Not implemented');
+    }
+}
+//# sourceMappingURL=aggregateQueue.js.map

package/dist/esm/helpers/aggregateQueue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"aggregateQueue.js","sourceRoot":"","sources":["../../../src/helpers/aggregateQueue.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,OAAO,cAAc;IACjC,OAAO,CAAU;IACjB,OAAO,CAAU;IAEjB,YAAY,MAAgB,EAAE,MAAgB;QAC5C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;IACvB,CAAC;IAED,GAAG,CAAC,KAAQ;QACV,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACvB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACzB,CAAC;IAED,MAAM;QACJ,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAA;IACpC,CAAC;CACF"}

package/dist/esm/helpers/asyncGeneratorQueue.d.ts

@@ -0,0 +1,8 @@
+import Queue from './queue.js';
+export default class AsyncGeneratorQueue<T> implements Queue<T> {
+    #private;
+    add(value: T): void;
+    consume(): AsyncGenerator<T>;
+    dispose(): void;
+}
+//# sourceMappingURL=asyncGeneratorQueue.d.ts.map

package/dist/esm/helpers/asyncGeneratorQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"asyncGeneratorQueue.d.ts","sourceRoot":"","sources":["../../../src/helpers/asyncGeneratorQueue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAA;AAE9B,MAAM,CAAC,OAAO,OAAO,mBAAmB,CAAC,CAAC,CAAE,YAAW,KAAK,CAAC,CAAC,CAAC;;IAK7D,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI;IAWZ,OAAO,IAAI,cAAc,CAAC,CAAC,CAAC;IAanC,OAAO,IAAI,IAAI;CAGhB"}

package/dist/esm/helpers/asyncGeneratorQueue.js

@@ -0,0 +1,32 @@
+export default class AsyncGeneratorQueue {
+    #buffer = [];
+    #resolveQueue = [];
+    #finished = false;
+    add(value) {
+        if (this.#resolveQueue.length > 0) {
+            // If there's a pending consumer, resolve immediately
+            const resolve = this.#resolveQueue.shift();
+            resolve(value);
+        }
+        else {
+            // Otherwise, add to the buffer
+            this.#buffer.push(value);
+        }
+    }
+    async *consume() {
+        while (!this.#finished ||
+            this.#resolveQueue.length > 0 ||
+            this.#buffer.length > 0) {
+            if (this.#buffer.length > 0) {
+                yield this.#buffer.shift(); // Yield available values
+            }
+            else {
+                yield await new Promise((resolve) => this.#resolveQueue.push(resolve));
+            }
+        }
+    }
+    dispose() {
+        this.#finished = true;
+    }
+}
+//# sourceMappingURL=asyncGeneratorQueue.js.map

package/dist/esm/helpers/asyncGeneratorQueue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"asyncGeneratorQueue.js","sourceRoot":"","sources":["../../../src/helpers/asyncGeneratorQueue.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,OAAO,mBAAmB;IACtC,OAAO,GAAQ,EAAE,CAAA;IACjB,aAAa,GAA2B,EAAE,CAAA;IAC1C,SAAS,GAAG,KAAK,CAAA;IAEjB,GAAG,CAAC,KAAQ;QACV,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,qDAAqD;YACrD,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,EAAG,CAAA;YAC3C,OAAO,CAAC,KAAK,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACN,+BAA+B;YAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,OAAO;QACZ,OACE,CAAC,IAAI,CAAC,SAAS;YACf,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EACvB,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAG,CAAA,CAAC,yBAAyB;YACvD,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAA;YAC3E,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO;QACL,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;IACvB,CAAC;CACF"}

package/dist/esm/helpers/batchedPool.d.ts

@@ -0,0 +1,12 @@
+export default class BatchedPool<T> {
+    #private;
+    constructor(params: {
+        concurrencyAndSizes: number[];
+        maxWaitTime?: number;
+        processFunction: (batch: T[]) => Promise<void>;
+    });
+    add(item: T): void;
+    getBatch(batchSize: number): T[];
+    disposeAsync(): Promise<void>;
+}
+//# sourceMappingURL=batchedPool.d.ts.map

package/dist/esm/helpers/batchedPool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchedPool.d.ts","sourceRoot":"","sources":["../../../src/helpers/batchedPool.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,OAAO,WAAW,CAAC,CAAC;;gBAUpB,MAAM,EAAE;QAClB,mBAAmB,EAAE,MAAM,EAAE,CAAA;QAC7B,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,eAAe,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KAC/C;IAOD,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI;IAIlB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE;IAc1B,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;CAgBpC"}

package/dist/esm/helpers/batchedPool.js

@@ -0,0 +1,42 @@
+export default class BatchedPool {
+    #queue = [];
+    #concurrencyAndSizes;
+    #processFunction;
+    #baseInterval;
+    #processingLoop;
+    #disposed = false;
+    constructor(params) {
+        this.#concurrencyAndSizes = params.concurrencyAndSizes;
+        this.#baseInterval = Math.min(params.maxWaitTime ?? 200, 200); // Initial batch time (ms)
+        this.#processFunction = params.processFunction;
+        this.#processingLoop = this.#loop();
+    }
+    add(item) {
+        this.#queue.push(item);
+    }
+    getBatch(batchSize) {
+        return this.#queue.splice(0, Math.min(batchSize, this.#queue.length));
+    }
+    async #runWorker(batchSize) {
+        while (!this.#disposed || this.#queue.length > 0) {
+            if (this.#queue.length > 0) {
+                const batch = this.getBatch(batchSize);
+                await this.#processFunction(batch);
+            }
+            await this.#delay(this.#baseInterval);
+        }
+    }
+    async disposeAsync() {
+        this.#disposed = true;
+        await this.#processingLoop;
+    }
+    async #loop() {
+        // Initialize workers
+        const workers = Array.from(this.#concurrencyAndSizes, (batchSize) => this.#runWorker(batchSize));
+        await Promise.all(workers);
+    }
+    #delay(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+}
+//# sourceMappingURL=batchedPool.js.map

package/dist/esm/helpers/batchedPool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchedPool.js","sourceRoot":"","sources":["../../../src/helpers/batchedPool.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,OAAO,WAAW;IAC9B,MAAM,GAAQ,EAAE,CAAA;IAChB,oBAAoB,CAAU;IAC9B,gBAAgB,CAA+B;IAE/C,aAAa,CAAQ;IAErB,eAAe,CAAe;IAC9B,SAAS,GAAG,KAAK,CAAA;IAEjB,YAAY,MAIX;QACC,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,mBAAmB,CAAA;QACtD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,GAAG,EAAE,GAAG,CAAC,CAAA,CAAC,0BAA0B;QACxF,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,eAAe,CAAA;QAC9C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,GAAG,CAAC,IAAO;QACT,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED,QAAQ,CAAC,SAAiB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;IACvE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,SAAiB;QAChC,OAAO,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;gBACtC,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAA;YACpC,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QACvC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,MAAM,IAAI,CAAC,eAAe,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,qBAAqB;QACrB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,SAAiB,EAAE,EAAE,CAC1E,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAC3B,CAAA;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,EAAU;QACf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;IAC1D,CAAC;CACF"}

package/dist/esm/helpers/batchingQueue.d.ts

@@ -0,0 +1,14 @@
+export default class BatchingQueue<T> {
+    #private;
+    constructor(params: {
+        batchSize: number;
+        maxWaitTime?: number;
+        processFunction: (batch: T[]) => Promise<void>;
+    });
+    disposeAsync(): Promise<void>;
+    add(key: string, item: T): void;
+    get(id: string): T | undefined;
+    count(): number;
+    isDisposed(): boolean;
+}
+//# sourceMappingURL=batchingQueue.d.ts.map

package/dist/esm/helpers/batchingQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchingQueue.d.ts","sourceRoot":"","sources":["../../../src/helpers/batchingQueue.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,OAAO,aAAa,CAAC,CAAC;;gBAYtB,MAAM,EAAE;QAClB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,eAAe,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KAC/C;IASK,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAKnC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI;IAI/B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAI9B,KAAK,IAAI,MAAM;IAIf,UAAU,IAAI,OAAO;CAwCtB"}

package/dist/esm/helpers/batchingQueue.js

@@ -0,0 +1,71 @@
+import KeyedQueue from './keyedQueue.js';
+export default class BatchingQueue {
+    #queue = new KeyedQueue();
+    #batchSize;
+    #processFunction;
+    #baseInterval;
+    #minInterval;
+    #maxInterval;
+    #processingLoop;
+    #disposed = false;
+    constructor(params) {
+        this.#batchSize = params.batchSize;
+        this.#baseInterval = Math.min(params.maxWaitTime ?? 200, 200); // Initial batch time (ms)
+        this.#minInterval = Math.min(params.maxWaitTime ?? 100, 100); // Minimum batch time
+        this.#maxInterval = Math.min(params.maxWaitTime ?? 3000, 3000); // Maximum batch time
+        this.#processFunction = params.processFunction;
+        this.#processingLoop = this.#loop();
+    }
+    async disposeAsync() {
+        this.#disposed = true;
+        await this.#processingLoop;
+    }
+    add(key, item) {
+        this.#queue.enqueue(key, item);
+    }
+    get(id) {
+        return this.#queue.get(id);
+    }
+    count() {
+        return this.#queue.size;
+    }
+    isDisposed() {
+        return this.#disposed;
+    }
+    #getBatch(batchSize) {
+        return this.#queue.spliceValues(0, Math.min(batchSize, this.#queue.size));
+    }
+    async #loop() {
+        let interval = this.#baseInterval;
+        while (!this.#disposed || this.#queue.size > 0) {
+            const startTime = performance.now();
+            if (this.#queue.size > 0) {
+                const batch = this.#getBatch(this.#batchSize);
+                //console.log('running with queue size of ' + this.#queue.length)
+                await this.#processFunction(batch);
+            }
+            if (this.#queue.size < this.#batchSize / 2) {
+                //refigure interval
+                const endTime = performance.now();
+                const duration = endTime - startTime;
+                if (duration > interval) {
+                    interval = Math.min(interval * 1.5, this.#maxInterval); // Increase if slow or empty
+                }
+                else {
+                    interval = Math.max(interval * 0.8, this.#minInterval); // Decrease if fast
+                }
+                /*console.log(
+                  'queue is waiting ' +
+                    interval / 1000 +
+                    ' with queue size of ' +
+                    this.#queue.length
+                )*/
+                await this.#delay(interval);
+            }
+        }
+    }
+    #delay(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+}
+//# sourceMappingURL=batchingQueue.js.map

package/dist/esm/helpers/batchingQueue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchingQueue.js","sourceRoot":"","sources":["../../../src/helpers/batchingQueue.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,iBAAiB,CAAA;AAExC,MAAM,CAAC,OAAO,OAAO,aAAa;IAChC,MAAM,GAA0B,IAAI,UAAU,EAAa,CAAA;IAC3D,UAAU,CAAQ;IAClB,gBAAgB,CAA+B;IAE/C,aAAa,CAAQ;IACrB,YAAY,CAAQ;IACpB,YAAY,CAAQ;IAEpB,eAAe,CAAe;IAC9B,SAAS,GAAG,KAAK,CAAA;IAEjB,YAAY,MAIX;QACC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,GAAG,EAAE,GAAG,CAAC,CAAA,CAAC,0BAA0B;QACxF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,GAAG,EAAE,GAAG,CAAC,CAAA,CAAC,qBAAqB;QAClF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,IAAI,CAAC,CAAA,CAAC,qBAAqB;QACpF,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,eAAe,CAAA;QAC9C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,MAAM,IAAI,CAAC,eAAe,CAAA;IAC5B,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,IAAO;QACtB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAChC,CAAC;IAED,GAAG,CAAC,EAAU;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;IACzB,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,SAAS,CAAA;IACvB,CAAC;IAED,SAAS,CAAC,SAAiB;QACzB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;IAC3E,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAA;QACjC,OAAO,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;YACnC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBAC7C,iEAAiE;gBACjE,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAA;YACpC,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;gBAC3C,mBAAmB;gBACnB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;gBACjC,MAAM,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAA;gBACpC,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;oBACxB,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA,CAAC,4BAA4B;gBACrF,CAAC;qBAAM,CAAC;oBACN,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA,CAAC,mBAAmB;gBAC5E,CAAC;gBACD;;;;;mBAKG;gBACH,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,CAAC,EAAU;QACf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;IAC1D,CAAC;CACF"}

package/dist/esm/helpers/bufferQueue.d.ts

@@ -0,0 +1,7 @@
+import Queue from './queue.js';
+export default class BufferQueue<T> implements Queue<T> {
+    #private;
+    add(value: T): void;
+    values(): T[];
+}
+//# sourceMappingURL=bufferQueue.d.ts.map

package/dist/esm/helpers/bufferQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bufferQueue.d.ts","sourceRoot":"","sources":["../../../src/helpers/bufferQueue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAA;AAE9B,MAAM,CAAC,OAAO,OAAO,WAAW,CAAC,CAAC,CAAE,YAAW,KAAK,CAAC,CAAC,CAAC;;IAErD,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI;IAInB,MAAM,IAAI,CAAC,EAAE;CAGd"}

package/dist/esm/helpers/bufferQueue.js

@@ -0,0 +1,10 @@
+export default class BufferQueue {
+    #buffer = [];
+    add(value) {
+        this.#buffer.push(value);
+    }
+    values() {
+        return this.#buffer;
+    }
+}
+//# sourceMappingURL=bufferQueue.js.map

package/dist/esm/helpers/bufferQueue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bufferQueue.js","sourceRoot":"","sources":["../../../src/helpers/bufferQueue.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,OAAO,WAAW;IAC9B,OAAO,GAAQ,EAAE,CAAA;IACjB,GAAG,CAAC,KAAQ;QACV,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC1B,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;CACF"}

package/dist/esm/helpers/cachePump.d.ts

@@ -0,0 +1,21 @@
+import { Database } from '../operations/interfaces.js';
+import { CacheOptions } from '../operations/options.js';
+import { Item } from '../types/types.js';
+import Queue from './queue.js';
+import { Downloader } from '../operations/interfaces.js';
+import { DefermentManager } from './defermentManager.js';
+import AsyncGeneratorQueue from './asyncGeneratorQueue.js';
+import { Pump } from './pump.js';
+export declare class CachePump implements Pump {
+    #private;
+    constructor(database: Database, gathered: AsyncGeneratorQueue<Item>, deferments: DefermentManager, options: CacheOptions);
+    add(item: Item): void;
+    disposeAsync(): Promise<void>;
+    pumpItems(params: {
+        ids: string[];
+        foundItems: Queue<Item>;
+        notFoundItems: Queue<string>;
+    }): Promise<void>;
+    gather(ids: string[], downloader: Downloader): AsyncGenerator<Item>;
+}
+//# sourceMappingURL=cachePump.d.ts.map

package/dist/esm/helpers/cachePump.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cachePump.d.ts","sourceRoot":"","sources":["../../../src/helpers/cachePump.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AACvD,OAAO,EAAgB,IAAI,EAAE,MAAM,mBAAmB,CAAA;AAEtD,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,mBAAmB,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,qBAAa,SAAU,YAAW,IAAI;;gBAWlC,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,mBAAmB,CAAC,IAAI,CAAC,EACnC,UAAU,EAAE,gBAAgB,EAC5B,OAAO,EAAE,YAAY;IASvB,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAYf,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B,SAAS,CAAC,MAAM,EAAE;QACtB,GAAG,EAAE,MAAM,EAAE,CAAA;QACb,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;QACvB,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;KAC7B,GAAG,OAAO,CAAC,IAAI,CAAC;IA0BV,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC;CAkB3E"}

package/dist/esm/helpers/cachePump.js

@@ -0,0 +1,73 @@
+import { TIME } from '@speckle/shared';
+import BatchingQueue from './batchingQueue.js';
+export class CachePump {
+    #writeQueue;
+    #database;
+    #logger;
+    #deferments;
+    #gathered;
+    #options;
+    constructor(database, gathered, deferments, options) {
+        this.#database = database;
+        this.#gathered = gathered;
+        this.#deferments = deferments;
+        this.#options = options;
+        this.#logger = options.logger || (() => { });
+    }
+    add(item) {
+        if (!this.#writeQueue) {
+            this.#writeQueue = new BatchingQueue({
+                batchSize: this.#options.maxCacheWriteSize,
+                maxWaitTime: this.#options.maxCacheBatchWriteWait,
+                processFunction: (batch) => this.#database.cacheSaveBatch({ batch })
+            });
+        }
+        this.#writeQueue.add(item.baseId, item);
+    }
+    async disposeAsync() {
+        await this.#writeQueue?.disposeAsync();
+    }
+    async pumpItems(params) {
+        const { ids, foundItems, notFoundItems } = params;
+        const maxCacheReadSize = this.#options.maxCacheReadSize;
+        for (let i = 0; i < ids.length;) {
+            if (this.#writeQueue?.isDisposed())
+                break;
+            if ((this.#writeQueue?.count() ?? 0) > this.#options.maxWriteQueueSize) {
+                this.#logger('pausing reads (# in write queue: ' + this.#writeQueue?.count() + ')');
+                await new Promise((resolve) => setTimeout(resolve, TIME.second)); // Pause for 1 second, protects against out of memory
+                continue;
+            }
+            const batch = ids.slice(i, i + maxCacheReadSize);
+            const cachedData = await this.#database.getAll(batch);
+            for (let i = 0; i < cachedData.length; i++) {
+                if (cachedData[i]) {
+                    foundItems.add(cachedData[i]);
+                }
+                else {
+                    notFoundItems.add(batch[i]);
+                }
+            }
+            i += maxCacheReadSize;
+        }
+    }
+    async *gather(ids, downloader) {
+        const total = ids.length;
+        const pumpPromise = this.pumpItems({
+            ids,
+            foundItems: this.#gathered,
+            notFoundItems: downloader
+        });
+        let count = 0;
+        for await (const item of this.#gathered.consume()) {
+            this.#deferments.undefer(item);
+            yield item;
+            count++;
+            if (count >= total) {
+                this.#gathered.dispose();
+            }
+        }
+        await pumpPromise;
+    }
+}
+//# sourceMappingURL=cachePump.js.map

package/dist/esm/helpers/cachePump.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cachePump.js","sourceRoot":"","sources":["../../../src/helpers/cachePump.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AAItC,OAAO,aAAa,MAAM,oBAAoB,CAAA;AAO9C,MAAM,OAAO,SAAS;IACpB,WAAW,CAAiC;IAC5C,SAAS,CAAU;IACnB,OAAO,CAAc;IACrB,WAAW,CAAkB;IAE7B,SAAS,CAA2B;IAEpC,QAAQ,CAAc;IAEtB,YACE,QAAkB,EAClB,QAAmC,EACnC,UAA4B,EAC5B,OAAqB;QAErB,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAA;QACzB,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAA;QACzB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAA;QAC7B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,GAAS,EAAE,GAAE,CAAC,CAAC,CAAA;IACnD,CAAC;IAED,GAAG,CAAC,IAAU;QACZ,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,GAAG,IAAI,aAAa,CAAC;gBACnC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,iBAAiB;gBAC1C,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,sBAAsB;gBACjD,eAAe,EAAE,CAAC,KAAa,EAAiB,EAAE,CAChD,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,CAAC;aAC3C,CAAC,CAAA;QACJ,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,WAAW,EAAE,YAAY,EAAE,CAAA;IACxC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAIf;QACC,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QACjD,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAA;QAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,GAAI,CAAC;YACjC,IAAI,IAAI,CAAC,WAAW,EAAE,UAAU,EAAE;gBAAE,MAAK;YACzC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;gBACvE,IAAI,CAAC,OAAO,CACV,mCAAmC,GAAG,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,GAAG,CACtE,CAAA;gBACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA,CAAC,qDAAqD;gBACtH,SAAQ;YACV,CAAC;YACD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAA;YAChD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;oBAClB,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC,CAAA;gBAChC,CAAC;qBAAM,CAAC;oBACN,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7B,CAAC;YACH,CAAC;YACD,CAAC,IAAI,gBAAgB,CAAA;QACvB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,GAAa,EAAE,UAAsB;QACjD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAA;QACxB,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;YACjC,GAAG;YACH,UAAU,EAAE,IAAI,CAAC,SAAS;YAC1B,aAAa,EAAE,UAAU;SAC1B,CAAC,CAAA;QACF,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAC9B,MAAM,IAAI,CAAA;YACV,KAAK,EAAE,CAAA;YACP,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;gBACnB,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAA;YAC1B,CAAC;QACH,CAAC;QACD,MAAM,WAAW,CAAA;IACnB,CAAC;CACF"}