@speckle/objectloader2 2.25.1 → 2.25.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commonjs/helpers/aggregateQueue.d.ts +8 -0
- package/dist/commonjs/helpers/aggregateQueue.d.ts.map +1 -0
- package/dist/commonjs/helpers/aggregateQueue.js +19 -0
- package/dist/commonjs/helpers/aggregateQueue.js.map +1 -0
- package/dist/commonjs/helpers/asyncGeneratorQueue.d.ts +8 -0
- package/dist/commonjs/helpers/asyncGeneratorQueue.d.ts.map +1 -0
- package/dist/commonjs/helpers/asyncGeneratorQueue.js +35 -0
- package/dist/commonjs/helpers/asyncGeneratorQueue.js.map +1 -0
- package/dist/commonjs/helpers/batchedPool.d.ts +12 -0
- package/dist/commonjs/helpers/batchedPool.d.ts.map +1 -0
- package/dist/commonjs/helpers/batchedPool.js +45 -0
- package/dist/commonjs/helpers/batchedPool.js.map +1 -0
- package/dist/commonjs/helpers/batchingQueue.d.ts +14 -0
- package/dist/commonjs/helpers/batchingQueue.d.ts.map +1 -0
- package/dist/commonjs/helpers/batchingQueue.js +77 -0
- package/dist/commonjs/helpers/batchingQueue.js.map +1 -0
- package/dist/commonjs/helpers/bufferQueue.d.ts +7 -0
- package/dist/commonjs/helpers/bufferQueue.d.ts.map +1 -0
- package/dist/commonjs/helpers/bufferQueue.js +13 -0
- package/dist/commonjs/helpers/bufferQueue.js.map +1 -0
- package/dist/commonjs/helpers/cachePump.d.ts +22 -0
- package/dist/commonjs/helpers/cachePump.d.ts.map +1 -0
- package/dist/commonjs/helpers/cachePump.js +86 -0
- package/dist/commonjs/helpers/cachePump.js.map +1 -0
- package/dist/commonjs/helpers/cacheReader.d.ts +14 -0
- package/dist/commonjs/helpers/cacheReader.d.ts.map +1 -0
- package/dist/commonjs/helpers/cacheReader.js +58 -0
- package/dist/commonjs/helpers/cacheReader.js.map +1 -0
- package/dist/commonjs/helpers/defermentManager.d.ts +28 -0
- package/dist/commonjs/helpers/defermentManager.d.ts.map +1 -0
- package/dist/commonjs/helpers/defermentManager.js +150 -0
- package/dist/commonjs/helpers/defermentManager.js.map +1 -0
- package/dist/commonjs/helpers/deferredBase.d.ts +19 -0
- package/dist/commonjs/helpers/deferredBase.d.ts.map +1 -0
- package/dist/commonjs/helpers/deferredBase.js +51 -0
- package/dist/commonjs/helpers/deferredBase.js.map +1 -0
- package/dist/commonjs/helpers/keyedQueue.d.ts +11 -0
- package/dist/commonjs/helpers/keyedQueue.d.ts.map +1 -0
- package/dist/commonjs/helpers/keyedQueue.js +41 -0
- package/dist/commonjs/helpers/keyedQueue.js.map +1 -0
- package/dist/commonjs/helpers/memoryPump.d.ts +15 -0
- package/dist/commonjs/helpers/memoryPump.d.ts.map +1 -0
- package/dist/commonjs/helpers/memoryPump.js +34 -0
- package/dist/commonjs/helpers/memoryPump.js.map +1 -0
- package/dist/commonjs/helpers/pump.d.ts +8 -0
- package/dist/commonjs/helpers/pump.d.ts.map +1 -0
- package/dist/commonjs/helpers/pump.js +3 -0
- package/dist/commonjs/helpers/pump.js.map +1 -0
- package/dist/commonjs/helpers/queue.d.ts +4 -0
- package/dist/commonjs/helpers/queue.d.ts.map +1 -0
- package/dist/commonjs/helpers/queue.js +3 -0
- package/dist/commonjs/helpers/queue.js.map +1 -0
- package/dist/commonjs/index.d.ts +3 -0
- package/dist/commonjs/index.d.ts.map +1 -0
- package/dist/commonjs/index.js +1 -3
- package/dist/commonjs/index.js.map +1 -0
- package/dist/commonjs/operations/databases/indexedDatabase.d.ts +27 -0
- package/dist/commonjs/operations/databases/indexedDatabase.d.ts.map +1 -0
- package/dist/commonjs/operations/databases/indexedDatabase.js +98 -0
- package/dist/commonjs/operations/databases/indexedDatabase.js.map +1 -0
- package/dist/commonjs/operations/databases/memoryDatabase.d.ts +13 -0
- package/dist/commonjs/operations/databases/memoryDatabase.d.ts.map +1 -0
- package/dist/commonjs/operations/databases/memoryDatabase.js +33 -0
- package/dist/commonjs/operations/databases/memoryDatabase.js.map +1 -0
- package/dist/commonjs/operations/downloaders/memoryDownloader.d.ts +16 -0
- package/dist/commonjs/operations/downloaders/memoryDownloader.d.ts.map +1 -0
- package/dist/commonjs/operations/downloaders/memoryDownloader.js +35 -0
- package/dist/commonjs/operations/downloaders/memoryDownloader.js.map +1 -0
- package/dist/commonjs/operations/downloaders/serverDownloader.d.ts +32 -0
- package/dist/commonjs/operations/downloaders/serverDownloader.d.ts.map +1 -0
- package/dist/commonjs/operations/downloaders/serverDownloader.js +169 -0
- package/dist/commonjs/operations/downloaders/serverDownloader.js.map +1 -0
- package/dist/commonjs/operations/interfaces.d.ts +19 -0
- package/dist/commonjs/operations/interfaces.d.ts.map +1 -0
- package/dist/commonjs/operations/interfaces.js +3 -0
- package/dist/commonjs/operations/interfaces.js.map +1 -0
- package/dist/commonjs/operations/objectLoader2.d.ts +16 -0
- package/dist/commonjs/operations/objectLoader2.d.ts.map +1 -0
- package/dist/commonjs/operations/objectLoader2.js +101 -0
- package/dist/commonjs/operations/objectLoader2.js.map +1 -0
- package/dist/commonjs/operations/objectLoader2Factory.d.ts +25 -0
- package/dist/commonjs/operations/objectLoader2Factory.d.ts.map +1 -0
- package/dist/commonjs/operations/objectLoader2Factory.js +68 -0
- package/dist/commonjs/operations/objectLoader2Factory.js.map +1 -0
- package/dist/commonjs/operations/options.d.ts +26 -0
- package/dist/commonjs/operations/options.d.ts.map +1 -0
- package/dist/commonjs/operations/options.js +3 -0
- package/dist/commonjs/operations/options.js.map +1 -0
- package/dist/commonjs/operations/traverser.d.ts +19 -0
- package/dist/commonjs/operations/traverser.d.ts.map +1 -0
- package/dist/commonjs/operations/traverser.js +96 -0
- package/dist/commonjs/operations/traverser.js.map +1 -0
- package/dist/commonjs/package.json +3 -0
- package/dist/commonjs/types/errors.d.ts +21 -0
- package/dist/commonjs/types/errors.d.ts.map +1 -0
- package/dist/commonjs/types/errors.js +28 -0
- package/dist/commonjs/types/errors.js.map +1 -0
- package/dist/commonjs/types/types.d.ts +25 -0
- package/dist/commonjs/types/types.d.ts.map +1 -0
- package/dist/commonjs/types/types.js +39 -0
- package/dist/commonjs/types/types.js.map +1 -0
- package/dist/esm/helpers/aggregateQueue.d.ts +8 -0
- package/dist/esm/helpers/aggregateQueue.d.ts.map +1 -0
- package/dist/esm/helpers/aggregateQueue.js +16 -0
- package/dist/esm/helpers/aggregateQueue.js.map +1 -0
- package/dist/esm/helpers/asyncGeneratorQueue.d.ts +8 -0
- package/dist/esm/helpers/asyncGeneratorQueue.d.ts.map +1 -0
- package/dist/esm/helpers/asyncGeneratorQueue.js +32 -0
- package/dist/esm/helpers/asyncGeneratorQueue.js.map +1 -0
- package/dist/esm/helpers/batchedPool.d.ts +12 -0
- package/dist/esm/helpers/batchedPool.d.ts.map +1 -0
- package/dist/esm/helpers/batchedPool.js +42 -0
- package/dist/esm/helpers/batchedPool.js.map +1 -0
- package/dist/esm/helpers/batchingQueue.d.ts +14 -0
- package/dist/esm/helpers/batchingQueue.d.ts.map +1 -0
- package/dist/esm/helpers/batchingQueue.js +71 -0
- package/dist/esm/helpers/batchingQueue.js.map +1 -0
- package/dist/esm/helpers/bufferQueue.d.ts +7 -0
- package/dist/esm/helpers/bufferQueue.d.ts.map +1 -0
- package/dist/esm/helpers/bufferQueue.js +10 -0
- package/dist/esm/helpers/bufferQueue.js.map +1 -0
- package/dist/esm/helpers/cachePump.d.ts +22 -0
- package/dist/esm/helpers/cachePump.d.ts.map +1 -0
- package/dist/esm/helpers/cachePump.js +79 -0
- package/dist/esm/helpers/cachePump.js.map +1 -0
- package/dist/esm/helpers/cacheReader.d.ts +14 -0
- package/dist/esm/helpers/cacheReader.d.ts.map +1 -0
- package/dist/esm/helpers/cacheReader.js +51 -0
- package/dist/esm/helpers/cacheReader.js.map +1 -0
- package/dist/esm/helpers/defermentManager.d.ts +28 -0
- package/dist/esm/helpers/defermentManager.d.ts.map +1 -0
- package/dist/esm/helpers/defermentManager.js +146 -0
- package/dist/esm/helpers/defermentManager.js.map +1 -0
- package/dist/esm/helpers/deferredBase.d.ts +19 -0
- package/dist/esm/helpers/deferredBase.d.ts.map +1 -0
- package/dist/esm/helpers/deferredBase.js +47 -0
- package/dist/esm/helpers/deferredBase.js.map +1 -0
- package/dist/esm/helpers/keyedQueue.d.ts +11 -0
- package/dist/esm/helpers/keyedQueue.d.ts.map +1 -0
- package/dist/esm/helpers/keyedQueue.js +38 -0
- package/dist/esm/helpers/keyedQueue.js.map +1 -0
- package/dist/esm/helpers/memoryPump.d.ts +15 -0
- package/dist/esm/helpers/memoryPump.d.ts.map +1 -0
- package/dist/esm/helpers/memoryPump.js +30 -0
- package/dist/esm/helpers/memoryPump.js.map +1 -0
- package/dist/esm/helpers/pump.d.ts +8 -0
- package/dist/esm/helpers/pump.d.ts.map +1 -0
- package/dist/esm/helpers/pump.js +2 -0
- package/dist/esm/helpers/pump.js.map +1 -0
- package/dist/esm/helpers/queue.d.ts +4 -0
- package/dist/esm/helpers/queue.d.ts.map +1 -0
- package/dist/esm/helpers/queue.js +2 -0
- package/dist/esm/helpers/queue.js.map +1 -0
- package/dist/esm/index.d.ts +3 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +0 -1
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/operations/databases/indexedDatabase.d.ts +27 -0
- package/dist/esm/operations/databases/indexedDatabase.d.ts.map +1 -0
- package/dist/esm/operations/databases/indexedDatabase.js +93 -0
- package/dist/esm/operations/databases/indexedDatabase.js.map +1 -0
- package/dist/esm/operations/databases/memoryDatabase.d.ts +13 -0
- package/dist/esm/operations/databases/memoryDatabase.d.ts.map +1 -0
- package/dist/esm/operations/databases/memoryDatabase.js +29 -0
- package/dist/esm/operations/databases/memoryDatabase.js.map +1 -0
- package/dist/esm/operations/downloaders/memoryDownloader.d.ts +16 -0
- package/dist/esm/operations/downloaders/memoryDownloader.d.ts.map +1 -0
- package/dist/esm/operations/downloaders/memoryDownloader.js +31 -0
- package/dist/esm/operations/downloaders/memoryDownloader.js.map +1 -0
- package/dist/esm/operations/downloaders/serverDownloader.d.ts +32 -0
- package/dist/esm/operations/downloaders/serverDownloader.d.ts.map +1 -0
- package/dist/esm/operations/downloaders/serverDownloader.js +163 -0
- package/dist/esm/operations/downloaders/serverDownloader.js.map +1 -0
- package/dist/esm/operations/interfaces.d.ts +19 -0
- package/dist/esm/operations/interfaces.d.ts.map +1 -0
- package/dist/esm/operations/interfaces.js +2 -0
- package/dist/esm/operations/interfaces.js.map +1 -0
- package/dist/esm/operations/objectLoader2.d.ts +16 -0
- package/dist/esm/operations/objectLoader2.d.ts.map +1 -0
- package/dist/esm/operations/objectLoader2.js +94 -0
- package/dist/esm/operations/objectLoader2.js.map +1 -0
- package/dist/esm/operations/objectLoader2Factory.d.ts +25 -0
- package/dist/esm/operations/objectLoader2Factory.d.ts.map +1 -0
- package/dist/esm/operations/objectLoader2Factory.js +61 -0
- package/dist/esm/operations/objectLoader2Factory.js.map +1 -0
- package/dist/esm/operations/options.d.ts +26 -0
- package/dist/esm/operations/options.d.ts.map +1 -0
- package/dist/esm/operations/options.js +2 -0
- package/dist/esm/operations/options.js.map +1 -0
- package/dist/esm/operations/traverser.d.ts +19 -0
- package/dist/esm/operations/traverser.d.ts.map +1 -0
- package/dist/esm/operations/traverser.js +93 -0
- package/dist/esm/operations/traverser.js.map +1 -0
- package/dist/esm/package.json +3 -0
- package/dist/esm/types/errors.d.ts +21 -0
- package/dist/esm/types/errors.d.ts.map +1 -0
- package/dist/esm/types/errors.js +23 -0
- package/dist/esm/types/errors.js.map +1 -0
- package/dist/esm/types/types.d.ts +25 -0
- package/dist/esm/types/types.d.ts.map +1 -0
- package/dist/esm/types/types.js +33 -0
- package/dist/esm/types/types.js.map +1 -0
- package/package.json +2 -2
- package/src/helpers/cachePump.disposal.spec.ts +3 -1
- package/src/helpers/cachePump.spec.ts +2 -1
- package/src/helpers/cachePump.ts +9 -1
- package/src/helpers/defermentManager.defermentTotals.spec.ts +53 -0
- package/src/helpers/defermentManager.ts +20 -1
- package/src/index.ts +0 -1
- package/src/operations/databases/__snapshots__/indexedDatabase.spec.ts.snap +13 -26
- package/src/operations/databases/indexedDatabase.spec.ts +23 -27
- package/src/operations/databases/indexedDatabase.ts +1 -25
- package/src/operations/databases/memoryDatabase.spec.ts +48 -0
- package/src/operations/databases/memoryDatabase.ts +0 -8
- package/src/operations/downloaders/__snapshots__/memoryDownloader.spec.ts.snap +12 -0
- package/src/operations/downloaders/memoryDownloader.spec.ts +49 -0
- package/src/operations/downloaders/memoryDownloader.ts +1 -1
- package/src/operations/downloaders/serverDownloader.ts +1 -1
- package/src/operations/interfaces.ts +0 -1
- package/src/operations/objectLoader2.ts +12 -4
- package/src/operations/objectLoader2Factory.ts +40 -18
- package/src/test/e2e.spec.ts +4 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aggregateQueue.d.ts","sourceRoot":"","sources":["../../../src/helpers/aggregateQueue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAA;AAE9B,MAAM,CAAC,OAAO,OAAO,cAAc,CAAC,CAAC,CAAE,YAAW,KAAK,CAAC,CAAC,CAAC;;gBAI5C,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;IAK9C,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI;IAKnB,MAAM,IAAI,CAAC,EAAE;CAGd"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
class AggregateQueue {
|
|
4
|
+
#queue1;
|
|
5
|
+
#queue2;
|
|
6
|
+
constructor(queue1, queue2) {
|
|
7
|
+
this.#queue1 = queue1;
|
|
8
|
+
this.#queue2 = queue2;
|
|
9
|
+
}
|
|
10
|
+
add(value) {
|
|
11
|
+
this.#queue1.add(value);
|
|
12
|
+
this.#queue2.add(value);
|
|
13
|
+
}
|
|
14
|
+
values() {
|
|
15
|
+
throw new Error('Not implemented');
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
exports.default = AggregateQueue;
|
|
19
|
+
//# sourceMappingURL=aggregateQueue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aggregateQueue.js","sourceRoot":"","sources":["../../../src/helpers/aggregateQueue.ts"],"names":[],"mappings":";;AAEA,MAAqB,cAAc;IACjC,OAAO,CAAU;IACjB,OAAO,CAAU;IAEjB,YAAY,MAAgB,EAAE,MAAgB;QAC5C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;IACvB,CAAC;IAED,GAAG,CAAC,KAAQ;QACV,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACvB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACzB,CAAC;IAED,MAAM;QACJ,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAA;IACpC,CAAC;CACF;AAjBD,iCAiBC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"asyncGeneratorQueue.d.ts","sourceRoot":"","sources":["../../../src/helpers/asyncGeneratorQueue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAA;AAE9B,MAAM,CAAC,OAAO,OAAO,mBAAmB,CAAC,CAAC,CAAE,YAAW,KAAK,CAAC,CAAC,CAAC;;IAK7D,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI;IAWZ,OAAO,IAAI,cAAc,CAAC,CAAC,CAAC;IAanC,OAAO,IAAI,IAAI;CAGhB"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
class AsyncGeneratorQueue {
|
|
4
|
+
#buffer = [];
|
|
5
|
+
#resolveQueue = [];
|
|
6
|
+
#finished = false;
|
|
7
|
+
add(value) {
|
|
8
|
+
if (this.#resolveQueue.length > 0) {
|
|
9
|
+
// If there's a pending consumer, resolve immediately
|
|
10
|
+
const resolve = this.#resolveQueue.shift();
|
|
11
|
+
resolve(value);
|
|
12
|
+
}
|
|
13
|
+
else {
|
|
14
|
+
// Otherwise, add to the buffer
|
|
15
|
+
this.#buffer.push(value);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
async *consume() {
|
|
19
|
+
while (!this.#finished ||
|
|
20
|
+
this.#resolveQueue.length > 0 ||
|
|
21
|
+
this.#buffer.length > 0) {
|
|
22
|
+
if (this.#buffer.length > 0) {
|
|
23
|
+
yield this.#buffer.shift(); // Yield available values
|
|
24
|
+
}
|
|
25
|
+
else {
|
|
26
|
+
yield await new Promise((resolve) => this.#resolveQueue.push(resolve));
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
dispose() {
|
|
31
|
+
this.#finished = true;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
exports.default = AsyncGeneratorQueue;
|
|
35
|
+
//# sourceMappingURL=asyncGeneratorQueue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"asyncGeneratorQueue.js","sourceRoot":"","sources":["../../../src/helpers/asyncGeneratorQueue.ts"],"names":[],"mappings":";;AAEA,MAAqB,mBAAmB;IACtC,OAAO,GAAQ,EAAE,CAAA;IACjB,aAAa,GAA2B,EAAE,CAAA;IAC1C,SAAS,GAAG,KAAK,CAAA;IAEjB,GAAG,CAAC,KAAQ;QACV,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,qDAAqD;YACrD,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,EAAG,CAAA;YAC3C,OAAO,CAAC,KAAK,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACN,+BAA+B;YAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,OAAO;QACZ,OACE,CAAC,IAAI,CAAC,SAAS;YACf,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EACvB,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAG,CAAA,CAAC,yBAAyB;YACvD,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAA;YAC3E,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO;QACL,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;IACvB,CAAC;CACF;AAhCD,sCAgCC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export default class BatchedPool<T> {
|
|
2
|
+
#private;
|
|
3
|
+
constructor(params: {
|
|
4
|
+
concurrencyAndSizes: number[];
|
|
5
|
+
maxWaitTime?: number;
|
|
6
|
+
processFunction: (batch: T[]) => Promise<void>;
|
|
7
|
+
});
|
|
8
|
+
add(item: T): void;
|
|
9
|
+
getBatch(batchSize: number): T[];
|
|
10
|
+
disposeAsync(): Promise<void>;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=batchedPool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batchedPool.d.ts","sourceRoot":"","sources":["../../../src/helpers/batchedPool.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,OAAO,WAAW,CAAC,CAAC;;gBAUpB,MAAM,EAAE;QAClB,mBAAmB,EAAE,MAAM,EAAE,CAAA;QAC7B,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,eAAe,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KAC/C;IAOD,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI;IAIlB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE;IAc1B,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;CAgBpC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
class BatchedPool {
|
|
4
|
+
#queue = [];
|
|
5
|
+
#concurrencyAndSizes;
|
|
6
|
+
#processFunction;
|
|
7
|
+
#baseInterval;
|
|
8
|
+
#processingLoop;
|
|
9
|
+
#disposed = false;
|
|
10
|
+
constructor(params) {
|
|
11
|
+
this.#concurrencyAndSizes = params.concurrencyAndSizes;
|
|
12
|
+
this.#baseInterval = Math.min(params.maxWaitTime ?? 200, 200); // Initial batch time (ms)
|
|
13
|
+
this.#processFunction = params.processFunction;
|
|
14
|
+
this.#processingLoop = this.#loop();
|
|
15
|
+
}
|
|
16
|
+
add(item) {
|
|
17
|
+
this.#queue.push(item);
|
|
18
|
+
}
|
|
19
|
+
getBatch(batchSize) {
|
|
20
|
+
return this.#queue.splice(0, Math.min(batchSize, this.#queue.length));
|
|
21
|
+
}
|
|
22
|
+
async #runWorker(batchSize) {
|
|
23
|
+
while (!this.#disposed || this.#queue.length > 0) {
|
|
24
|
+
if (this.#queue.length > 0) {
|
|
25
|
+
const batch = this.getBatch(batchSize);
|
|
26
|
+
await this.#processFunction(batch);
|
|
27
|
+
}
|
|
28
|
+
await this.#delay(this.#baseInterval);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async disposeAsync() {
|
|
32
|
+
this.#disposed = true;
|
|
33
|
+
await this.#processingLoop;
|
|
34
|
+
}
|
|
35
|
+
async #loop() {
|
|
36
|
+
// Initialize workers
|
|
37
|
+
const workers = Array.from(this.#concurrencyAndSizes, (batchSize) => this.#runWorker(batchSize));
|
|
38
|
+
await Promise.all(workers);
|
|
39
|
+
}
|
|
40
|
+
#delay(ms) {
|
|
41
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
exports.default = BatchedPool;
|
|
45
|
+
//# sourceMappingURL=batchedPool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batchedPool.js","sourceRoot":"","sources":["../../../src/helpers/batchedPool.ts"],"names":[],"mappings":";;AAAA,MAAqB,WAAW;IAC9B,MAAM,GAAQ,EAAE,CAAA;IAChB,oBAAoB,CAAU;IAC9B,gBAAgB,CAA+B;IAE/C,aAAa,CAAQ;IAErB,eAAe,CAAe;IAC9B,SAAS,GAAG,KAAK,CAAA;IAEjB,YAAY,MAIX;QACC,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,mBAAmB,CAAA;QACtD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,GAAG,EAAE,GAAG,CAAC,CAAA,CAAC,0BAA0B;QACxF,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,eAAe,CAAA;QAC9C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,GAAG,CAAC,IAAO;QACT,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED,QAAQ,CAAC,SAAiB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;IACvE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,SAAiB;QAChC,OAAO,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;gBACtC,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAA;YACpC,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QACvC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,MAAM,IAAI,CAAC,eAAe,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,qBAAqB;QACrB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,SAAiB,EAAE,EAAE,CAC1E,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAC3B,CAAA;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,EAAU;QACf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;IAC1D,CAAC;CACF;AAvDD,8BAuDC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export default class BatchingQueue<T> {
|
|
2
|
+
#private;
|
|
3
|
+
constructor(params: {
|
|
4
|
+
batchSize: number;
|
|
5
|
+
maxWaitTime?: number;
|
|
6
|
+
processFunction: (batch: T[]) => Promise<void>;
|
|
7
|
+
});
|
|
8
|
+
disposeAsync(): Promise<void>;
|
|
9
|
+
add(key: string, item: T): void;
|
|
10
|
+
get(id: string): T | undefined;
|
|
11
|
+
count(): number;
|
|
12
|
+
isDisposed(): boolean;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=batchingQueue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batchingQueue.d.ts","sourceRoot":"","sources":["../../../src/helpers/batchingQueue.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,OAAO,aAAa,CAAC,CAAC;;gBAYtB,MAAM,EAAE;QAClB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,eAAe,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KAC/C;IASK,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAKnC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI;IAI/B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS;IAI9B,KAAK,IAAI,MAAM;IAIf,UAAU,IAAI,OAAO;CAwCtB"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const keyedQueue_js_1 = __importDefault(require("./keyedQueue.js"));
|
|
7
|
+
class BatchingQueue {
|
|
8
|
+
#queue = new keyedQueue_js_1.default();
|
|
9
|
+
#batchSize;
|
|
10
|
+
#processFunction;
|
|
11
|
+
#baseInterval;
|
|
12
|
+
#minInterval;
|
|
13
|
+
#maxInterval;
|
|
14
|
+
#processingLoop;
|
|
15
|
+
#disposed = false;
|
|
16
|
+
constructor(params) {
|
|
17
|
+
this.#batchSize = params.batchSize;
|
|
18
|
+
this.#baseInterval = Math.min(params.maxWaitTime ?? 200, 200); // Initial batch time (ms)
|
|
19
|
+
this.#minInterval = Math.min(params.maxWaitTime ?? 100, 100); // Minimum batch time
|
|
20
|
+
this.#maxInterval = Math.min(params.maxWaitTime ?? 3000, 3000); // Maximum batch time
|
|
21
|
+
this.#processFunction = params.processFunction;
|
|
22
|
+
this.#processingLoop = this.#loop();
|
|
23
|
+
}
|
|
24
|
+
async disposeAsync() {
|
|
25
|
+
this.#disposed = true;
|
|
26
|
+
await this.#processingLoop;
|
|
27
|
+
}
|
|
28
|
+
add(key, item) {
|
|
29
|
+
this.#queue.enqueue(key, item);
|
|
30
|
+
}
|
|
31
|
+
get(id) {
|
|
32
|
+
return this.#queue.get(id);
|
|
33
|
+
}
|
|
34
|
+
count() {
|
|
35
|
+
return this.#queue.size;
|
|
36
|
+
}
|
|
37
|
+
isDisposed() {
|
|
38
|
+
return this.#disposed;
|
|
39
|
+
}
|
|
40
|
+
#getBatch(batchSize) {
|
|
41
|
+
return this.#queue.spliceValues(0, Math.min(batchSize, this.#queue.size));
|
|
42
|
+
}
|
|
43
|
+
async #loop() {
|
|
44
|
+
let interval = this.#baseInterval;
|
|
45
|
+
while (!this.#disposed || this.#queue.size > 0) {
|
|
46
|
+
const startTime = performance.now();
|
|
47
|
+
if (this.#queue.size > 0) {
|
|
48
|
+
const batch = this.#getBatch(this.#batchSize);
|
|
49
|
+
//console.log('running with queue size of ' + this.#queue.length)
|
|
50
|
+
await this.#processFunction(batch);
|
|
51
|
+
}
|
|
52
|
+
if (this.#queue.size < this.#batchSize / 2) {
|
|
53
|
+
//refigure interval
|
|
54
|
+
const endTime = performance.now();
|
|
55
|
+
const duration = endTime - startTime;
|
|
56
|
+
if (duration > interval) {
|
|
57
|
+
interval = Math.min(interval * 1.5, this.#maxInterval); // Increase if slow or empty
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
interval = Math.max(interval * 0.8, this.#minInterval); // Decrease if fast
|
|
61
|
+
}
|
|
62
|
+
/*console.log(
|
|
63
|
+
'queue is waiting ' +
|
|
64
|
+
interval / 1000 +
|
|
65
|
+
' with queue size of ' +
|
|
66
|
+
this.#queue.length
|
|
67
|
+
)*/
|
|
68
|
+
await this.#delay(interval);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
#delay(ms) {
|
|
73
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
exports.default = BatchingQueue;
|
|
77
|
+
//# sourceMappingURL=batchingQueue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batchingQueue.js","sourceRoot":"","sources":["../../../src/helpers/batchingQueue.ts"],"names":[],"mappings":";;;;;AAAA,oEAAwC;AAExC,MAAqB,aAAa;IAChC,MAAM,GAA0B,IAAI,uBAAU,EAAa,CAAA;IAC3D,UAAU,CAAQ;IAClB,gBAAgB,CAA+B;IAE/C,aAAa,CAAQ;IACrB,YAAY,CAAQ;IACpB,YAAY,CAAQ;IAEpB,eAAe,CAAe;IAC9B,SAAS,GAAG,KAAK,CAAA;IAEjB,YAAY,MAIX;QACC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,GAAG,EAAE,GAAG,CAAC,CAAA,CAAC,0BAA0B;QACxF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,GAAG,EAAE,GAAG,CAAC,CAAA,CAAC,qBAAqB;QAClF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,IAAI,CAAC,CAAA,CAAC,qBAAqB;QACpF,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,eAAe,CAAA;QAC9C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,MAAM,IAAI,CAAC,eAAe,CAAA;IAC5B,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,IAAO;QACtB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAChC,CAAC;IAED,GAAG,CAAC,EAAU;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;IACzB,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,SAAS,CAAA;IACvB,CAAC;IAED,SAAS,CAAC,SAAiB;QACzB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;IAC3E,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAA;QACjC,OAAO,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;YACnC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBAC7C,iEAAiE;gBACjE,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAA;YACpC,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;gBAC3C,mBAAmB;gBACnB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;gBACjC,MAAM,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAA;gBACpC,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;oBACxB,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA,CAAC,4BAA4B;gBACrF,CAAC;qBAAM,CAAC;oBACN,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA,CAAC,mBAAmB;gBAC5E,CAAC;gBACD;;;;;mBAKG;gBACH,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,CAAC,EAAU;QACf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;IAC1D,CAAC;CACF;AAlFD,gCAkFC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bufferQueue.d.ts","sourceRoot":"","sources":["../../../src/helpers/bufferQueue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAA;AAE9B,MAAM,CAAC,OAAO,OAAO,WAAW,CAAC,CAAC,CAAE,YAAW,KAAK,CAAC,CAAC,CAAC;;IAErD,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI;IAInB,MAAM,IAAI,CAAC,EAAE;CAGd"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
class BufferQueue {
|
|
4
|
+
#buffer = [];
|
|
5
|
+
add(value) {
|
|
6
|
+
this.#buffer.push(value);
|
|
7
|
+
}
|
|
8
|
+
values() {
|
|
9
|
+
return this.#buffer;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
exports.default = BufferQueue;
|
|
13
|
+
//# sourceMappingURL=bufferQueue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bufferQueue.js","sourceRoot":"","sources":["../../../src/helpers/bufferQueue.ts"],"names":[],"mappings":";;AAEA,MAAqB,WAAW;IAC9B,OAAO,GAAQ,EAAE,CAAA;IACjB,GAAG,CAAC,KAAQ;QACV,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC1B,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;CACF;AATD,8BASC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { Database } from '../operations/interfaces.js';
|
|
2
|
+
import { CacheOptions } from '../operations/options.js';
|
|
3
|
+
import { Item } from '../types/types.js';
|
|
4
|
+
import Queue from './queue.js';
|
|
5
|
+
import { Downloader } from '../operations/interfaces.js';
|
|
6
|
+
import { DefermentManager } from './defermentManager.js';
|
|
7
|
+
import AsyncGeneratorQueue from './asyncGeneratorQueue.js';
|
|
8
|
+
import { Pump } from './pump.js';
|
|
9
|
+
export declare class CachePump implements Pump {
|
|
10
|
+
#private;
|
|
11
|
+
constructor(database: Database, gathered: AsyncGeneratorQueue<Item>, deferments: DefermentManager, options: CacheOptions);
|
|
12
|
+
add(item: Item): void;
|
|
13
|
+
disposeAsync(): Promise<void>;
|
|
14
|
+
get isDisposed(): boolean;
|
|
15
|
+
pumpItems(params: {
|
|
16
|
+
ids: string[];
|
|
17
|
+
foundItems: Queue<Item>;
|
|
18
|
+
notFoundItems: Queue<string>;
|
|
19
|
+
}): Promise<void>;
|
|
20
|
+
gather(ids: string[], downloader: Downloader): AsyncGenerator<Item>;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=cachePump.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cachePump.d.ts","sourceRoot":"","sources":["../../../src/helpers/cachePump.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AACvD,OAAO,EAAgB,IAAI,EAAE,MAAM,mBAAmB,CAAA;AAEtD,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,mBAAmB,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,qBAAa,SAAU,YAAW,IAAI;;gBAalC,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,mBAAmB,CAAC,IAAI,CAAC,EACnC,UAAU,EAAE,gBAAgB,EAC5B,OAAO,EAAE,YAAY;IASvB,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAYf,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAMnC,IAAI,UAAU,IAAI,OAAO,CAExB;IAEK,SAAS,CAAC,MAAM,EAAE;QACtB,GAAG,EAAE,MAAM,EAAE,CAAA;QACb,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;QACvB,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;KAC7B,GAAG,OAAO,CAAC,IAAI,CAAC;IA0BV,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC;CAkB3E"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.CachePump = void 0;
|
|
7
|
+
const shared_1 = require("@speckle/shared");
|
|
8
|
+
const batchingQueue_js_1 = __importDefault(require("./batchingQueue.js"));
|
|
9
|
+
class CachePump {
|
|
10
|
+
#writeQueue;
|
|
11
|
+
#database;
|
|
12
|
+
#logger;
|
|
13
|
+
#deferments;
|
|
14
|
+
#gathered;
|
|
15
|
+
#options;
|
|
16
|
+
#disposed = false;
|
|
17
|
+
constructor(database, gathered, deferments, options) {
|
|
18
|
+
this.#database = database;
|
|
19
|
+
this.#gathered = gathered;
|
|
20
|
+
this.#deferments = deferments;
|
|
21
|
+
this.#options = options;
|
|
22
|
+
this.#logger = options.logger || (() => { });
|
|
23
|
+
}
|
|
24
|
+
add(item) {
|
|
25
|
+
if (!this.#writeQueue) {
|
|
26
|
+
this.#writeQueue = new batchingQueue_js_1.default({
|
|
27
|
+
batchSize: this.#options.maxCacheWriteSize,
|
|
28
|
+
maxWaitTime: this.#options.maxCacheBatchWriteWait,
|
|
29
|
+
processFunction: (batch) => this.#database.cacheSaveBatch({ batch })
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
this.#writeQueue.add(item.baseId, item);
|
|
33
|
+
}
|
|
34
|
+
async disposeAsync() {
|
|
35
|
+
await this.#writeQueue?.disposeAsync();
|
|
36
|
+
await this.#database.disposeAsync();
|
|
37
|
+
this.#disposed = true;
|
|
38
|
+
}
|
|
39
|
+
get isDisposed() {
|
|
40
|
+
return this.#disposed;
|
|
41
|
+
}
|
|
42
|
+
async pumpItems(params) {
|
|
43
|
+
const { ids, foundItems, notFoundItems } = params;
|
|
44
|
+
const maxCacheReadSize = this.#options.maxCacheReadSize;
|
|
45
|
+
for (let i = 0; i < ids.length;) {
|
|
46
|
+
if (this.isDisposed)
|
|
47
|
+
break;
|
|
48
|
+
if ((this.#writeQueue?.count() ?? 0) > this.#options.maxWriteQueueSize) {
|
|
49
|
+
this.#logger('pausing reads (# in write queue: ' + this.#writeQueue?.count() + ')');
|
|
50
|
+
await new Promise((resolve) => setTimeout(resolve, shared_1.TIME.second)); // Pause for 1 second, protects against out of memory
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const batch = ids.slice(i, i + maxCacheReadSize);
|
|
54
|
+
const cachedData = await this.#database.getAll(batch);
|
|
55
|
+
for (let i = 0; i < cachedData.length; i++) {
|
|
56
|
+
if (cachedData[i]) {
|
|
57
|
+
foundItems.add(cachedData[i]);
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
notFoundItems.add(batch[i]);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
i += maxCacheReadSize;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
async *gather(ids, downloader) {
|
|
67
|
+
const total = ids.length;
|
|
68
|
+
const pumpPromise = this.pumpItems({
|
|
69
|
+
ids,
|
|
70
|
+
foundItems: this.#gathered,
|
|
71
|
+
notFoundItems: downloader
|
|
72
|
+
});
|
|
73
|
+
let count = 0;
|
|
74
|
+
for await (const item of this.#gathered.consume()) {
|
|
75
|
+
this.#deferments.undefer(item);
|
|
76
|
+
yield item;
|
|
77
|
+
count++;
|
|
78
|
+
if (count >= total) {
|
|
79
|
+
this.#gathered.dispose();
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
await pumpPromise;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
exports.CachePump = CachePump;
|
|
86
|
+
//# sourceMappingURL=cachePump.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cachePump.js","sourceRoot":"","sources":["../../../src/helpers/cachePump.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAsC;AAItC,0EAA8C;AAO9C,MAAa,SAAS;IACpB,WAAW,CAAiC;IAC5C,SAAS,CAAU;IACnB,OAAO,CAAc;IACrB,WAAW,CAAkB;IAE7B,SAAS,CAA2B;IAEpC,QAAQ,CAAc;IAEtB,SAAS,GAAG,KAAK,CAAA;IAEjB,YACE,QAAkB,EAClB,QAAmC,EACnC,UAA4B,EAC5B,OAAqB;QAErB,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAA;QACzB,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAA;QACzB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAA;QAC7B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,GAAS,EAAE,GAAE,CAAC,CAAC,CAAA;IACnD,CAAC;IAED,GAAG,CAAC,IAAU;QACZ,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,GAAG,IAAI,0BAAa,CAAC;gBACnC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,iBAAiB;gBAC1C,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,sBAAsB;gBACjD,eAAe,EAAE,CAAC,KAAa,EAAiB,EAAE,CAChD,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,CAAC;aAC3C,CAAC,CAAA;QACJ,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,WAAW,EAAE,YAAY,EAAE,CAAA;QACtC,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,CAAA;QACnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;IACvB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,SAAS,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAIf;QACC,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QACjD,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAA;QAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,GAAI,CAAC;YACjC,IAAI,IAAI,CAAC,UAAU;gBAAE,MAAK;YAC1B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;gBACvE,IAAI,CAAC,OAAO,CACV,mCAAmC,GAAG,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,GAAG,CACtE,CAAA;gBACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,aAAI,CAAC,MAAM,CAAC,CAAC,CAAA,CAAC,qDAAqD;gBACtH,SAAQ;YACV,CAAC;YACD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAA;YAChD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;oBAClB,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC,CAAA;gBAChC,CAAC;qBAAM,CAAC;oBACN,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7B,CAAC;YACH,CAAC;YACD,CAAC,IAAI,gBAAgB,CAAA;QACvB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,GAAa,EAAE,UAAsB;QACjD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAA;QACxB,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;YACjC,GAAG;YACH,UAAU,EAAE,IAAI,CAAC,SAAS;YAC1B,aAAa,EAAE,UAAU;SAC1B,CAAC,CAAA;QACF,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAC9B,MAAM,IAAI,CAAA;YACV,KAAK,EAAE,CAAA;YACP,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;gBACnB,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAA;YAC1B,CAAC;QACH,CAAC;QACD,MAAM,WAAW,CAAA;IACnB,CAAC;CACF;AA/FD,8BA+FC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Database } from '../operations/interfaces.js';
|
|
2
|
+
import { CacheOptions } from '../operations/options.js';
|
|
3
|
+
import { Base, Item } from '../types/types.js';
|
|
4
|
+
import { DefermentManager } from './defermentManager.js';
|
|
5
|
+
export declare class CacheReader {
|
|
6
|
+
#private;
|
|
7
|
+
constructor(database: Database, defermentManager: DefermentManager, options: CacheOptions);
|
|
8
|
+
getObject(params: {
|
|
9
|
+
id: string;
|
|
10
|
+
}): Promise<Base>;
|
|
11
|
+
getAll(keys: string[]): Promise<(Item | undefined)[]>;
|
|
12
|
+
disposeAsync(): Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=cacheReader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cacheReader.d.ts","sourceRoot":"","sources":["../../../src/helpers/cacheReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AACvD,OAAO,EAAE,IAAI,EAAgB,IAAI,EAAE,MAAM,mBAAmB,CAAA;AAE5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AAExD,qBAAa,WAAW;;gBAQpB,QAAQ,EAAE,QAAQ,EAClB,gBAAgB,EAAE,gBAAgB,EAClC,OAAO,EAAE,YAAY;IAQjB,SAAS,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBhD,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC;IAgBrD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;CAGpC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.CacheReader = void 0;
|
|
7
|
+
const batchingQueue_js_1 = __importDefault(require("./batchingQueue.js"));
|
|
8
|
+
class CacheReader {
|
|
9
|
+
#database;
|
|
10
|
+
#defermentManager;
|
|
11
|
+
#logger;
|
|
12
|
+
#options;
|
|
13
|
+
#readQueue;
|
|
14
|
+
constructor(database, defermentManager, options) {
|
|
15
|
+
this.#database = database;
|
|
16
|
+
this.#defermentManager = defermentManager;
|
|
17
|
+
this.#options = options;
|
|
18
|
+
this.#logger = options.logger || (() => { });
|
|
19
|
+
}
|
|
20
|
+
async getObject(params) {
|
|
21
|
+
if (!this.#defermentManager.isDeferred(params.id)) {
|
|
22
|
+
this.#getItem(params.id);
|
|
23
|
+
}
|
|
24
|
+
return await this.#defermentManager.defer({ id: params.id });
|
|
25
|
+
}
|
|
26
|
+
#getItem(id) {
|
|
27
|
+
if (!this.#readQueue) {
|
|
28
|
+
this.#readQueue = new batchingQueue_js_1.default({
|
|
29
|
+
batchSize: this.#options.maxCacheReadSize,
|
|
30
|
+
maxWaitTime: this.#options.maxCacheBatchReadWait,
|
|
31
|
+
processFunction: this.#processBatch
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
if (!this.#readQueue.get(id)) {
|
|
35
|
+
this.#readQueue.add(id, id);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
async getAll(keys) {
|
|
39
|
+
return this.#database.getAll(keys);
|
|
40
|
+
}
|
|
41
|
+
#processBatch = async (batch) => {
|
|
42
|
+
const items = await this.#database.getAll(batch);
|
|
43
|
+
for (let i = 0; i < items.length; i++) {
|
|
44
|
+
if (items[i]) {
|
|
45
|
+
this.#defermentManager.undefer(items[i]);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
//this is okay!
|
|
49
|
+
//this.#logger(`Item ${batch[i]} not found in cache`)
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
async disposeAsync() {
|
|
54
|
+
await this.#readQueue?.disposeAsync();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
exports.CacheReader = CacheReader;
|
|
58
|
+
//# sourceMappingURL=cacheReader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cacheReader.js","sourceRoot":"","sources":["../../../src/helpers/cacheReader.ts"],"names":[],"mappings":";;;;;;AAGA,0EAA8C;AAG9C,MAAa,WAAW;IACtB,SAAS,CAAU;IACnB,iBAAiB,CAAkB;IACnC,OAAO,CAAc;IACrB,QAAQ,CAAc;IACtB,UAAU,CAAmC;IAE7C,YACE,QAAkB,EAClB,gBAAkC,EAClC,OAAqB;QAErB,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAA;QACzB,IAAI,CAAC,iBAAiB,GAAG,gBAAgB,CAAA;QACzC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,GAAS,EAAE,GAAE,CAAC,CAAC,CAAA;IACnD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAsB;QACpC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC1B,CAAC;QACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAA;IAC9D,CAAC;IAED,QAAQ,CAAC,EAAU;QACjB,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,IAAI,0BAAa,CAAC;gBAClC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,gBAAgB;gBACzC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,qBAAqB;gBAChD,eAAe,EAAE,IAAI,CAAC,aAAa;aACpC,CAAC,CAAA;QACJ,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAC7B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAc;QACzB,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC;IAED,aAAa,GAAG,KAAK,EAAE,KAAe,EAAiB,EAAE;QACvD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBACb,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;YAC3C,CAAC;iBAAM,CAAC;gBACN,eAAe;gBACf,qDAAqD;YACvD,CAAC;QACH,CAAC;IACH,CAAC,CAAA;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,UAAU,EAAE,YAAY,EAAE,CAAA;IACvC,CAAC;CACF;AAzDD,kCAyDC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { DeferredBase } from './deferredBase.js';
|
|
2
|
+
import { Base, Item } from '../types/types.js';
|
|
3
|
+
import { DefermentManagerOptions } from '../operations/options.js';
|
|
4
|
+
export declare class DefermentManager {
|
|
5
|
+
private options;
|
|
6
|
+
private deferments;
|
|
7
|
+
private timer?;
|
|
8
|
+
private logger;
|
|
9
|
+
private currentSize;
|
|
10
|
+
private disposed;
|
|
11
|
+
private totalDefermentRequests;
|
|
12
|
+
constructor(options: DefermentManagerOptions);
|
|
13
|
+
private now;
|
|
14
|
+
isDeferred(id: string): boolean;
|
|
15
|
+
get(id: string): DeferredBase | undefined;
|
|
16
|
+
defer(params: {
|
|
17
|
+
id: string;
|
|
18
|
+
}): Promise<Base>;
|
|
19
|
+
private trackDefermentRequest;
|
|
20
|
+
undefer(item: Item): void;
|
|
21
|
+
private resetGlobalTimer;
|
|
22
|
+
dispose(): void;
|
|
23
|
+
private clearDeferments;
|
|
24
|
+
private cleanDeferments;
|
|
25
|
+
compareMaybeBasesBySize(a: Item | undefined, b: Item | undefined): number;
|
|
26
|
+
compareMaybe(a: number | undefined, b: number | undefined): number;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=defermentManager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"defermentManager.d.ts","sourceRoot":"","sources":["../../../src/helpers/defermentManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,IAAI,EAAgB,IAAI,EAAE,MAAM,mBAAmB,CAAA;AAC5D,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAA;AAElE,qBAAa,gBAAgB;IAUf,OAAO,CAAC,OAAO;IAT3B,OAAO,CAAC,UAAU,CAAuC;IACzD,OAAO,CAAC,KAAK,CAAC,CAA+B;IAC7C,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,WAAW,CAAI;IACvB,OAAO,CAAC,QAAQ,CAAQ;IAGxB,OAAO,CAAC,sBAAsB,CAAiC;gBAE3C,OAAO,EAAE,uBAAuB;IAKpD,OAAO,CAAC,GAAG;IAIX,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAI/B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAKnC,KAAK,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlD,OAAO,CAAC,qBAAqB;IAS7B,OAAO,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAgBzB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,IAAI,IAAI;IAUf,OAAO,CAAC,eAAe;IAavB,OAAO,CAAC,eAAe;IAwCvB,uBAAuB,CAAC,CAAC,EAAE,IAAI,GAAG,SAAS,EAAE,CAAC,EAAE,IAAI,GAAG,SAAS,GAAG,MAAM;IAOzE,YAAY,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM;CAMnE"}
|