npm - @milaboratories/pl-tree - Versions diffs - 1.8.44 → 1.8.46 - Mend

@milaboratories/pl-tree 1.8.44 → 1.8.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (7) hide show

package/dist/sync.cjs CHANGED Viewed

@@ -1,7 +1,6 @@
 'use strict';
 var plClient = require('@milaboratories/pl-client');
-var Denque = require('denque');
 var tsHelpers = require('@milaboratories/ts-helpers');
 /** Given the current tree state, build the request object to pass to
@@ -57,39 +56,36 @@ async function loadTreeState(tx, loadingRequest, stats) {
     if (stats)
         stats.requests++;
     const { seedResources, finalResources, pruningFunction } = loadingRequest;
-    // Main idea of using a queue here is that responses will arrive in the same order as they were
-    // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.
-    // In such a way logic become linear without recursion, and at the same time deal with data
-    // as soon as it arrives.
-    const pending = new Denque();
+    // Limits the number of concurrent gRPC fetches to bound peak memory
+    // from in-flight request/response buffers.
+    const limiter = new tsHelpers.ConcurrencyLimitingExecutor(100);
+    // Promises of resource states, in the order they were requested.
+    const pending = [];
     // vars to calculate number of roundtrips for stats
     let roundTripToggle = true;
     let numberOfRoundTrips = 0;
-    // tracking resources we already requested
+    // tracking resources we already requested or queued
     const requested = new Set();
+    /** Mark a resource for fetching. Deduplicates and respects final-resource set. */
     const requestState = (rid) => {
         if (plClient.isNullResourceId(rid) || requested.has(rid))
             return;
-        // separate check to collect stats
         if (finalResources.has(rid)) {
             if (stats)
                 stats.finalResourcesSkipped++;
             return;
         }
-        // adding the id, so we will not request it's state again if somebody else
-        // references the same resource
         requested.add(rid);
-        // requesting resource and all kv records
-        const resourceData = tx.getResourceDataIfExists(rid, true);
-        const kvData = tx.listKeyValuesIfResourceExists(rid);
-        // counting round-trip (begin)
-        const addRT = roundTripToggle;
-        if (roundTripToggle)
-            roundTripToggle = false;
-        // pushing combined promise
-        pending.push((async () => {
+        pending.push(limiter.run(async () => {
+            const resourceData = tx.getResourceDataIfExists(rid, true);
+            const kvData = tx.listKeyValuesIfResourceExists(rid);
+            // counting round-trip (begin)
+            const addRT = roundTripToggle;
+            if (roundTripToggle)
+                roundTripToggle = false;
             const [resource, kv] = await Promise.all([resourceData, kvData]);
-            // counting round-trip, actually incrementing counter and returning toggle back, so the next request can acquire it
+            // counting round-trip, actually incrementing counter and returning toggle back,
+            // so the next request can acquire it
             if (addRT) {
                 numberOfRoundTrips++;
                 roundTripToggle = true;
@@ -99,19 +95,14 @@ async function loadTreeState(tx, loadingRequest, stats) {
             if (kv === undefined)
                 throw new Error("Inconsistent replies");
             return { ...resource, kv };
-        })());
+        }));
     };
     // sending seed requests
     seedResources.forEach((rid) => requestState(rid));
     const result = [];
-    while (true) {
-        // taking next pending request
-        const nextResourcePromise = pending.shift();
-        if (nextResourcePromise === undefined)
-            // this means we have no pending requests and traversal is over
-            break;
-        // at this point we pause and wait for the nest requested resource state to arrive
-        let nextResource = await nextResourcePromise;
+    for (let i = 0; i < pending.length; i++) {
+        // at this point we pause and wait for the next requested resource state to arrive
+        let nextResource = await pending[i];
         if (nextResource === undefined)
             // ignoring resources that were not found (this may happen for seed resource ids)
             continue;
@@ -123,7 +114,7 @@ async function loadTreeState(tx, loadingRequest, stats) {
                 stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;
             nextResource = { ...nextResource, fields: fieldsAfterPruning };
         }
-        // continue traversal over the referenced resource
+        // continue traversal over the referenced resources
         requestState(nextResource.error);
         for (const field of nextResource.fields) {
             requestState(field.value);

package/dist/sync.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"sync.cjs","sources":["../src/sync.ts"],"sourcesContent":["import type {\n FieldData,\n OptionalResourceId,\n PlTransaction,\n ResourceId,\n} from \"@milaboratories/pl-client\";\nimport { isNullResourceId } from \"@milaboratories/pl-client\";\nimport Denque from \"denque\";\nimport type { ExtendedResourceData, PlTreeState } from \"./state\";\nimport { msToHumanReadable } from \"@milaboratories/ts-helpers\";\n\n/** Applied to list of fields in resource data. /\nexport type PruningFunction = (resource: ExtendedResourceData) => FieldData[];\n\nexport interface TreeLoadingRequest {\n /* Resource to prime the traversal algorithm. It is ok, if some of them\n * doesn't exist anymore. Should not contain elements from final resource\n * set. /\n readonly seedResources: ResourceId[];\n\n /* Resource ids for which state is already known and not expected to change.\n * Algorithm will not continue traversal over those ids, and states will not\n * be retrieved for them. /\n readonly finalResources: Set<ResourceId>;\n\n /* This function is applied to each resource data field list, before\n * using it continue traversal. This modification also is applied to\n * output data to make result self-consistent in terms that it will contain\n * all referenced resources, this is required to be able to pass it to tree\n * to update the state. /\n readonly pruningFunction?: PruningFunction;\n}\n\n/* Given the current tree state, build the request object to pass to\n * {@link loadTreeState} to load updated state. /\nexport function constructTreeLoadingRequest(\n tree: PlTreeState,\n pruningFunction?: PruningFunction,\n): TreeLoadingRequest {\n const seedResources: ResourceId[] = [];\n const finalResources = new Set<ResourceId>();\n tree.forEachResource((res) => {\n if (res.finalState) finalResources.add(res.id);\n else seedResources.push(res.id);\n });\n\n // if tree is empty, seeding tree reconstruction from the specified root\n if (seedResources.length === 0 && finalResources.size === 0) seedResources.push(tree.root);\n\n return { seedResources, finalResources, pruningFunction };\n}\n\nexport type TreeLoadingStat = {\n requests: number;\n roundTrips: number;\n retrievedResources: number;\n retrievedFields: number;\n retrievedKeyValues: number;\n retrievedResourceDataBytes: number;\n retrievedKeyValueBytes: number;\n prunedFields: number;\n finalResourcesSkipped: number;\n millisSpent: number;\n};\n\nexport function initialTreeLoadingStat(): TreeLoadingStat {\n return {\n requests: 0,\n roundTrips: 0,\n retrievedResources: 0,\n retrievedFields: 0,\n retrievedKeyValues: 0,\n retrievedResourceDataBytes: 0,\n retrievedKeyValueBytes: 0,\n prunedFields: 0,\n finalResourcesSkipped: 0,\n millisSpent: 0,\n };\n}\n\nexport function formatTreeLoadingStat(stat: TreeLoadingStat): string {\n let result = `Requests: ${stat.requests}\\n`;\n result += `Total time: ${msToHumanReadable(stat.millisSpent)}\\n`;\n result += `Round-trips: ${stat.roundTrips}\\n`;\n result += `Resources: ${stat.retrievedResources}\\n`;\n result += `Fields: ${stat.retrievedFields}\\n`;\n result += `KV: ${stat.retrievedKeyValues}\\n`;\n result += `Data Bytes: ${stat.retrievedResourceDataBytes}\\n`;\n result += `KV Bytes: ${stat.retrievedKeyValueBytes}\\n`;\n result += `Pruned fields: ${stat.prunedFields}\\n`;\n result += `Final resources skipped: ${stat.finalResourcesSkipped}`;\n return result;\n}\n\n/* Given the transaction (preferably read-only) and loading request, executes\n * the tree traversal algorithm, and collects fresh states of resources\n * to update the tree state. */\nexport async function loadTreeState(\n tx: PlTransaction,\n loadingRequest: TreeLoadingRequest,\n stats?: TreeLoadingStat,\n): Promise<ExtendedResourceData[]> {\n // saving start timestamp to add time spent in this function to the stats at the end of the method\n const startTimestamp = Date.now();\n\n // counting the request\n if (stats) stats.requests++;\n\n const { seedResources, finalResources, pruningFunction } = loadingRequest;\n\n // Main idea of using a queue here is that responses will arrive in the same order as they were\n // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.\n // In such a way logic become linear without recursion, and at the same time deal with data\n // as soon as it arrives.\n\n const pending = new Denque<Promise<ExtendedResourceData \| undefined>>();\n\n // vars to calculate number of roundtrips for stats\n let roundTripToggle: boolean = true;\n let numberOfRoundTrips = 0;\n\n // tracking resources we already requested\n const requested = new Set<ResourceId>();\n const requestState = (rid: OptionalResourceId) => {\n if (isNullResourceId(rid) \|\| requested.has(rid)) return;\n\n // separate check to collect stats\n if (finalResources.has(rid)) {\n if (stats) stats.finalResourcesSkipped++;\n return;\n }\n\n // adding the id, so we will not request it's state again if somebody else\n // references the same resource\n requested.add(rid);\n\n // requesting resource and all kv records\n const resourceData = tx.getResourceDataIfExists(rid, true);\n const kvData = tx.listKeyValuesIfResourceExists(rid);\n\n // counting round-trip (begin)\n const addRT = roundTripToggle;\n if (roundTripToggle) roundTripToggle = false;\n\n // pushing combined promise\n pending.push(\n (async () => {\n const [resource, kv] = await Promise.all([resourceData, kvData]);\n\n // counting round-trip, actually incrementing counter and returning toggle back, so the next request can acquire it\n if (addRT) {\n numberOfRoundTrips++;\n roundTripToggle = true;\n }\n\n if (resource === undefined) return undefined;\n\n if (kv === undefined) throw new Error(\"Inconsistent replies\");\n\n return { ...resource, kv };\n })(),\n );\n };\n\n // sending seed requests\n seedResources.forEach((rid) => requestState(rid));\n\n const result: ExtendedResourceData[] = [];\n while (true) {\n // taking next pending request\n const nextResourcePromise = pending.shift();\n if (nextResourcePromise === undefined)\n // this means we have no pending requests and traversal is over\n break;\n\n // at this point we pause and wait for the nest requested resource state to arrive\n let nextResource = await nextResourcePromise;\n if (nextResource === undefined)\n // ignoring resources that were not found (this may happen for seed resource ids)\n continue;\n\n if (pruningFunction !== undefined) {\n // apply field pruning, if requested\n const fieldsAfterPruning = pruningFunction(nextResource);\n // collecting stats\n if (stats) stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;\n nextResource = { ...nextResource, fields: fieldsAfterPruning };\n }\n\n // continue traversal over the referenced resource\n requestState(nextResource.error);\n for (const field of nextResource.fields) {\n requestState(field.value);\n requestState(field.error);\n }\n\n // collecting stats\n if (stats) {\n stats.retrievedResources++;\n stats.retrievedFields += nextResource.fields.length;\n stats.retrievedKeyValues += nextResource.kv.length;\n stats.retrievedResourceDataBytes += nextResource.data?.length ?? 0;\n for (const kv of nextResource.kv) stats.retrievedKeyValueBytes += kv.value.length;\n }\n\n // aggregating the state\n result.push(nextResource);\n }\n\n // adding the time we spent in this method to stats\n if (stats) {\n stats.millisSpent += Date.now() - startTimestamp;\n stats.roundTrips += numberOfRoundTrips;\n }\n\n return result;\n}\n"],"names":["msToHumanReadable","isNullResourceId"],"mappings":";;;;;;AAiCA;AACkD;AAC5C,SAAU,2BAA2B,CACzC,IAAiB,EACjB,eAAiC,EAAA;IAEjC,MAAM,aAAa,GAAiB,EAAE;AACtC,IAAA,MAAM,cAAc,GAAG,IAAI,GAAG,EAAc;AAC5C,IAAA,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,KAAI;QAC3B,IAAI,GAAG,CAAC,UAAU;AAAE,YAAA,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;;AACzC,YAAA,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACjC,IAAA,CAAC,CAAC;;IAGF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC;AAAE,QAAA,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAE1F,IAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE;AAC3D;SAegB,sBAAsB,GAAA;IACpC,OAAO;AACL,QAAA,QAAQ,EAAE,CAAC;AACX,QAAA,UAAU,EAAE,CAAC;AACb,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,eAAe,EAAE,CAAC;AAClB,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,0BAA0B,EAAE,CAAC;AAC7B,QAAA,sBAAsB,EAAE,CAAC;AACzB,QAAA,YAAY,EAAE,CAAC;AACf,QAAA,qBAAqB,EAAE,CAAC;AACxB,QAAA,WAAW,EAAE,CAAC;KACf;AACH;AAEM,SAAU,qBAAqB,CAAC,IAAqB,EAAA;AACzD,IAAA,IAAI,MAAM,GAAG,CAAA,UAAA,EAAa,IAAI,CAAC,QAAQ,IAAI;IAC3C,MAAM,IAAI,eAAeA,2BAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA,EAAA,CAAI;AAChE,IAAA,MAAM,IAAI,CAAA,aAAA,EAAgB,IAAI,CAAC,UAAU,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,kBAAkB,IAAI;AACnD,IAAA,MAAM,IAAI,CAAA,QAAA,EAAW,IAAI,CAAC,eAAe,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,IAAA,EAAO,IAAI,CAAC,kBAAkB,IAAI;AAC5C,IAAA,MAAM,IAAI,CAAA,YAAA,EAAe,IAAI,CAAC,0BAA0B,IAAI;AAC5D,IAAA,MAAM,IAAI,CAAA,UAAA,EAAa,IAAI,CAAC,sBAAsB,IAAI;AACtD,IAAA,MAAM,IAAI,CAAA,eAAA,EAAkB,IAAI,CAAC,YAAY,IAAI;AACjD,IAAA,MAAM,IAAI,CAAA,yBAAA,EAA4B,IAAI,CAAC,qBAAqB,EAAE;AAClE,IAAA,OAAO,MAAM;AACf;AAEA;;AAE+B;AACxB,eAAe,aAAa,CACjC,EAAiB,EACjB,cAAkC,EAClC,KAAuB,EAAA;;AAGvB,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE;;AAGjC,IAAA,IAAI,KAAK;QAAE,KAAK,CAAC,QAAQ,EAAE;IAE3B,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,cAAc;;;;;AAOzE,IAAA,MAAM,OAAO,GAAG,IAAI,MAAM,EAA6C;;IAGvE,IAAI,eAAe,GAAY,IAAI;IACnC,IAAI,kBAAkB,GAAG,CAAC;;AAG1B,IAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc;AACvC,IAAA,MAAM,YAAY,GAAG,CAAC,GAAuB,KAAI;QAC/C,IAAIC,yBAAgB,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE;;AAGjD,QAAA,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;AAC3B,YAAA,IAAI,KAAK;gBAAE,KAAK,CAAC,qBAAqB,EAAE;YACxC;QACF;;;AAIA,QAAA,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;;QAGlB,MAAM,YAAY,GAAG,EAAE,CAAC,uBAAuB,CAAC,GAAG,EAAE,IAAI,CAAC;QAC1D,MAAM,MAAM,GAAG,EAAE,CAAC,6BAA6B,CAAC,GAAG,CAAC;;QAGpD,MAAM,KAAK,GAAG,eAAe;AAC7B,QAAA,IAAI,eAAe;YAAE,eAAe,GAAG,KAAK;;AAG5C,QAAA,OAAO,CAAC,IAAI,CACV,CAAC,YAAW;AACV,YAAA,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;;YAGhE,IAAI,KAAK,EAAE;AACT,gBAAA,kBAAkB,EAAE;gBACpB,eAAe,GAAG,IAAI;YACxB;YAEA,IAAI,QAAQ,KAAK,SAAS;AAAE,gBAAA,OAAO,SAAS;YAE5C,IAAI,EAAE,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC;AAE7D,YAAA,OAAO,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE;QAC5B,CAAC,GAAG,CACL;AACH,IAAA,CAAC;;AAGD,IAAA,aAAa,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC;IAEjD,MAAM,MAAM,GAA2B,EAAE;IACzC,OAAO,IAAI,EAAE;;AAEX,QAAA,MAAM,mBAAmB,GAAG,OAAO,CAAC,KAAK,EAAE;QAC3C,IAAI,mBAAmB,KAAK,SAAS;;YAEnC;;AAGF,QAAA,IAAI,YAAY,GAAG,MAAM,mBAAmB;QAC5C,IAAI,YAAY,KAAK,SAAS;;YAE5B;AAEF,QAAA,IAAI,eAAe,KAAK,SAAS,EAAE;;AAEjC,YAAA,MAAM,kBAAkB,GAAG,eAAe,CAAC,YAAY,CAAC;;AAExD,YAAA,IAAI,KAAK;AAAE,gBAAA,KAAK,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM;YACvF,YAAY,GAAG,EAAE,GAAG,YAAY,EAAE,MAAM,EAAE,kBAAkB,EAAE;QAChE;;AAGA,QAAA,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC;AAChC,QAAA,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE;AACvC,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;QAC3B;;QAGA,IAAI,KAAK,EAAE;YACT,KAAK,CAAC,kBAAkB,EAAE;YAC1B,KAAK,CAAC,eAAe,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM;YACnD,KAAK,CAAC,kBAAkB,IAAI,YAAY,CAAC,EAAE,CAAC,MAAM;YAClD,KAAK,CAAC,0BAA0B,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;AAClE,YAAA,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,EAAE;gBAAE,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM;QACnF;;AAGA,QAAA,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;IAC3B;;IAGA,IAAI,KAAK,EAAE;QACT,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;AAChD,QAAA,KAAK,CAAC,UAAU,IAAI,kBAAkB;IACxC;AAEA,IAAA,OAAO,MAAM;AACf;;;;;;;"}
1	+ {"version":3,"file":"sync.cjs","sources":["../src/sync.ts"],"sourcesContent":["import type {\n FieldData,\n OptionalResourceId,\n PlTransaction,\n ResourceId,\n} from \"@milaboratories/pl-client\";\nimport { isNullResourceId } from \"@milaboratories/pl-client\";\nimport type { ExtendedResourceData, PlTreeState } from \"./state\";\nimport { ConcurrencyLimitingExecutor, msToHumanReadable } from \"@milaboratories/ts-helpers\";\n\n/** Applied to list of fields in resource data. /\nexport type PruningFunction = (resource: ExtendedResourceData) => FieldData[];\n\nexport interface TreeLoadingRequest {\n /* Resource to prime the traversal algorithm. It is ok, if some of them\n * doesn't exist anymore. Should not contain elements from final resource\n * set. /\n readonly seedResources: ResourceId[];\n\n /* Resource ids for which state is already known and not expected to change.\n * Algorithm will not continue traversal over those ids, and states will not\n * be retrieved for them. /\n readonly finalResources: Set<ResourceId>;\n\n /* This function is applied to each resource data field list, before\n * using it continue traversal. This modification also is applied to\n * output data to make result self-consistent in terms that it will contain\n * all referenced resources, this is required to be able to pass it to tree\n * to update the state. /\n readonly pruningFunction?: PruningFunction;\n}\n\n/* Given the current tree state, build the request object to pass to\n * {@link loadTreeState} to load updated state. /\nexport function constructTreeLoadingRequest(\n tree: PlTreeState,\n pruningFunction?: PruningFunction,\n): TreeLoadingRequest {\n const seedResources: ResourceId[] = [];\n const finalResources = new Set<ResourceId>();\n tree.forEachResource((res) => {\n if (res.finalState) finalResources.add(res.id);\n else seedResources.push(res.id);\n });\n\n // if tree is empty, seeding tree reconstruction from the specified root\n if (seedResources.length === 0 && finalResources.size === 0) seedResources.push(tree.root);\n\n return { seedResources, finalResources, pruningFunction };\n}\n\nexport type TreeLoadingStat = {\n requests: number;\n roundTrips: number;\n retrievedResources: number;\n retrievedFields: number;\n retrievedKeyValues: number;\n retrievedResourceDataBytes: number;\n retrievedKeyValueBytes: number;\n prunedFields: number;\n finalResourcesSkipped: number;\n millisSpent: number;\n};\n\nexport function initialTreeLoadingStat(): TreeLoadingStat {\n return {\n requests: 0,\n roundTrips: 0,\n retrievedResources: 0,\n retrievedFields: 0,\n retrievedKeyValues: 0,\n retrievedResourceDataBytes: 0,\n retrievedKeyValueBytes: 0,\n prunedFields: 0,\n finalResourcesSkipped: 0,\n millisSpent: 0,\n };\n}\n\nexport function formatTreeLoadingStat(stat: TreeLoadingStat): string {\n let result = `Requests: ${stat.requests}\\n`;\n result += `Total time: ${msToHumanReadable(stat.millisSpent)}\\n`;\n result += `Round-trips: ${stat.roundTrips}\\n`;\n result += `Resources: ${stat.retrievedResources}\\n`;\n result += `Fields: ${stat.retrievedFields}\\n`;\n result += `KV: ${stat.retrievedKeyValues}\\n`;\n result += `Data Bytes: ${stat.retrievedResourceDataBytes}\\n`;\n result += `KV Bytes: ${stat.retrievedKeyValueBytes}\\n`;\n result += `Pruned fields: ${stat.prunedFields}\\n`;\n result += `Final resources skipped: ${stat.finalResourcesSkipped}`;\n return result;\n}\n\n/* Given the transaction (preferably read-only) and loading request, executes\n * the tree traversal algorithm, and collects fresh states of resources\n * to update the tree state. /\nexport async function loadTreeState(\n tx: PlTransaction,\n loadingRequest: TreeLoadingRequest,\n stats?: TreeLoadingStat,\n): Promise<ExtendedResourceData[]> {\n // saving start timestamp to add time spent in this function to the stats at the end of the method\n const startTimestamp = Date.now();\n\n // counting the request\n if (stats) stats.requests++;\n\n const { seedResources, finalResources, pruningFunction } = loadingRequest;\n\n // Limits the number of concurrent gRPC fetches to bound peak memory\n // from in-flight request/response buffers.\n const limiter = new ConcurrencyLimitingExecutor(100);\n\n // Promises of resource states, in the order they were requested.\n const pending: Promise<ExtendedResourceData \| undefined>[] = [];\n\n // vars to calculate number of roundtrips for stats\n let roundTripToggle: boolean = true;\n let numberOfRoundTrips = 0;\n\n // tracking resources we already requested or queued\n const requested = new Set<ResourceId>();\n\n /* Mark a resource for fetching. Deduplicates and respects final-resource set. */\n const requestState = (rid: OptionalResourceId) => {\n if (isNullResourceId(rid) \|\| requested.has(rid)) return;\n\n if (finalResources.has(rid)) {\n if (stats) stats.finalResourcesSkipped++;\n return;\n }\n\n requested.add(rid);\n\n pending.push(\n limiter.run(async () => {\n const resourceData = tx.getResourceDataIfExists(rid, true);\n const kvData = tx.listKeyValuesIfResourceExists(rid);\n\n // counting round-trip (begin)\n const addRT = roundTripToggle;\n if (roundTripToggle) roundTripToggle = false;\n\n const [resource, kv] = await Promise.all([resourceData, kvData]);\n\n // counting round-trip, actually incrementing counter and returning toggle back,\n // so the next request can acquire it\n if (addRT) {\n numberOfRoundTrips++;\n roundTripToggle = true;\n }\n\n if (resource === undefined) return undefined;\n if (kv === undefined) throw new Error(\"Inconsistent replies\");\n\n return { ...resource, kv };\n }),\n );\n };\n\n // sending seed requests\n seedResources.forEach((rid) => requestState(rid));\n\n const result: ExtendedResourceData[] = [];\n for (let i = 0; i < pending.length; i++) {\n // at this point we pause and wait for the next requested resource state to arrive\n let nextResource = await pending[i];\n if (nextResource === undefined)\n // ignoring resources that were not found (this may happen for seed resource ids)\n continue;\n\n if (pruningFunction !== undefined) {\n // apply field pruning, if requested\n const fieldsAfterPruning = pruningFunction(nextResource);\n // collecting stats\n if (stats) stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;\n nextResource = { ...nextResource, fields: fieldsAfterPruning };\n }\n\n // continue traversal over the referenced resources\n requestState(nextResource.error);\n for (const field of nextResource.fields) {\n requestState(field.value);\n requestState(field.error);\n }\n\n // collecting stats\n if (stats) {\n stats.retrievedResources++;\n stats.retrievedFields += nextResource.fields.length;\n stats.retrievedKeyValues += nextResource.kv.length;\n stats.retrievedResourceDataBytes += nextResource.data?.length ?? 0;\n for (const kv of nextResource.kv) stats.retrievedKeyValueBytes += kv.value.length;\n }\n\n // aggregating the state\n result.push(nextResource);\n }\n\n // adding the time we spent in this method to stats\n if (stats) {\n stats.millisSpent += Date.now() - startTimestamp;\n stats.roundTrips += numberOfRoundTrips;\n }\n\n return result;\n}\n"],"names":["msToHumanReadable","ConcurrencyLimitingExecutor","isNullResourceId"],"mappings":";;;;;AAgCA;AACkD;AAC5C,SAAU,2BAA2B,CACzC,IAAiB,EACjB,eAAiC,EAAA;IAEjC,MAAM,aAAa,GAAiB,EAAE;AACtC,IAAA,MAAM,cAAc,GAAG,IAAI,GAAG,EAAc;AAC5C,IAAA,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,KAAI;QAC3B,IAAI,GAAG,CAAC,UAAU;AAAE,YAAA,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;;AACzC,YAAA,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACjC,IAAA,CAAC,CAAC;;IAGF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC;AAAE,QAAA,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAE1F,IAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE;AAC3D;SAegB,sBAAsB,GAAA;IACpC,OAAO;AACL,QAAA,QAAQ,EAAE,CAAC;AACX,QAAA,UAAU,EAAE,CAAC;AACb,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,eAAe,EAAE,CAAC;AAClB,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,0BAA0B,EAAE,CAAC;AAC7B,QAAA,sBAAsB,EAAE,CAAC;AACzB,QAAA,YAAY,EAAE,CAAC;AACf,QAAA,qBAAqB,EAAE,CAAC;AACxB,QAAA,WAAW,EAAE,CAAC;KACf;AACH;AAEM,SAAU,qBAAqB,CAAC,IAAqB,EAAA;AACzD,IAAA,IAAI,MAAM,GAAG,CAAA,UAAA,EAAa,IAAI,CAAC,QAAQ,IAAI;IAC3C,MAAM,IAAI,eAAeA,2BAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA,EAAA,CAAI;AAChE,IAAA,MAAM,IAAI,CAAA,aAAA,EAAgB,IAAI,CAAC,UAAU,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,kBAAkB,IAAI;AACnD,IAAA,MAAM,IAAI,CAAA,QAAA,EAAW,IAAI,CAAC,eAAe,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,IAAA,EAAO,IAAI,CAAC,kBAAkB,IAAI;AAC5C,IAAA,MAAM,IAAI,CAAA,YAAA,EAAe,IAAI,CAAC,0BAA0B,IAAI;AAC5D,IAAA,MAAM,IAAI,CAAA,UAAA,EAAa,IAAI,CAAC,sBAAsB,IAAI;AACtD,IAAA,MAAM,IAAI,CAAA,eAAA,EAAkB,IAAI,CAAC,YAAY,IAAI;AACjD,IAAA,MAAM,IAAI,CAAA,yBAAA,EAA4B,IAAI,CAAC,qBAAqB,EAAE;AAClE,IAAA,OAAO,MAAM;AACf;AAEA;;AAE+B;AACxB,eAAe,aAAa,CACjC,EAAiB,EACjB,cAAkC,EAClC,KAAuB,EAAA;;AAGvB,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE;;AAGjC,IAAA,IAAI,KAAK;QAAE,KAAK,CAAC,QAAQ,EAAE;IAE3B,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,cAAc;;;AAIzE,IAAA,MAAM,OAAO,GAAG,IAAIC,qCAA2B,CAAC,GAAG,CAAC;;IAGpD,MAAM,OAAO,GAAgD,EAAE;;IAG/D,IAAI,eAAe,GAAY,IAAI;IACnC,IAAI,kBAAkB,GAAG,CAAC;;AAG1B,IAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc;;AAGvC,IAAA,MAAM,YAAY,GAAG,CAAC,GAAuB,KAAI;QAC/C,IAAIC,yBAAgB,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE;AAEjD,QAAA,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;AAC3B,YAAA,IAAI,KAAK;gBAAE,KAAK,CAAC,qBAAqB,EAAE;YACxC;QACF;AAEA,QAAA,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;QAElB,OAAO,CAAC,IAAI,CACV,OAAO,CAAC,GAAG,CAAC,YAAW;YACrB,MAAM,YAAY,GAAG,EAAE,CAAC,uBAAuB,CAAC,GAAG,EAAE,IAAI,CAAC;YAC1D,MAAM,MAAM,GAAG,EAAE,CAAC,6BAA6B,CAAC,GAAG,CAAC;;YAGpD,MAAM,KAAK,GAAG,eAAe;AAC7B,YAAA,IAAI,eAAe;gBAAE,eAAe,GAAG,KAAK;AAE5C,YAAA,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;;;YAIhE,IAAI,KAAK,EAAE;AACT,gBAAA,kBAAkB,EAAE;gBACpB,eAAe,GAAG,IAAI;YACxB;YAEA,IAAI,QAAQ,KAAK,SAAS;AAAE,gBAAA,OAAO,SAAS;YAC5C,IAAI,EAAE,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC;AAE7D,YAAA,OAAO,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE;QAC5B,CAAC,CAAC,CACH;AACH,IAAA,CAAC;;AAGD,IAAA,aAAa,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC;IAEjD,MAAM,MAAM,GAA2B,EAAE;AACzC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;;AAEvC,QAAA,IAAI,YAAY,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;QACnC,IAAI,YAAY,KAAK,SAAS;;YAE5B;AAEF,QAAA,IAAI,eAAe,KAAK,SAAS,EAAE;;AAEjC,YAAA,MAAM,kBAAkB,GAAG,eAAe,CAAC,YAAY,CAAC;;AAExD,YAAA,IAAI,KAAK;AAAE,gBAAA,KAAK,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM;YACvF,YAAY,GAAG,EAAE,GAAG,YAAY,EAAE,MAAM,EAAE,kBAAkB,EAAE;QAChE;;AAGA,QAAA,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC;AAChC,QAAA,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE;AACvC,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;QAC3B;;QAGA,IAAI,KAAK,EAAE;YACT,KAAK,CAAC,kBAAkB,EAAE;YAC1B,KAAK,CAAC,eAAe,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM;YACnD,KAAK,CAAC,kBAAkB,IAAI,YAAY,CAAC,EAAE,CAAC,MAAM;YAClD,KAAK,CAAC,0BAA0B,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;AAClE,YAAA,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,EAAE;gBAAE,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM;QACnF;;AAGA,QAAA,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;IAC3B;;IAGA,IAAI,KAAK,EAAE;QACT,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;AAChD,QAAA,KAAK,CAAC,UAAU,IAAI,kBAAkB;IACxC;AAEA,IAAA,OAAO,MAAM;AACf;;;;;;;"}

package/dist/sync.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../src/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EAET,aAAa,EACb,UAAU,EACX,MAAM,2BAA2B,CAAC;~~AAGnC~~,OAAO,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAGjE,kDAAkD;AAClD,MAAM,MAAM,eAAe,GAAG,CAAC,QAAQ,EAAE,oBAAoB,KAAK,SAAS,EAAE,CAAC;AAE9E,MAAM,WAAW,kBAAkB;IACjC;;cAEU;IACV,QAAQ,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;IAErC;;gCAE4B;IAC5B,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAEzC;;;;8BAI0B;IAC1B,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;CAC5C;AAED;kDACkD;AAClD,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,WAAW,EACjB,eAAe,CAAC,EAAE,eAAe,GAChC,kBAAkB,CAYpB;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0BAA0B,EAAE,MAAM,CAAC;IACnC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,wBAAgB,sBAAsB,IAAI,eAAe,CAaxD;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,CAYnE;AAED;;+BAE+B;AAC/B,wBAAsB,aAAa,CACjC,EAAE,EAAE,aAAa,EACjB,cAAc,EAAE,kBAAkB,EAClC,KAAK,CAAC,EAAE,eAAe,GACtB,OAAO,CAAC,oBAAoB,EAAE,CAAC,~~CAmHjC~~"}
1	+ {"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../src/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EAET,aAAa,EACb,UAAU,EACX,MAAM,2BAA2B,CAAC;AAEnC,OAAO,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAGjE,kDAAkD;AAClD,MAAM,MAAM,eAAe,GAAG,CAAC,QAAQ,EAAE,oBAAoB,KAAK,SAAS,EAAE,CAAC;AAE9E,MAAM,WAAW,kBAAkB;IACjC;;cAEU;IACV,QAAQ,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;IAErC;;gCAE4B;IAC5B,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAEzC;;;;8BAI0B;IAC1B,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;CAC5C;AAED;kDACkD;AAClD,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,WAAW,EACjB,eAAe,CAAC,EAAE,eAAe,GAChC,kBAAkB,CAYpB;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0BAA0B,EAAE,MAAM,CAAC;IACnC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,wBAAgB,sBAAsB,IAAI,eAAe,CAaxD;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,CAYnE;AAED;;+BAE+B;AAC/B,wBAAsB,aAAa,CACjC,EAAE,EAAE,aAAa,EACjB,cAAc,EAAE,kBAAkB,EAClC,KAAK,CAAC,EAAE,eAAe,GACtB,OAAO,CAAC,oBAAoB,EAAE,CAAC,CA0GjC"}

package/dist/sync.js CHANGED Viewed

@@ -1,6 +1,5 @@
 import { isNullResourceId } from '@milaboratories/pl-client';
-import Denque from 'denque';
-import { msToHumanReadable } from '@milaboratories/ts-helpers';
+import { msToHumanReadable, ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';
 /** Given the current tree state, build the request object to pass to
  * {@link loadTreeState} to load updated state. */
@@ -55,39 +54,36 @@ async function loadTreeState(tx, loadingRequest, stats) {
     if (stats)
         stats.requests++;
     const { seedResources, finalResources, pruningFunction } = loadingRequest;
-    // Main idea of using a queue here is that responses will arrive in the same order as they were
-    // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.
-    // In such a way logic become linear without recursion, and at the same time deal with data
-    // as soon as it arrives.
-    const pending = new Denque();
+    // Limits the number of concurrent gRPC fetches to bound peak memory
+    // from in-flight request/response buffers.
+    const limiter = new ConcurrencyLimitingExecutor(100);
+    // Promises of resource states, in the order they were requested.
+    const pending = [];
     // vars to calculate number of roundtrips for stats
     let roundTripToggle = true;
     let numberOfRoundTrips = 0;
-    // tracking resources we already requested
+    // tracking resources we already requested or queued
     const requested = new Set();
+    /** Mark a resource for fetching. Deduplicates and respects final-resource set. */
     const requestState = (rid) => {
         if (isNullResourceId(rid) || requested.has(rid))
             return;
-        // separate check to collect stats
         if (finalResources.has(rid)) {
             if (stats)
                 stats.finalResourcesSkipped++;
             return;
         }
-        // adding the id, so we will not request it's state again if somebody else
-        // references the same resource
         requested.add(rid);
-        // requesting resource and all kv records
-        const resourceData = tx.getResourceDataIfExists(rid, true);
-        const kvData = tx.listKeyValuesIfResourceExists(rid);
-        // counting round-trip (begin)
-        const addRT = roundTripToggle;
-        if (roundTripToggle)
-            roundTripToggle = false;
-        // pushing combined promise
-        pending.push((async () => {
+        pending.push(limiter.run(async () => {
+            const resourceData = tx.getResourceDataIfExists(rid, true);
+            const kvData = tx.listKeyValuesIfResourceExists(rid);
+            // counting round-trip (begin)
+            const addRT = roundTripToggle;
+            if (roundTripToggle)
+                roundTripToggle = false;
             const [resource, kv] = await Promise.all([resourceData, kvData]);
-            // counting round-trip, actually incrementing counter and returning toggle back, so the next request can acquire it
+            // counting round-trip, actually incrementing counter and returning toggle back,
+            // so the next request can acquire it
             if (addRT) {
                 numberOfRoundTrips++;
                 roundTripToggle = true;
@@ -97,19 +93,14 @@ async function loadTreeState(tx, loadingRequest, stats) {
             if (kv === undefined)
                 throw new Error("Inconsistent replies");
             return { ...resource, kv };
-        })());
+        }));
     };
     // sending seed requests
     seedResources.forEach((rid) => requestState(rid));
     const result = [];
-    while (true) {
-        // taking next pending request
-        const nextResourcePromise = pending.shift();
-        if (nextResourcePromise === undefined)
-            // this means we have no pending requests and traversal is over
-            break;
-        // at this point we pause and wait for the nest requested resource state to arrive
-        let nextResource = await nextResourcePromise;
+    for (let i = 0; i < pending.length; i++) {
+        // at this point we pause and wait for the next requested resource state to arrive
+        let nextResource = await pending[i];
         if (nextResource === undefined)
             // ignoring resources that were not found (this may happen for seed resource ids)
             continue;
@@ -121,7 +112,7 @@ async function loadTreeState(tx, loadingRequest, stats) {
                 stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;
             nextResource = { ...nextResource, fields: fieldsAfterPruning };
         }
-        // continue traversal over the referenced resource
+        // continue traversal over the referenced resources
         requestState(nextResource.error);
         for (const field of nextResource.fields) {
             requestState(field.value);

package/dist/sync.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"sync.js","sources":["../src/sync.ts"],"sourcesContent":["import type {\n FieldData,\n OptionalResourceId,\n PlTransaction,\n ResourceId,\n} from \"@milaboratories/pl-client\";\nimport { isNullResourceId } from \"@milaboratories/pl-client\";\nimport Denque from \"denque\";\nimport type { ExtendedResourceData, PlTreeState } from \"./state\";\nimport { msToHumanReadable } from \"@milaboratories/ts-helpers\";\n\n/** Applied to list of fields in resource data. /\nexport type PruningFunction = (resource: ExtendedResourceData) => FieldData[];\n\nexport interface TreeLoadingRequest {\n /* Resource to prime the traversal algorithm. It is ok, if some of them\n * doesn't exist anymore. Should not contain elements from final resource\n * set. /\n readonly seedResources: ResourceId[];\n\n /* Resource ids for which state is already known and not expected to change.\n * Algorithm will not continue traversal over those ids, and states will not\n * be retrieved for them. /\n readonly finalResources: Set<ResourceId>;\n\n /* This function is applied to each resource data field list, before\n * using it continue traversal. This modification also is applied to\n * output data to make result self-consistent in terms that it will contain\n * all referenced resources, this is required to be able to pass it to tree\n * to update the state. /\n readonly pruningFunction?: PruningFunction;\n}\n\n/* Given the current tree state, build the request object to pass to\n * {@link loadTreeState} to load updated state. /\nexport function constructTreeLoadingRequest(\n tree: PlTreeState,\n pruningFunction?: PruningFunction,\n): TreeLoadingRequest {\n const seedResources: ResourceId[] = [];\n const finalResources = new Set<ResourceId>();\n tree.forEachResource((res) => {\n if (res.finalState) finalResources.add(res.id);\n else seedResources.push(res.id);\n });\n\n // if tree is empty, seeding tree reconstruction from the specified root\n if (seedResources.length === 0 && finalResources.size === 0) seedResources.push(tree.root);\n\n return { seedResources, finalResources, pruningFunction };\n}\n\nexport type TreeLoadingStat = {\n requests: number;\n roundTrips: number;\n retrievedResources: number;\n retrievedFields: number;\n retrievedKeyValues: number;\n retrievedResourceDataBytes: number;\n retrievedKeyValueBytes: number;\n prunedFields: number;\n finalResourcesSkipped: number;\n millisSpent: number;\n};\n\nexport function initialTreeLoadingStat(): TreeLoadingStat {\n return {\n requests: 0,\n roundTrips: 0,\n retrievedResources: 0,\n retrievedFields: 0,\n retrievedKeyValues: 0,\n retrievedResourceDataBytes: 0,\n retrievedKeyValueBytes: 0,\n prunedFields: 0,\n finalResourcesSkipped: 0,\n millisSpent: 0,\n };\n}\n\nexport function formatTreeLoadingStat(stat: TreeLoadingStat): string {\n let result = `Requests: ${stat.requests}\\n`;\n result += `Total time: ${msToHumanReadable(stat.millisSpent)}\\n`;\n result += `Round-trips: ${stat.roundTrips}\\n`;\n result += `Resources: ${stat.retrievedResources}\\n`;\n result += `Fields: ${stat.retrievedFields}\\n`;\n result += `KV: ${stat.retrievedKeyValues}\\n`;\n result += `Data Bytes: ${stat.retrievedResourceDataBytes}\\n`;\n result += `KV Bytes: ${stat.retrievedKeyValueBytes}\\n`;\n result += `Pruned fields: ${stat.prunedFields}\\n`;\n result += `Final resources skipped: ${stat.finalResourcesSkipped}`;\n return result;\n}\n\n/* Given the transaction (preferably read-only) and loading request, executes\n * the tree traversal algorithm, and collects fresh states of resources\n * to update the tree state. */\nexport async function loadTreeState(\n tx: PlTransaction,\n loadingRequest: TreeLoadingRequest,\n stats?: TreeLoadingStat,\n): Promise<ExtendedResourceData[]> {\n // saving start timestamp to add time spent in this function to the stats at the end of the method\n const startTimestamp = Date.now();\n\n // counting the request\n if (stats) stats.requests++;\n\n const { seedResources, finalResources, pruningFunction } = loadingRequest;\n\n // Main idea of using a queue here is that responses will arrive in the same order as they were\n // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.\n // In such a way logic become linear without recursion, and at the same time deal with data\n // as soon as it arrives.\n\n const pending = new Denque<Promise<ExtendedResourceData \| undefined>>();\n\n // vars to calculate number of roundtrips for stats\n let roundTripToggle: boolean = true;\n let numberOfRoundTrips = 0;\n\n // tracking resources we already requested\n const requested = new Set<ResourceId>();\n const requestState = (rid: OptionalResourceId) => {\n if (isNullResourceId(rid) \|\| requested.has(rid)) return;\n\n // separate check to collect stats\n if (finalResources.has(rid)) {\n if (stats) stats.finalResourcesSkipped++;\n return;\n }\n\n // adding the id, so we will not request it's state again if somebody else\n // references the same resource\n requested.add(rid);\n\n // requesting resource and all kv records\n const resourceData = tx.getResourceDataIfExists(rid, true);\n const kvData = tx.listKeyValuesIfResourceExists(rid);\n\n // counting round-trip (begin)\n const addRT = roundTripToggle;\n if (roundTripToggle) roundTripToggle = false;\n\n // pushing combined promise\n pending.push(\n (async () => {\n const [resource, kv] = await Promise.all([resourceData, kvData]);\n\n // counting round-trip, actually incrementing counter and returning toggle back, so the next request can acquire it\n if (addRT) {\n numberOfRoundTrips++;\n roundTripToggle = true;\n }\n\n if (resource === undefined) return undefined;\n\n if (kv === undefined) throw new Error(\"Inconsistent replies\");\n\n return { ...resource, kv };\n })(),\n );\n };\n\n // sending seed requests\n seedResources.forEach((rid) => requestState(rid));\n\n const result: ExtendedResourceData[] = [];\n while (true) {\n // taking next pending request\n const nextResourcePromise = pending.shift();\n if (nextResourcePromise === undefined)\n // this means we have no pending requests and traversal is over\n break;\n\n // at this point we pause and wait for the nest requested resource state to arrive\n let nextResource = await nextResourcePromise;\n if (nextResource === undefined)\n // ignoring resources that were not found (this may happen for seed resource ids)\n continue;\n\n if (pruningFunction !== undefined) {\n // apply field pruning, if requested\n const fieldsAfterPruning = pruningFunction(nextResource);\n // collecting stats\n if (stats) stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;\n nextResource = { ...nextResource, fields: fieldsAfterPruning };\n }\n\n // continue traversal over the referenced resource\n requestState(nextResource.error);\n for (const field of nextResource.fields) {\n requestState(field.value);\n requestState(field.error);\n }\n\n // collecting stats\n if (stats) {\n stats.retrievedResources++;\n stats.retrievedFields += nextResource.fields.length;\n stats.retrievedKeyValues += nextResource.kv.length;\n stats.retrievedResourceDataBytes += nextResource.data?.length ?? 0;\n for (const kv of nextResource.kv) stats.retrievedKeyValueBytes += kv.value.length;\n }\n\n // aggregating the state\n result.push(nextResource);\n }\n\n // adding the time we spent in this method to stats\n if (stats) {\n stats.millisSpent += Date.now() - startTimestamp;\n stats.roundTrips += numberOfRoundTrips;\n }\n\n return result;\n}\n"],"names":[],"mappings":";;;;AAiCA;AACkD;AAC5C,SAAU,2BAA2B,CACzC,IAAiB,EACjB,eAAiC,EAAA;IAEjC,MAAM,aAAa,GAAiB,EAAE;AACtC,IAAA,MAAM,cAAc,GAAG,IAAI,GAAG,EAAc;AAC5C,IAAA,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,KAAI;QAC3B,IAAI,GAAG,CAAC,UAAU;AAAE,YAAA,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;;AACzC,YAAA,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACjC,IAAA,CAAC,CAAC;;IAGF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC;AAAE,QAAA,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAE1F,IAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE;AAC3D;SAegB,sBAAsB,GAAA;IACpC,OAAO;AACL,QAAA,QAAQ,EAAE,CAAC;AACX,QAAA,UAAU,EAAE,CAAC;AACb,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,eAAe,EAAE,CAAC;AAClB,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,0BAA0B,EAAE,CAAC;AAC7B,QAAA,sBAAsB,EAAE,CAAC;AACzB,QAAA,YAAY,EAAE,CAAC;AACf,QAAA,qBAAqB,EAAE,CAAC;AACxB,QAAA,WAAW,EAAE,CAAC;KACf;AACH;AAEM,SAAU,qBAAqB,CAAC,IAAqB,EAAA;AACzD,IAAA,IAAI,MAAM,GAAG,CAAA,UAAA,EAAa,IAAI,CAAC,QAAQ,IAAI;IAC3C,MAAM,IAAI,eAAe,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA,EAAA,CAAI;AAChE,IAAA,MAAM,IAAI,CAAA,aAAA,EAAgB,IAAI,CAAC,UAAU,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,kBAAkB,IAAI;AACnD,IAAA,MAAM,IAAI,CAAA,QAAA,EAAW,IAAI,CAAC,eAAe,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,IAAA,EAAO,IAAI,CAAC,kBAAkB,IAAI;AAC5C,IAAA,MAAM,IAAI,CAAA,YAAA,EAAe,IAAI,CAAC,0BAA0B,IAAI;AAC5D,IAAA,MAAM,IAAI,CAAA,UAAA,EAAa,IAAI,CAAC,sBAAsB,IAAI;AACtD,IAAA,MAAM,IAAI,CAAA,eAAA,EAAkB,IAAI,CAAC,YAAY,IAAI;AACjD,IAAA,MAAM,IAAI,CAAA,yBAAA,EAA4B,IAAI,CAAC,qBAAqB,EAAE;AAClE,IAAA,OAAO,MAAM;AACf;AAEA;;AAE+B;AACxB,eAAe,aAAa,CACjC,EAAiB,EACjB,cAAkC,EAClC,KAAuB,EAAA;;AAGvB,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE;;AAGjC,IAAA,IAAI,KAAK;QAAE,KAAK,CAAC,QAAQ,EAAE;IAE3B,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,cAAc;;;;;AAOzE,IAAA,MAAM,OAAO,GAAG,IAAI,MAAM,EAA6C;;IAGvE,IAAI,eAAe,GAAY,IAAI;IACnC,IAAI,kBAAkB,GAAG,CAAC;;AAG1B,IAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc;AACvC,IAAA,MAAM,YAAY,GAAG,CAAC,GAAuB,KAAI;QAC/C,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE;;AAGjD,QAAA,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;AAC3B,YAAA,IAAI,KAAK;gBAAE,KAAK,CAAC,qBAAqB,EAAE;YACxC;QACF;;;AAIA,QAAA,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;;QAGlB,MAAM,YAAY,GAAG,EAAE,CAAC,uBAAuB,CAAC,GAAG,EAAE,IAAI,CAAC;QAC1D,MAAM,MAAM,GAAG,EAAE,CAAC,6BAA6B,CAAC,GAAG,CAAC;;QAGpD,MAAM,KAAK,GAAG,eAAe;AAC7B,QAAA,IAAI,eAAe;YAAE,eAAe,GAAG,KAAK;;AAG5C,QAAA,OAAO,CAAC,IAAI,CACV,CAAC,YAAW;AACV,YAAA,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;;YAGhE,IAAI,KAAK,EAAE;AACT,gBAAA,kBAAkB,EAAE;gBACpB,eAAe,GAAG,IAAI;YACxB;YAEA,IAAI,QAAQ,KAAK,SAAS;AAAE,gBAAA,OAAO,SAAS;YAE5C,IAAI,EAAE,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC;AAE7D,YAAA,OAAO,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE;QAC5B,CAAC,GAAG,CACL;AACH,IAAA,CAAC;;AAGD,IAAA,aAAa,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC;IAEjD,MAAM,MAAM,GAA2B,EAAE;IACzC,OAAO,IAAI,EAAE;;AAEX,QAAA,MAAM,mBAAmB,GAAG,OAAO,CAAC,KAAK,EAAE;QAC3C,IAAI,mBAAmB,KAAK,SAAS;;YAEnC;;AAGF,QAAA,IAAI,YAAY,GAAG,MAAM,mBAAmB;QAC5C,IAAI,YAAY,KAAK,SAAS;;YAE5B;AAEF,QAAA,IAAI,eAAe,KAAK,SAAS,EAAE;;AAEjC,YAAA,MAAM,kBAAkB,GAAG,eAAe,CAAC,YAAY,CAAC;;AAExD,YAAA,IAAI,KAAK;AAAE,gBAAA,KAAK,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM;YACvF,YAAY,GAAG,EAAE,GAAG,YAAY,EAAE,MAAM,EAAE,kBAAkB,EAAE;QAChE;;AAGA,QAAA,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC;AAChC,QAAA,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE;AACvC,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;QAC3B;;QAGA,IAAI,KAAK,EAAE;YACT,KAAK,CAAC,kBAAkB,EAAE;YAC1B,KAAK,CAAC,eAAe,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM;YACnD,KAAK,CAAC,kBAAkB,IAAI,YAAY,CAAC,EAAE,CAAC,MAAM;YAClD,KAAK,CAAC,0BAA0B,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;AAClE,YAAA,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,EAAE;gBAAE,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM;QACnF;;AAGA,QAAA,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;IAC3B;;IAGA,IAAI,KAAK,EAAE;QACT,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;AAChD,QAAA,KAAK,CAAC,UAAU,IAAI,kBAAkB;IACxC;AAEA,IAAA,OAAO,MAAM;AACf;;;;"}
1	+ {"version":3,"file":"sync.js","sources":["../src/sync.ts"],"sourcesContent":["import type {\n FieldData,\n OptionalResourceId,\n PlTransaction,\n ResourceId,\n} from \"@milaboratories/pl-client\";\nimport { isNullResourceId } from \"@milaboratories/pl-client\";\nimport type { ExtendedResourceData, PlTreeState } from \"./state\";\nimport { ConcurrencyLimitingExecutor, msToHumanReadable } from \"@milaboratories/ts-helpers\";\n\n/** Applied to list of fields in resource data. /\nexport type PruningFunction = (resource: ExtendedResourceData) => FieldData[];\n\nexport interface TreeLoadingRequest {\n /* Resource to prime the traversal algorithm. It is ok, if some of them\n * doesn't exist anymore. Should not contain elements from final resource\n * set. /\n readonly seedResources: ResourceId[];\n\n /* Resource ids for which state is already known and not expected to change.\n * Algorithm will not continue traversal over those ids, and states will not\n * be retrieved for them. /\n readonly finalResources: Set<ResourceId>;\n\n /* This function is applied to each resource data field list, before\n * using it continue traversal. This modification also is applied to\n * output data to make result self-consistent in terms that it will contain\n * all referenced resources, this is required to be able to pass it to tree\n * to update the state. /\n readonly pruningFunction?: PruningFunction;\n}\n\n/* Given the current tree state, build the request object to pass to\n * {@link loadTreeState} to load updated state. /\nexport function constructTreeLoadingRequest(\n tree: PlTreeState,\n pruningFunction?: PruningFunction,\n): TreeLoadingRequest {\n const seedResources: ResourceId[] = [];\n const finalResources = new Set<ResourceId>();\n tree.forEachResource((res) => {\n if (res.finalState) finalResources.add(res.id);\n else seedResources.push(res.id);\n });\n\n // if tree is empty, seeding tree reconstruction from the specified root\n if (seedResources.length === 0 && finalResources.size === 0) seedResources.push(tree.root);\n\n return { seedResources, finalResources, pruningFunction };\n}\n\nexport type TreeLoadingStat = {\n requests: number;\n roundTrips: number;\n retrievedResources: number;\n retrievedFields: number;\n retrievedKeyValues: number;\n retrievedResourceDataBytes: number;\n retrievedKeyValueBytes: number;\n prunedFields: number;\n finalResourcesSkipped: number;\n millisSpent: number;\n};\n\nexport function initialTreeLoadingStat(): TreeLoadingStat {\n return {\n requests: 0,\n roundTrips: 0,\n retrievedResources: 0,\n retrievedFields: 0,\n retrievedKeyValues: 0,\n retrievedResourceDataBytes: 0,\n retrievedKeyValueBytes: 0,\n prunedFields: 0,\n finalResourcesSkipped: 0,\n millisSpent: 0,\n };\n}\n\nexport function formatTreeLoadingStat(stat: TreeLoadingStat): string {\n let result = `Requests: ${stat.requests}\\n`;\n result += `Total time: ${msToHumanReadable(stat.millisSpent)}\\n`;\n result += `Round-trips: ${stat.roundTrips}\\n`;\n result += `Resources: ${stat.retrievedResources}\\n`;\n result += `Fields: ${stat.retrievedFields}\\n`;\n result += `KV: ${stat.retrievedKeyValues}\\n`;\n result += `Data Bytes: ${stat.retrievedResourceDataBytes}\\n`;\n result += `KV Bytes: ${stat.retrievedKeyValueBytes}\\n`;\n result += `Pruned fields: ${stat.prunedFields}\\n`;\n result += `Final resources skipped: ${stat.finalResourcesSkipped}`;\n return result;\n}\n\n/* Given the transaction (preferably read-only) and loading request, executes\n * the tree traversal algorithm, and collects fresh states of resources\n * to update the tree state. /\nexport async function loadTreeState(\n tx: PlTransaction,\n loadingRequest: TreeLoadingRequest,\n stats?: TreeLoadingStat,\n): Promise<ExtendedResourceData[]> {\n // saving start timestamp to add time spent in this function to the stats at the end of the method\n const startTimestamp = Date.now();\n\n // counting the request\n if (stats) stats.requests++;\n\n const { seedResources, finalResources, pruningFunction } = loadingRequest;\n\n // Limits the number of concurrent gRPC fetches to bound peak memory\n // from in-flight request/response buffers.\n const limiter = new ConcurrencyLimitingExecutor(100);\n\n // Promises of resource states, in the order they were requested.\n const pending: Promise<ExtendedResourceData \| undefined>[] = [];\n\n // vars to calculate number of roundtrips for stats\n let roundTripToggle: boolean = true;\n let numberOfRoundTrips = 0;\n\n // tracking resources we already requested or queued\n const requested = new Set<ResourceId>();\n\n /* Mark a resource for fetching. Deduplicates and respects final-resource set. */\n const requestState = (rid: OptionalResourceId) => {\n if (isNullResourceId(rid) \|\| requested.has(rid)) return;\n\n if (finalResources.has(rid)) {\n if (stats) stats.finalResourcesSkipped++;\n return;\n }\n\n requested.add(rid);\n\n pending.push(\n limiter.run(async () => {\n const resourceData = tx.getResourceDataIfExists(rid, true);\n const kvData = tx.listKeyValuesIfResourceExists(rid);\n\n // counting round-trip (begin)\n const addRT = roundTripToggle;\n if (roundTripToggle) roundTripToggle = false;\n\n const [resource, kv] = await Promise.all([resourceData, kvData]);\n\n // counting round-trip, actually incrementing counter and returning toggle back,\n // so the next request can acquire it\n if (addRT) {\n numberOfRoundTrips++;\n roundTripToggle = true;\n }\n\n if (resource === undefined) return undefined;\n if (kv === undefined) throw new Error(\"Inconsistent replies\");\n\n return { ...resource, kv };\n }),\n );\n };\n\n // sending seed requests\n seedResources.forEach((rid) => requestState(rid));\n\n const result: ExtendedResourceData[] = [];\n for (let i = 0; i < pending.length; i++) {\n // at this point we pause and wait for the next requested resource state to arrive\n let nextResource = await pending[i];\n if (nextResource === undefined)\n // ignoring resources that were not found (this may happen for seed resource ids)\n continue;\n\n if (pruningFunction !== undefined) {\n // apply field pruning, if requested\n const fieldsAfterPruning = pruningFunction(nextResource);\n // collecting stats\n if (stats) stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;\n nextResource = { ...nextResource, fields: fieldsAfterPruning };\n }\n\n // continue traversal over the referenced resources\n requestState(nextResource.error);\n for (const field of nextResource.fields) {\n requestState(field.value);\n requestState(field.error);\n }\n\n // collecting stats\n if (stats) {\n stats.retrievedResources++;\n stats.retrievedFields += nextResource.fields.length;\n stats.retrievedKeyValues += nextResource.kv.length;\n stats.retrievedResourceDataBytes += nextResource.data?.length ?? 0;\n for (const kv of nextResource.kv) stats.retrievedKeyValueBytes += kv.value.length;\n }\n\n // aggregating the state\n result.push(nextResource);\n }\n\n // adding the time we spent in this method to stats\n if (stats) {\n stats.millisSpent += Date.now() - startTimestamp;\n stats.roundTrips += numberOfRoundTrips;\n }\n\n return result;\n}\n"],"names":[],"mappings":";;;AAgCA;AACkD;AAC5C,SAAU,2BAA2B,CACzC,IAAiB,EACjB,eAAiC,EAAA;IAEjC,MAAM,aAAa,GAAiB,EAAE;AACtC,IAAA,MAAM,cAAc,GAAG,IAAI,GAAG,EAAc;AAC5C,IAAA,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,KAAI;QAC3B,IAAI,GAAG,CAAC,UAAU;AAAE,YAAA,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;;AACzC,YAAA,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACjC,IAAA,CAAC,CAAC;;IAGF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC;AAAE,QAAA,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAE1F,IAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE;AAC3D;SAegB,sBAAsB,GAAA;IACpC,OAAO;AACL,QAAA,QAAQ,EAAE,CAAC;AACX,QAAA,UAAU,EAAE,CAAC;AACb,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,eAAe,EAAE,CAAC;AAClB,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,0BAA0B,EAAE,CAAC;AAC7B,QAAA,sBAAsB,EAAE,CAAC;AACzB,QAAA,YAAY,EAAE,CAAC;AACf,QAAA,qBAAqB,EAAE,CAAC;AACxB,QAAA,WAAW,EAAE,CAAC;KACf;AACH;AAEM,SAAU,qBAAqB,CAAC,IAAqB,EAAA;AACzD,IAAA,IAAI,MAAM,GAAG,CAAA,UAAA,EAAa,IAAI,CAAC,QAAQ,IAAI;IAC3C,MAAM,IAAI,eAAe,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA,EAAA,CAAI;AAChE,IAAA,MAAM,IAAI,CAAA,aAAA,EAAgB,IAAI,CAAC,UAAU,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,kBAAkB,IAAI;AACnD,IAAA,MAAM,IAAI,CAAA,QAAA,EAAW,IAAI,CAAC,eAAe,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,IAAA,EAAO,IAAI,CAAC,kBAAkB,IAAI;AAC5C,IAAA,MAAM,IAAI,CAAA,YAAA,EAAe,IAAI,CAAC,0BAA0B,IAAI;AAC5D,IAAA,MAAM,IAAI,CAAA,UAAA,EAAa,IAAI,CAAC,sBAAsB,IAAI;AACtD,IAAA,MAAM,IAAI,CAAA,eAAA,EAAkB,IAAI,CAAC,YAAY,IAAI;AACjD,IAAA,MAAM,IAAI,CAAA,yBAAA,EAA4B,IAAI,CAAC,qBAAqB,EAAE;AAClE,IAAA,OAAO,MAAM;AACf;AAEA;;AAE+B;AACxB,eAAe,aAAa,CACjC,EAAiB,EACjB,cAAkC,EAClC,KAAuB,EAAA;;AAGvB,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE;;AAGjC,IAAA,IAAI,KAAK;QAAE,KAAK,CAAC,QAAQ,EAAE;IAE3B,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,cAAc;;;AAIzE,IAAA,MAAM,OAAO,GAAG,IAAI,2BAA2B,CAAC,GAAG,CAAC;;IAGpD,MAAM,OAAO,GAAgD,EAAE;;IAG/D,IAAI,eAAe,GAAY,IAAI;IACnC,IAAI,kBAAkB,GAAG,CAAC;;AAG1B,IAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc;;AAGvC,IAAA,MAAM,YAAY,GAAG,CAAC,GAAuB,KAAI;QAC/C,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE;AAEjD,QAAA,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;AAC3B,YAAA,IAAI,KAAK;gBAAE,KAAK,CAAC,qBAAqB,EAAE;YACxC;QACF;AAEA,QAAA,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;QAElB,OAAO,CAAC,IAAI,CACV,OAAO,CAAC,GAAG,CAAC,YAAW;YACrB,MAAM,YAAY,GAAG,EAAE,CAAC,uBAAuB,CAAC,GAAG,EAAE,IAAI,CAAC;YAC1D,MAAM,MAAM,GAAG,EAAE,CAAC,6BAA6B,CAAC,GAAG,CAAC;;YAGpD,MAAM,KAAK,GAAG,eAAe;AAC7B,YAAA,IAAI,eAAe;gBAAE,eAAe,GAAG,KAAK;AAE5C,YAAA,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;;;YAIhE,IAAI,KAAK,EAAE;AACT,gBAAA,kBAAkB,EAAE;gBACpB,eAAe,GAAG,IAAI;YACxB;YAEA,IAAI,QAAQ,KAAK,SAAS;AAAE,gBAAA,OAAO,SAAS;YAC5C,IAAI,EAAE,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC;AAE7D,YAAA,OAAO,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE;QAC5B,CAAC,CAAC,CACH;AACH,IAAA,CAAC;;AAGD,IAAA,aAAa,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC;IAEjD,MAAM,MAAM,GAA2B,EAAE;AACzC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;;AAEvC,QAAA,IAAI,YAAY,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;QACnC,IAAI,YAAY,KAAK,SAAS;;YAE5B;AAEF,QAAA,IAAI,eAAe,KAAK,SAAS,EAAE;;AAEjC,YAAA,MAAM,kBAAkB,GAAG,eAAe,CAAC,YAAY,CAAC;;AAExD,YAAA,IAAI,KAAK;AAAE,gBAAA,KAAK,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM;YACvF,YAAY,GAAG,EAAE,GAAG,YAAY,EAAE,MAAM,EAAE,kBAAkB,EAAE;QAChE;;AAGA,QAAA,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC;AAChC,QAAA,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE;AACvC,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;QAC3B;;QAGA,IAAI,KAAK,EAAE;YACT,KAAK,CAAC,kBAAkB,EAAE;YAC1B,KAAK,CAAC,eAAe,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM;YACnD,KAAK,CAAC,kBAAkB,IAAI,YAAY,CAAC,EAAE,CAAC,MAAM;YAClD,KAAK,CAAC,0BAA0B,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;AAClE,YAAA,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,EAAE;gBAAE,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM;QACnF;;AAGA,QAAA,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;IAC3B;;IAGA,IAAI,KAAK,EAAE;QACT,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;AAChD,QAAA,KAAK,CAAC,UAAU,IAAI,kBAAkB;IACxC;AAEA,IAAA,OAAO,MAAM;AACf;;;;"}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-tree",
-  "version": "1.8.44",
+  "version": "1.8.46",
   "description": "Reactive pl tree state",
   "files": [
     "./dist/**/*",
@@ -17,22 +17,21 @@
     }
   },
   "dependencies": {
-    "denque": "^2.1.0",
     "utility-types": "^3.11.0",
     "zod": "~3.23.8",
     "@milaboratories/computable": "2.8.5",
-    "@milaboratories/pl-client": "2.17.6",
     "@milaboratories/ts-helpers": "1.7.2",
-    "@milaboratories/pl-errors": "1.1.68"
+    "@milaboratories/pl-errors": "1.1.69",
+    "@milaboratories/pl-client": "2.17.7"
   },
   "devDependencies": {
     "@types/node": "~24.5.2",
-    "@vitest/coverage-istanbul": "^4.0.16",
+    "@vitest/coverage-istanbul": "^4.0.18",
     "typescript": "~5.6.3",
-    "vitest": "^4.0.16",
-    "@milaboratories/build-configs": "1.4.4",
+    "vitest": "^4.0.18",
+    "@milaboratories/build-configs": "1.5.0",
     "@milaboratories/ts-configs": "1.2.1",
-    "@milaboratories/ts-builder": "1.2.10"
+    "@milaboratories/ts-builder": "1.2.11"
   },
   "engines": {
     "node": ">=22.19.0"

package/src/sync.ts CHANGED Viewed

@@ -5,9 +5,8 @@ import type {
   ResourceId,
 } from "@milaboratories/pl-client";
 import { isNullResourceId } from "@milaboratories/pl-client";
-import Denque from "denque";
 import type { ExtendedResourceData, PlTreeState } from "./state";
-import { msToHumanReadable } from "@milaboratories/ts-helpers";
+import { ConcurrencyLimitingExecutor, msToHumanReadable } from "@milaboratories/ts-helpers";
 /** Applied to list of fields in resource data. */
 export type PruningFunction = (resource: ExtendedResourceData) => FieldData[];
@@ -108,57 +107,54 @@ export async function loadTreeState(
   const { seedResources, finalResources, pruningFunction } = loadingRequest;
-  // Main idea of using a queue here is that responses will arrive in the same order as they were
-  // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.
-  // In such a way logic become linear without recursion, and at the same time deal with data
-  // as soon as it arrives.
+  // Limits the number of concurrent gRPC fetches to bound peak memory
+  // from in-flight request/response buffers.
+  const limiter = new ConcurrencyLimitingExecutor(100);
-  const pending = new Denque<Promise<ExtendedResourceData | undefined>>();
+  // Promises of resource states, in the order they were requested.
+  const pending: Promise<ExtendedResourceData | undefined>[] = [];
   // vars to calculate number of roundtrips for stats
   let roundTripToggle: boolean = true;
   let numberOfRoundTrips = 0;
-  // tracking resources we already requested
+  // tracking resources we already requested or queued
   const requested = new Set<ResourceId>();
+  /** Mark a resource for fetching. Deduplicates and respects final-resource set. */
   const requestState = (rid: OptionalResourceId) => {
     if (isNullResourceId(rid) || requested.has(rid)) return;
-    // separate check to collect stats
     if (finalResources.has(rid)) {
       if (stats) stats.finalResourcesSkipped++;
       return;
     }
-    // adding the id, so we will not request it's state again if somebody else
-    // references the same resource
     requested.add(rid);
-    // requesting resource and all kv records
-    const resourceData = tx.getResourceDataIfExists(rid, true);
-    const kvData = tx.listKeyValuesIfResourceExists(rid);
+    pending.push(
+      limiter.run(async () => {
+        const resourceData = tx.getResourceDataIfExists(rid, true);
+        const kvData = tx.listKeyValuesIfResourceExists(rid);
-    // counting round-trip (begin)
-    const addRT = roundTripToggle;
-    if (roundTripToggle) roundTripToggle = false;
+        // counting round-trip (begin)
+        const addRT = roundTripToggle;
+        if (roundTripToggle) roundTripToggle = false;
-    // pushing combined promise
-    pending.push(
-      (async () => {
         const [resource, kv] = await Promise.all([resourceData, kvData]);
-        // counting round-trip, actually incrementing counter and returning toggle back, so the next request can acquire it
+        // counting round-trip, actually incrementing counter and returning toggle back,
+        // so the next request can acquire it
         if (addRT) {
           numberOfRoundTrips++;
           roundTripToggle = true;
         }
         if (resource === undefined) return undefined;
         if (kv === undefined) throw new Error("Inconsistent replies");
         return { ...resource, kv };
-      })(),
+      }),
     );
   };
@@ -166,15 +162,9 @@ export async function loadTreeState(
   seedResources.forEach((rid) => requestState(rid));
   const result: ExtendedResourceData[] = [];
-  while (true) {
-    // taking next pending request
-    const nextResourcePromise = pending.shift();
-    if (nextResourcePromise === undefined)
-      // this means we have no pending requests and traversal is over
-      break;
-    // at this point we pause and wait for the nest requested resource state to arrive
-    let nextResource = await nextResourcePromise;
+  for (let i = 0; i < pending.length; i++) {
+    // at this point we pause and wait for the next requested resource state to arrive
+    let nextResource = await pending[i];
     if (nextResource === undefined)
       // ignoring resources that were not found (this may happen for seed resource ids)
       continue;
@@ -187,7 +177,7 @@ export async function loadTreeState(
       nextResource = { ...nextResource, fields: fieldsAfterPruning };
     }
-    // continue traversal over the referenced resource
+    // continue traversal over the referenced resources
     requestState(nextResource.error);
     for (const field of nextResource.fields) {
       requestState(field.value);