@milaboratories/pl-tree 1.8.45 → 1.8.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/sync.cjs +22 -29
  2. package/dist/sync.cjs.map +1 -1
  3. package/dist/sync.d.ts.map +1 -1
  4. package/dist/sync.js +23 -30
  5. package/dist/sync.js.map +1 -1
  6. package/package.json +3 -3
  7. package/src/sync.ts +24 -32
package/dist/sync.cjs CHANGED
@@ -1,7 +1,7 @@
1
1
  'use strict';
2
2
 
3
- var plClient = require('@milaboratories/pl-client');
4
3
  var Denque = require('denque');
4
+ var plClient = require('@milaboratories/pl-client');
5
5
  var tsHelpers = require('@milaboratories/ts-helpers');
6
6
 
7
7
  /** Given the current tree state, build the request object to pass to
@@ -57,39 +57,36 @@ async function loadTreeState(tx, loadingRequest, stats) {
57
57
  if (stats)
58
58
  stats.requests++;
59
59
  const { seedResources, finalResources, pruningFunction } = loadingRequest;
60
- // Main idea of using a queue here is that responses will arrive in the same order as they were
61
- // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.
62
- // In such a way logic become linear without recursion, and at the same time deal with data
63
- // as soon as it arrives.
60
+ // Limits the number of concurrent gRPC fetches to bound peak memory
61
+ // from in-flight request/response buffers.
62
+ const limiter = new tsHelpers.ConcurrencyLimitingExecutor(100);
63
+ // Promises of resource states, in the order they were requested.
64
64
  const pending = new Denque();
65
65
  // vars to calculate number of roundtrips for stats
66
66
  let roundTripToggle = true;
67
67
  let numberOfRoundTrips = 0;
68
- // tracking resources we already requested
68
+ // tracking resources we already requested or queued
69
69
  const requested = new Set();
70
+ /** Mark a resource for fetching. Deduplicates and respects final-resource set. */
70
71
  const requestState = (rid) => {
71
72
  if (plClient.isNullResourceId(rid) || requested.has(rid))
72
73
  return;
73
- // separate check to collect stats
74
74
  if (finalResources.has(rid)) {
75
75
  if (stats)
76
76
  stats.finalResourcesSkipped++;
77
77
  return;
78
78
  }
79
- // adding the id, so we will not request it's state again if somebody else
80
- // references the same resource
81
79
  requested.add(rid);
82
- // requesting resource and all kv records
83
- const resourceData = tx.getResourceDataIfExists(rid, true);
84
- const kvData = tx.listKeyValuesIfResourceExists(rid);
85
- // counting round-trip (begin)
86
- const addRT = roundTripToggle;
87
- if (roundTripToggle)
88
- roundTripToggle = false;
89
- // pushing combined promise
90
- pending.push((async () => {
80
+ pending.push(limiter.run(async () => {
81
+ const resourceData = tx.getResourceDataIfExists(rid, true);
82
+ const kvData = tx.listKeyValuesIfResourceExists(rid);
83
+ // counting round-trip (begin)
84
+ const addRT = roundTripToggle;
85
+ if (roundTripToggle)
86
+ roundTripToggle = false;
91
87
  const [resource, kv] = await Promise.all([resourceData, kvData]);
92
- // counting round-trip, actually incrementing counter and returning toggle back, so the next request can acquire it
88
+ // counting round-trip, actually incrementing counter and returning toggle back,
89
+ // so the next request can acquire it
93
90
  if (addRT) {
94
91
  numberOfRoundTrips++;
95
92
  roundTripToggle = true;
@@ -99,19 +96,15 @@ async function loadTreeState(tx, loadingRequest, stats) {
99
96
  if (kv === undefined)
100
97
  throw new Error("Inconsistent replies");
101
98
  return { ...resource, kv };
102
- })());
99
+ }));
103
100
  };
104
101
  // sending seed requests
105
102
  seedResources.forEach((rid) => requestState(rid));
106
103
  const result = [];
107
- while (true) {
108
- // taking next pending request
109
- const nextResourcePromise = pending.shift();
110
- if (nextResourcePromise === undefined)
111
- // this means we have no pending requests and traversal is over
112
- break;
113
- // at this point we pause and wait for the nest requested resource state to arrive
114
- let nextResource = await nextResourcePromise;
104
+ let nextPromise;
105
+ while ((nextPromise = pending.shift()) !== undefined) {
106
+ // at this point we pause and wait for the next requested resource state to arrive
107
+ let nextResource = await nextPromise;
115
108
  if (nextResource === undefined)
116
109
  // ignoring resources that were not found (this may happen for seed resource ids)
117
110
  continue;
@@ -123,7 +116,7 @@ async function loadTreeState(tx, loadingRequest, stats) {
123
116
  stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;
124
117
  nextResource = { ...nextResource, fields: fieldsAfterPruning };
125
118
  }
126
- // continue traversal over the referenced resource
119
+ // continue traversal over the referenced resources
127
120
  requestState(nextResource.error);
128
121
  for (const field of nextResource.fields) {
129
122
  requestState(field.value);
package/dist/sync.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"sync.cjs","sources":["../src/sync.ts"],"sourcesContent":["import type {\n FieldData,\n OptionalResourceId,\n PlTransaction,\n ResourceId,\n} from \"@milaboratories/pl-client\";\nimport { isNullResourceId } from \"@milaboratories/pl-client\";\nimport Denque from \"denque\";\nimport type { ExtendedResourceData, PlTreeState } from \"./state\";\nimport { msToHumanReadable } from \"@milaboratories/ts-helpers\";\n\n/** Applied to list of fields in resource data. */\nexport type PruningFunction = (resource: ExtendedResourceData) => FieldData[];\n\nexport interface TreeLoadingRequest {\n /** Resource to prime the traversal algorithm. It is ok, if some of them\n * doesn't exist anymore. Should not contain elements from final resource\n * set. */\n readonly seedResources: ResourceId[];\n\n /** Resource ids for which state is already known and not expected to change.\n * Algorithm will not continue traversal over those ids, and states will not\n * be retrieved for them. */\n readonly finalResources: Set<ResourceId>;\n\n /** This function is applied to each resource data field list, before\n * using it continue traversal. This modification also is applied to\n * output data to make result self-consistent in terms that it will contain\n * all referenced resources, this is required to be able to pass it to tree\n * to update the state. */\n readonly pruningFunction?: PruningFunction;\n}\n\n/** Given the current tree state, build the request object to pass to\n * {@link loadTreeState} to load updated state. */\nexport function constructTreeLoadingRequest(\n tree: PlTreeState,\n pruningFunction?: PruningFunction,\n): TreeLoadingRequest {\n const seedResources: ResourceId[] = [];\n const finalResources = new Set<ResourceId>();\n tree.forEachResource((res) => {\n if (res.finalState) finalResources.add(res.id);\n else seedResources.push(res.id);\n });\n\n // if tree is empty, seeding tree reconstruction from the specified root\n if (seedResources.length === 0 && finalResources.size === 0) seedResources.push(tree.root);\n\n return { seedResources, finalResources, pruningFunction };\n}\n\nexport type TreeLoadingStat = {\n requests: number;\n roundTrips: number;\n retrievedResources: number;\n retrievedFields: number;\n retrievedKeyValues: number;\n retrievedResourceDataBytes: number;\n retrievedKeyValueBytes: number;\n prunedFields: number;\n finalResourcesSkipped: number;\n millisSpent: number;\n};\n\nexport function initialTreeLoadingStat(): TreeLoadingStat {\n return {\n requests: 0,\n roundTrips: 0,\n retrievedResources: 0,\n retrievedFields: 0,\n retrievedKeyValues: 0,\n retrievedResourceDataBytes: 0,\n retrievedKeyValueBytes: 0,\n prunedFields: 0,\n finalResourcesSkipped: 0,\n millisSpent: 0,\n };\n}\n\nexport function formatTreeLoadingStat(stat: TreeLoadingStat): string {\n let result = `Requests: ${stat.requests}\\n`;\n result += `Total time: ${msToHumanReadable(stat.millisSpent)}\\n`;\n result += `Round-trips: ${stat.roundTrips}\\n`;\n result += `Resources: ${stat.retrievedResources}\\n`;\n result += `Fields: ${stat.retrievedFields}\\n`;\n result += `KV: ${stat.retrievedKeyValues}\\n`;\n result += `Data Bytes: ${stat.retrievedResourceDataBytes}\\n`;\n result += `KV Bytes: ${stat.retrievedKeyValueBytes}\\n`;\n result += `Pruned fields: ${stat.prunedFields}\\n`;\n result += `Final resources skipped: ${stat.finalResourcesSkipped}`;\n return result;\n}\n\n/** Given the transaction (preferably read-only) and loading request, executes\n * the tree traversal algorithm, and collects fresh states of resources\n * to update the tree state. */\nexport async function loadTreeState(\n tx: PlTransaction,\n loadingRequest: TreeLoadingRequest,\n stats?: TreeLoadingStat,\n): Promise<ExtendedResourceData[]> {\n // saving start timestamp to add time spent in this function to the stats at the end of the method\n const startTimestamp = Date.now();\n\n // counting the request\n if (stats) stats.requests++;\n\n const { seedResources, finalResources, pruningFunction } = loadingRequest;\n\n // Main idea of using a queue here is that responses will arrive in the same order as they were\n // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.\n // In such a way logic become linear without recursion, and at the same time deal with data\n // as soon as it arrives.\n\n const pending = new Denque<Promise<ExtendedResourceData | undefined>>();\n\n // vars to calculate number of roundtrips for stats\n let roundTripToggle: boolean = true;\n let numberOfRoundTrips = 0;\n\n // tracking resources we already requested\n const requested = new Set<ResourceId>();\n const requestState = (rid: OptionalResourceId) => {\n if (isNullResourceId(rid) || requested.has(rid)) return;\n\n // separate check to collect stats\n if (finalResources.has(rid)) {\n if (stats) stats.finalResourcesSkipped++;\n return;\n }\n\n // adding the id, so we will not request it's state again if somebody else\n // references the same resource\n requested.add(rid);\n\n // requesting resource and all kv records\n const resourceData = tx.getResourceDataIfExists(rid, true);\n const kvData = tx.listKeyValuesIfResourceExists(rid);\n\n // counting round-trip (begin)\n const addRT = roundTripToggle;\n if (roundTripToggle) roundTripToggle = false;\n\n // pushing combined promise\n pending.push(\n (async () => {\n const [resource, kv] = await Promise.all([resourceData, kvData]);\n\n // counting round-trip, actually incrementing counter and returning toggle back, so the next request can acquire it\n if (addRT) {\n numberOfRoundTrips++;\n roundTripToggle = true;\n }\n\n if (resource === undefined) return undefined;\n\n if (kv === undefined) throw new Error(\"Inconsistent replies\");\n\n return { ...resource, kv };\n })(),\n );\n };\n\n // sending seed requests\n seedResources.forEach((rid) => requestState(rid));\n\n const result: ExtendedResourceData[] = [];\n while (true) {\n // taking next pending request\n const nextResourcePromise = pending.shift();\n if (nextResourcePromise === undefined)\n // this means we have no pending requests and traversal is over\n break;\n\n // at this point we pause and wait for the nest requested resource state to arrive\n let nextResource = await nextResourcePromise;\n if (nextResource === undefined)\n // ignoring resources that were not found (this may happen for seed resource ids)\n continue;\n\n if (pruningFunction !== undefined) {\n // apply field pruning, if requested\n const fieldsAfterPruning = pruningFunction(nextResource);\n // collecting stats\n if (stats) stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;\n nextResource = { ...nextResource, fields: fieldsAfterPruning };\n }\n\n // continue traversal over the referenced resource\n requestState(nextResource.error);\n for (const field of nextResource.fields) {\n requestState(field.value);\n requestState(field.error);\n }\n\n // collecting stats\n if (stats) {\n stats.retrievedResources++;\n stats.retrievedFields += nextResource.fields.length;\n stats.retrievedKeyValues += nextResource.kv.length;\n stats.retrievedResourceDataBytes += nextResource.data?.length ?? 0;\n for (const kv of nextResource.kv) stats.retrievedKeyValueBytes += kv.value.length;\n }\n\n // aggregating the state\n result.push(nextResource);\n }\n\n // adding the time we spent in this method to stats\n if (stats) {\n stats.millisSpent += Date.now() - startTimestamp;\n stats.roundTrips += numberOfRoundTrips;\n }\n\n return result;\n}\n"],"names":["msToHumanReadable","isNullResourceId"],"mappings":";;;;;;AAiCA;AACkD;AAC5C,SAAU,2BAA2B,CACzC,IAAiB,EACjB,eAAiC,EAAA;IAEjC,MAAM,aAAa,GAAiB,EAAE;AACtC,IAAA,MAAM,cAAc,GAAG,IAAI,GAAG,EAAc;AAC5C,IAAA,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,KAAI;QAC3B,IAAI,GAAG,CAAC,UAAU;AAAE,YAAA,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;;AACzC,YAAA,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACjC,IAAA,CAAC,CAAC;;IAGF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC;AAAE,QAAA,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAE1F,IAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE;AAC3D;SAegB,sBAAsB,GAAA;IACpC,OAAO;AACL,QAAA,QAAQ,EAAE,CAAC;AACX,QAAA,UAAU,EAAE,CAAC;AACb,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,eAAe,EAAE,CAAC;AAClB,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,0BAA0B,EAAE,CAAC;AAC7B,QAAA,sBAAsB,EAAE,CAAC;AACzB,QAAA,YAAY,EAAE,CAAC;AACf,QAAA,qBAAqB,EAAE,CAAC;AACxB,QAAA,WAAW,EAAE,CAAC;KACf;AACH;AAEM,SAAU,qBAAqB,CAAC,IAAqB,EAAA;AACzD,IAAA,IAAI,MAAM,GAAG,CAAA,UAAA,EAAa,IAAI,CAAC,QAAQ,IAAI;IAC3C,MAAM,IAAI,eAAeA,2BAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA,EAAA,CAAI;AAChE,IAAA,MAAM,IAAI,CAAA,aAAA,EAAgB,IAAI,CAAC,UAAU,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,kBAAkB,IAAI;AACnD,IAAA,MAAM,IAAI,CAAA,QAAA,EAAW,IAAI,CAAC,eAAe,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,IAAA,EAAO,IAAI,CAAC,kBAAkB,IAAI;AAC5C,IAAA,MAAM,IAAI,CAAA,YAAA,EAAe,IAAI,CAAC,0BAA0B,IAAI;AAC5D,IAAA,MAAM,IAAI,CAAA,UAAA,EAAa,IAAI,CAAC,sBAAsB,IAAI;AACtD,IAAA,MAAM,IAAI,CAAA,eAAA,EAAkB,IAAI,CAAC,YAAY,IAAI;AACjD,IAAA,MAAM,IAAI,CAAA,yBAAA,EAA4B,IAAI,CAAC,qBAAqB,EAAE;AAClE,IAAA,OAAO,MAAM;AACf;AAEA;;AAE+B;AACxB,eAAe,aAAa,CACjC,EAAiB,EACjB,cAAkC,EAClC,KAAuB,EAAA;;AAGvB,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE;;AAGjC,IAAA,IAAI,KAAK;QAAE,KAAK,CAAC,QAAQ,EAAE;IAE3B,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,cAAc;;;;;AAOzE,IAAA,MAAM,OAAO,GAAG,IAAI,MAAM,EAA6C;;IAGvE,IAAI,eAAe,GAAY,IAAI;IACnC,IAAI,kBAAkB,GAAG,CAAC;;AAG1B,IAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc;AACvC,IAAA,MAAM,YAAY,GAAG,CAAC,GAAuB,KAAI;QAC/C,IAAIC,yBAAgB,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE;;AAGjD,QAAA,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;AAC3B,YAAA,IAAI,KAAK;gBAAE,KAAK,CAAC,qBAAqB,EAAE;YACxC;QACF;;;AAIA,QAAA,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;;QAGlB,MAAM,YAAY,GAAG,EAAE,CAAC,uBAAuB,CAAC,GAAG,EAAE,IAAI,CAAC;QAC1D,MAAM,MAAM,GAAG,EAAE,CAAC,6BAA6B,CAAC,GAAG,CAAC;;QAGpD,MAAM,KAAK,GAAG,eAAe;AAC7B,QAAA,IAAI,eAAe;YAAE,eAAe,GAAG,KAAK;;AAG5C,QAAA,OAAO,CAAC,IAAI,CACV,CAAC,YAAW;AACV,YAAA,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;;YAGhE,IAAI,KAAK,EAAE;AACT,gBAAA,kBAAkB,EAAE;gBACpB,eAAe,GAAG,IAAI;YACxB;YAEA,IAAI,QAAQ,KAAK,SAAS;AAAE,gBAAA,OAAO,SAAS;YAE5C,IAAI,EAAE,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC;AAE7D,YAAA,OAAO,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE;QAC5B,CAAC,GAAG,CACL;AACH,IAAA,CAAC;;AAGD,IAAA,aAAa,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC;IAEjD,MAAM,MAAM,GAA2B,EAAE;IACzC,OAAO,IAAI,EAAE;;AAEX,QAAA,MAAM,mBAAmB,GAAG,OAAO,CAAC,KAAK,EAAE;QAC3C,IAAI,mBAAmB,KAAK,SAAS;;YAEnC;;AAGF,QAAA,IAAI,YAAY,GAAG,MAAM,mBAAmB;QAC5C,IAAI,YAAY,KAAK,SAAS;;YAE5B;AAEF,QAAA,IAAI,eAAe,KAAK,SAAS,EAAE;;AAEjC,YAAA,MAAM,kBAAkB,GAAG,eAAe,CAAC,YAAY,CAAC;;AAExD,YAAA,IAAI,KAAK;AAAE,gBAAA,KAAK,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM;YACvF,YAAY,GAAG,EAAE,GAAG,YAAY,EAAE,MAAM,EAAE,kBAAkB,EAAE;QAChE;;AAGA,QAAA,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC;AAChC,QAAA,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE;AACvC,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;QAC3B;;QAGA,IAAI,KAAK,EAAE;YACT,KAAK,CAAC,kBAAkB,EAAE;YAC1B,KAAK,CAAC,eAAe,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM;YACnD,KAAK,CAAC,kBAAkB,IAAI,YAAY,CAAC,EAAE,CAAC,MAAM;YAClD,KAAK,CAAC,0BAA0B,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;AAClE,YAAA,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,EAAE;gBAAE,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM;QACnF;;AAGA,QAAA,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;IAC3B;;IAGA,IAAI,KAAK,EAAE;QACT,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;AAChD,QAAA,KAAK,CAAC,UAAU,IAAI,kBAAkB;IACxC;AAEA,IAAA,OAAO,MAAM;AACf;;;;;;;"}
1
+ {"version":3,"file":"sync.cjs","sources":["../src/sync.ts"],"sourcesContent":["import type {\n FieldData,\n OptionalResourceId,\n PlTransaction,\n ResourceId,\n} from \"@milaboratories/pl-client\";\nimport Denque from \"denque\";\nimport { isNullResourceId } from \"@milaboratories/pl-client\";\nimport type { ExtendedResourceData, PlTreeState } from \"./state\";\nimport { ConcurrencyLimitingExecutor, msToHumanReadable } from \"@milaboratories/ts-helpers\";\n\n/** Applied to list of fields in resource data. */\nexport type PruningFunction = (resource: ExtendedResourceData) => FieldData[];\n\nexport interface TreeLoadingRequest {\n /** Resource to prime the traversal algorithm. It is ok, if some of them\n * doesn't exist anymore. Should not contain elements from final resource\n * set. */\n readonly seedResources: ResourceId[];\n\n /** Resource ids for which state is already known and not expected to change.\n * Algorithm will not continue traversal over those ids, and states will not\n * be retrieved for them. */\n readonly finalResources: Set<ResourceId>;\n\n /** This function is applied to each resource data field list, before\n * using it continue traversal. This modification also is applied to\n * output data to make result self-consistent in terms that it will contain\n * all referenced resources, this is required to be able to pass it to tree\n * to update the state. */\n readonly pruningFunction?: PruningFunction;\n}\n\n/** Given the current tree state, build the request object to pass to\n * {@link loadTreeState} to load updated state. */\nexport function constructTreeLoadingRequest(\n tree: PlTreeState,\n pruningFunction?: PruningFunction,\n): TreeLoadingRequest {\n const seedResources: ResourceId[] = [];\n const finalResources = new Set<ResourceId>();\n tree.forEachResource((res) => {\n if (res.finalState) finalResources.add(res.id);\n else seedResources.push(res.id);\n });\n\n // if tree is empty, seeding tree reconstruction from the specified root\n if (seedResources.length === 0 && finalResources.size === 0) seedResources.push(tree.root);\n\n return { seedResources, finalResources, pruningFunction };\n}\n\nexport type TreeLoadingStat = {\n requests: number;\n roundTrips: number;\n retrievedResources: number;\n retrievedFields: number;\n retrievedKeyValues: number;\n retrievedResourceDataBytes: number;\n retrievedKeyValueBytes: number;\n prunedFields: number;\n finalResourcesSkipped: number;\n millisSpent: number;\n};\n\nexport function initialTreeLoadingStat(): TreeLoadingStat {\n return {\n requests: 0,\n roundTrips: 0,\n retrievedResources: 0,\n retrievedFields: 0,\n retrievedKeyValues: 0,\n retrievedResourceDataBytes: 0,\n retrievedKeyValueBytes: 0,\n prunedFields: 0,\n finalResourcesSkipped: 0,\n millisSpent: 0,\n };\n}\n\nexport function formatTreeLoadingStat(stat: TreeLoadingStat): string {\n let result = `Requests: ${stat.requests}\\n`;\n result += `Total time: ${msToHumanReadable(stat.millisSpent)}\\n`;\n result += `Round-trips: ${stat.roundTrips}\\n`;\n result += `Resources: ${stat.retrievedResources}\\n`;\n result += `Fields: ${stat.retrievedFields}\\n`;\n result += `KV: ${stat.retrievedKeyValues}\\n`;\n result += `Data Bytes: ${stat.retrievedResourceDataBytes}\\n`;\n result += `KV Bytes: ${stat.retrievedKeyValueBytes}\\n`;\n result += `Pruned fields: ${stat.prunedFields}\\n`;\n result += `Final resources skipped: ${stat.finalResourcesSkipped}`;\n return result;\n}\n\n/** Given the transaction (preferably read-only) and loading request, executes\n * the tree traversal algorithm, and collects fresh states of resources\n * to update the tree state. */\nexport async function loadTreeState(\n tx: PlTransaction,\n loadingRequest: TreeLoadingRequest,\n stats?: TreeLoadingStat,\n): Promise<ExtendedResourceData[]> {\n // saving start timestamp to add time spent in this function to the stats at the end of the method\n const startTimestamp = Date.now();\n\n // counting the request\n if (stats) stats.requests++;\n\n const { seedResources, finalResources, pruningFunction } = loadingRequest;\n\n // Limits the number of concurrent gRPC fetches to bound peak memory\n // from in-flight request/response buffers.\n const limiter = new ConcurrencyLimitingExecutor(100);\n\n // Promises of resource states, in the order they were requested.\n const pending = new Denque<Promise<ExtendedResourceData | undefined>>();\n\n // vars to calculate number of roundtrips for stats\n let roundTripToggle: boolean = true;\n let numberOfRoundTrips = 0;\n\n // tracking resources we already requested or queued\n const requested = new Set<ResourceId>();\n\n /** Mark a resource for fetching. Deduplicates and respects final-resource set. */\n const requestState = (rid: OptionalResourceId) => {\n if (isNullResourceId(rid) || requested.has(rid)) return;\n\n if (finalResources.has(rid)) {\n if (stats) stats.finalResourcesSkipped++;\n return;\n }\n\n requested.add(rid);\n\n pending.push(\n limiter.run(async () => {\n const resourceData = tx.getResourceDataIfExists(rid, true);\n const kvData = tx.listKeyValuesIfResourceExists(rid);\n\n // counting round-trip (begin)\n const addRT = roundTripToggle;\n if (roundTripToggle) roundTripToggle = false;\n\n const [resource, kv] = await Promise.all([resourceData, kvData]);\n\n // counting round-trip, actually incrementing counter and returning toggle back,\n // so the next request can acquire it\n if (addRT) {\n numberOfRoundTrips++;\n roundTripToggle = true;\n }\n\n if (resource === undefined) return undefined;\n if (kv === undefined) throw new Error(\"Inconsistent replies\");\n\n return { ...resource, kv };\n }),\n );\n };\n\n // sending seed requests\n seedResources.forEach((rid) => requestState(rid));\n\n const result: ExtendedResourceData[] = [];\n let nextPromise: Promise<ExtendedResourceData | undefined> | undefined;\n while ((nextPromise = pending.shift()) !== undefined) {\n // at this point we pause and wait for the next requested resource state to arrive\n let nextResource = await nextPromise;\n if (nextResource === undefined)\n // ignoring resources that were not found (this may happen for seed resource ids)\n continue;\n\n if (pruningFunction !== undefined) {\n // apply field pruning, if requested\n const fieldsAfterPruning = pruningFunction(nextResource);\n // collecting stats\n if (stats) stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;\n nextResource = { ...nextResource, fields: fieldsAfterPruning };\n }\n\n // continue traversal over the referenced resources\n requestState(nextResource.error);\n for (const field of nextResource.fields) {\n requestState(field.value);\n requestState(field.error);\n }\n\n // collecting stats\n if (stats) {\n stats.retrievedResources++;\n stats.retrievedFields += nextResource.fields.length;\n stats.retrievedKeyValues += nextResource.kv.length;\n stats.retrievedResourceDataBytes += nextResource.data?.length ?? 0;\n for (const kv of nextResource.kv) stats.retrievedKeyValueBytes += kv.value.length;\n }\n\n // aggregating the state\n result.push(nextResource);\n }\n\n // adding the time we spent in this method to stats\n if (stats) {\n stats.millisSpent += Date.now() - startTimestamp;\n stats.roundTrips += numberOfRoundTrips;\n }\n\n return result;\n}\n"],"names":["msToHumanReadable","ConcurrencyLimitingExecutor","isNullResourceId"],"mappings":";;;;;;AAiCA;AACkD;AAC5C,SAAU,2BAA2B,CACzC,IAAiB,EACjB,eAAiC,EAAA;IAEjC,MAAM,aAAa,GAAiB,EAAE;AACtC,IAAA,MAAM,cAAc,GAAG,IAAI,GAAG,EAAc;AAC5C,IAAA,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,KAAI;QAC3B,IAAI,GAAG,CAAC,UAAU;AAAE,YAAA,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;;AACzC,YAAA,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACjC,IAAA,CAAC,CAAC;;IAGF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC;AAAE,QAAA,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAE1F,IAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE;AAC3D;SAegB,sBAAsB,GAAA;IACpC,OAAO;AACL,QAAA,QAAQ,EAAE,CAAC;AACX,QAAA,UAAU,EAAE,CAAC;AACb,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,eAAe,EAAE,CAAC;AAClB,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,0BAA0B,EAAE,CAAC;AAC7B,QAAA,sBAAsB,EAAE,CAAC;AACzB,QAAA,YAAY,EAAE,CAAC;AACf,QAAA,qBAAqB,EAAE,CAAC;AACxB,QAAA,WAAW,EAAE,CAAC;KACf;AACH;AAEM,SAAU,qBAAqB,CAAC,IAAqB,EAAA;AACzD,IAAA,IAAI,MAAM,GAAG,CAAA,UAAA,EAAa,IAAI,CAAC,QAAQ,IAAI;IAC3C,MAAM,IAAI,eAAeA,2BAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA,EAAA,CAAI;AAChE,IAAA,MAAM,IAAI,CAAA,aAAA,EAAgB,IAAI,CAAC,UAAU,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,kBAAkB,IAAI;AACnD,IAAA,MAAM,IAAI,CAAA,QAAA,EAAW,IAAI,CAAC,eAAe,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,IAAA,EAAO,IAAI,CAAC,kBAAkB,IAAI;AAC5C,IAAA,MAAM,IAAI,CAAA,YAAA,EAAe,IAAI,CAAC,0BAA0B,IAAI;AAC5D,IAAA,MAAM,IAAI,CAAA,UAAA,EAAa,IAAI,CAAC,sBAAsB,IAAI;AACtD,IAAA,MAAM,IAAI,CAAA,eAAA,EAAkB,IAAI,CAAC,YAAY,IAAI;AACjD,IAAA,MAAM,IAAI,CAAA,yBAAA,EAA4B,IAAI,CAAC,qBAAqB,EAAE;AAClE,IAAA,OAAO,MAAM;AACf;AAEA;;AAE+B;AACxB,eAAe,aAAa,CACjC,EAAiB,EACjB,cAAkC,EAClC,KAAuB,EAAA;;AAGvB,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE;;AAGjC,IAAA,IAAI,KAAK;QAAE,KAAK,CAAC,QAAQ,EAAE;IAE3B,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,cAAc;;;AAIzE,IAAA,MAAM,OAAO,GAAG,IAAIC,qCAA2B,CAAC,GAAG,CAAC;;AAGpD,IAAA,MAAM,OAAO,GAAG,IAAI,MAAM,EAA6C;;IAGvE,IAAI,eAAe,GAAY,IAAI;IACnC,IAAI,kBAAkB,GAAG,CAAC;;AAG1B,IAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc;;AAGvC,IAAA,MAAM,YAAY,GAAG,CAAC,GAAuB,KAAI;QAC/C,IAAIC,yBAAgB,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE;AAEjD,QAAA,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;AAC3B,YAAA,IAAI,KAAK;gBAAE,KAAK,CAAC,qBAAqB,EAAE;YACxC;QACF;AAEA,QAAA,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;QAElB,OAAO,CAAC,IAAI,CACV,OAAO,CAAC,GAAG,CAAC,YAAW;YACrB,MAAM,YAAY,GAAG,EAAE,CAAC,uBAAuB,CAAC,GAAG,EAAE,IAAI,CAAC;YAC1D,MAAM,MAAM,GAAG,EAAE,CAAC,6BAA6B,CAAC,GAAG,CAAC;;YAGpD,MAAM,KAAK,GAAG,eAAe;AAC7B,YAAA,IAAI,eAAe;gBAAE,eAAe,GAAG,KAAK;AAE5C,YAAA,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;;;YAIhE,IAAI,KAAK,EAAE;AACT,gBAAA,kBAAkB,EAAE;gBACpB,eAAe,GAAG,IAAI;YACxB;YAEA,IAAI,QAAQ,KAAK,SAAS;AAAE,gBAAA,OAAO,SAAS;YAC5C,IAAI,EAAE,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC;AAE7D,YAAA,OAAO,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE;QAC5B,CAAC,CAAC,CACH;AACH,IAAA,CAAC;;AAGD,IAAA,aAAa,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC;IAEjD,MAAM,MAAM,GAA2B,EAAE;AACzC,IAAA,IAAI,WAAkE;IACtE,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,MAAM,SAAS,EAAE;;AAEpD,QAAA,IAAI,YAAY,GAAG,MAAM,WAAW;QACpC,IAAI,YAAY,KAAK,SAAS;;YAE5B;AAEF,QAAA,IAAI,eAAe,KAAK,SAAS,EAAE;;AAEjC,YAAA,MAAM,kBAAkB,GAAG,eAAe,CAAC,YAAY,CAAC;;AAExD,YAAA,IAAI,KAAK;AAAE,gBAAA,KAAK,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM;YACvF,YAAY,GAAG,EAAE,GAAG,YAAY,EAAE,MAAM,EAAE,kBAAkB,EAAE;QAChE;;AAGA,QAAA,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC;AAChC,QAAA,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE;AACvC,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;QAC3B;;QAGA,IAAI,KAAK,EAAE;YACT,KAAK,CAAC,kBAAkB,EAAE;YAC1B,KAAK,CAAC,eAAe,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM;YACnD,KAAK,CAAC,kBAAkB,IAAI,YAAY,CAAC,EAAE,CAAC,MAAM;YAClD,KAAK,CAAC,0BAA0B,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;AAClE,YAAA,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,EAAE;gBAAE,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM;QACnF;;AAGA,QAAA,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;IAC3B;;IAGA,IAAI,KAAK,EAAE;QACT,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;AAChD,QAAA,KAAK,CAAC,UAAU,IAAI,kBAAkB;IACxC;AAEA,IAAA,OAAO,MAAM;AACf;;;;;;;"}
package/dist/sync.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../src/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EAET,aAAa,EACb,UAAU,EACX,MAAM,2BAA2B,CAAC;AAGnC,OAAO,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAGjE,kDAAkD;AAClD,MAAM,MAAM,eAAe,GAAG,CAAC,QAAQ,EAAE,oBAAoB,KAAK,SAAS,EAAE,CAAC;AAE9E,MAAM,WAAW,kBAAkB;IACjC;;cAEU;IACV,QAAQ,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;IAErC;;gCAE4B;IAC5B,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAEzC;;;;8BAI0B;IAC1B,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;CAC5C;AAED;kDACkD;AAClD,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,WAAW,EACjB,eAAe,CAAC,EAAE,eAAe,GAChC,kBAAkB,CAYpB;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0BAA0B,EAAE,MAAM,CAAC;IACnC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,wBAAgB,sBAAsB,IAAI,eAAe,CAaxD;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,CAYnE;AAED;;+BAE+B;AAC/B,wBAAsB,aAAa,CACjC,EAAE,EAAE,aAAa,EACjB,cAAc,EAAE,kBAAkB,EAClC,KAAK,CAAC,EAAE,eAAe,GACtB,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAmHjC"}
1
+ {"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../src/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EAET,aAAa,EACb,UAAU,EACX,MAAM,2BAA2B,CAAC;AAGnC,OAAO,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAGjE,kDAAkD;AAClD,MAAM,MAAM,eAAe,GAAG,CAAC,QAAQ,EAAE,oBAAoB,KAAK,SAAS,EAAE,CAAC;AAE9E,MAAM,WAAW,kBAAkB;IACjC;;cAEU;IACV,QAAQ,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;IAErC;;gCAE4B;IAC5B,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAEzC;;;;8BAI0B;IAC1B,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;CAC5C;AAED;kDACkD;AAClD,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,WAAW,EACjB,eAAe,CAAC,EAAE,eAAe,GAChC,kBAAkB,CAYpB;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0BAA0B,EAAE,MAAM,CAAC;IACnC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,wBAAgB,sBAAsB,IAAI,eAAe,CAaxD;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,CAYnE;AAED;;+BAE+B;AAC/B,wBAAsB,aAAa,CACjC,EAAE,EAAE,aAAa,EACjB,cAAc,EAAE,kBAAkB,EAClC,KAAK,CAAC,EAAE,eAAe,GACtB,OAAO,CAAC,oBAAoB,EAAE,CAAC,CA2GjC"}
package/dist/sync.js CHANGED
@@ -1,6 +1,6 @@
1
- import { isNullResourceId } from '@milaboratories/pl-client';
2
1
  import Denque from 'denque';
3
- import { msToHumanReadable } from '@milaboratories/ts-helpers';
2
+ import { isNullResourceId } from '@milaboratories/pl-client';
3
+ import { msToHumanReadable, ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';
4
4
 
5
5
  /** Given the current tree state, build the request object to pass to
6
6
  * {@link loadTreeState} to load updated state. */
@@ -55,39 +55,36 @@ async function loadTreeState(tx, loadingRequest, stats) {
55
55
  if (stats)
56
56
  stats.requests++;
57
57
  const { seedResources, finalResources, pruningFunction } = loadingRequest;
58
- // Main idea of using a queue here is that responses will arrive in the same order as they were
59
- // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.
60
- // In such a way logic become linear without recursion, and at the same time deal with data
61
- // as soon as it arrives.
58
+ // Limits the number of concurrent gRPC fetches to bound peak memory
59
+ // from in-flight request/response buffers.
60
+ const limiter = new ConcurrencyLimitingExecutor(100);
61
+ // Promises of resource states, in the order they were requested.
62
62
  const pending = new Denque();
63
63
  // vars to calculate number of roundtrips for stats
64
64
  let roundTripToggle = true;
65
65
  let numberOfRoundTrips = 0;
66
- // tracking resources we already requested
66
+ // tracking resources we already requested or queued
67
67
  const requested = new Set();
68
+ /** Mark a resource for fetching. Deduplicates and respects final-resource set. */
68
69
  const requestState = (rid) => {
69
70
  if (isNullResourceId(rid) || requested.has(rid))
70
71
  return;
71
- // separate check to collect stats
72
72
  if (finalResources.has(rid)) {
73
73
  if (stats)
74
74
  stats.finalResourcesSkipped++;
75
75
  return;
76
76
  }
77
- // adding the id, so we will not request it's state again if somebody else
78
- // references the same resource
79
77
  requested.add(rid);
80
- // requesting resource and all kv records
81
- const resourceData = tx.getResourceDataIfExists(rid, true);
82
- const kvData = tx.listKeyValuesIfResourceExists(rid);
83
- // counting round-trip (begin)
84
- const addRT = roundTripToggle;
85
- if (roundTripToggle)
86
- roundTripToggle = false;
87
- // pushing combined promise
88
- pending.push((async () => {
78
+ pending.push(limiter.run(async () => {
79
+ const resourceData = tx.getResourceDataIfExists(rid, true);
80
+ const kvData = tx.listKeyValuesIfResourceExists(rid);
81
+ // counting round-trip (begin)
82
+ const addRT = roundTripToggle;
83
+ if (roundTripToggle)
84
+ roundTripToggle = false;
89
85
  const [resource, kv] = await Promise.all([resourceData, kvData]);
90
- // counting round-trip, actually incrementing counter and returning toggle back, so the next request can acquire it
86
+ // counting round-trip, actually incrementing counter and returning toggle back,
87
+ // so the next request can acquire it
91
88
  if (addRT) {
92
89
  numberOfRoundTrips++;
93
90
  roundTripToggle = true;
@@ -97,19 +94,15 @@ async function loadTreeState(tx, loadingRequest, stats) {
97
94
  if (kv === undefined)
98
95
  throw new Error("Inconsistent replies");
99
96
  return { ...resource, kv };
100
- })());
97
+ }));
101
98
  };
102
99
  // sending seed requests
103
100
  seedResources.forEach((rid) => requestState(rid));
104
101
  const result = [];
105
- while (true) {
106
- // taking next pending request
107
- const nextResourcePromise = pending.shift();
108
- if (nextResourcePromise === undefined)
109
- // this means we have no pending requests and traversal is over
110
- break;
111
- // at this point we pause and wait for the nest requested resource state to arrive
112
- let nextResource = await nextResourcePromise;
102
+ let nextPromise;
103
+ while ((nextPromise = pending.shift()) !== undefined) {
104
+ // at this point we pause and wait for the next requested resource state to arrive
105
+ let nextResource = await nextPromise;
113
106
  if (nextResource === undefined)
114
107
  // ignoring resources that were not found (this may happen for seed resource ids)
115
108
  continue;
@@ -121,7 +114,7 @@ async function loadTreeState(tx, loadingRequest, stats) {
121
114
  stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;
122
115
  nextResource = { ...nextResource, fields: fieldsAfterPruning };
123
116
  }
124
- // continue traversal over the referenced resource
117
+ // continue traversal over the referenced resources
125
118
  requestState(nextResource.error);
126
119
  for (const field of nextResource.fields) {
127
120
  requestState(field.value);
package/dist/sync.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"sync.js","sources":["../src/sync.ts"],"sourcesContent":["import type {\n FieldData,\n OptionalResourceId,\n PlTransaction,\n ResourceId,\n} from \"@milaboratories/pl-client\";\nimport { isNullResourceId } from \"@milaboratories/pl-client\";\nimport Denque from \"denque\";\nimport type { ExtendedResourceData, PlTreeState } from \"./state\";\nimport { msToHumanReadable } from \"@milaboratories/ts-helpers\";\n\n/** Applied to list of fields in resource data. */\nexport type PruningFunction = (resource: ExtendedResourceData) => FieldData[];\n\nexport interface TreeLoadingRequest {\n /** Resource to prime the traversal algorithm. It is ok, if some of them\n * doesn't exist anymore. Should not contain elements from final resource\n * set. */\n readonly seedResources: ResourceId[];\n\n /** Resource ids for which state is already known and not expected to change.\n * Algorithm will not continue traversal over those ids, and states will not\n * be retrieved for them. */\n readonly finalResources: Set<ResourceId>;\n\n /** This function is applied to each resource data field list, before\n * using it continue traversal. This modification also is applied to\n * output data to make result self-consistent in terms that it will contain\n * all referenced resources, this is required to be able to pass it to tree\n * to update the state. */\n readonly pruningFunction?: PruningFunction;\n}\n\n/** Given the current tree state, build the request object to pass to\n * {@link loadTreeState} to load updated state. */\nexport function constructTreeLoadingRequest(\n tree: PlTreeState,\n pruningFunction?: PruningFunction,\n): TreeLoadingRequest {\n const seedResources: ResourceId[] = [];\n const finalResources = new Set<ResourceId>();\n tree.forEachResource((res) => {\n if (res.finalState) finalResources.add(res.id);\n else seedResources.push(res.id);\n });\n\n // if tree is empty, seeding tree reconstruction from the specified root\n if (seedResources.length === 0 && finalResources.size === 0) seedResources.push(tree.root);\n\n return { seedResources, finalResources, pruningFunction };\n}\n\nexport type TreeLoadingStat = {\n requests: number;\n roundTrips: number;\n retrievedResources: number;\n retrievedFields: number;\n retrievedKeyValues: number;\n retrievedResourceDataBytes: number;\n retrievedKeyValueBytes: number;\n prunedFields: number;\n finalResourcesSkipped: number;\n millisSpent: number;\n};\n\nexport function initialTreeLoadingStat(): TreeLoadingStat {\n return {\n requests: 0,\n roundTrips: 0,\n retrievedResources: 0,\n retrievedFields: 0,\n retrievedKeyValues: 0,\n retrievedResourceDataBytes: 0,\n retrievedKeyValueBytes: 0,\n prunedFields: 0,\n finalResourcesSkipped: 0,\n millisSpent: 0,\n };\n}\n\nexport function formatTreeLoadingStat(stat: TreeLoadingStat): string {\n let result = `Requests: ${stat.requests}\\n`;\n result += `Total time: ${msToHumanReadable(stat.millisSpent)}\\n`;\n result += `Round-trips: ${stat.roundTrips}\\n`;\n result += `Resources: ${stat.retrievedResources}\\n`;\n result += `Fields: ${stat.retrievedFields}\\n`;\n result += `KV: ${stat.retrievedKeyValues}\\n`;\n result += `Data Bytes: ${stat.retrievedResourceDataBytes}\\n`;\n result += `KV Bytes: ${stat.retrievedKeyValueBytes}\\n`;\n result += `Pruned fields: ${stat.prunedFields}\\n`;\n result += `Final resources skipped: ${stat.finalResourcesSkipped}`;\n return result;\n}\n\n/** Given the transaction (preferably read-only) and loading request, executes\n * the tree traversal algorithm, and collects fresh states of resources\n * to update the tree state. */\nexport async function loadTreeState(\n tx: PlTransaction,\n loadingRequest: TreeLoadingRequest,\n stats?: TreeLoadingStat,\n): Promise<ExtendedResourceData[]> {\n // saving start timestamp to add time spent in this function to the stats at the end of the method\n const startTimestamp = Date.now();\n\n // counting the request\n if (stats) stats.requests++;\n\n const { seedResources, finalResources, pruningFunction } = loadingRequest;\n\n // Main idea of using a queue here is that responses will arrive in the same order as they were\n // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.\n // In such a way logic become linear without recursion, and at the same time deal with data\n // as soon as it arrives.\n\n const pending = new Denque<Promise<ExtendedResourceData | undefined>>();\n\n // vars to calculate number of roundtrips for stats\n let roundTripToggle: boolean = true;\n let numberOfRoundTrips = 0;\n\n // tracking resources we already requested\n const requested = new Set<ResourceId>();\n const requestState = (rid: OptionalResourceId) => {\n if (isNullResourceId(rid) || requested.has(rid)) return;\n\n // separate check to collect stats\n if (finalResources.has(rid)) {\n if (stats) stats.finalResourcesSkipped++;\n return;\n }\n\n // adding the id, so we will not request it's state again if somebody else\n // references the same resource\n requested.add(rid);\n\n // requesting resource and all kv records\n const resourceData = tx.getResourceDataIfExists(rid, true);\n const kvData = tx.listKeyValuesIfResourceExists(rid);\n\n // counting round-trip (begin)\n const addRT = roundTripToggle;\n if (roundTripToggle) roundTripToggle = false;\n\n // pushing combined promise\n pending.push(\n (async () => {\n const [resource, kv] = await Promise.all([resourceData, kvData]);\n\n // counting round-trip, actually incrementing counter and returning toggle back, so the next request can acquire it\n if (addRT) {\n numberOfRoundTrips++;\n roundTripToggle = true;\n }\n\n if (resource === undefined) return undefined;\n\n if (kv === undefined) throw new Error(\"Inconsistent replies\");\n\n return { ...resource, kv };\n })(),\n );\n };\n\n // sending seed requests\n seedResources.forEach((rid) => requestState(rid));\n\n const result: ExtendedResourceData[] = [];\n while (true) {\n // taking next pending request\n const nextResourcePromise = pending.shift();\n if (nextResourcePromise === undefined)\n // this means we have no pending requests and traversal is over\n break;\n\n // at this point we pause and wait for the nest requested resource state to arrive\n let nextResource = await nextResourcePromise;\n if (nextResource === undefined)\n // ignoring resources that were not found (this may happen for seed resource ids)\n continue;\n\n if (pruningFunction !== undefined) {\n // apply field pruning, if requested\n const fieldsAfterPruning = pruningFunction(nextResource);\n // collecting stats\n if (stats) stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;\n nextResource = { ...nextResource, fields: fieldsAfterPruning };\n }\n\n // continue traversal over the referenced resource\n requestState(nextResource.error);\n for (const field of nextResource.fields) {\n requestState(field.value);\n requestState(field.error);\n }\n\n // collecting stats\n if (stats) {\n stats.retrievedResources++;\n stats.retrievedFields += nextResource.fields.length;\n stats.retrievedKeyValues += nextResource.kv.length;\n stats.retrievedResourceDataBytes += nextResource.data?.length ?? 0;\n for (const kv of nextResource.kv) stats.retrievedKeyValueBytes += kv.value.length;\n }\n\n // aggregating the state\n result.push(nextResource);\n }\n\n // adding the time we spent in this method to stats\n if (stats) {\n stats.millisSpent += Date.now() - startTimestamp;\n stats.roundTrips += numberOfRoundTrips;\n }\n\n return result;\n}\n"],"names":[],"mappings":";;;;AAiCA;AACkD;AAC5C,SAAU,2BAA2B,CACzC,IAAiB,EACjB,eAAiC,EAAA;IAEjC,MAAM,aAAa,GAAiB,EAAE;AACtC,IAAA,MAAM,cAAc,GAAG,IAAI,GAAG,EAAc;AAC5C,IAAA,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,KAAI;QAC3B,IAAI,GAAG,CAAC,UAAU;AAAE,YAAA,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;;AACzC,YAAA,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACjC,IAAA,CAAC,CAAC;;IAGF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC;AAAE,QAAA,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAE1F,IAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE;AAC3D;SAegB,sBAAsB,GAAA;IACpC,OAAO;AACL,QAAA,QAAQ,EAAE,CAAC;AACX,QAAA,UAAU,EAAE,CAAC;AACb,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,eAAe,EAAE,CAAC;AAClB,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,0BAA0B,EAAE,CAAC;AAC7B,QAAA,sBAAsB,EAAE,CAAC;AACzB,QAAA,YAAY,EAAE,CAAC;AACf,QAAA,qBAAqB,EAAE,CAAC;AACxB,QAAA,WAAW,EAAE,CAAC;KACf;AACH;AAEM,SAAU,qBAAqB,CAAC,IAAqB,EAAA;AACzD,IAAA,IAAI,MAAM,GAAG,CAAA,UAAA,EAAa,IAAI,CAAC,QAAQ,IAAI;IAC3C,MAAM,IAAI,eAAe,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA,EAAA,CAAI;AAChE,IAAA,MAAM,IAAI,CAAA,aAAA,EAAgB,IAAI,CAAC,UAAU,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,kBAAkB,IAAI;AACnD,IAAA,MAAM,IAAI,CAAA,QAAA,EAAW,IAAI,CAAC,eAAe,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,IAAA,EAAO,IAAI,CAAC,kBAAkB,IAAI;AAC5C,IAAA,MAAM,IAAI,CAAA,YAAA,EAAe,IAAI,CAAC,0BAA0B,IAAI;AAC5D,IAAA,MAAM,IAAI,CAAA,UAAA,EAAa,IAAI,CAAC,sBAAsB,IAAI;AACtD,IAAA,MAAM,IAAI,CAAA,eAAA,EAAkB,IAAI,CAAC,YAAY,IAAI;AACjD,IAAA,MAAM,IAAI,CAAA,yBAAA,EAA4B,IAAI,CAAC,qBAAqB,EAAE;AAClE,IAAA,OAAO,MAAM;AACf;AAEA;;AAE+B;AACxB,eAAe,aAAa,CACjC,EAAiB,EACjB,cAAkC,EAClC,KAAuB,EAAA;;AAGvB,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE;;AAGjC,IAAA,IAAI,KAAK;QAAE,KAAK,CAAC,QAAQ,EAAE;IAE3B,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,cAAc;;;;;AAOzE,IAAA,MAAM,OAAO,GAAG,IAAI,MAAM,EAA6C;;IAGvE,IAAI,eAAe,GAAY,IAAI;IACnC,IAAI,kBAAkB,GAAG,CAAC;;AAG1B,IAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc;AACvC,IAAA,MAAM,YAAY,GAAG,CAAC,GAAuB,KAAI;QAC/C,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE;;AAGjD,QAAA,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;AAC3B,YAAA,IAAI,KAAK;gBAAE,KAAK,CAAC,qBAAqB,EAAE;YACxC;QACF;;;AAIA,QAAA,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;;QAGlB,MAAM,YAAY,GAAG,EAAE,CAAC,uBAAuB,CAAC,GAAG,EAAE,IAAI,CAAC;QAC1D,MAAM,MAAM,GAAG,EAAE,CAAC,6BAA6B,CAAC,GAAG,CAAC;;QAGpD,MAAM,KAAK,GAAG,eAAe;AAC7B,QAAA,IAAI,eAAe;YAAE,eAAe,GAAG,KAAK;;AAG5C,QAAA,OAAO,CAAC,IAAI,CACV,CAAC,YAAW;AACV,YAAA,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;;YAGhE,IAAI,KAAK,EAAE;AACT,gBAAA,kBAAkB,EAAE;gBACpB,eAAe,GAAG,IAAI;YACxB;YAEA,IAAI,QAAQ,KAAK,SAAS;AAAE,gBAAA,OAAO,SAAS;YAE5C,IAAI,EAAE,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC;AAE7D,YAAA,OAAO,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE;QAC5B,CAAC,GAAG,CACL;AACH,IAAA,CAAC;;AAGD,IAAA,aAAa,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC;IAEjD,MAAM,MAAM,GAA2B,EAAE;IACzC,OAAO,IAAI,EAAE;;AAEX,QAAA,MAAM,mBAAmB,GAAG,OAAO,CAAC,KAAK,EAAE;QAC3C,IAAI,mBAAmB,KAAK,SAAS;;YAEnC;;AAGF,QAAA,IAAI,YAAY,GAAG,MAAM,mBAAmB;QAC5C,IAAI,YAAY,KAAK,SAAS;;YAE5B;AAEF,QAAA,IAAI,eAAe,KAAK,SAAS,EAAE;;AAEjC,YAAA,MAAM,kBAAkB,GAAG,eAAe,CAAC,YAAY,CAAC;;AAExD,YAAA,IAAI,KAAK;AAAE,gBAAA,KAAK,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM;YACvF,YAAY,GAAG,EAAE,GAAG,YAAY,EAAE,MAAM,EAAE,kBAAkB,EAAE;QAChE;;AAGA,QAAA,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC;AAChC,QAAA,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE;AACvC,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;QAC3B;;QAGA,IAAI,KAAK,EAAE;YACT,KAAK,CAAC,kBAAkB,EAAE;YAC1B,KAAK,CAAC,eAAe,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM;YACnD,KAAK,CAAC,kBAAkB,IAAI,YAAY,CAAC,EAAE,CAAC,MAAM;YAClD,KAAK,CAAC,0BAA0B,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;AAClE,YAAA,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,EAAE;gBAAE,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM;QACnF;;AAGA,QAAA,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;IAC3B;;IAGA,IAAI,KAAK,EAAE;QACT,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;AAChD,QAAA,KAAK,CAAC,UAAU,IAAI,kBAAkB;IACxC;AAEA,IAAA,OAAO,MAAM;AACf;;;;"}
1
+ {"version":3,"file":"sync.js","sources":["../src/sync.ts"],"sourcesContent":["import type {\n FieldData,\n OptionalResourceId,\n PlTransaction,\n ResourceId,\n} from \"@milaboratories/pl-client\";\nimport Denque from \"denque\";\nimport { isNullResourceId } from \"@milaboratories/pl-client\";\nimport type { ExtendedResourceData, PlTreeState } from \"./state\";\nimport { ConcurrencyLimitingExecutor, msToHumanReadable } from \"@milaboratories/ts-helpers\";\n\n/** Applied to list of fields in resource data. */\nexport type PruningFunction = (resource: ExtendedResourceData) => FieldData[];\n\nexport interface TreeLoadingRequest {\n /** Resource to prime the traversal algorithm. It is ok, if some of them\n * doesn't exist anymore. Should not contain elements from final resource\n * set. */\n readonly seedResources: ResourceId[];\n\n /** Resource ids for which state is already known and not expected to change.\n * Algorithm will not continue traversal over those ids, and states will not\n * be retrieved for them. */\n readonly finalResources: Set<ResourceId>;\n\n /** This function is applied to each resource data field list, before\n * using it continue traversal. This modification also is applied to\n * output data to make result self-consistent in terms that it will contain\n * all referenced resources, this is required to be able to pass it to tree\n * to update the state. */\n readonly pruningFunction?: PruningFunction;\n}\n\n/** Given the current tree state, build the request object to pass to\n * {@link loadTreeState} to load updated state. */\nexport function constructTreeLoadingRequest(\n tree: PlTreeState,\n pruningFunction?: PruningFunction,\n): TreeLoadingRequest {\n const seedResources: ResourceId[] = [];\n const finalResources = new Set<ResourceId>();\n tree.forEachResource((res) => {\n if (res.finalState) finalResources.add(res.id);\n else seedResources.push(res.id);\n });\n\n // if tree is empty, seeding tree reconstruction from the specified root\n if (seedResources.length === 0 && finalResources.size === 0) seedResources.push(tree.root);\n\n return { seedResources, finalResources, pruningFunction };\n}\n\nexport type TreeLoadingStat = {\n requests: number;\n roundTrips: number;\n retrievedResources: number;\n retrievedFields: number;\n retrievedKeyValues: number;\n retrievedResourceDataBytes: number;\n retrievedKeyValueBytes: number;\n prunedFields: number;\n finalResourcesSkipped: number;\n millisSpent: number;\n};\n\nexport function initialTreeLoadingStat(): TreeLoadingStat {\n return {\n requests: 0,\n roundTrips: 0,\n retrievedResources: 0,\n retrievedFields: 0,\n retrievedKeyValues: 0,\n retrievedResourceDataBytes: 0,\n retrievedKeyValueBytes: 0,\n prunedFields: 0,\n finalResourcesSkipped: 0,\n millisSpent: 0,\n };\n}\n\nexport function formatTreeLoadingStat(stat: TreeLoadingStat): string {\n let result = `Requests: ${stat.requests}\\n`;\n result += `Total time: ${msToHumanReadable(stat.millisSpent)}\\n`;\n result += `Round-trips: ${stat.roundTrips}\\n`;\n result += `Resources: ${stat.retrievedResources}\\n`;\n result += `Fields: ${stat.retrievedFields}\\n`;\n result += `KV: ${stat.retrievedKeyValues}\\n`;\n result += `Data Bytes: ${stat.retrievedResourceDataBytes}\\n`;\n result += `KV Bytes: ${stat.retrievedKeyValueBytes}\\n`;\n result += `Pruned fields: ${stat.prunedFields}\\n`;\n result += `Final resources skipped: ${stat.finalResourcesSkipped}`;\n return result;\n}\n\n/** Given the transaction (preferably read-only) and loading request, executes\n * the tree traversal algorithm, and collects fresh states of resources\n * to update the tree state. */\nexport async function loadTreeState(\n tx: PlTransaction,\n loadingRequest: TreeLoadingRequest,\n stats?: TreeLoadingStat,\n): Promise<ExtendedResourceData[]> {\n // saving start timestamp to add time spent in this function to the stats at the end of the method\n const startTimestamp = Date.now();\n\n // counting the request\n if (stats) stats.requests++;\n\n const { seedResources, finalResources, pruningFunction } = loadingRequest;\n\n // Limits the number of concurrent gRPC fetches to bound peak memory\n // from in-flight request/response buffers.\n const limiter = new ConcurrencyLimitingExecutor(100);\n\n // Promises of resource states, in the order they were requested.\n const pending = new Denque<Promise<ExtendedResourceData | undefined>>();\n\n // vars to calculate number of roundtrips for stats\n let roundTripToggle: boolean = true;\n let numberOfRoundTrips = 0;\n\n // tracking resources we already requested or queued\n const requested = new Set<ResourceId>();\n\n /** Mark a resource for fetching. Deduplicates and respects final-resource set. */\n const requestState = (rid: OptionalResourceId) => {\n if (isNullResourceId(rid) || requested.has(rid)) return;\n\n if (finalResources.has(rid)) {\n if (stats) stats.finalResourcesSkipped++;\n return;\n }\n\n requested.add(rid);\n\n pending.push(\n limiter.run(async () => {\n const resourceData = tx.getResourceDataIfExists(rid, true);\n const kvData = tx.listKeyValuesIfResourceExists(rid);\n\n // counting round-trip (begin)\n const addRT = roundTripToggle;\n if (roundTripToggle) roundTripToggle = false;\n\n const [resource, kv] = await Promise.all([resourceData, kvData]);\n\n // counting round-trip, actually incrementing counter and returning toggle back,\n // so the next request can acquire it\n if (addRT) {\n numberOfRoundTrips++;\n roundTripToggle = true;\n }\n\n if (resource === undefined) return undefined;\n if (kv === undefined) throw new Error(\"Inconsistent replies\");\n\n return { ...resource, kv };\n }),\n );\n };\n\n // sending seed requests\n seedResources.forEach((rid) => requestState(rid));\n\n const result: ExtendedResourceData[] = [];\n let nextPromise: Promise<ExtendedResourceData | undefined> | undefined;\n while ((nextPromise = pending.shift()) !== undefined) {\n // at this point we pause and wait for the next requested resource state to arrive\n let nextResource = await nextPromise;\n if (nextResource === undefined)\n // ignoring resources that were not found (this may happen for seed resource ids)\n continue;\n\n if (pruningFunction !== undefined) {\n // apply field pruning, if requested\n const fieldsAfterPruning = pruningFunction(nextResource);\n // collecting stats\n if (stats) stats.prunedFields += nextResource.fields.length - fieldsAfterPruning.length;\n nextResource = { ...nextResource, fields: fieldsAfterPruning };\n }\n\n // continue traversal over the referenced resources\n requestState(nextResource.error);\n for (const field of nextResource.fields) {\n requestState(field.value);\n requestState(field.error);\n }\n\n // collecting stats\n if (stats) {\n stats.retrievedResources++;\n stats.retrievedFields += nextResource.fields.length;\n stats.retrievedKeyValues += nextResource.kv.length;\n stats.retrievedResourceDataBytes += nextResource.data?.length ?? 0;\n for (const kv of nextResource.kv) stats.retrievedKeyValueBytes += kv.value.length;\n }\n\n // aggregating the state\n result.push(nextResource);\n }\n\n // adding the time we spent in this method to stats\n if (stats) {\n stats.millisSpent += Date.now() - startTimestamp;\n stats.roundTrips += numberOfRoundTrips;\n }\n\n return result;\n}\n"],"names":[],"mappings":";;;;AAiCA;AACkD;AAC5C,SAAU,2BAA2B,CACzC,IAAiB,EACjB,eAAiC,EAAA;IAEjC,MAAM,aAAa,GAAiB,EAAE;AACtC,IAAA,MAAM,cAAc,GAAG,IAAI,GAAG,EAAc;AAC5C,IAAA,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,KAAI;QAC3B,IAAI,GAAG,CAAC,UAAU;AAAE,YAAA,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;;AACzC,YAAA,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACjC,IAAA,CAAC,CAAC;;IAGF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC;AAAE,QAAA,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAE1F,IAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE;AAC3D;SAegB,sBAAsB,GAAA;IACpC,OAAO;AACL,QAAA,QAAQ,EAAE,CAAC;AACX,QAAA,UAAU,EAAE,CAAC;AACb,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,eAAe,EAAE,CAAC;AAClB,QAAA,kBAAkB,EAAE,CAAC;AACrB,QAAA,0BAA0B,EAAE,CAAC;AAC7B,QAAA,sBAAsB,EAAE,CAAC;AACzB,QAAA,YAAY,EAAE,CAAC;AACf,QAAA,qBAAqB,EAAE,CAAC;AACxB,QAAA,WAAW,EAAE,CAAC;KACf;AACH;AAEM,SAAU,qBAAqB,CAAC,IAAqB,EAAA;AACzD,IAAA,IAAI,MAAM,GAAG,CAAA,UAAA,EAAa,IAAI,CAAC,QAAQ,IAAI;IAC3C,MAAM,IAAI,eAAe,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA,EAAA,CAAI;AAChE,IAAA,MAAM,IAAI,CAAA,aAAA,EAAgB,IAAI,CAAC,UAAU,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,WAAA,EAAc,IAAI,CAAC,kBAAkB,IAAI;AACnD,IAAA,MAAM,IAAI,CAAA,QAAA,EAAW,IAAI,CAAC,eAAe,IAAI;AAC7C,IAAA,MAAM,IAAI,CAAA,IAAA,EAAO,IAAI,CAAC,kBAAkB,IAAI;AAC5C,IAAA,MAAM,IAAI,CAAA,YAAA,EAAe,IAAI,CAAC,0BAA0B,IAAI;AAC5D,IAAA,MAAM,IAAI,CAAA,UAAA,EAAa,IAAI,CAAC,sBAAsB,IAAI;AACtD,IAAA,MAAM,IAAI,CAAA,eAAA,EAAkB,IAAI,CAAC,YAAY,IAAI;AACjD,IAAA,MAAM,IAAI,CAAA,yBAAA,EAA4B,IAAI,CAAC,qBAAqB,EAAE;AAClE,IAAA,OAAO,MAAM;AACf;AAEA;;AAE+B;AACxB,eAAe,aAAa,CACjC,EAAiB,EACjB,cAAkC,EAClC,KAAuB,EAAA;;AAGvB,IAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE;;AAGjC,IAAA,IAAI,KAAK;QAAE,KAAK,CAAC,QAAQ,EAAE;IAE3B,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,cAAc;;;AAIzE,IAAA,MAAM,OAAO,GAAG,IAAI,2BAA2B,CAAC,GAAG,CAAC;;AAGpD,IAAA,MAAM,OAAO,GAAG,IAAI,MAAM,EAA6C;;IAGvE,IAAI,eAAe,GAAY,IAAI;IACnC,IAAI,kBAAkB,GAAG,CAAC;;AAG1B,IAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAc;;AAGvC,IAAA,MAAM,YAAY,GAAG,CAAC,GAAuB,KAAI;QAC/C,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE;AAEjD,QAAA,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;AAC3B,YAAA,IAAI,KAAK;gBAAE,KAAK,CAAC,qBAAqB,EAAE;YACxC;QACF;AAEA,QAAA,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;QAElB,OAAO,CAAC,IAAI,CACV,OAAO,CAAC,GAAG,CAAC,YAAW;YACrB,MAAM,YAAY,GAAG,EAAE,CAAC,uBAAuB,CAAC,GAAG,EAAE,IAAI,CAAC;YAC1D,MAAM,MAAM,GAAG,EAAE,CAAC,6BAA6B,CAAC,GAAG,CAAC;;YAGpD,MAAM,KAAK,GAAG,eAAe;AAC7B,YAAA,IAAI,eAAe;gBAAE,eAAe,GAAG,KAAK;AAE5C,YAAA,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;;;YAIhE,IAAI,KAAK,EAAE;AACT,gBAAA,kBAAkB,EAAE;gBACpB,eAAe,GAAG,IAAI;YACxB;YAEA,IAAI,QAAQ,KAAK,SAAS;AAAE,gBAAA,OAAO,SAAS;YAC5C,IAAI,EAAE,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC;AAE7D,YAAA,OAAO,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE;QAC5B,CAAC,CAAC,CACH;AACH,IAAA,CAAC;;AAGD,IAAA,aAAa,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC;IAEjD,MAAM,MAAM,GAA2B,EAAE;AACzC,IAAA,IAAI,WAAkE;IACtE,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,MAAM,SAAS,EAAE;;AAEpD,QAAA,IAAI,YAAY,GAAG,MAAM,WAAW;QACpC,IAAI,YAAY,KAAK,SAAS;;YAE5B;AAEF,QAAA,IAAI,eAAe,KAAK,SAAS,EAAE;;AAEjC,YAAA,MAAM,kBAAkB,GAAG,eAAe,CAAC,YAAY,CAAC;;AAExD,YAAA,IAAI,KAAK;AAAE,gBAAA,KAAK,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM;YACvF,YAAY,GAAG,EAAE,GAAG,YAAY,EAAE,MAAM,EAAE,kBAAkB,EAAE;QAChE;;AAGA,QAAA,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC;AAChC,QAAA,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE;AACvC,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;AACzB,YAAA,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC;QAC3B;;QAGA,IAAI,KAAK,EAAE;YACT,KAAK,CAAC,kBAAkB,EAAE;YAC1B,KAAK,CAAC,eAAe,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM;YACnD,KAAK,CAAC,kBAAkB,IAAI,YAAY,CAAC,EAAE,CAAC,MAAM;YAClD,KAAK,CAAC,0BAA0B,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;AAClE,YAAA,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,EAAE;gBAAE,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM;QACnF;;AAGA,QAAA,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;IAC3B;;IAGA,IAAI,KAAK,EAAE;QACT,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;AAChD,QAAA,KAAK,CAAC,UAAU,IAAI,kBAAkB;IACxC;AAEA,IAAA,OAAO,MAAM;AACf;;;;"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@milaboratories/pl-tree",
3
- "version": "1.8.45",
3
+ "version": "1.8.47",
4
4
  "description": "Reactive pl tree state",
5
5
  "files": [
6
6
  "./dist/**/*",
@@ -30,9 +30,9 @@
30
30
  "@vitest/coverage-istanbul": "^4.0.18",
31
31
  "typescript": "~5.6.3",
32
32
  "vitest": "^4.0.18",
33
- "@milaboratories/ts-configs": "1.2.1",
33
+ "@milaboratories/ts-builder": "1.2.11",
34
34
  "@milaboratories/build-configs": "1.5.0",
35
- "@milaboratories/ts-builder": "1.2.11"
35
+ "@milaboratories/ts-configs": "1.2.1"
36
36
  },
37
37
  "engines": {
38
38
  "node": ">=22.19.0"
package/src/sync.ts CHANGED
@@ -4,10 +4,10 @@ import type {
4
4
  PlTransaction,
5
5
  ResourceId,
6
6
  } from "@milaboratories/pl-client";
7
- import { isNullResourceId } from "@milaboratories/pl-client";
8
7
  import Denque from "denque";
8
+ import { isNullResourceId } from "@milaboratories/pl-client";
9
9
  import type { ExtendedResourceData, PlTreeState } from "./state";
10
- import { msToHumanReadable } from "@milaboratories/ts-helpers";
10
+ import { ConcurrencyLimitingExecutor, msToHumanReadable } from "@milaboratories/ts-helpers";
11
11
 
12
12
  /** Applied to list of fields in resource data. */
13
13
  export type PruningFunction = (resource: ExtendedResourceData) => FieldData[];
@@ -108,57 +108,54 @@ export async function loadTreeState(
108
108
 
109
109
  const { seedResources, finalResources, pruningFunction } = loadingRequest;
110
110
 
111
- // Main idea of using a queue here is that responses will arrive in the same order as they were
112
- // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.
113
- // In such a way logic become linear without recursion, and at the same time deal with data
114
- // as soon as it arrives.
111
+ // Limits the number of concurrent gRPC fetches to bound peak memory
112
+ // from in-flight request/response buffers.
113
+ const limiter = new ConcurrencyLimitingExecutor(100);
115
114
 
115
+ // Promises of resource states, in the order they were requested.
116
116
  const pending = new Denque<Promise<ExtendedResourceData | undefined>>();
117
117
 
118
118
  // vars to calculate number of roundtrips for stats
119
119
  let roundTripToggle: boolean = true;
120
120
  let numberOfRoundTrips = 0;
121
121
 
122
- // tracking resources we already requested
122
+ // tracking resources we already requested or queued
123
123
  const requested = new Set<ResourceId>();
124
+
125
+ /** Mark a resource for fetching. Deduplicates and respects final-resource set. */
124
126
  const requestState = (rid: OptionalResourceId) => {
125
127
  if (isNullResourceId(rid) || requested.has(rid)) return;
126
128
 
127
- // separate check to collect stats
128
129
  if (finalResources.has(rid)) {
129
130
  if (stats) stats.finalResourcesSkipped++;
130
131
  return;
131
132
  }
132
133
 
133
- // adding the id, so we will not request it's state again if somebody else
134
- // references the same resource
135
134
  requested.add(rid);
136
135
 
137
- // requesting resource and all kv records
138
- const resourceData = tx.getResourceDataIfExists(rid, true);
139
- const kvData = tx.listKeyValuesIfResourceExists(rid);
136
+ pending.push(
137
+ limiter.run(async () => {
138
+ const resourceData = tx.getResourceDataIfExists(rid, true);
139
+ const kvData = tx.listKeyValuesIfResourceExists(rid);
140
140
 
141
- // counting round-trip (begin)
142
- const addRT = roundTripToggle;
143
- if (roundTripToggle) roundTripToggle = false;
141
+ // counting round-trip (begin)
142
+ const addRT = roundTripToggle;
143
+ if (roundTripToggle) roundTripToggle = false;
144
144
 
145
- // pushing combined promise
146
- pending.push(
147
- (async () => {
148
145
  const [resource, kv] = await Promise.all([resourceData, kvData]);
149
146
 
150
- // counting round-trip, actually incrementing counter and returning toggle back, so the next request can acquire it
147
+ // counting round-trip, actually incrementing counter and returning toggle back,
148
+ // so the next request can acquire it
151
149
  if (addRT) {
152
150
  numberOfRoundTrips++;
153
151
  roundTripToggle = true;
154
152
  }
155
153
 
156
154
  if (resource === undefined) return undefined;
157
-
158
155
  if (kv === undefined) throw new Error("Inconsistent replies");
159
156
 
160
157
  return { ...resource, kv };
161
- })(),
158
+ }),
162
159
  );
163
160
  };
164
161
 
@@ -166,15 +163,10 @@ export async function loadTreeState(
166
163
  seedResources.forEach((rid) => requestState(rid));
167
164
 
168
165
  const result: ExtendedResourceData[] = [];
169
- while (true) {
170
- // taking next pending request
171
- const nextResourcePromise = pending.shift();
172
- if (nextResourcePromise === undefined)
173
- // this means we have no pending requests and traversal is over
174
- break;
175
-
176
- // at this point we pause and wait for the nest requested resource state to arrive
177
- let nextResource = await nextResourcePromise;
166
+ let nextPromise: Promise<ExtendedResourceData | undefined> | undefined;
167
+ while ((nextPromise = pending.shift()) !== undefined) {
168
+ // at this point we pause and wait for the next requested resource state to arrive
169
+ let nextResource = await nextPromise;
178
170
  if (nextResource === undefined)
179
171
  // ignoring resources that were not found (this may happen for seed resource ids)
180
172
  continue;
@@ -187,7 +179,7 @@ export async function loadTreeState(
187
179
  nextResource = { ...nextResource, fields: fieldsAfterPruning };
188
180
  }
189
181
 
190
- // continue traversal over the referenced resource
182
+ // continue traversal over the referenced resources
191
183
  requestState(nextResource.error);
192
184
  for (const field of nextResource.fields) {
193
185
  requestState(field.value);