@milaboratories/pl-tree 1.3.17 → 1.3.19

package/src/sync.ts

@@ -1,12 +1,14 @@
 import {
   FieldData,
   isNullResourceId,
+  NullResourceId,
   OptionalResourceId,
   PlTransaction,
   ResourceId
 } from '@milaboratories/pl-client';
 import Denque from 'denque';
 import { ExtendedResourceData, PlTreeState } from './state';
+import { msToHumanReadable } from '@milaboratories/ts-helpers';
 
 /** Applied to list of fields in resource data. */
 export type PruningFunction = (resource: ExtendedResourceData) => FieldData[];
@@ -49,13 +51,59 @@ export function constructTreeLoadingRequest(
   return { seedResources, finalResources, pruningFunction };
 }
 
+export type TreeLoadingStat = {
+  requests: number;
+  roundtrips: number;
+  retrievedResources: number;
+  retrievedFields: number;
+  retrievedKeyValues: number;
+  retrievedResourceDataBytes: number;
+  prunnedFields: number;
+  finalResourcesSkipped: number;
+  millisSpent: number;
+};
+
+export function initialTreeLoadingStat(): TreeLoadingStat {
+  return {
+    requests: 0,
+    roundtrips: 0,
+    retrievedResources: 0,
+    retrievedFields: 0,
+    retrievedKeyValues: 0,
+    retrievedResourceDataBytes: 0,
+    prunnedFields: 0,
+    finalResourcesSkipped: 0,
+    millisSpent: 0
+  };
+}
+
+export function formatTreeLoadingStat(stat: TreeLoadingStat): string {
+  let result = `Requests: ${stat.requests}\n`;
+  result += `Total time: ${msToHumanReadable(stat.millisSpent)}\n`;
+  result += `Roundtrips: ${stat.roundtrips}\n`;
+  result += `Resources: ${stat.retrievedResources}\n`;
+  result += `Fields: ${stat.retrievedFields}\n`;
+  result += `KV: ${stat.retrievedKeyValues}\n`;
+  result += `Data Bytes: ${stat.retrievedResourceDataBytes}\n`;
+  result += `Pruned fields: ${stat.prunnedFields}\n`;
+  result += `Final resources skipped: ${stat.finalResourcesSkipped}`;
+  return result;
+}
+
 /** Given the transaction (preferably read-only) and loading request, executes
  * the tree traversal algorithm, and collects fresh states of resources
  * to update the tree state. */
 export async function loadTreeState(
   tx: PlTransaction,
-  loadingRequest: TreeLoadingRequest
+  loadingRequest: TreeLoadingRequest,
+  stats?: TreeLoadingStat
 ): Promise<ExtendedResourceData[]> {
+  // saving start timestamp to add time spent in this function to the stats at the end of the method
+  const startTimestamp = Date.now();
+
+  // countind the request
+  if (stats) stats.requests++;
+
   const { seedResources, finalResources, pruningFunction } = loadingRequest;
 
   // Main idea of using a queue here is that responses will arrive in the same order as they were
@@ -65,10 +113,20 @@ export async function loadTreeState(
 
   const pending = new Denque<Promise<ExtendedResourceData | undefined>>();
 
+  // vars to calculate number of roundtrips for stats
+  let roundtripToggle: boolean = true;
+  let numberOfRoundtrips = 0;
+
   // tracking resources we already requested
   const requested = new Set<ResourceId>();
   const requestState = (rid: OptionalResourceId) => {
-    if (isNullResourceId(rid) || requested.has(rid) || finalResources.has(rid)) return;
+    if (isNullResourceId(rid) || requested.has(rid)) return;
+
+    // separate check to collect stats
+    if (finalResources.has(rid)) {
+      if (stats) stats.finalResourcesSkipped++;
+      return;
+    }
 
     // adding the id, so we will not request it's state again if somebody else
     // references the same resource
@@ -78,11 +136,20 @@ export async function loadTreeState(
     const resourceData = tx.getResourceDataIfExists(rid, true);
     const kvData = tx.listKeyValuesIfResourceExists(rid);
 
+    // counting roundrip (begin)
+    const addRT = roundtripToggle;
+    if (roundtripToggle) roundtripToggle = false;
+
     // pushing combined promise
     pending.push(
       (async () => {
-        const resource = await resourceData;
-        const kv = await kvData;
+        const [resource, kv] = await Promise.all([resourceData, kvData]);
+
+        // counting roundrip, actually incrementing counter and returning toggle back, so the next request can acquire it
+        if (addRT) {
+          numberOfRoundtrips++;
+          roundtripToggle = true;
+        }
 
         if (resource === undefined) return undefined;
 
@@ -110,9 +177,13 @@ export async function loadTreeState(
       // ignoring resources that were not found (this may happen for seed resource ids)
       continue;
 
-    // apply field pruning, if requested
-    if (pruningFunction !== undefined)
-      nextResource = { ...nextResource, fields: pruningFunction(nextResource) };
+    if (pruningFunction !== undefined) {
+      // apply field pruning, if requested
+      const fieldsAfterPruning = pruningFunction(nextResource);
+      // collecting stats
+      if (stats) stats.prunnedFields += nextResource.fields.length - fieldsAfterPruning.length;
+      nextResource = { ...nextResource, fields: fieldsAfterPruning };
+    }
 
     // continue traversal over the referenced resource
     requestState(nextResource.error);
@@ -121,9 +192,23 @@ export async function loadTreeState(
       requestState(field.error);
     }
 
+    // collecting stats
+    if (stats) {
+      stats.retrievedResources++;
+      stats.retrievedFields += nextResource.fields.length;
+      stats.retrievedKeyValues += nextResource.kv.length;
+      stats.retrievedResourceDataBytes += nextResource.data?.length ?? 0;
+    }
+
     // aggregating the state
     result.push(nextResource);
   }
 
+  // adding the time we spent in this method to stats
+  if (stats) {
+    stats.millisSpent += Date.now() - startTimestamp;
+    stats.roundtrips += numberOfRoundtrips;
+  }
+
   return result;
 }

package/src/synchronized_tree.test.ts

@@ -1,7 +1,9 @@
+import { test, expect } from '@jest/globals';
 import { field, TestHelpers } from '@milaboratories/pl-client';
 import { TestStructuralResourceType1 } from './test_utils';
 import { Computable } from '@milaboratories/computable';
 import { SynchronizedTreeState } from './synchronized_tree';
+import { ConsoleLoggerAdapter } from '@milaboratories/ts-helpers';
 
 test('simple synchronized tree test', async () => {
   await TestHelpers.withTempRoot(async (pl) => {
@@ -18,10 +20,16 @@ test('simple synchronized tree test', async () => {
       { sync: true }
     );
 
-    const treeState = await SynchronizedTreeState.init(pl, r1, {
-      stopPollingDelay: 10,
-      pollingInterval: 10
-    });
+    const treeState = await SynchronizedTreeState.init(
+      pl,
+      r1,
+      {
+        stopPollingDelay: 10,
+        pollingInterval: 10,
+        logStat: 'cumulative'
+      },
+      new ConsoleLoggerAdapter(require('console'))
+    );
 
     const theComputable = Computable.make((c) =>
       c.accessor(treeState.entry()).node().traverse('a', 'b')?.getDataAsString()
@@ -113,10 +121,16 @@ test('synchronized tree test with KV', async () => {
       { sync: true }
     );
 
-    const treeState = await SynchronizedTreeState.init(pl, r1, {
-      stopPollingDelay: 10,
-      pollingInterval: 10
-    });
+    const treeState = await SynchronizedTreeState.init(
+      pl,
+      r1,
+      {
+        stopPollingDelay: 10,
+        pollingInterval: 10,
+        logStat: 'cumulative'
+      },
+      new ConsoleLoggerAdapter(require('console'))
+    );
 
     const theComputable = Computable.make((c) =>
       c.accessor(treeState.entry()).node().traverse('a')?.getKeyValueAsString('b', true)
@@ -182,10 +196,16 @@ test('termination test', async () => {
       { sync: true }
     );
 
-    const treeState = await SynchronizedTreeState.init(pl, r1, {
-      stopPollingDelay: 10,
-      pollingInterval: 10
-    });
+    const treeState = await SynchronizedTreeState.init(
+      pl,
+      r1,
+      {
+        stopPollingDelay: 10,
+        pollingInterval: 10,
+        logStat: 'cumulative'
+      },
+      new ConsoleLoggerAdapter(require('console'))
+    );
 
     const entry = treeState.entry();
     const theComputable = Computable.make((c) =>

package/src/synchronized_tree.ts

@@ -2,10 +2,18 @@ import { PollingComputableHooks } from '@milaboratories/computable';
 import { PlTreeEntry } from './accessors';
 import { isTimeoutOrCancelError, PlClient, ResourceId } from '@milaboratories/pl-client';
 import { FinalPredicate, PlTreeState, TreeStateUpdateError } from './state';
-import { constructTreeLoadingRequest, loadTreeState, PruningFunction } from './sync';
+import {
+  constructTreeLoadingRequest,
+  initialTreeLoadingStat,
+  loadTreeState,
+  PruningFunction,
+  TreeLoadingStat
+} from './sync';
 import * as tp from 'node:timers/promises';
 import { MiLogger } from '@milaboratories/ts-helpers';
 
+type StatLoggingMode = 'cumulative' | 'per-request';
+
 export type SynchronizedTreeOps = {
   finalPredicate?: FinalPredicate;
   pruning?: PruningFunction;
@@ -14,6 +22,9 @@ export type SynchronizedTreeOps = {
   pollingInterval: number;
   /** For how long to continue polling after the last derived value access */
   stopPollingDelay: number;
+
+  /** If one of the values, tree will log stats of each polling request */
+  logStat?: StatLoggingMode;
 };
 
 type ScheduledRefresh = {
@@ -26,6 +37,7 @@ export class SynchronizedTreeState {
   private state: PlTreeState;
   private readonly pollingInterval: number;
   private readonly pruning?: PruningFunction;
+  private readonly logStat?: StatLoggingMode;
   private readonly hooks: PollingComputableHooks;
   private readonly abortController = new AbortController();
 
@@ -35,10 +47,11 @@ export class SynchronizedTreeState {
     ops: SynchronizedTreeOps,
     private readonly logger?: MiLogger
   ) {
-    const { finalPredicate, pruning, pollingInterval, stopPollingDelay } = ops;
+    const { finalPredicate, pruning, pollingInterval, stopPollingDelay, logStat } = ops;
     this.pruning = pruning;
     this.pollingInterval = pollingInterval;
     this.finalPredicate = finalPredicate;
+    this.logStat = logStat;
     this.state = new PlTreeState(root, finalPredicate);
     this.hooks = new PollingComputableHooks(
       () => this.startUpdating(),
@@ -91,11 +104,11 @@ export class SynchronizedTreeState {
   private currentLoop: Promise<void> | undefined = undefined;
 
   /** Executed from the main loop, and initialization procedure. */
-  private async refresh(): Promise<void> {
+  private async refresh(stats?: TreeLoadingStat): Promise<void> {
     if (this.terminated) throw new Error('tree synchronization is terminated');
     const request = constructTreeLoadingRequest(this.state, this.pruning);
     const data = await this.pl.withReadTx('ReadingTree', async (tx) => {
-      return await loadTreeState(tx, request);
+      return await loadTreeState(tx, request, stats);
     });
     this.state.updateFromResourceData(data, true);
   }
@@ -104,6 +117,9 @@ export class SynchronizedTreeState {
   private terminated = false;
 
   private async mainLoop() {
+    // will hold request stats
+    let stat = this.logStat ? initialTreeLoadingStat() : undefined;
+
     while (true) {
       if (!this.keepRunning) break;
 
@@ -117,12 +133,21 @@ export class SynchronizedTreeState {
       }
 
       try {
+        // resetting stats if we were asked to collect non-cumulative stats
+        if (this.logStat === 'per-request') stat = initialTreeLoadingStat();
+
         // actual tree synchronization
-        await this.refresh();
+        await this.refresh(stat);
+
+        // logging stats if we were asked to
+        if (stat && this.logger) this.logger.info(`Tree stat (success): ${JSON.stringify(stat)}`);
 
         // notifying that we got new state
         if (toNotify !== undefined) for (const n of toNotify) n.resolve();
       } catch (e: any) {
+        // logging stats if we were asked to (even if error occured)
+        if (stat && this.logger) this.logger.info(`Tree stat (error): ${JSON.stringify(stat)}`);
+
         // notifying that we failed to refresh the state
         if (toNotify !== undefined) for (const n of toNotify) n.reject(e);
 
@@ -181,8 +206,13 @@ export class SynchronizedTreeState {
     await this.currentLoop;
   }
 
-  public static async init(pl: PlClient, root: ResourceId, ops: SynchronizedTreeOps) {
-    const tree = new SynchronizedTreeState(pl, root, ops);
+  public static async init(
+    pl: PlClient,
+    root: ResourceId,
+    ops: SynchronizedTreeOps,
+    logger?: MiLogger
+  ) {
+    const tree = new SynchronizedTreeState(pl, root, ops, logger);
     await tree.refresh();
     return tree;
   }