@milaboratories/pl-tree 1.13.3 → 1.13.5

package/src/state.ts

@@ -68,6 +68,49 @@ export type ResourceDataWithFinalState = ResourceData & {
   finalState: boolean;
 };
 
+/** Cross-cycle re-fetch duplication counters, populated by {@link PlTreeState.updateFromResourceData}.
+ * Separates genuinely new/changed resources from redundant re-fetches of unchanged ones
+ * (the delta-skip opportunity: intra-cycle dedup already happens in the loader, so all waste here
+ * is cross-cycle). */
+export interface ResourceUpdateStat {
+  /** Resources seen for the first time in this update. */
+  resourcesNew: number;
+  /** Resources already held whose state actually changed. */
+  resourcesChanged: number;
+  /** Resources already held that came back unchanged: a pure duplicate re-fetch. */
+  resourcesUnchanged: number;
+  /** data + KV bytes of the unchanged bucket: wasted downlink. */
+  bytesUnchanged: number;
+  /** Changed resources whose immutable metadata (type, kind, field set) did not change,
+   * i.e. metadata re-streamed even though only a value or flag flipped. */
+  metadataStableChanged: number;
+  /** Per-resource fetches spent on unchanged resources (BFS only; streaming re-sends them
+   * inside one stream, so this stays 0). */
+  bfsRequestsWasted: number;
+  /** Whether the current load used backend streaming; set by the loader, read here to
+   * attribute {@link bfsRequestsWasted}. */
+  usedStreaming: boolean;
+  // Breakdown of what changed among resourcesChanged (a resource can bump several).
+  /** New fields appeared on a held resource. */
+  fieldsAdded: number;
+  /** Dynamic fields disappeared from a held resource. */
+  fieldsRemoved: number;
+  /** Existing field value pointer changed (a new value/result was attached). */
+  fieldsChanged: number;
+  /** KV entries added, changed, or deleted. */
+  kvChanged: number;
+  /** resourceReady flipped. */
+  readyFlips: number;
+  /** inputsLocked or outputsLocked transitioned. */
+  locksChanged: number;
+  /** An error resource was attached. */
+  errorsAttached: number;
+  /** originalResourceId resolved (duplicate -> original). */
+  duplicatesResolved: number;
+  /** Held resources that transitioned to final this update. */
+  resourcesMarkedFinal: number;
+}
+
 /** Never store instances of this class, always get fresh instance from {@link PlTreeState} */
 export class PlTreeResource implements ResourceDataWithFinalState {
   /** Tracks number of other resources referencing this resource. Used to perform garbage collection in tree patching procedure */
@@ -451,7 +494,11 @@ export class PlTreeState {
     return res;
   }
 
-  updateFromResourceData(resourceData: ExtendedResourceData[], allowOrphanInputs: boolean = false) {
+  updateFromResourceData(
+    resourceData: ExtendedResourceData[],
+    opts: { allowOrphanInputs?: boolean; stat?: ResourceUpdateStat } = {},
+  ) {
+    const { allowOrphanInputs = false, stat } = opts;
     this.checkValid();
 
     // All resources for which recount should be incremented, first are aggregated in this list
@@ -461,6 +508,11 @@ export class PlTreeState {
     // patching / creating resources
     for (const rd of resourceData) {
       let resource = this.resources.get(rd.id);
+      const held = resource !== undefined;
+      let changed = false;
+      // Structural/metadata change (new or removed field). type and kind are readonly, so
+      // they never change; this flag isolates value/flag-only changes from real metadata churn.
+      let metadataChanged = false;
 
       const statBeforeMutation = resource?.basicState;
       const unexpectedTransitionError = (reason: string): never => {
@@ -480,7 +532,6 @@ export class PlTreeState {
         if (resource.finalState)
           unexpectedTransitionError("resource state can\t be updated after it is marked as final");
 
-        let changed = false;
         // updating resource version, even if it was not changed
         resource.version += 1;
 
@@ -494,6 +545,7 @@ export class PlTreeState {
             `originalResourceId changed for ${resourceIdToString(resource.id)}`,
           );
           changed = true;
+          if (stat) stat.duplicatesResolved++;
         }
 
         // error
@@ -506,6 +558,7 @@ export class PlTreeState {
             `error changed for ${resourceIdToString(resource.id)}`,
           );
           changed = true;
+          if (stat) stat.errorsAttached++;
         }
 
         // updating fields
@@ -552,6 +605,8 @@ export class PlTreeState {
             resource.fieldsMap.set(fd.name, field);
 
             changed = true;
+            metadataChanged = true;
+            if (stat) stat.fieldsAdded++;
           } else {
             // change of old field
 
@@ -596,6 +651,7 @@ export class PlTreeState {
                 `field ${fd.name} value changed in ${resourceIdToString(resource.id)}`,
               );
               changed = true;
+              if (stat) stat.fieldsChanged++;
             }
 
             // field error
@@ -640,6 +696,8 @@ export class PlTreeState {
               `dynamic field ${fieldName} removed from ${resourceIdToString(resource!.id)}`,
             );
             fields.delete(fieldName);
+            metadataChanged = true;
+            if (stat) stat.fieldsRemoved++;
 
             if (isNotNullSignedResourceId(field.value)) decrementRefs.push(field.value);
             if (isNotNullSignedResourceId(field.error)) decrementRefs.push(field.error);
@@ -658,6 +716,7 @@ export class PlTreeState {
             `inputs locked for ${resourceIdToString(resource.id)}`,
           );
           changed = true;
+          if (stat) stat.locksChanged++;
         }
 
         // outputsLocked
@@ -669,6 +728,7 @@ export class PlTreeState {
             `outputs locked for ${resourceIdToString(resource.id)}`,
           );
           changed = true;
+          if (stat) stat.locksChanged++;
         }
 
         // ready flag
@@ -684,6 +744,7 @@ export class PlTreeState {
             `ready flag changed to ${rd.resourceReady} for ${resourceIdToString(resource.id)}`,
           );
           changed = true;
+          if (stat) stat.readyFlips++;
         }
 
         // syncing kv
@@ -695,12 +756,16 @@ export class PlTreeState {
               kv.key,
               `kv added for ${resourceIdToString(resource.id)}: ${kv.key}`,
             );
+            changed = true;
+            if (stat) stat.kvChanged++;
           } else if (Buffer.compare(current, kv.value) !== 0) {
             resource.kv.set(kv.key, kv.value);
             notEmpty(resource.kvChangedPerKey).markChanged(
               kv.key,
               `kv changed for ${resourceIdToString(resource.id)}: ${kv.key}`,
             );
+            changed = true;
+            if (stat) stat.kvChanged++;
           }
         }
 
@@ -716,6 +781,8 @@ export class PlTreeState {
                 key,
                 `kv deleted for ${resourceIdToString(resource!.id)}: ${key}`,
               );
+              changed = true;
+              if (stat) stat.kvChanged++;
             }
           });
         }
@@ -723,7 +790,10 @@ export class PlTreeState {
         if (changed) {
           // if resource was changed, updating resource data version
           resource.dataVersion = resource.version;
-          if (this.isFinalPredicate(resource)) resource.markFinal();
+          if (this.isFinalPredicate(resource)) {
+            resource.markFinal();
+            if (stat) stat.resourcesMarkedFinal++;
+          }
         }
       } else {
         // creating new resource
@@ -756,6 +826,19 @@ export class PlTreeState {
         this.resources.set(resource.id, resource);
         this.resourcesAdded.markChanged(`new resource ${resourceIdToString(resource.id)} added`);
       }
+
+      if (stat) {
+        if (!held) stat.resourcesNew++;
+        else if (changed) {
+          stat.resourcesChanged++;
+          if (!metadataChanged) stat.metadataStableChanged++;
+        } else {
+          stat.resourcesUnchanged++;
+          stat.bytesUnchanged += rd.data?.length ?? 0;
+          for (const kv of rd.kv) stat.bytesUnchanged += kv.value.length;
+          if (!stat.usedStreaming) stat.bfsRequestsWasted++;
+        }
+      }
     }
 
     // applying refCount increments

package/src/sync.ts

@@ -8,7 +8,7 @@ import type {
 } from "@milaboratories/pl-client";
 import Denque from "denque";
 import { hasCapability, isNullSignedResourceId } from "@milaboratories/pl-client";
-import type { ExtendedResourceData, PlTreeState } from "./state";
+import type { ExtendedResourceData, PlTreeState, ResourceUpdateStat } from "./state";
 import { ConcurrencyLimitingExecutor, msToHumanReadable } from "@milaboratories/ts-helpers";
 
 /** Applied to list of fields in resource data. */
@@ -70,7 +70,7 @@ export function constructTreeLoadingRequest(
   };
 }
 
-export type TreeLoadingStat = {
+export type TreeLoadingStat = ResourceUpdateStat & {
   requests: number;
   roundTrips: number;
   retrievedResources: number;
@@ -85,6 +85,20 @@ export type TreeLoadingStat = {
   stopMarkersSkipped: number;
   /** Number of follow-up resourceTree() calls issued to resolve unknown stop markers. */
   stopMarkerFollowUpRoundTrips: number;
+  /** Streaming (resourceTree) path: resourceTree() streams consumed (1, or 2 with a follow-up). */
+  streamRounds: number;
+  /** Streaming path: resource frames received. */
+  resourceFrames: number;
+  /** Streaming path: stopMarker frames received (both skipped and follow-up). */
+  stopMarkerFrames: number;
+  /** Streaming path: stop markers that were not final locally and triggered a follow-up fetch. */
+  stopMarkersFollowUp: number;
+  /** Streaming path: frames where the backend stopped traversal (final or traverseWasStopped). */
+  traverseWasStoppedCount: number;
+  /** BFS path: resource states actually requested from the backend this cycle (after intra-cycle dedup). */
+  bfsResourcesRequested: number;
+  /** BFS path: requested resources that no longer exist (undefined reply). */
+  bfsResourcesNotFound: number;
 };
 
 export function initialTreeLoadingStat(): TreeLoadingStat {
@@ -101,23 +115,54 @@ export function initialTreeLoadingStat(): TreeLoadingStat {
     millisSpent: 0,
     stopMarkersSkipped: 0,
     stopMarkerFollowUpRoundTrips: 0,
+    streamRounds: 0,
+    resourceFrames: 0,
+    stopMarkerFrames: 0,
+    stopMarkersFollowUp: 0,
+    traverseWasStoppedCount: 0,
+    bfsResourcesRequested: 0,
+    bfsResourcesNotFound: 0,
+    resourcesNew: 0,
+    resourcesChanged: 0,
+    resourcesUnchanged: 0,
+    bytesUnchanged: 0,
+    metadataStableChanged: 0,
+    bfsRequestsWasted: 0,
+    usedStreaming: false,
+    fieldsAdded: 0,
+    fieldsRemoved: 0,
+    fieldsChanged: 0,
+    kvChanged: 0,
+    readyFlips: 0,
+    locksChanged: 0,
+    errorsAttached: 0,
+    duplicatesResolved: 0,
+    resourcesMarkedFinal: 0,
   };
 }
 
 export function formatTreeLoadingStat(stat: TreeLoadingStat): string {
-  let result = `Requests: ${stat.requests}\n`;
-  result += `Total time: ${msToHumanReadable(stat.millisSpent)}\n`;
-  result += `Round-trips: ${stat.roundTrips}\n`;
-  result += `Resources: ${stat.retrievedResources}\n`;
-  result += `Fields: ${stat.retrievedFields}\n`;
-  result += `KV: ${stat.retrievedKeyValues}\n`;
-  result += `Data Bytes: ${stat.retrievedResourceDataBytes}\n`;
-  result += `KV Bytes: ${stat.retrievedKeyValueBytes}\n`;
-  result += `Pruned fields: ${stat.prunedFields}\n`;
-  result += `Final resources skipped: ${stat.finalResourcesSkipped}\n`;
-  result += `Stop markers skipped: ${stat.stopMarkersSkipped}\n`;
-  result += `Stop marker follow-up round-trips: ${stat.stopMarkerFollowUpRoundTrips}`;
-  return result;
+  return `Requests: ${stat.requests}
+Total time: ${msToHumanReadable(stat.millisSpent)}
+Round-trips: ${stat.roundTrips}
+Resources: ${stat.retrievedResources}
+Fields: ${stat.retrievedFields}
+KV: ${stat.retrievedKeyValues}
+Data Bytes: ${stat.retrievedResourceDataBytes}
+KV Bytes: ${stat.retrievedKeyValueBytes}
+Pruned fields: ${stat.prunedFields}
+Final resources skipped: ${stat.finalResourcesSkipped}
+Stop markers skipped: ${stat.stopMarkersSkipped}
+Stop marker follow-up round-trips: ${stat.stopMarkerFollowUpRoundTrips}
+New resources: ${stat.resourcesNew}
+Changed resources: ${stat.resourcesChanged}
+Unchanged (duplicate re-fetch) resources: ${stat.resourcesUnchanged}
+Unchanged bytes (wasted downlink): ${stat.bytesUnchanged}
+Changed with stable metadata: ${stat.metadataStableChanged}
+BFS fetches wasted on unchanged: ${stat.bfsRequestsWasted}
+Used streaming: ${stat.usedStreaming}
+[streaming] rounds: ${stat.streamRounds}, resource frames: ${stat.resourceFrames}, stop-marker frames: ${stat.stopMarkerFrames}, stop->follow-up: ${stat.stopMarkersFollowUp}, traverse-stopped: ${stat.traverseWasStoppedCount}
+[bfs] resources requested: ${stat.bfsResourcesRequested}, not found: ${stat.bfsResourcesNotFound}`;
 }
 
 function supportsResourceTreeTraversal(capabilities: readonly string[] = []): boolean {
@@ -164,6 +209,7 @@ async function loadTreeStateViaBfs(
     }
 
     requested.add(rid);
+    if (stats) stats.bfsResourcesRequested++;
 
     pending.push(
       limiter.run(async () => {
@@ -183,7 +229,10 @@ async function loadTreeStateViaBfs(
           roundTripToggle = true;
         }
 
-        if (resource === undefined) return undefined;
+        if (resource === undefined) {
+          if (stats) stats.bfsResourcesNotFound++;
+          return undefined;
+        }
         if (kv === undefined) throw new Error("Inconsistent replies");
 
         return { ...resource, kv };
@@ -247,15 +296,21 @@ async function processResourceTreeStream(
   // should make a decision: has it already loaded the resource or should it be requested for get the latest state?
   for await (const frame of treeItems) {
     if (frame.frameKind === "stopMarker") {
+      if (stats) stats.stopMarkerFrames++;
       if (finalResources.has(frame.id)) {
         if (stats) stats.stopMarkersSkipped++;
         continue;
       }
+      if (stats) stats.stopMarkersFollowUp++;
       followUpSeeds.push(frame.id);
       continue;
     }
 
     // Normal resource frame.
+    if (stats) {
+      stats.resourceFrames++;
+      if (frame.traverseWasStopped) stats.traverseWasStoppedCount++;
+    }
     if (finalResources.has(frame.id)) {
       if (stats) stats.finalResourcesSkipped++;
       continue;
@@ -317,16 +372,30 @@ async function loadTreeStateViaResourceTree(
     pruningFunction,
     stats,
   );
-  if (stats) stats.roundTrips++;
+  if (stats) {
+    stats.roundTrips++;
+    stats.streamRounds++;
+  }
 
-  // Client must request full resource tree in case when stop-marker seeds are returned,
-  // to ensure all resources are loaded and stop-marker frames are processed.
-  if (followUpSeeds.length > 0) {
-    const followUpItems = tx.resourceTree(followUpSeeds, {
+  // Resolve stop-marker seeds by fetching them (see the note below on why the stop rule is not
+  // reapplied). Loop in case a fetch still yields stop markers; dedup fetched ids so shared refs
+  // or diamonds cannot loop forever (the id set is finite, so the loop terminates). A referenced
+  // resource left unloaded would make updateFromResourceData throw "orphan resource".
+  let pendingSeeds = followUpSeeds;
+  const fetchedSeeds = new Set<SignedResourceId>();
+  while (pendingSeeds.length > 0) {
+    const roundSeeds = pendingSeeds.filter((id) => !fetchedSeeds.has(id));
+    if (roundSeeds.length === 0) break;
+    for (const id of roundSeeds) fetchedSeeds.add(id);
+
+    // No traverseStopRules here on purpose: the seeds are the stop-marked resources, and the
+    // stop rule (finality-based) would re-flag them as stop markers instead of loading their
+    // state, leaving resources that reference them as orphans. The retry must fetch them plainly.
+    const followUpItems = tx.resourceTree(roundSeeds, {
       includeKv: true,
       fieldFilter,
     });
-    const { result: followUpResult } = await processResourceTreeStream(
+    const { result: followUpResult, followUpSeeds: nextSeeds } = await processResourceTreeStream(
       followUpItems,
       finalResources,
       pruningFunction,
@@ -335,11 +404,13 @@ async function loadTreeStateViaResourceTree(
     result.push(...followUpResult);
     if (stats) {
       logger?.info?.(
-        `loadTreeStateViaResourceTree: follow-up request for ${followUpSeeds.length} stop-marker seeds: ${JSON.stringify(followUpSeeds)}`,
+        `loadTreeStateViaResourceTree: follow-up request for ${roundSeeds.length} stop-marker seeds: ${JSON.stringify(roundSeeds)}`,
       );
       stats.roundTrips++;
+      stats.streamRounds++;
       stats.stopMarkerFollowUpRoundTrips++;
     }
+    pendingSeeds = nextSeeds;
   }
 
   return result;
@@ -364,6 +435,8 @@ export async function loadTreeState(
       mode === "backend-streaming" ||
       (mode === "auto" && supportsResourceTreeTraversal(capabilities));
 
+    if (stats) stats.usedStreaming = wantsStreaming && supportsResourceTreeTraversal(capabilities);
+
     if (wantsStreaming && !supportsResourceTreeTraversal(capabilities)) {
       const msg =
         "traversalMode=backend-streaming but backend lacks treeFilter:v2 capability; falling back to BFS";

package/src/synchronized_tree.ts

@@ -276,7 +276,7 @@ export class SynchronizedTreeState {
       },
       txOps,
     );
-    this.state.updateFromResourceData(data, true);
+    this.state.updateFromResourceData(data, { allowOrphanInputs: true, stat: stats });
   }
 
   /** Discovery sync for shared-type seeds: re-polls `ListUserResources` (gRPC-only) and