@milaboratories/pl-tree 1.14.4 → 1.15.1

package/src/state.ts

@@ -513,6 +513,53 @@ export class PlTreeState {
     const incrementRefs: SignedResourceId[] = [];
     const decrementRefs: SignedResourceId[] = [];
 
+    // `resourcesAdded` is notified once, after the loop: markChanged detaches every watcher on
+    // its first call and no watcher can attach in the middle of this synchronous loop, so a
+    // single notification covers every resource added here. It also builds the marker string
+    // just once, which is worth doing because resourceIdToString on a signed id parses a
+    // BigInt and hex-decodes a Buffer.
+    let addedCount = 0;
+    let firstAdded: SignedResourceId | undefined;
+
+    // Diagnostic context for unexpectedTransitionError, refreshed once per resource. It lives
+    // out here so the loop body allocates neither a closure nor a snapshot per resource on the
+    // path where nothing goes wrong. Only the mutable half of BasicResourceData needs
+    // capturing: id, kind, type and data are readonly on PlTreeResource, so they are read back
+    // from the resource when the message is built.
+    let errRd: ExtendedResourceData;
+    let errRes: PlTreeResource | undefined;
+    let errOriginalResourceId: OptionalSignedResourceId = NullSignedResourceId;
+    let errError: OptionalSignedResourceId = NullSignedResourceId;
+    let errInputsLocked = false;
+    let errOutputsLocked = false;
+    let errResourceReady = false;
+    let errFinal = false;
+    const unexpectedTransitionError = (reason: string): never => {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const { fields, ...rdWithoutFields } = errRd;
+      const statBeforeMutation: BasicResourceData | undefined =
+        errRes === undefined
+          ? undefined
+          : {
+              id: errRes.id,
+              kind: errRes.kind,
+              type: errRes.type,
+              data: errRes.data,
+              resourceReady: errResourceReady,
+              inputsLocked: errInputsLocked,
+              outputsLocked: errOutputsLocked,
+              error: errError,
+              originalResourceId: errOriginalResourceId,
+              final: errFinal,
+            };
+      this.invalidateTree();
+      throw new TreeStateUpdateError(
+        `Unexpected resource state transition (${reason}): ${stringifyWithResourceId(
+          rdWithoutFields,
+        )} -> ${stringifyWithResourceId(statBeforeMutation)}`,
+      );
+    };
+
     // patching / creating resources
     for (const rd of resourceData) {
       let resource = this.resources.get(rd.id);
@@ -522,20 +569,17 @@ export class PlTreeState {
       // they never change; this flag isolates value/flag-only changes from real metadata churn.
       let metadataChanged = false;
 
-      const statBeforeMutation = resource?.basicState;
-      const unexpectedTransitionError = (reason: string): never => {
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        const { fields, ...rdWithoutFields } = rd;
-        this.invalidateTree();
-        throw new TreeStateUpdateError(
-          `Unexpected resource state transition (${reason}): ${stringifyWithResourceId(
-            rdWithoutFields,
-          )} -> ${stringifyWithResourceId(statBeforeMutation)}`,
-        );
-      };
+      errRd = rd;
+      errRes = resource;
 
       if (resource !== undefined) {
         // updating existing resource
+        errOriginalResourceId = resource.originalResourceId;
+        errError = resource.error;
+        errInputsLocked = resource.inputsLocked;
+        errOutputsLocked = resource.outputsLocked;
+        errResourceReady = resource.resourceReady;
+        errFinal = resource.finalFlag;
 
         if (resource.finalState)
           unexpectedTransitionError("resource state can\t be updated after it is marked as final");
@@ -569,9 +613,27 @@ export class PlTreeState {
           if (stat) stat.errorsAttached++;
         }
 
-        // updating fields
+        // updating fields.
+        //
+        // Fields normally arrive in the order they are already stored, so the stored ones are
+        // walked in lockstep with the incoming ones and matched by direct name comparison
+        // rather than through a fieldsMap lookup: every field name in a poll response is
+        // freshly decoded from protobuf, so V8 has no cached hash for any of them.
+        //
+        // Any divergence (reordering, an insertion, a removal) abandons the walk, and the rest
+        // of the resource resolves through fieldsMap.get, which is always correct. The walk is
+        // abandoned before any fieldsMap.set, so it never observes an insertion made through
+        // the live iterator.
+        let walk: MapIterator<PlTreeField> | undefined =
+          rd.fields.length > 0 ? resource.fieldsMap.values() : undefined;
         for (const fd of rd.fields) {
-          let field = resource.fieldsMap.get(fd.name);
+          let field: PlTreeField | undefined;
+          if (walk !== undefined) {
+            const next = walk.next();
+            if (next.done === true || next.value.name !== fd.name) walk = undefined;
+            else field = next.value;
+          }
+          if (field === undefined) field = resource.fieldsMap.get(fd.name);
 
           if (!field) {
             // new field
@@ -755,11 +817,23 @@ export class PlTreeState {
           if (stat) stat.readyFlips++;
         }
 
-        // syncing kv
+        // syncing kv. Same lockstep walk as the fields above, for the same reason: kv keys
+        // arrive in a stable order and are freshly decoded strings. The walk is only set up
+        // when there is something to compare against, so a resource with no kv allocates no
+        // iterator.
+        let kvWalk: MapIterator<[string, Uint8Array]> | undefined =
+          rd.kv.length > 0 ? resource.kv.entries() : undefined;
         for (const kv of rd.kv) {
-          const current = resource.kv.get(kv.key);
+          let current: Uint8Array | undefined;
+          if (kvWalk !== undefined) {
+            const next = kvWalk.next();
+            if (next.done === true || next.value[0] !== kv.key) kvWalk = undefined;
+            else current = next.value[1];
+          }
+          if (current === undefined) current = resource.kv.get(kv.key);
           if (current === undefined) {
             resource.kv.set(kv.key, kv.value);
+            kvWalk = undefined;
             notEmpty(resource.kvChangedPerKey).markChanged(
               kv.key,
               `kv added for ${resourceIdToString(resource.id)}: ${kv.key}`,
@@ -832,7 +906,8 @@ export class PlTreeState {
 
         // adding the resource to the heap
         this.resources.set(resource.id, resource);
-        this.resourcesAdded.markChanged(`new resource ${resourceIdToString(resource.id)} added`);
+        if (addedCount === 0) firstAdded = resource.id;
+        addedCount++;
       }
 
       if (stat) {
@@ -849,6 +924,13 @@ export class PlTreeState {
       }
     }
 
+    if (firstAdded !== undefined)
+      this.resourcesAdded.markChanged(
+        addedCount === 1
+          ? `new resource ${resourceIdToString(firstAdded)} added`
+          : `${addedCount} new resources added, first ${resourceIdToString(firstAdded)}`,
+      );
+
     // applying refCount increments
     for (const rid of incrementRefs) {
       const res = this.resources.get(rid);

package/src/sync.test.ts

@@ -6,7 +6,12 @@ import {
   TestHelpers,
 } from "@milaboratories/pl-client";
 import { PlTreeState } from "./state";
-import { constructTreeLoadingRequest, initialTreeLoadingStat, loadTreeState } from "./sync";
+import {
+  constructTreeLoadingRequest,
+  initialTreeLoadingStat,
+  loadTreeState,
+  resolveTreeLoadingAlgorithm,
+} from "./sync";
 import type { TraversalMode } from "./sync";
 import { Computable } from "@milaboratories/computable";
 import { TestStructuralResourceType1 } from "./test_utils";
@@ -476,6 +481,37 @@ test("stop-marker-unknown-triggers-followup: unknown marker fetched in follow-up
   expect(callCount).toBe(2);
 });
 
+test("stop-marker-followup-large: a very large follow-up round does not overflow the stack", async () => {
+  // Regression: result.push(...followUpResult) threw RangeError once the follow-up round
+  // exceeded the argument limit (~200k elements on Node 26).
+  const followUpSize = 300_000;
+  let callCount = 0;
+  const tx = {
+    resourceTree: () => {
+      callCount++;
+      if (callCount === 1) {
+        return (async function* () {
+          yield makeStopMarker("NG:0x1");
+        })();
+      } else {
+        return (async function* () {
+          for (let i = 0; i < followUpSize; i++) yield makeFullResource(`NG:0x${i.toString(16)}`);
+        })();
+      }
+    },
+  } as unknown as Parameters<typeof loadTreeState>[0];
+
+  const request = {
+    seedResources: ["NG:0x99"],
+    finalResources: new Set<string>(),
+  } as unknown as Parameters<typeof loadTreeState>[1];
+
+  const result = await loadTreeState(tx, request, undefined, stopMarkerCaps);
+
+  expect(result).toHaveLength(followUpSize);
+  expect(callCount).toBe(2);
+});
+
 test("legacy-backend-compat: no stop markers in stream → no follow-up call", async () => {
   // Old backends emit only full-resource frames; pendingFollowUp stays empty
   // and the follow-up loop body never executes.
@@ -541,3 +577,71 @@ test("orphan-invariant-preserved: error referent streamed alongside stop marker"
   expect(ids).toEqual(["NG:0x1", "NG:0xE"]);
   // No throw means the invariant held throughout loadTreeState
 });
+
+//
+// Algorithm selection. These pin the contract that a capable backend is polled with a token
+// by default, and that an unavailable preference degrades rather than throwing.
+//
+
+test("auto prefers delta, then streaming, then BFS", () => {
+  expect(resolveTreeLoadingAlgorithm("auto", ["treeChangedSince:v1", "treeFilter:v2"])).toBe(
+    "backend-delta",
+  );
+  // Delta needs no treeFilter:v2 of its own.
+  expect(resolveTreeLoadingAlgorithm("auto", ["treeChangedSince:v1"])).toBe("backend-delta");
+  expect(resolveTreeLoadingAlgorithm("auto", ["treeFilter:v2"])).toBe("backend-streaming");
+  expect(resolveTreeLoadingAlgorithm("auto", [])).toBe("client-bfs");
+});
+
+test("an explicit mode is honoured over what auto would pick", () => {
+  const capable = ["treeChangedSince:v1", "treeFilter:v2"];
+  expect(resolveTreeLoadingAlgorithm("client-bfs", capable)).toBe("client-bfs");
+  expect(resolveTreeLoadingAlgorithm("backend-streaming", capable)).toBe("backend-streaming");
+  expect(resolveTreeLoadingAlgorithm("backend-delta", capable)).toBe("backend-delta");
+});
+
+test("backend-delta degrades to the best available path, with a warning, never a throw", () => {
+  const warnings: string[] = [];
+  const logger = { warn: (m: string) => warnings.push(m) };
+
+  expect(resolveTreeLoadingAlgorithm("backend-delta", ["treeFilter:v2"], logger)).toBe(
+    "backend-streaming",
+  );
+  expect(resolveTreeLoadingAlgorithm("backend-delta", [], logger)).toBe("client-bfs");
+
+  expect(warnings).toHaveLength(2);
+  for (const w of warnings) expect(w).toContain("treeChangedSince:v1");
+});
+
+test("loadTreeState routes into the delta path and passes the token through", async () => {
+  const received: { seeds?: string[]; token?: Uint8Array; stopRules?: unknown } = {};
+  const tx = {
+    resourceTree: (seeds: string[], opts: Record<string, unknown>) => {
+      received.seeds = seeds;
+      received.token = opts.changedSinceToken as Uint8Array;
+      received.stopRules = opts.traverseStopRules;
+      return (async function* () {})();
+    },
+  } as unknown as Parameters<typeof loadTreeState>[0];
+
+  const request = {
+    seedResources: ["NG:0x1"],
+    finalResources: new Set<string>(),
+    roots: ["NG:0x1"],
+    knownResources: new Set<string>(["NG:0x1"]),
+    changedSinceToken: new Uint8Array([9]),
+  } as unknown as Parameters<typeof loadTreeState>[1];
+
+  const stat = initialTreeLoadingStat();
+  // Via the mode, not by calling loadDeltaTreeState directly: this is the only test that
+  // proves the dispatch in loadTreeState reaches delta at all.
+  await loadTreeState(tx, request, stat, ["treeChangedSince:v1"], "auto", { warn: () => {} });
+
+  expect(received.seeds).toEqual(["NG:0x1"]);
+  expect(received.token).toEqual(new Uint8Array([9]));
+  expect(received.stopRules).toBeUndefined();
+  // True for delta as well as streaming: it marks a backend path, which is what state.ts
+  // reads to decide the BFS-only wasted-fetch attribution.
+  expect(stat.usedStreaming).toBe(true);
+  expect(stat.deltaSeedsSent).toBe(1);
+});

package/src/sync.ts

@@ -10,6 +10,7 @@ import Denque from "denque";
 import { hasCapability, isNullSignedResourceId } from "@milaboratories/pl-client";
 import type { ExtendedResourceData, PlTreeState, ResourceUpdateStat } from "./state";
 import { ConcurrencyLimitingExecutor, msToHumanReadable } from "@milaboratories/ts-helpers";
+import { loadDeltaTreeState } from "./delta_sync";
 
 /** Applied to list of fields in resource data. */
 export type PruningFunction = (resource: ExtendedResourceData) => FieldData[];
@@ -31,24 +32,81 @@ export interface TreeLoadingRequest {
   /** ResourceTree field filter passed to the backend when supported. */
   readonly fieldFilter?: Filter;
 
-  /** ResourceTree traversal stop rules passed to the backend when supported. */
+  /** ResourceTree traversal stop rules passed to the backend when supported.
+   * @deprecated prune with {@link changedSinceToken} instead; ignored by the backend under one. */
   readonly traverseStopRules?: Filter;
+
+  /** The tree's roots. Delta seeds at these only when the non-final frontier is empty;
+   * resourceTree requires at least one seed. */
+  readonly roots: readonly SignedResourceId[];
+
+  /** Every id the mirror currently holds, final or not. Delta uses it to tell a reference it
+   * must resolve from one already satisfied locally. */
+  readonly knownResources: ReadonlySet<SignedResourceId>;
+
+  /** Change token from the transaction this request will run in, for a delta walk. Absent
+   * means "send the full tree", which is also what a token the server cannot use gets. */
+  readonly changedSinceToken?: Uint8Array;
 }
 
 /** Controls which tree-loading path is used.
- * - `"auto"` (default): use backend streaming when the backend advertises `treeFilter:v2`,
- *   fall back to client-side BFS otherwise.
+ * - `"auto"` (default): use delta polling when the backend advertises `treeChangedSince:v1`,
+ *   else backend streaming when it advertises `treeFilter:v2`, else client-side BFS.
  * - `"client-bfs"`: always use client-side BFS, even on capable backends.
  * - `"backend-streaming"`: always prefer backend streaming; if the capability is absent,
  *   logs a warning and falls back to BFS (never throws).
+ * - `"backend-delta"`: always prefer delta polling, which hands the backend the transaction's
+ *   change token and takes only what changed since it; if `treeChangedSince:v1` is absent,
+ *   logs a warning and falls back to the best available path (never throws).
  */
-export type TraversalMode = "auto" | "client-bfs" | "backend-streaming";
+export type TraversalMode = "auto" | "client-bfs" | "backend-streaming" | "backend-delta";
+
+/** A concrete loading algorithm: a {@link TraversalMode} with `"auto"` and any unsupported
+ * preference already resolved against the server's capabilities. */
+export type TreeLoadingAlgorithmName = "client-bfs" | "backend-streaming" | "backend-delta";
+
+/** Resolves a traversal mode into the algorithm a tree will run. A tree calls this once, when
+ * it is made, and keeps the answer for its whole life, so the choice (and the fallback warning
+ * for a preference the backend cannot serve) happens once rather than on every poll. */
+export function resolveTreeLoadingAlgorithm(
+  mode: TraversalMode,
+  capabilities: readonly string[] = [],
+  logger?: { warn: (msg: string) => void },
+): TreeLoadingAlgorithmName {
+  const streaming = supportsResourceTreeTraversal(capabilities);
+  const delta = supportsTreeDelta(capabilities);
+  switch (mode) {
+    case "client-bfs":
+      return "client-bfs";
+    case "backend-delta":
+      if (delta) return "backend-delta";
+      (logger ?? console).warn(
+        "traversalMode=backend-delta but backend lacks treeChangedSince:v1 capability; falling back to " +
+          (streaming ? "backend-streaming" : "client-bfs"),
+      );
+      return streaming ? "backend-streaming" : "client-bfs";
+    case "backend-streaming":
+      if (streaming) return "backend-streaming";
+      (logger ?? console).warn(
+        "traversalMode=backend-streaming but backend lacks treeFilter:v2 capability; falling back to BFS",
+      );
+      return "client-bfs";
+    case "auto":
+      // Delta first: its cost tracks what changed rather than what the tree holds. Streaming
+      // is the fallback for a backend that can shape a walk but not date one.
+      if (delta) return "backend-delta";
+      return streaming ? "backend-streaming" : "client-bfs";
+  }
+}
 
 /** Given the current tree state, build the request object to pass to
  * {@link loadTreeState} to load updated state. */
 export function constructTreeLoadingRequest(
   tree: PlTreeState,
-  options: Pick<TreeLoadingRequest, "pruningFunction" | "fieldFilter" | "traverseStopRules"> = {},
+  options: Pick<
+    TreeLoadingRequest,
+    "pruningFunction" | "fieldFilter" | "traverseStopRules" | "changedSinceToken"
+  > = {},
 ): TreeLoadingRequest {
   const seedResources: SignedResourceId[] = [];
   const finalResources = new Set<SignedResourceId>();
@@ -64,9 +122,12 @@ export function constructTreeLoadingRequest(
   return {
     seedResources,
     finalResources,
+    roots: [...tree.roots],
+    knownResources: materialized,
     pruningFunction: options.pruningFunction,
     fieldFilter: options.fieldFilter,
     traverseStopRules: options.traverseStopRules,
+    changedSinceToken: options.changedSinceToken,
   };
 }
 
@@ -85,9 +146,10 @@ export type TreeLoadingStat = ResourceUpdateStat & {
   stopMarkersSkipped: number;
   /** Number of follow-up resourceTree() calls issued to resolve unknown stop markers. */
   stopMarkerFollowUpRoundTrips: number;
-  /** Streaming (resourceTree) path: resourceTree() streams consumed (1, or 2 with a follow-up). */
+  /** Backend paths: resourceTree() streams consumed. Streaming spends 1, or 2 with a
+   * follow-up; delta spends 1 plus one per resolution round. */
   streamRounds: number;
-  /** Streaming path: resource frames received. */
+  /** Backend paths: resource frames received. */
   resourceFrames: number;
   /** Streaming path: stopMarker frames received (both skipped and follow-up). */
   stopMarkerFrames: number;
@@ -99,6 +161,16 @@ export type TreeLoadingStat = ResourceUpdateStat & {
   bfsResourcesRequested: number;
   /** BFS path: requested resources that no longer exist (undefined reply). */
   bfsResourcesNotFound: number;
+  /** Delta path: seed ids handed to the backend, summed over every round of the poll.
+   * This is what seeding the whole frontier costs on the uplink. */
+  deltaSeedsSent: number;
+  /** Delta path: extra rounds spent resolving references a delta body pointed at but the
+   * response did not carry. */
+  deltaResolutionRounds: number;
+  /** Delta path: polls that sent a token and got back a response the size of the whole
+   * mirror, which is what a refused token looks like from here - rejection is silent, so this
+   * is the only tell. Heuristic: a genuinely large change set trips it too. */
+  deltaSuspectedFullAnswers: number;
 };
 
 export function initialTreeLoadingStat(): TreeLoadingStat {
@@ -122,6 +194,9 @@ export function initialTreeLoadingStat(): TreeLoadingStat {
     traverseWasStoppedCount: 0,
     bfsResourcesRequested: 0,
     bfsResourcesNotFound: 0,
+    deltaSeedsSent: 0,
+    deltaResolutionRounds: 0,
+    deltaSuspectedFullAnswers: 0,
     resourcesNew: 0,
     resourcesChanged: 0,
     resourcesUnchanged: 0,
@@ -161,15 +236,20 @@ Unchanged bytes (wasted downlink): ${stat.bytesUnchanged}
 Changed with stable metadata: ${stat.metadataStableChanged}
 BFS fetches wasted on unchanged: ${stat.bfsRequestsWasted}
 Used streaming: ${stat.usedStreaming}
-[streaming] rounds: ${stat.streamRounds}, resource frames: ${stat.resourceFrames}, stop-marker frames: ${stat.stopMarkerFrames}, stop->follow-up: ${stat.stopMarkersFollowUp}, traverse-stopped: ${stat.traverseWasStoppedCount}
-[bfs] resources requested: ${stat.bfsResourcesRequested}, not found: ${stat.bfsResourcesNotFound}`;
+[backend] rounds: ${stat.streamRounds}, resource frames: ${stat.resourceFrames}, stop-marker frames: ${stat.stopMarkerFrames}, stop->follow-up: ${stat.stopMarkersFollowUp}, traverse-stopped: ${stat.traverseWasStoppedCount}
+[bfs] resources requested: ${stat.bfsResourcesRequested}, not found: ${stat.bfsResourcesNotFound}
+[delta] seeds sent: ${stat.deltaSeedsSent}, resolution rounds: ${stat.deltaResolutionRounds}, suspected full answers: ${stat.deltaSuspectedFullAnswers}`;
 }
 
-function supportsResourceTreeTraversal(capabilities: readonly string[] = []): boolean {
+export function supportsResourceTreeTraversal(capabilities: readonly string[] = []): boolean {
   return hasCapability(capabilities, "treeFilter:v2");
 }
 
-function collectStatsForResource(resource: ExtendedResourceData, stats?: TreeLoadingStat) {
+function supportsTreeDelta(capabilities: readonly string[] = []): boolean {
+  return hasCapability(capabilities, "treeChangedSince:v1");
+}
+
+export function collectStatsForResource(resource: ExtendedResourceData, stats?: TreeLoadingStat) {
   if (!stats) return;
   stats.retrievedResources++;
   stats.retrievedFields += resource.fields.length;
@@ -366,7 +446,7 @@ async function loadTreeStateViaResourceTree(
     traverseStopRules,
   });
 
-  const { result, followUpSeeds } = await processResourceTreeStream(
+  let { result, followUpSeeds } = await processResourceTreeStream(
     treeItems,
     finalResources,
     pruningFunction,
@@ -401,7 +481,8 @@ async function loadTreeStateViaResourceTree(
       pruningFunction,
       stats,
     );
-    result.push(...followUpResult);
+    // spreading fails due to exceeding stack argument size
+    result = result.concat(followUpResult);
     if (stats) {
       logger?.info?.(
         `loadTreeStateViaResourceTree: follow-up request for ${roundSeeds.length} stop-marker seeds: ${JSON.stringify(roundSeeds)}`,
@@ -431,23 +512,23 @@ export async function loadTreeState(
   if (stats) stats.requests++;
 
   try {
-    const wantsStreaming =
-      mode === "backend-streaming" ||
-      (mode === "auto" && supportsResourceTreeTraversal(capabilities));
-
-    if (stats) stats.usedStreaming = wantsStreaming && supportsResourceTreeTraversal(capabilities);
-
-    if (wantsStreaming && !supportsResourceTreeTraversal(capabilities)) {
-      const msg =
-        "traversalMode=backend-streaming but backend lacks treeFilter:v2 capability; falling back to BFS";
-      if (logger) logger.warn(msg);
-      else console.warn(msg);
-      return await loadTreeStateViaBfs(tx, loadingRequest, stats);
+    // A tree passes its pinned algorithm here, which resolves to itself. The mode form is
+    // kept for callers that run a single load.
+    const algorithm = resolveTreeLoadingAlgorithm(mode, capabilities, logger);
+    // Both backend paths, not just streaming: state.ts reads this to attribute a BFS-only
+    // wasted-fetch counter, and leaving it false for delta reports phantom waste.
+    if (stats) stats.usedStreaming = algorithm !== "client-bfs";
+
+    switch (algorithm) {
+      case "backend-delta":
+        return await loadDeltaTreeState(tx, loadingRequest, stats, logger);
+      case "backend-streaming":
+        return await loadTreeStateViaResourceTree(tx, loadingRequest, stats, logger);
+      case "client-bfs":
+        return await loadTreeStateViaBfs(tx, loadingRequest, stats);
+      default:
+        throw new Error(`unknown tree loading algorithm: ${algorithm as string}`);
     }
-
-    return wantsStreaming
-      ? await loadTreeStateViaResourceTree(tx, loadingRequest, stats, logger)
-      : await loadTreeStateViaBfs(tx, loadingRequest, stats);
   } finally {
     if (stats) stats.millisSpent += Date.now() - startTimestamp;
   }