@milaboratories/pl-tree 1.15.0 → 1.15.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-tree",
-  "version": "1.15.0",
+  "version": "1.15.2",
   "description": "Reactive pl tree state",
   "files": [
     "./dist/**/*",
@@ -21,9 +21,9 @@
     "denque": "^2.1.0",
     "utility-types": "^3.11.0",
     "zod": "~3.25.76",
-    "@milaboratories/pl-errors": "1.4.41",
-    "@milaboratories/pl-client": "3.17.0",
     "@milaboratories/computable": "2.9.8",
+    "@milaboratories/pl-client": "3.17.1",
+    "@milaboratories/pl-errors": "1.4.42",
     "@milaboratories/ts-helpers": "1.8.6"
   },
   "devDependencies": {
@@ -31,9 +31,9 @@
     "@vitest/coverage-istanbul": "^4.1.3",
     "typescript": "7.0.2",
     "vitest": "^4.1.3",
-    "@milaboratories/build-configs": "2.0.1",
-    "@milaboratories/ts-configs": "1.4.0",
-    "@milaboratories/ts-builder": "1.7.2"
+    "@milaboratories/build-configs": "2.0.2",
+    "@milaboratories/ts-builder": "1.7.2",
+    "@milaboratories/ts-configs": "1.4.0"
   },
   "engines": {
     "node": ">=22.19.0"

package/src/state.test.ts

@@ -1,5 +1,7 @@
 import { expect, test } from "vitest";
 import { Computable } from "@milaboratories/computable";
+import type { Watcher } from "@milaboratories/computable";
+import type { FieldData } from "@milaboratories/pl-client";
 import {
   createSignedResourceId,
   DefaultFinalResourceDataPredicate,
@@ -20,6 +22,15 @@ import {
 
 const rid = createSignedResourceId;
 
+/** Minimal Watcher for tests that read tree state directly, outside a Computable. */
+class NoopWatcher implements Watcher {
+  isChanged = false;
+  markChanged(): void {
+    this.isChanged = true;
+  }
+}
+const w = () => new NoopWatcher();
+
 test("simple tree test 1", async () => {
   const tree = new PlTreeState(TestDynamicRootId1, DefaultFinalResourceDataPredicate);
   const entry = tree.entry();
@@ -317,3 +328,149 @@ test("exception - ready without locks 2", () => {
     ]),
   ).toThrow(/ready without input or output lock/);
 });
+
+// The field and kv update loops walk the stored entries in lockstep with the incoming ones
+// and only fall back to a hash lookup once the two orders diverge. These cover the
+// divergence shapes: reordering, an insertion in the middle, a removal. A reorder that
+// changes nothing must invalidate nothing and must not disturb refCounts, which is what
+// pins the fallback: mistaking a reordered field for a new one still leaves the right field
+// set behind, but double-counts the reference and fires spurious change notifications.
+
+const rootRes = (fields: FieldData[]) => [{ ...TestDynamicRootState1, fields }];
+
+test("a pure field reorder changes nothing and invalidates nothing", () => {
+  const tree = new PlTreeState(TestDynamicRootId1, DefaultFinalResourceDataPredicate);
+  tree.updateFromResourceData(rootRes([dField("a"), dField("b"), dField("c")]));
+
+  const watcher = w();
+  const root = tree.get(watcher, TestDynamicRootId1);
+  root.listDynamicFields(watcher);
+  for (const name of ["a", "b", "c"]) root.getField(watcher, name, () => {});
+  expect(watcher.isChanged).toStrictEqual(false);
+
+  tree.updateFromResourceData(rootRes([dField("c"), dField("b"), dField("a")]));
+
+  expect(watcher.isChanged).toStrictEqual(false);
+  expect(
+    tree
+      .get(w(), TestDynamicRootId1)
+      .fields.map((f) => f.name)
+      .sort(),
+  ).toStrictEqual(["a", "b", "c"]);
+});
+
+test("field inserted mid-order, then removed, is tracked", () => {
+  const tree = new PlTreeState(TestDynamicRootId1, DefaultFinalResourceDataPredicate);
+  const names = () =>
+    tree
+      .get(w(), TestDynamicRootId1)
+      .fields.map((f) => f.name)
+      .sort();
+
+  tree.updateFromResourceData(rootRes([dField("a"), dField("c")]));
+  expect(names()).toStrictEqual(["a", "c"]);
+
+  // "b" appears between two fields that are already stored: the walk diverges at "b"
+  tree.updateFromResourceData(rootRes([dField("a"), dField("b"), dField("c")]));
+  expect(names()).toStrictEqual(["a", "b", "c"]);
+
+  // and disappears again, in a shuffled order
+  tree.updateFromResourceData(rootRes([dField("c"), dField("a")]));
+  expect(names()).toStrictEqual(["a", "c"]);
+});
+
+test("reordering fields does not inflate refCounts", () => {
+  const tree = new PlTreeState(TestDynamicRootId1, DefaultFinalResourceDataPredicate);
+  const leaf = (n: bigint) => ({
+    ...TestValueResourceState1,
+    id: rid(n),
+    data: new TextEncoder().encode(`v${n}`),
+  });
+
+  tree.updateFromResourceData([
+    {
+      ...TestDynamicRootState1,
+      fields: [dField("a", rid(1n)), dField("b", rid(2n))],
+    },
+    leaf(1n),
+    leaf(2n),
+  ]);
+
+  // reorder and repoint in one update: the two fields swap targets
+  tree.updateFromResourceData([
+    {
+      ...TestDynamicRootState1,
+      fields: [dField("b", rid(1n)), dField("a", rid(2n))],
+    },
+    leaf(1n),
+    leaf(2n),
+  ]);
+
+  const byName = new Map(tree.get(w(), TestDynamicRootId1).fields.map((f) => [f.name, f.value]));
+  expect(byName.get("a")).toStrictEqual(rid(2n));
+  expect(byName.get("b")).toStrictEqual(rid(1n));
+
+  // dropping "a" must collect leaf 2: its refCount has to be exactly 1, so a reorder that
+  // was mistaken for an insertion (and double-incremented) would leave it alive here
+  tree.updateFromResourceData(rootRes([dField("b", rid(1n))]));
+  expect(tree.get(w(), rid(1n)).getDataAsString()).toStrictEqual("v1");
+  expect(() => tree.get(w(), rid(2n))).toThrow(/not found/);
+});
+
+test("kv reordering neither invalidates watchers nor loses entries", () => {
+  const tree = new PlTreeState(TestDynamicRootId1, DefaultFinalResourceDataPredicate);
+  const kvOf = (entries: [string, string][]) => [
+    {
+      ...TestDynamicRootState1,
+      fields: [],
+      kv: entries.map(([key, value]) => ({ key, value: Buffer.from(value) })),
+    },
+  ];
+  const read = (key: string) => tree.get(w(), TestDynamicRootId1).getKeyValueString(w(), key);
+
+  tree.updateFromResourceData(
+    kvOf([
+      ["k1", "one"],
+      ["k2", "two"],
+    ]),
+  );
+  expect([read("k1"), read("k2")]).toStrictEqual(["one", "two"]);
+
+  // reversed order, same values: no watcher may fire
+  const watcher = w();
+  const root = tree.get(watcher, TestDynamicRootId1);
+  root.getKeyValue(watcher, "k1");
+  root.getKeyValue(watcher, "k2");
+  tree.updateFromResourceData(
+    kvOf([
+      ["k2", "two"],
+      ["k1", "one"],
+    ]),
+  );
+  expect(watcher.isChanged).toStrictEqual(false);
+
+  // a key inserted before the stored ones, so the walk diverges immediately
+  tree.updateFromResourceData(
+    kvOf([
+      ["k0", "zero"],
+      ["k2", "two"],
+      ["k1", "ONE"],
+    ]),
+  );
+  expect([read("k0"), read("k1"), read("k2")]).toStrictEqual(["zero", "ONE", "two"]);
+
+  // and a deletion
+  tree.updateFromResourceData(kvOf([["k1", "ONE"]]));
+  expect([read("k0"), read("k1"), read("k2")]).toStrictEqual([undefined, "ONE", undefined]);
+});
+
+test("removal of a typed field still throws after a reorder", () => {
+  const tree = new PlTreeState(TestDynamicRootId1, DefaultFinalResourceDataPredicate);
+
+  tree.updateFromResourceData(rootRes([iField("a"), iField("b"), dField("c")]));
+
+  // reordered, and the input field "b" is gone: the removal scan must still see it
+  expect(() => tree.updateFromResourceData(rootRes([dField("c"), iField("a")]))).toThrow(
+    /removal of Input field b/,
+  );
+});

package/src/state.ts

@@ -513,6 +513,53 @@ export class PlTreeState {
     const incrementRefs: SignedResourceId[] = [];
     const decrementRefs: SignedResourceId[] = [];
 
+    // `resourcesAdded` is notified once, after the loop: markChanged detaches every watcher on
+    // its first call and no watcher can attach in the middle of this synchronous loop, so a
+    // single notification covers every resource added here. It also builds the marker string
+    // just once, which is worth doing because resourceIdToString on a signed id parses a
+    // BigInt and hex-decodes a Buffer.
+    let addedCount = 0;
+    let firstAdded: SignedResourceId | undefined;
+
+    // Diagnostic context for unexpectedTransitionError, refreshed once per resource. It lives
+    // out here so the loop body allocates neither a closure nor a snapshot per resource on the
+    // path where nothing goes wrong. Only the mutable half of BasicResourceData needs
+    // capturing: id, kind, type and data are readonly on PlTreeResource, so they are read back
+    // from the resource when the message is built.
+    let errRd: ExtendedResourceData;
+    let errRes: PlTreeResource | undefined;
+    let errOriginalResourceId: OptionalSignedResourceId = NullSignedResourceId;
+    let errError: OptionalSignedResourceId = NullSignedResourceId;
+    let errInputsLocked = false;
+    let errOutputsLocked = false;
+    let errResourceReady = false;
+    let errFinal = false;
+    const unexpectedTransitionError = (reason: string): never => {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const { fields, ...rdWithoutFields } = errRd;
+      const statBeforeMutation: BasicResourceData | undefined =
+        errRes === undefined
+          ? undefined
+          : {
+              id: errRes.id,
+              kind: errRes.kind,
+              type: errRes.type,
+              data: errRes.data,
+              resourceReady: errResourceReady,
+              inputsLocked: errInputsLocked,
+              outputsLocked: errOutputsLocked,
+              error: errError,
+              originalResourceId: errOriginalResourceId,
+              final: errFinal,
+            };
+      this.invalidateTree();
+      throw new TreeStateUpdateError(
+        `Unexpected resource state transition (${reason}): ${stringifyWithResourceId(
+          rdWithoutFields,
+        )} -> ${stringifyWithResourceId(statBeforeMutation)}`,
+      );
+    };
+
     // patching / creating resources
     for (const rd of resourceData) {
       let resource = this.resources.get(rd.id);
@@ -522,20 +569,17 @@ export class PlTreeState {
       // they never change; this flag isolates value/flag-only changes from real metadata churn.
       let metadataChanged = false;
 
-      const statBeforeMutation = resource?.basicState;
-      const unexpectedTransitionError = (reason: string): never => {
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        const { fields, ...rdWithoutFields } = rd;
-        this.invalidateTree();
-        throw new TreeStateUpdateError(
-          `Unexpected resource state transition (${reason}): ${stringifyWithResourceId(
-            rdWithoutFields,
-          )} -> ${stringifyWithResourceId(statBeforeMutation)}`,
-        );
-      };
+      errRd = rd;
+      errRes = resource;
 
       if (resource !== undefined) {
         // updating existing resource
+        errOriginalResourceId = resource.originalResourceId;
+        errError = resource.error;
+        errInputsLocked = resource.inputsLocked;
+        errOutputsLocked = resource.outputsLocked;
+        errResourceReady = resource.resourceReady;
+        errFinal = resource.finalFlag;
 
         if (resource.finalState)
           unexpectedTransitionError("resource state can\t be updated after it is marked as final");
@@ -569,9 +613,27 @@ export class PlTreeState {
           if (stat) stat.errorsAttached++;
         }
 
-        // updating fields
+        // updating fields.
+        //
+        // Fields normally arrive in the order they are already stored, so the stored ones are
+        // walked in lockstep with the incoming ones and matched by direct name comparison
+        // rather than through a fieldsMap lookup: every field name in a poll response is
+        // freshly decoded from protobuf, so V8 has no cached hash for any of them.
+        //
+        // Any divergence (reordering, an insertion, a removal) abandons the walk, and the rest
+        // of the resource resolves through fieldsMap.get, which is always correct. The walk is
+        // abandoned before any fieldsMap.set, so it never observes an insertion made through
+        // the live iterator.
+        let walk: MapIterator<PlTreeField> | undefined =
+          rd.fields.length > 0 ? resource.fieldsMap.values() : undefined;
         for (const fd of rd.fields) {
-          let field = resource.fieldsMap.get(fd.name);
+          let field: PlTreeField | undefined;
+          if (walk !== undefined) {
+            const next = walk.next();
+            if (next.done === true || next.value.name !== fd.name) walk = undefined;
+            else field = next.value;
+          }
+          if (field === undefined) field = resource.fieldsMap.get(fd.name);
 
           if (!field) {
             // new field
@@ -755,11 +817,23 @@ export class PlTreeState {
           if (stat) stat.readyFlips++;
         }
 
-        // syncing kv
+        // syncing kv. Same lockstep walk as the fields above, for the same reason: kv keys
+        // arrive in a stable order and are freshly decoded strings. The walk is only set up
+        // when there is something to compare against, so a resource with no kv allocates no
+        // iterator.
+        let kvWalk: MapIterator<[string, Uint8Array]> | undefined =
+          rd.kv.length > 0 ? resource.kv.entries() : undefined;
         for (const kv of rd.kv) {
-          const current = resource.kv.get(kv.key);
+          let current: Uint8Array | undefined;
+          if (kvWalk !== undefined) {
+            const next = kvWalk.next();
+            if (next.done === true || next.value[0] !== kv.key) kvWalk = undefined;
+            else current = next.value[1];
+          }
+          if (current === undefined) current = resource.kv.get(kv.key);
           if (current === undefined) {
             resource.kv.set(kv.key, kv.value);
+            kvWalk = undefined;
             notEmpty(resource.kvChangedPerKey).markChanged(
               kv.key,
               `kv added for ${resourceIdToString(resource.id)}: ${kv.key}`,
@@ -832,7 +906,8 @@ export class PlTreeState {
 
         // adding the resource to the heap
         this.resources.set(resource.id, resource);
-        this.resourcesAdded.markChanged(`new resource ${resourceIdToString(resource.id)} added`);
+        if (addedCount === 0) firstAdded = resource.id;
+        addedCount++;
       }
 
       if (stat) {
@@ -849,6 +924,13 @@ export class PlTreeState {
       }
     }
 
+    if (firstAdded !== undefined)
+      this.resourcesAdded.markChanged(
+        addedCount === 1
+          ? `new resource ${resourceIdToString(firstAdded)} added`
+          : `${addedCount} new resources added, first ${resourceIdToString(firstAdded)}`,
+      );
+
     // applying refCount increments
     for (const rid of incrementRefs) {
       const res = this.resources.get(rid);

package/src/sync.test.ts

@@ -481,6 +481,37 @@ test("stop-marker-unknown-triggers-followup: unknown marker fetched in follow-up
   expect(callCount).toBe(2);
 });
 
+test("stop-marker-followup-large: a very large follow-up round does not overflow the stack", async () => {
+  // Regression: result.push(...followUpResult) threw RangeError once the follow-up round
+  // exceeded the argument limit (~200k elements on Node 26).
+  const followUpSize = 300_000;
+  let callCount = 0;
+  const tx = {
+    resourceTree: () => {
+      callCount++;
+      if (callCount === 1) {
+        return (async function* () {
+          yield makeStopMarker("NG:0x1");
+        })();
+      } else {
+        return (async function* () {
+          for (let i = 0; i < followUpSize; i++) yield makeFullResource(`NG:0x${i.toString(16)}`);
+        })();
+      }
+    },
+  } as unknown as Parameters<typeof loadTreeState>[0];
+
+  const request = {
+    seedResources: ["NG:0x99"],
+    finalResources: new Set<string>(),
+  } as unknown as Parameters<typeof loadTreeState>[1];
+
+  const result = await loadTreeState(tx, request, undefined, stopMarkerCaps);
+
+  expect(result).toHaveLength(followUpSize);
+  expect(callCount).toBe(2);
+});
+
 test("legacy-backend-compat: no stop markers in stream → no follow-up call", async () => {
   // Old backends emit only full-resource frames; pendingFollowUp stays empty
   // and the follow-up loop body never executes.

package/src/sync.ts

@@ -446,7 +446,7 @@ async function loadTreeStateViaResourceTree(
     traverseStopRules,
   });
 
-  const { result, followUpSeeds } = await processResourceTreeStream(
+  let { result, followUpSeeds } = await processResourceTreeStream(
     treeItems,
     finalResources,
     pruningFunction,
@@ -481,7 +481,8 @@ async function loadTreeStateViaResourceTree(
       pruningFunction,
       stats,
     );
-    result.push(...followUpResult);
+    // spreading fails due to exceeding stack argument size
+    result = result.concat(followUpResult);
     if (stats) {
       logger?.info?.(
         `loadTreeStateViaResourceTree: follow-up request for ${roundSeeds.length} stop-marker seeds: ${JSON.stringify(roundSeeds)}`,