@milaboratories/pl-tree 1.15.7 → 1.16.1

package/src/synchronized_tree.ts

@@ -13,6 +13,7 @@ import {
   isUnauthenticated,
   isTimeoutOrCancelError,
   isUnimplementedError,
+  resourceIdToString,
 } from "@milaboratories/pl-client";
 import type { ExtendedResourceData } from "./state";
 import { PlTreeState, TreeStateUpdateError } from "./state";
@@ -32,7 +33,12 @@ import {
 import type { PersistedTree } from "./persisted_tree";
 import { captureTreeState, restoreTreeState } from "./persisted_tree";
 import * as tp from "node:timers/promises";
-import type { MiLogger } from "@milaboratories/ts-helpers";
+import type {
+  InfiniteRetryOptions,
+  InfiniteRetryState,
+  MiLogger,
+} from "@milaboratories/ts-helpers";
+import { createInfiniteRetryState, nextInfiniteRetryState } from "@milaboratories/ts-helpers";
 
 /** Hard floor between consecutive tree-refresh calls.
  * Applies even when {@link scheduleOnNextState} has woken the loop early,
@@ -60,6 +66,17 @@ const RTT_POLL_FACTOR = 2;
  * rather than slow, and spacing polls further only delays noticing it recovered. */
 const MAX_RTT_POLL_INTERVAL_MS = 30_000;
 
+/** Waits between rebuilds after consecutive {@link TreeStateUpdateError}s. The first retry
+ * comes after the ordinary floor, so a one-off inconsistency heals as fast as a normal poll;
+ * a persistent one settles at one full read per {@link MAX_POLLING_INTERVAL_MS}. */
+const REBUILD_RETRY: InfiniteRetryOptions = {
+  type: "exponentialWithMaxDelayBackoff",
+  initialDelay: MIN_POLLING_INTERVAL_MS,
+  maxDelay: MAX_POLLING_INTERVAL_MS,
+  backoffMultiplier: 2,
+  jitter: 0.2,
+};
+
 type StatLoggingMode = "cumulative" | "per-request";
 
 export type SynchronizedTreeOps = {
@@ -92,6 +109,13 @@ export type SynchronizedTreeOps = {
   /** Controls which tree-loading path to use.  Default `"auto"`. */
   traversalMode?: TraversalMode;
 
+  /** Treat every explicit root as never final, whatever its type, so every poll seeds it and,
+   * once it is held, checks that it still exists. Without it a root the predicate calls final is never re-read,
+   * and a deleted final root is never noticed. Roots discovered for shared-type seeds are not
+   * covered: a discovered resource may already be held final, and discovery itself drops a root
+   * that is gone. */
+  rootsNeverFinal?: boolean;
+
   /** A previously persisted mirror to seed the tree with, before its first refresh, so that
    * refresh transfers only what changed while the tree was gone.
    *
@@ -146,13 +170,7 @@ const DISCOVERY_INTERVAL_MS = 3_000;
  * `resourcesUnchanged` is excluded by design, since a cycle that only re-fetched unchanged
  * state is exactly the idle case the backoff exists for. */
 function countedChanges(stat: TreeLoadingStat): number {
-  // `fieldsRemoved` is included despite being a per-field count, because it is the one change
-  // that never shows up in `resourcesChanged`: the removed-dynamic-field branch in
-  // `updateFromResourceData` does not set its `changed` flag, so a cycle that only dropped a
-  // field (and garbage-collected whatever it pointed at) otherwise reads as an idle cycle.
-  // That double-counts a resource that both changed and lost a field, which is harmless here:
-  // every caller compares this against an earlier value rather than reading it as a total.
-  return stat.resourcesNew + stat.resourcesChanged + stat.resourcesMarkedFinal + stat.fieldsRemoved;
+  return stat.resourcesNew + stat.resourcesChanged + stat.resourcesMarkedFinal + stat.rootsDropped;
 }
 
 /** The poll-cadence policy, as a pure function of the last cycle's outcome.
@@ -243,6 +261,7 @@ export class SynchronizedTreeState {
       pollingInterval,
       stopPollingDelay,
       logStat,
+      rootsNeverFinal,
     } = ops;
     this.pruning = pruning;
     this.fieldFilter = fieldFilter;
@@ -256,7 +275,15 @@ export class SynchronizedTreeState {
     logger?.info(`tree loading algorithm: ${this.algorithm} (traversalMode=${this.traversalMode})`);
     this.pollingInterval = pollingInterval;
     this.effectivePollingInterval = pollingInterval;
-    this.finalPredicate = finalPredicateOverride ?? pl.finalPredicate;
+    const basePredicate = finalPredicateOverride ?? pl.finalPredicate;
+    // Explicit roots only: they are fixed before the first resource is ever evaluated, so no
+    // root can have been marked final before the option applied to it.
+    const explicitRoots = new Set(
+      seeds.flatMap((s): SignedResourceId[] => (s.kind === "resource" ? [s.root] : [])),
+    );
+    this.finalPredicate = rootsNeverFinal
+      ? (r) => !explicitRoots.has(r.id) && basePredicate(r)
+      : basePredicate;
     this.logStat = logStat;
 
     this.explicitRoots = seeds
@@ -467,7 +494,8 @@ export class SynchronizedTreeState {
     // adds roots later via setRoots(), which schedules the next refresh. Explicit-root trees
     // never hit this (their root set is non-empty by construction).
     if (request.seedResources.length === 0 && request.finalResources.size === 0) return;
-    const { data, nextToken } = await this.pl.withReadTx(
+    const checkedRoots = this.state.nonFinalRoots();
+    const { data, nextToken, rootsExist } = await this.pl.withReadTx(
       "ReadingTree",
       async (tx) => {
         // Started, not awaited, before the walk. The token dates the transaction rather than
@@ -477,6 +505,12 @@ export class SynchronizedTreeState {
         // because requests pipeline on one bidi stream and withReadTx does not await the open.
         const tokenPromise =
           this.algorithm === "backend-delta" ? tx.getNextSinceToken() : undefined;
+        // A walk seeded at a deleted resource yields nothing and no error, so a deleted root
+        // would stay in the mirror for good. Checked in the same transaction, pipelined the
+        // same way as the token. The catch only keeps a rejection that lands while the walk is
+        // still running from being reported as unhandled; it is awaited below.
+        const existence = Promise.all(checkedRoots.map((rid) => tx.resourceExists(rid)));
+        existence.catch(() => {});
         const data = await loadTreeState(
           tx,
           request,
@@ -485,14 +519,32 @@ export class SynchronizedTreeState {
           this.algorithm,
           this.logger,
         );
-        return { data, nextToken: await tokenPromise };
+        return { data, nextToken: await tokenPromise, rootsExist: await existence };
       },
       txOps,
     );
     this.state.updateFromResourceData(data, { allowOrphanInputs: true, stat: stats });
 
+    // Repeated until nothing more drops: a deleted root held by another deleted root is
+    // refused (still referenced) until its holder has been dropped.
+    let gone = checkedRoots.filter((_, i) => !rootsExist[i]);
+    let dropped = true;
+    while (dropped) {
+      dropped = false;
+      gone = gone.filter((rid) => {
+        if (!this.state.dropDeletedRoot(rid)) return true;
+        dropped = true;
+        if (stats) stats.rootsDropped++;
+        this.logger?.warn(
+          `tree root ${resourceIdToString(rid)} no longer exists; dropped from the tree`,
+        );
+        return false;
+      });
+    }
+
     // Only with the whole batch applied: advancing past a partial apply loses the dropped
-    // resources for good. A throw above leaves the old token, so the next poll re-reads it.
+    // resources for good. A throw above leaves the old token; for an update error the loop then
+    // rebuilds the mirror and discards it.
     if (nextToken !== undefined) this.deltaToken = nextToken;
     else if (this.algorithm === "backend-delta") this.demoteFromDelta();
   }
@@ -574,6 +626,9 @@ export class SynchronizedTreeState {
     // paces the discovery poll for shared-type seeds; 0 forces discovery on the first pass.
     let lastDiscovery = 0;
 
+    // Set while the tree is being rebuilt after consecutive update errors; spaces the rebuilds.
+    let rebuildRetry: InfiniteRetryState | undefined;
+
     while (true) {
       if (!this.keepRunning || this.terminated) break;
 
@@ -615,6 +670,8 @@ export class SynchronizedTreeState {
           );
         lastUpdate = Date.now();
 
+        rebuildRetry = undefined;
+
         // notifying that we got new state
         if (toNotify !== undefined) for (const n of toNotify) n.resolve();
       } catch (e: any) {
@@ -630,8 +687,20 @@ export class SynchronizedTreeState {
 
         // catching tree update errors, as they may leave our tree in inconsistent state
         if (e instanceof TreeStateUpdateError) {
+          rebuildRetry =
+            rebuildRetry === undefined
+              ? createInfiniteRetryState(REBUILD_RETRY)
+              : nextInfiniteRetryState(rebuildRetry);
+
           // important error logging, this should never happen
-          this.logger?.error(e);
+          this.logger?.error(
+            new Error(
+              `tree rebuilt after an update error; next read in ${Math.round(rebuildRetry.nextDelay)}ms`,
+              {
+                cause: e,
+              },
+            ),
+          );
 
           // marking everybody who used previous state as changed
           this.state.invalidateTree("stat update error");
@@ -640,14 +709,15 @@ export class SynchronizedTreeState {
           // The new mirror holds nothing, so the old token would skip everything.
           this.discardDeltaToken("tree rebuilt after update error");
 
-          // scheduling state update without delay
-          continue;
-
           // unfortunately external observer may still see tree in its default
           // empty state, though this is best we can do in this exceptional
           // situation, and hope on caching layers inside computables to present
           // some stale state until we reconstruct the tree again
-        } else this.logger?.warn(e);
+        } else {
+          // Not an inconsistency: the ordinary cadence applies, not the rebuild backoff.
+          rebuildRetry = undefined;
+          this.logger?.warn(e);
+        }
       }
 
       if (!this.keepRunning || this.terminated) break;
@@ -665,10 +735,16 @@ export class SynchronizedTreeState {
 
       if (!this.keepRunning || this.terminated) break;
 
-      // Phase 2: optional remainder up to pollingInterval — interruptible by
-      // scheduleOnNextState so that an external nudge wakes the loop promptly.
+      // Phase 2: the interruptible remainder — up to pollingInterval, or, while rebuilding, up
+      // to the rebuild backoff, so a persistent update error cannot become a hot loop of full
+      // reads. The polling interval does not apply while rebuilding: readers see the empty
+      // rebuilt tree until the next read. A nudge (scheduleOnNextState) cuts either short; the
+      // floor above still bounds nudged reads.
       if (this.scheduledOnNextState.length === 0) {
-        const remaining = Math.max(0, this.effectivePollingInterval - MIN_POLLING_INTERVAL_MS);
+        const remaining = Math.max(
+          0,
+          (rebuildRetry?.nextDelay ?? this.effectivePollingInterval) - MIN_POLLING_INTERVAL_MS,
+        );
         if (remaining > 0) {
           try {
             this.currentLoopDelayInterrupt = new AbortController();
@@ -711,6 +787,12 @@ export class SynchronizedTreeState {
     this.terminated = true;
     this.abortController.abort();
 
+    // Refreshes still queued would otherwise never settle: the loop takes them only at the top
+    // of an iteration, and a terminated loop runs no further iteration.
+    const pending = this.scheduledOnNextState;
+    this.scheduledOnNextState = [];
+    for (const n of pending) n.reject(new Error("tree synchronization is terminated"));
+
     if (this.currentLoop === undefined) return;
     await this.currentLoop;