@remnic/core 9.69.38 → 9.69.40

package/src/recall-state-view.ts

@@ -8,6 +8,13 @@
  *
  * Change-intent morphology lives here (do not grow intent.ts). A superseded
  * memory is admitted only when its successor is also in the candidate set.
+ *
+ * #2859 — chain identities are namespace-qualified (`namespace\0id`): the
+ * same id in two namespaces never cross-anchors. The reverse chain derives
+ * from the successor's `supersedes` back-pointer when a predecessor lacks
+ * `supersededBy`, and asOf snapshot labels use the temporal validity
+ * boundary (predecessor `invalidAt` / successor `validAt`), not the
+ * `supersededAt` write time.
  */
 import { coerceBooleanLike } from "./connectors/coerce.js";
 
@@ -17,14 +24,24 @@ export interface StateViewChain {
   predecessorId: string;
   successorId: string;
   supersededAt?: string;
+  /** #2859 — namespace scope; a chain link never crosses namespaces. */
+  namespace?: string;
 }
 
 export interface StateViewResult {
   id?: string;
   docid?: string;
   status?: string;
+  /** #2859 — namespace that owns this row; qualifies every chain identity. */
+  namespace?: string;
+  /** #2859 — back-pointer: id of the row this one replaced. */
+  supersedes?: string;
   supersededBy?: string;
   supersededAt?: string;
+  /** #2859 — fact validity start (frontmatter `valid_at`). */
+  validAt?: string;
+  /** #2859 — fact validity end (frontmatter `invalid_at`). */
+  invalidAt?: string;
   stateLabel?: StateLabel;
 }
 
@@ -66,32 +83,94 @@ export function resultStateViewId(result: StateViewResult): string {
   return result.id ?? result.docid ?? "";
 }
 
-function buildSuccessorMap(results: readonly StateViewResult[], chains: readonly StateViewChain[]): Map<string, string> {
+/**
+ * #2859 — namespace-qualified chain identity. Namespace fanout can surface
+ * the same id from two namespaces; every successor/admission match runs on
+ * `namespace\0id` keys so a row in one namespace can never anchor (or be
+ * anchored by) an identically-named row in another.
+ */
+export function stateViewKey(namespace: string | undefined, id: string): string {
+  return `${typeof namespace === "string" ? namespace : ""}\u0000${id}`;
+}
+
+export function resultStateViewKey(result: StateViewResult): string {
+  const id = resultStateViewId(result);
+  return id ? stateViewKey(result.namespace, id) : "";
+}
+
+/** Inverse of {@link stateViewKey}: the id after the namespace separator. */
+function stateViewIdFromKey(key: string): string {
+  const sep = key.indexOf("\u0000");
+  return sep === -1 ? key : key.slice(sep + 1);
+}
+
+/**
+ * predecessorKey → successorKey, namespace-qualified. Precedence: explicit
+ * chains, then successor rows' `supersedes` back-pointers (#2859 — derives
+ * the reverse chain when the predecessor lacks `supersededBy`), then the
+ * predecessor's own `supersededBy` (authoritative when present).
+ */
+function buildSuccessorMap(
+  results: readonly StateViewResult[],
+  chains: readonly StateViewChain[],
+): Map<string, string> {
   const byPred = new Map<string, string>();
+  const link = (predKey: string, succKey: string, overwrite: boolean): void => {
+    if (!predKey || !succKey) return;
+    if (overwrite || !byPred.has(predKey)) byPred.set(predKey, succKey);
+  };
   for (const chain of chains) {
     if (chain.predecessorId && chain.successorId) {
-      byPred.set(chain.predecessorId, chain.successorId);
+      link(
+        stateViewKey(chain.namespace, chain.predecessorId),
+        stateViewKey(chain.namespace, chain.successorId),
+        false,
+      );
+    }
+  }
+  for (const result of results) {
+    const key = resultStateViewKey(result);
+    if (key && result.supersedes) {
+      link(stateViewKey(result.namespace, result.supersedes), key, false);
     }
   }
   for (const result of results) {
-    const id = resultStateViewId(result);
-    if (id && result.supersededBy) byPred.set(id, result.supersededBy);
+    const key = resultStateViewKey(result);
+    if (key && result.supersededBy) {
+      link(key, stateViewKey(result.namespace, result.supersededBy), true);
+    }
   }
   return byPred;
 }
 
+function successorLinkOf(
+  result: StateViewResult,
+  byPred: ReadonlyMap<string, string>,
+): { superseded: boolean; successorKey: string | undefined } {
+  const key = resultStateViewKey(result);
+  return {
+    superseded:
+      result.status === "superseded" ||
+      Boolean(result.supersededBy) ||
+      byPred.has(key),
+    successorKey: result.supersededBy
+      ? stateViewKey(result.namespace, result.supersededBy)
+      : byPred.get(key),
+  };
+}
+
 function labelFor(
-  id: string,
+  key: string,
   result: StateViewResult,
   byPred: Map<string, string>,
   admittedIds: ReadonlySet<string>,
   asOfMs: number | null,
 ): StateLabel {
-  const successorId = result.supersededBy ?? byPred.get(id);
-  const isPred = shouldWidenSuperseded(successorId, admittedIds);
+  const { successorKey } = successorLinkOf(result, byPred);
+  const isPred = shouldWidenSuperseded(successorKey, admittedIds);
   let isSucc = false;
   for (const [predId, succId] of byPred) {
-    if (succId === id && admittedIds.has(predId)) {
+    if (succId === key && admittedIds.has(predId)) {
       isSucc = true;
       break;
     }
@@ -101,29 +180,38 @@ function labelFor(
   if (isSucc) return "current";
   if (result.status === "superseded") {
     // #1952 asOf mode: the successor is legitimately absent under the
-    // historical pin. The row was the live fact at the snapshot when the
-    // recorded supersession postdates the pin ("current" relative to
-    // the snapshot); already replaced at the pin is "historical".
+    // historical pin (a linked successor takes the pair branches above).
+    // The row was the live fact at the snapshot when the temporal
+    // validity boundary postdates the pin ("current" relative to the
+    // snapshot); already replaced at the pin is "historical". #2859: the
+    // boundary is the fact's validity flip — `invalidAt` (the supersession
+    // pipeline sets it to the successor's valid_at) — not the
+    // `supersededAt` write time; legacy rows without a window fall back
+    // to supersededAt.
     return asOfMs === null ? "historical" : snapshotStateLabel(result, asOfMs);
   }
   return "current";
 }
 
 function snapshotStateLabel(result: StateViewResult, asOfMs: number): StateLabel {
-  const supersededAtMs = Date.parse(result.supersededAt ?? "");
-  return Number.isFinite(supersededAtMs) && supersededAtMs <= asOfMs
-    ? "historical"
-    : "current";
+  const invalidAtMs = Date.parse(result.invalidAt ?? "");
+  const boundaryMs = Number.isFinite(invalidAtMs)
+    ? invalidAtMs
+    : Date.parse(result.supersededAt ?? "");
+  return Number.isFinite(boundaryMs) && boundaryMs <= asOfMs ? "historical" : "current";
 }
 
 export function annotateStateView<T extends StateViewResult>(
   results: T[],
   query: string,
   chains: readonly StateViewChain[],
-  options: { enabled?: boolean; asOfMs?: number } = {},
+  options: { enabled?: boolean; asOfMs?: number; changeIntent?: boolean } = {},
 ): T[] {
   const enabled = options.enabled === true;
-  if (!enabled || !isChangeOrientedQuery(query)) return results;
+  // #2893 — changeIntent:true marks intent as already classified from the
+  // ORIGINAL prompt; `query` may be cron-normalized (intent signal
+  // truncated or stop-worded away) and must not be re-checked then.
+  if (!enabled || (options.changeIntent !== true && !isChangeOrientedQuery(query))) return results;
   // #1952 asOf mode: under a historical pin the pipeline's isValidAsOf
   // gate already decided validity at the pinned instant, so a predecessor
   // must not be discarded merely because its successor is absent under
@@ -136,26 +224,17 @@ export function annotateStateView<T extends StateViewResult>(
   const byPred = buildSuccessorMap(results, chains);
   const candidateIds = new Set<string>();
   for (const result of results) {
-    const id = resultStateViewId(result);
-    if (id) candidateIds.add(id);
+    const key = resultStateViewKey(result);
+    if (key) candidateIds.add(key);
   }
-
-  const isSupersededRow = (result: T): { superseded: boolean; successorId: string | undefined } => {
-    const id = resultStateViewId(result);
-    return {
-      superseded: result.status === "superseded" || Boolean(result.supersededBy) || byPred.has(id),
-      successorId: result.supersededBy ?? byPred.get(id),
-    };
-  };
-
   let admitted: T[];
   if (asOfMs !== null) {
     admitted = results.slice();
   } else {
     admitted = [];
     for (const result of results) {
-      const { superseded, successorId } = isSupersededRow(result);
-      if (!superseded || shouldWidenSuperseded(successorId, candidateIds)) {
+      const { superseded, successorKey } = successorLinkOf(result, byPred);
+      if (!superseded || shouldWidenSuperseded(successorKey, candidateIds)) {
         admitted.push(result);
       }
     }
@@ -163,24 +242,119 @@ export function annotateStateView<T extends StateViewResult>(
     // #1952 contract: superseded never appears without its successor. A
     // successor can itself be dropped (filtered elsewhere, or transitively
     // orphaned), so reconcile to a fixpoint instead of trusting the first pass.
-    let admittedIds = new Set(admitted.map(resultStateViewId).filter(Boolean));
+    let admittedIds = new Set(admitted.map(resultStateViewKey).filter(Boolean));
     for (let changed = true; changed; ) {
       changed = false;
       for (let i = admitted.length - 1; i >= 0; i -= 1) {
         const row = admitted[i]!;
-        const { superseded, successorId } = isSupersededRow(row);
-        if (superseded && !shouldWidenSuperseded(successorId, admittedIds)) {
+        const { superseded, successorKey } = successorLinkOf(row, byPred);
+        if (superseded && !shouldWidenSuperseded(successorKey, admittedIds)) {
           admitted.splice(i, 1);
           changed = true;
         }
       }
-      if (changed) admittedIds = new Set(admitted.map(resultStateViewId).filter(Boolean));
+      if (changed) admittedIds = new Set(admitted.map(resultStateViewKey).filter(Boolean));
     }
   }
 
-  const admittedIds = new Set(admitted.map(resultStateViewId).filter(Boolean));
+  const admittedIds = new Set(admitted.map(resultStateViewKey).filter(Boolean));
   return admitted.map((result) => {
-    const id = resultStateViewId(result);
-    return { ...result, stateLabel: labelFor(id, result, byPred, admittedIds, asOfMs) };
+    const key = resultStateViewKey(result);
+    const { successorKey } = successorLinkOf(result, byPred);
+    const stateLabel = labelFor(key, result, byPred, admittedIds, asOfMs);
+    if (result.supersededBy || !successorKey) return { ...result, stateLabel };
+    return { ...result, stateLabel, supersededBy: stateViewIdFromKey(successorKey) };
   });
 }
+
+/**
+ * #2859 — drop superseded rows whose successor is not in the pool BEFORE
+ * the user cap/MMR runs, so post-cap orphan removal can never underfill
+ * the result set. Reconciles to a fixpoint (a dropped successor orphans
+ * its own predecessors). Rows with no successor links pass through in
+ * order. ponytail: repeated full-pool passes are O(n²) on one long chain;
+ * recall pools stay small, memoize per-key if pools ever grow.
+ */
+export function reconcileStateViewPairs<T extends StateViewResult>(results: readonly T[]): T[] {
+  const byPred = buildSuccessorMap(results, []);
+  let kept = results.slice();
+  for (let changed = true; changed; ) {
+    changed = false;
+    const keptKeys = new Set(kept.map(resultStateViewKey).filter(Boolean));
+    const next: T[] = [];
+    for (const row of kept) {
+      const { superseded, successorKey } = successorLinkOf(row, byPred);
+      if (superseded && !shouldWidenSuperseded(successorKey, keptKeys)) {
+        changed = true;
+        continue;
+      }
+      next.push(row);
+    }
+    if (changed) kept = next;
+  }
+  return kept;
+}
+
+/**
+ * #2859 — packet-counting cap. A predecessor/successor chain admitted
+ * together is ONE complete evidence packet: it consumes a single slot of
+ * the user cap, and the slice never splits a packet at the boundary (no
+ * underfill from post-cap orphan removal). Rows keep their incoming
+ * (MMR) order; packets are ordered by first appearance. Call
+ * reconcileStateViewPairs first so orphans never consume a slot.
+ */
+export function capStateViewPackets<T extends StateViewResult>(
+  ordered: readonly T[],
+  limit: number,
+): T[] {
+  const safeLimit =
+    typeof limit === "number" && Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 0;
+  if (safeLimit === 0) return [];
+  if (ordered.length === 0) return [];
+
+  const byPred = buildSuccessorMap(ordered, []);
+  // Union-find over row keys; blank-key rows become per-index singletons.
+  const parent = new Map<string, string>();
+  const root = (key: string): string => {
+    let current = key;
+    for (;;) {
+      const up = parent.get(current);
+      if (up === undefined || up === current) {
+        parent.set(current, current);
+        return current;
+      }
+      current = up;
+    }
+  };
+  const union = (a: string, b: string): void => {
+    const ra = root(a);
+    const rb = root(b);
+    if (ra !== rb) parent.set(ra, rb);
+  };
+  const keys: string[] = [];
+  const keyToIndex = new Map<string, number>();
+  for (let i = 0; i < ordered.length; i += 1) {
+    // ponytail: blank-key rows must stay distinct singletons; index-keyed.
+    const key = resultStateViewKey(ordered[i]!) || `#idx${i}`;
+    keys.push(key);
+    if (!parent.has(key)) parent.set(key, key);
+    const first = keyToIndex.get(key);
+    if (first !== undefined) union(keys[first]!, key);
+    else keyToIndex.set(key, i);
+  }
+  for (const [predKey, succKey] of byPred) {
+    const predIdx = keyToIndex.get(predKey);
+    const succIdx = keyToIndex.get(succKey);
+    if (predIdx !== undefined && succIdx !== undefined) union(keys[predIdx]!, keys[succIdx]!);
+  }
+  // Packets ordered by first appearance; admit whole packets until the cap.
+  const selectedRoots = new Set<string>();
+  for (let i = 0; i < ordered.length && selectedRoots.size < safeLimit; i += 1) {
+    selectedRoots.add(root(keys[i]!));
+  }
+  const selected: T[] = [];
+  for (let i = 0; i < ordered.length; i += 1) {
+    if (selectedRoots.has(root(keys[i]!))) selected.push(ordered[i]!);
+  }
+  return selected;
+}

package/src/reconcile-identity-cache.test.ts

@@ -3,9 +3,15 @@ import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import test from "node:test";
+import * as fsp from "node:fs/promises";
 
 import { createHash } from "node:crypto";
-import { type ServerIdentityCacheEntry, planServerIdentityCacheWrite } from "./access-offline-manifest.js";
+import {
+  type ServerIdentityCacheEntry,
+  mergeServerIdentityCacheEntries,
+  planServerIdentityCacheWrite,
+  retainExcludedClassifications,
+} from "./access-offline-manifest.js";
 import { isInternalRemnicStatePath } from "./offline-sync.js";
 import { convergeIdentityCachePath } from "./reconcile/cursor.js";
 import {
@@ -375,4 +381,79 @@ test("a thrown memory-file read is not cached as a reusable negative", async ()
   assert.equal(reads, 1, "a failed read must not become a warm negative hit");
 });
 
+test("the write plan retains never-yielded excluded paths on a completed walk", () => {
+  const persisted = new Map<string, ServerIdentityCacheEntry>([
+    ["facts/gone.md", { path: "facts/gone.md", sha256: "aa" }],
+    ["facts/private.md", { path: "facts/private.md", sha256: "", statIdentity: "s", excluded: true }],
+  ]);
+  const plan = planServerIdentityCacheWrite({
+    persisted,
+    yieldedPaths: new Set([]),
+    retainedPaths: new Set(["facts/private.md"]),
+    streamCompleted: true,
+    cacheDirty: false,
+  });
+  assert.ok(plan.shouldWrite, "the vanished path still forces a write");
+  assert.deepEqual(
+    plan.entries.map((entry) => entry.path),
+    ["facts/private.md"],
+    "an excluded file's entry survives the deleted-path prune; a vanished path's does not"
+  );
+});
+
+test("excluded classifications persist for never-yielded paths without retaining unseen ones", () => {
+  const persisted = new Map<string, ServerIdentityCacheEntry>([
+    ["facts/deleted.md", { path: "facts/deleted.md", sha256: "aa" }],
+  ]);
+  const classifications = new Map([
+    ["facts/private.md", { statIdentity: "1:1:1:1:1", excluded: true }],
+    ["facts/yielded.md", { statIdentity: "2:2:2:2:2", excluded: false }],
+  ]);
+  const { retained, changed } = retainExcludedClassifications({
+    persisted,
+    classifications,
+    yieldedPaths: new Set(["facts/yielded.md"]),
+  });
+  assert.deepEqual([...retained], ["facts/private.md"]);
+  assert.equal(changed, true);
+  assert.equal(persisted.get("facts/private.md")?.excluded, true);
+  assert.equal(typeof persisted.get("facts/private.md")?.statIdentity, "string");
+  assert.ok(!retained.has("facts/deleted.md"), "a path the walk never saw must not be retained");
+});
+
+test("a concurrent server merge preserves the other writer's classification", async () => {
+  const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "server-identity-merge-"));
+  try {
+    const cachePath = path.join(dir, "identity.json");
+    await fs.promises.writeFile(
+      cachePath,
+      JSON.stringify({
+        citationTemplate: citationTemplateFingerprint(undefined),
+        files: [
+          { path: "facts/a.md", sha256: "aa", statIdentity: "1:2:3:4:5", excluded: false },
+          { path: "facts/dropped.md", sha256: "dd", statIdentity: "9:9:9:9:9", excluded: true },
+        ],
+      })
+    );
+    const merged = await mergeServerIdentityCacheEntries(
+      fsp,
+      cachePath,
+      [{ path: "facts/b.md", sha256: "bb" }],
+      undefined,
+      new Set(["facts/dropped.md"])
+    );
+    const byPath = new Map(merged.map((entry) => [entry.path, entry]));
+    assert.equal(
+      byPath.get("facts/a.md")?.statIdentity,
+      "1:2:3:4:5",
+      "the other writer's classification survives the merge"
+    );
+    assert.equal(byPath.get("facts/a.md")?.excluded, false);
+    assert.ok(byPath.has("facts/b.md"), "this writer's entry survives too");
+    assert.equal(byPath.has("facts/dropped.md"), false, "a dropped path is still not resurrected");
+  } finally {
+    await fs.promises.rm(dir, { recursive: true, force: true });
+  }
+});
+
 

package/src/search/port.ts

@@ -91,7 +91,7 @@ export interface SearchBackendStatus {
  */
 export interface SearchBackend {
   // ── Lifecycle ──
-  probe(): Promise<boolean>;
+  probe(execution?: SearchExecutionOptions): Promise<boolean>;
   /**
    * Optional non-mutating availability probe for health/readiness checks.
    * Implementations must avoid auto-upgrades, collection creation, daemon

package/src/storage/cas-revision-store.test.ts

@@ -226,6 +226,7 @@ test("an evidenced gated recovery defers a reserve-only marker to the path-locke
       status: "present",
       revision: b.pendingRevision,
       committedDigest: store.digestContent("landed bytes"),
+      committedSemanticFingerprint: store.digestContent("landed bytes"),
     });
   });
 });
@@ -258,6 +259,7 @@ test("recoverPendingRevision publishes a crashed reservation whose write landed
       status: "present",
       revision: b.pendingRevision,
       committedDigest: store.digestContent("new durable bytes"),
+      committedSemanticFingerprint: store.digestContent("new durable bytes"),
     });
     assert.equal(await store.recoverPendingRevision(target), "committed", "idempotent");
   });
@@ -361,6 +363,7 @@ test("beginRevisionTransaction with expectedContent persists expectedDigest in p
       status: "present",
       revision: txn2.pendingRevision,
       committedDigest: store.digestContent(expectedContent),
+      committedSemanticFingerprint: store.digestContent(expectedContent),
     });
   });
 });
@@ -425,6 +428,7 @@ test("legacy pending shards without writeLanded treat expectedDigest as already
       status: "present",
       revision: "2026-08-01T00:00:00.000Z",
       committedDigest: expectedDigest,
+      committedSemanticFingerprint: expectedDigest,
     });
   });
 });