@remnic/core 9.69.39 → 9.69.41

package/src/orchestration/extraction-persist-promotion.test.ts

@@ -757,7 +757,229 @@ test("buildMergedTargetPromotionPayload and promotion refuse when interleaving w
   );
 });
 
-test("buildMergedTargetPromotionPayload: unavailable snapshot digest refuses when a committed digest exists (#2813 P1)", async () => {
+// ── #2870: access-only telemetry vs semantic metadata in the promotion binding ──
+
+type RawFileMutator = (raw: string) => string;
+
+const withFrontmatterLine = (line: string): RawFileMutator => (raw) =>
+  raw.replace("\n---\n", `\n${line}\n---\n`);
+
+/** Rewrite the durable file directly — a writer that changes semantics
+ * WITHOUT minting a CAS receipt (hand edit, legacy tool). Exactly the
+ * writer class the semantic-fingerprint clause must catch. */
+async function rewriteDurableWithoutReceipt(
+  storage: StorageManager,
+  memoryId: string,
+  mutate: RawFileMutator,
+): Promise<void> {
+  const row = await storage.getMemoryByIdIncludingArchived(memoryId);
+  assert.ok(row);
+  const before = await readFile(row.path, "utf8");
+  await writeFile(row.path, mutate(before), "utf8");
+}
+async function flushAccess(
+  storage: StorageManager,
+  memoryId: string,
+  newCount: number,
+): Promise<number> {
+  return await storage.flushAccessTracking([
+    { memoryId, newCount, lastAccessed: "2026-08-23T00:00:00.000Z" },
+  ]);
+}
+
+async function runPromotion(
+  s: { source: StorageManager; router: { storageFor: (namespace: string) => Promise<StorageManager> } },
+  payload: MergedTargetPromotionPayload | null,
+  sourceMemoryId: string,
+): Promise<number> {
+  let promotions = 0;
+  await promoteAndReconcileMergedTarget({
+    promote: async () => {
+      promotions += 1;
+      return `copy-${promotions}`;
+    },
+    config: parseConfig({ memoryDir: s.source.dir, sharedNamespace: "shared" }),
+    getStorageRouter: () => s.router,
+    scopeProfileWritePlan: null,
+    sourceStorage: s.source,
+    sourceMemoryId,
+    mergedPromotion: payload,
+    normalize: (content: string) => content,
+  });
+  return promotions;
+}
+
+test("access flush between merge commit and payload build still promotes (#2870)", async () => {
+  const s = await makeStorages();
+  const target = await s.source.writeMemory("fact", PRE_MERGE_BODY, {
+    source: "test",
+    confidence: 0.5,
+  });
+  await commitWriterMerge(s.source, target.id, A_MERGED_BODY);
+  const committed = await s.source.getMemoryByIdIncludingArchived(target.id);
+  assert.ok(committed);
+  const shardPath = casShardPath(s.source.dir, committed.path);
+  const receiptBefore = JSON.parse(await readFile(shardPath, "utf8")) as { revision: string };
+  const bytesBefore = await readFile(committed.path, "utf8");
+
+  assert.equal(await flushAccess(s.source, target.id, 3), 1, "the access flush landed");
+
+  // Prove the interleaving is real: bytes changed, receipt did not re-mint.
+  assert.notEqual(await readFile(committed.path, "utf8"), bytesBefore, "the flush rewrote the durable file");
+  const receiptAfter = JSON.parse(await readFile(shardPath, "utf8")) as { revision: string };
+  assert.equal(receiptAfter.revision, receiptBefore.revision, "an access flush mints no receipt");
+
+  const { payload, readFailed } = await buildMergedTargetPromotionPayload(s.source, {
+    targetId: target.id,
+    mergedContent: A_MERGED_BODY,
+    provenancePatched: true,
+  });
+  assert.equal(readFailed, false);
+  assert.ok(payload, "an access-only flush must not refuse the payload");
+  assert.ok(payload?.committedSemanticFingerprint, "the payload carries the semantic binding");
+
+  assert.equal(await runPromotion(s, payload, target.id), 1, "the flushed-but-unchanged semantics still promote");
+});
+
+test("access flush between payload build and promotion still promotes (#2870)", async () => {
+  const s = await makeStorages();
+  const target = await s.source.writeMemory("fact", PRE_MERGE_BODY, {
+    source: "test",
+    confidence: 0.5,
+  });
+  await commitWriterMerge(s.source, target.id, A_MERGED_BODY);
+  const { payload } = await buildMergedTargetPromotionPayload(s.source, {
+    targetId: target.id,
+    mergedContent: A_MERGED_BODY,
+    provenancePatched: true,
+  });
+  assert.ok(payload);
+  assert.equal(await flushAccess(s.source, target.id, 7), 1, "the access flush lands after the build");
+
+  assert.equal(await runPromotion(s, payload, target.id), 1, "the final guard confirms the flushed record's semantics");
+});
+
+test("payload build refuses a semantic mutation that minted no receipt (#2870)", async () => {
+  const s = await makeStorages();
+  const target = await s.source.writeMemory("fact", PRE_MERGE_BODY, {
+    source: "test",
+    confidence: 0.5,
+  });
+  await commitWriterMerge(s.source, target.id, A_MERGED_BODY);
+  await rewriteDurableWithoutReceipt(
+    s.source,
+    target.id,
+    (raw) => raw.replace(/^confidence: .*$/m, "confidence: 0.99"),
+  );
+  const { payload, readFailed } = await buildMergedTargetPromotionPayload(s.source, {
+    targetId: target.id,
+    mergedContent: A_MERGED_BODY,
+    provenancePatched: true,
+  });
+  assert.equal(payload, null, "a same-body confidence change must refuse the payload");
+  assert.equal(readFailed, true, "the refusal is an unknown outcome, not a no-promotion verdict");
+});
+
+test("final guard refuses each semantic field mutation that minted no receipt (#2870)", async () => {
+  const mutations: Array<[label: string, mutate: RawFileMutator]> = [
+    ["confidence", (raw) => raw.replace(/^confidence: .*$/m, "confidence: 0.99")],
+    ["tags", (raw) => raw.replace(/^tags: .*$/m, 'tags: ["mutated"]')],
+    ["status", withFrontmatterLine("status: superseded")],
+    ["provenance", withFrontmatterLine("provenance: unverified")],
+    ["tool scope", withFrontmatterLine("toolScoped: true")],
+    ["body", (raw) => `${raw}\nmutated body suffix`],
+  ];
+  for (const [label, mutate] of mutations) {
+    const s = await makeStorages();
+    const target = await s.source.writeMemory("fact", PRE_MERGE_BODY, {
+      source: "test",
+      confidence: 0.5,
+    });
+    await commitWriterMerge(s.source, target.id, A_MERGED_BODY);
+    const { payload } = await buildMergedTargetPromotionPayload(s.source, {
+      targetId: target.id,
+      mergedContent: A_MERGED_BODY,
+      provenancePatched: true,
+    });
+    assert.ok(payload, `payload builds before the ${label} mutation`);
+    await rewriteDurableWithoutReceipt(s.source, target.id, mutate);
+
+    assert.equal(
+      await runPromotion(s, payload, target.id),
+      0,
+      `a ${label} mutation without a receipt must refuse promotion`,
+    );
+  }
+});
+
+test("a PENDING receipt marker between build and promotion abandons it (#2870)", async () => {
+  const s = await makeStorages();
+  const target = await s.source.writeMemory("fact", PRE_MERGE_BODY, {
+    source: "test",
+    confidence: 0.5,
+  });
+  await commitWriterMerge(s.source, target.id, A_MERGED_BODY);
+  const { payload } = await buildMergedTargetPromotionPayload(s.source, {
+    targetId: target.id,
+    mergedContent: A_MERGED_BODY,
+    provenancePatched: true,
+  });
+  assert.ok(payload);
+  const committed = await s.source.getMemoryByIdIncludingArchived(target.id);
+  assert.ok(committed);
+  const shardPath = casShardPath(s.source.dir, committed.path);
+  const relative = path.relative(s.source.dir, committed.path).split(path.sep).join("/");
+  // A pending marker whose expected digest matches nothing: lazy recovery
+  // cannot decide it, so the receipt reads UNAVAILABLE and the promotion
+  // must refuse rather than confirm against an unknown identity.
+  await writeFile(
+    shardPath,
+    `${JSON.stringify({
+      version: 1,
+      path: relative,
+      revision: "2026-08-23T00:00:00.001Z",
+      state: "pending",
+      previous: payload.committedRevision ?? "2026-08-23T00:00:00.000Z",
+      baselineDigest: "0".repeat(64),
+      expectedDigest: "1".repeat(64),
+    })}\n`,
+    "utf8",
+  );
+
+  assert.equal(
+    await runPromotion(s, payload, target.id),
+    0,
+    "a pending (recovery-needed) receipt must not confirm the payload",
+  );
+});
+
+test("a legacy shard without a stored semantic fingerprint keeps the conservative digest refusal (#2870)", async () => {
+  const s = await makeStorages();
+  const target = await s.source.writeMemory("fact", PRE_MERGE_BODY, {
+    source: "test",
+    confidence: 0.5,
+  });
+  await commitWriterMerge(s.source, target.id, A_MERGED_BODY);
+  const committed = await s.source.getMemoryByIdIncludingArchived(target.id);
+  assert.ok(committed);
+  // Downgrade the shard to its pre-#2870 shape: committed digest, no
+  // semantic fingerprint.
+  const shardPath = casShardPath(s.source.dir, committed.path);
+  const shard = JSON.parse(await readFile(shardPath, "utf8")) as Record<string, unknown>;
+  delete shard.committedSemanticFingerprint;
+  await writeFile(shardPath, `${JSON.stringify(shard)}\n`, "utf8");
+
+  assert.equal(await flushAccess(s.source, target.id, 4), 1, "the access flush lands on the legacy shard");
+  const { payload, readFailed } = await buildMergedTargetPromotionPayload(s.source, {
+    targetId: target.id,
+    mergedContent: A_MERGED_BODY,
+    provenancePatched: true,
+  });
+  assert.equal(payload, null, "legacy shards keep the full-digest binding until the next semantic write");
+  assert.equal(readFailed, true);
+});
+
+test("legacy receipt with unavailable snapshot digest refuses promotion (#2813 P1, #2870)", async () => {
   const s = await makeStorages();
   const target = await s.source.writeMemory("fact", PRE_MERGE_BODY, { source: "test" });
   await commitWriterMerge(s.source, target.id, A_MERGED_BODY);
@@ -769,6 +991,10 @@ test("buildMergedTargetPromotionPayload: unavailable snapshot digest refuses whe
     status.status === "present" && status.committedDigest,
     "the standing receipt carries a committed digest",
   );
+  const shardPath = casShardPath(s.source.dir, committedRow.path);
+  const shard = JSON.parse(await readFile(shardPath, "utf8")) as Record<string, unknown>;
+  delete shard.committedSemanticFingerprint;
+  await writeFile(shardPath, `${JSON.stringify(shard)}\n`, "utf8");
 
   s.source.readDurableFileDigest = async () => null;
   const built = await buildMergedTargetPromotionPayload(s.source, {
@@ -776,7 +1002,7 @@ test("buildMergedTargetPromotionPayload: unavailable snapshot digest refuses whe
     mergedContent: A_MERGED_BODY,
     provenancePatched: true,
   });
-  assert.equal(built.payload, null, "an unreadable snapshot digest must refuse the promotion");
+  assert.equal(built.payload, null, "an unreadable snapshot digest must refuse the legacy receipt");
   assert.equal(built.readFailed, true, "unavailable snapshot is unknown, not an absent revision");
 
   const legacyStore = await makeStorages();

package/src/orchestration/extraction-persist-promotion.ts

@@ -22,7 +22,7 @@ import {
 } from "../capabilities.js";
 import type { MemoryFile, PluginConfig } from "../types.js";
 import type { ResolvedScopeProfilePlan } from "../namespaces/scope-profiles.js";
-import { readPromotionReceiptStatus, type MergedTargetPromotionPayload } from "./semantic-merge-promotion-payload.js";
+import { readDurableSemanticFingerprint, readPromotionReceiptStatus, type MergedTargetPromotionPayload } from "./semantic-merge-promotion-payload.js";
 
 export const confidenceTierOrder = [
   "explicit",
@@ -395,28 +395,31 @@ export async function promoteAndReconcileMergedTarget(args: {
       );
       return;
     }
-    // #2813 (P1 A, P1 B): confirm the cached payload against a TRUTHFUL receipt
-    // read and digest check. An unavailable sidecar or digest mismatch abandons the
-    // promotion — the previous undefined-vs-undefined comparison confirmed any
-    // payload whose receipt identity had become unreadable. A genuinely absent receipt
-    // still compares equal and promotes.
+    // #2813 (P1 A, P1 B) + #2870: confirm the cached payload against a
+    // TRUTHFUL receipt read and a SEMANTIC fingerprint check. An
+    // unavailable sidecar or revision mismatch abandons the promotion —
+    // the previous undefined-vs-undefined comparison confirmed any payload
+    // whose receipt identity had become unreadable. A genuinely absent
+    // receipt still compares equal and promotes. The file check binds
+    // SEMANTIC identity, not bytes: an access-tracking flush between build
+    // and promotion rewrites only `accessCount`/`lastAccessed` (mints no
+    // receipt, changes no semantic line), so it stays compatible and the
+    // promotion proceeds; any body/confidence/provenance/scope/tags/status
+    // change makes the fingerprint differ and refuses.
     const standingReceipt = await readPromotionReceiptStatus(args.sourceStorage, current.path);
-    const standingDigest =
-      typeof args.sourceStorage.readDurableFileDigest === "function"
-        ? await args.sourceStorage.readDurableFileDigest(current.path).catch(() => null)
-        : null;
+    const standingSemanticFingerprint = await readDurableSemanticFingerprint(current.path);
     if (
       !standingReceipt.available ||
       (standingReceipt.revision ?? undefined) !== (payload.committedRevision ?? undefined) ||
       (payload.committedDigest !== undefined &&
         standingReceipt.committedDigest !== undefined &&
         standingReceipt.committedDigest !== payload.committedDigest) ||
-      (payload.committedDigest !== undefined &&
-        standingDigest !== null &&
-        standingDigest !== payload.committedDigest)
+      (payload.committedSemanticFingerprint !== undefined &&
+        standingSemanticFingerprint !== null &&
+        standingSemanticFingerprint !== payload.committedSemanticFingerprint)
     ) {
       log.warn(
-        `persistExtraction: merged-target promotion abandoned for ${args.sourceMemoryId} — the committed record's CAS receipt or digest ${standingReceipt.available ? "does not match the cached payload's" : "is unreadable"}, so the cached body is not confirmed`,
+        `persistExtraction: merged-target promotion abandoned for ${args.sourceMemoryId} — the committed record's CAS receipt or semantic fingerprint ${standingReceipt.available ? "does not match the cached payload's" : "is unreadable"}, so the cached body is not confirmed`,
       );
       return;
     }

package/src/orchestration/extraction-persist.ts

@@ -2545,7 +2545,7 @@ export class ExtractionPersistCoordinator {
       const semanticMerge = await applySemanticMergeAtPersist(this.deps, {
         storage: targetStorage, content: fact.content, category: writeCategory, sources: fact.sources, sourceConnector: extractionSourceConnector,
         incomingMetadata: { tags: [...fact.tags, ...injectionScreenTags], entityRef: fact.entityRef, structuredAttributes: fact.structuredAttributes, validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt, biTemporal: biTemporal !== undefined, importanceScore: importance.score, confidence: fact.confidence, provenanceStrength: fact.provenance, toolScoped: factToolScoped, subject: factSubject, origin, faithfulness: faithfulnessFm, memoryKind }, skip: contradictionDetected || faithfulnessEnforceStatus === "pending_review" || batchBackendUnavailable || novelty.decision === "add",
-        incomingLinks: links.length > 0 ? links : undefined, incomingCitedContent: citedFactContent,
+        incomingLinks: links.length > 0 ? links : undefined, incomingCitedContent: citedFactContent, incomingCitationContext: citationContextBase,
         targetHasPromotedCopies: (targetId) => promotedCopyProbe.check(targetStorage, targetId),
       });
       if (semanticMerge.action === "created" && semanticMerge.reason === "backend_unavailable") batchBackendUnavailable = true; // arms the batch short circuit for the remaining facts

package/src/orchestration/orchestrator-helpers.test.ts

@@ -0,0 +1,104 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  qmdStartupCollectionCheckWithTimeout,
+  type SearchCollectionState,
+} from "./orchestrator-helpers.js";
+import {
+  qmdStartupCollectionChecksWithTimeout,
+  startupDiscoveryWithTimeout,
+} from "./startup-collection-checks.js";
+
+test("individual startup collection check aborts and returns unknown when it never settles", async (t) => {
+  t.mock.timers.enable({ apis: ["setTimeout", "Date"], now: 0 });
+  const originalTimeout = process.env.REMNIC_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS;
+  const controller = new AbortController();
+  const neverSettles = new Promise<SearchCollectionState>(() => undefined);
+
+  try {
+    process.env.REMNIC_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS = "1000";
+
+    const pending = qmdStartupCollectionCheckWithTimeout(
+      neverSettles,
+      controller,
+      "default",
+    );
+    t.mock.timers.tick(1000);
+    const state = await pending;
+
+    assert.equal(state, "unknown");
+    assert.equal(controller.signal.aborted, true);
+  } finally {
+    if (originalTimeout === undefined) delete process.env.REMNIC_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS;
+    else process.env.REMNIC_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS = originalTimeout;
+  }
+});
+
+test("startup collection batch falls back to configured namespaces when checks never settle", async () => {
+  const neverSettles = new Promise<{ namespace: string; state: SearchCollectionState }>(() => undefined);
+
+  const states = await qmdStartupCollectionChecksWithTimeout(
+    [neverSettles],
+    ["default", "configured"],
+    1,
+  );
+
+  assert.deepEqual(states, [
+    { namespace: "default", state: "unknown" },
+    { namespace: "configured", state: "unknown" },
+  ]);
+});
+
+test("startup collection batch preserves settled collection states", async () => {
+  const states = await qmdStartupCollectionChecksWithTimeout(
+    [Promise.resolve({ namespace: "catalog-only", state: "present" as const })],
+    ["default"],
+    50,
+  );
+
+  assert.deepEqual(states, [{ namespace: "catalog-only", state: "present" }]);
+});
+
+test("startup collection batch preserves settled states when other checks time out", async () => {
+  const neverSettles = new Promise<{ namespace: string; state: SearchCollectionState }>(() => undefined);
+  const states = await qmdStartupCollectionChecksWithTimeout(
+    [
+      Promise.resolve({ namespace: "default", state: "missing" as const }),
+      neverSettles,
+    ],
+    ["default", "dynamic"],
+    1,
+  );
+
+  assert.deepEqual(states, [
+    { namespace: "default", state: "missing" },
+    { namespace: "dynamic", state: "unknown" },
+  ]);
+});
+
+test("generic startup discovery falls back without assuming QMD namespaces", async () => {
+  const discovery = await startupDiscoveryWithTimeout(
+    () => new Promise<{ records: string[] }>(() => undefined),
+    { records: ["configured"] },
+    1,
+  );
+
+  assert.deepEqual(discovery, {
+    value: { records: ["configured"] },
+    complete: false,
+  });
+});
+
+test("generic startup discovery preserves a complete dynamic namespace set", async () => {
+  const discovery = await startupDiscoveryWithTimeout(
+    async () => ["default", "catalog-only"],
+    ["default"],
+    50,
+  );
+
+  assert.deepEqual(discovery, {
+    value: ["default", "catalog-only"],
+    complete: true,
+  });
+});