@remnic/core 9.69.89 → 9.69.90

package/src/recall-retrieval-failure.test.ts

@@ -13,18 +13,20 @@ import { mkdtemp, rm } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import test from "node:test";
+import { createRecallCorpusVersionCapture } from "./access-recall-corpus-versions.js";
 
 import { EngramMcpServer } from "./access-mcp.js";
 import { EngramAccessService } from "./access-service.js";
 import type { EngramAccessRecallRequest, EngramAccessRecallResponse } from "./access-service.js";
 import { parseConfig } from "./config.js";
 import { Orchestrator } from "./orchestrator.js";
+import { resolveScopePlan } from "./scopes/scope-plan.js";
 import {
   MEMORY_CONTEXT_UNAVAILABLE_NOTE,
   type RecallContextComposition,
 } from "./recall-context-composition.js";
 import type { SearchBackend, SearchDegradation, SearchExecutionOptions } from "./search/port.js";
-import { DEFAULT_RECALL_DISCLOSURE } from "./types.js";
+import { DEFAULT_RECALL_DISCLOSURE, type PluginConfig } from "./types.js";
 
 const TIMEOUT_DEGRADATION: SearchDegradation = {
   backend: "qmd",
@@ -34,10 +36,15 @@ const TIMEOUT_DEGRADATION: SearchDegradation = {
 
 const QUERY = "what writing rules apply here?";
 
-function searchBackend(observed: { calls: number }, degrade: boolean): SearchBackend {
-  const search = (execution?: SearchExecutionOptions) => {
+function searchBackend(
+  observed: { calls: number },
+  degrade: boolean,
+  onSearch?: () => Promise<void>,
+): SearchBackend {
+  const search = async (execution?: SearchExecutionOptions) => {
     observed.calls += 1;
     if (degrade) execution?.onDegradation?.(TIMEOUT_DEGRADATION);
+    await onSearch?.();
     return [];
   };
   const backend: Partial<SearchBackend> = {
@@ -80,6 +87,7 @@ async function withOrchestrator(
   prefix: string,
   degrade: boolean,
   run: (orchestrator: Orchestrator, observed: { calls: number }) => Promise<void>,
+  configOverrides: Partial<PluginConfig> = {},
 ): Promise<void> {
   const memoryDir = await mkdtemp(path.join(os.tmpdir(), prefix));
   const orchestrator = new Orchestrator(
@@ -88,6 +96,7 @@ async function withOrchestrator(
       workspaceDir: memoryDir,
       qmdEnabled: true,
       embeddingFallbackEnabled: false,
+      ...configOverrides,
     }),
   );
   const observed = { calls: 0 };
@@ -101,7 +110,7 @@ async function withOrchestrator(
     await run(orchestrator, observed);
   } finally {
     await orchestrator.destroy();
-    await rm(memoryDir, { recursive: true, force: true });
+    await rm(memoryDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   }
 }
 
@@ -137,6 +146,443 @@ test("genuine empty retrieval stays marker-free at the orchestrator", async () =
   });
 });
 
+test("recall reports storage corpus versions captured before retrieval, not index application", async () => {
+  await withOrchestrator("remnic-recall-version-", false, async (orchestrator, observed) => {
+    const storage = await orchestrator.getStorage("default");
+    const beforeRetrieval = storage.getMemoryCorpusVersion();
+    let writeDuringSearch = true;
+    let indexUpdates = 0;
+    const backend = searchBackend(observed, false, async () => {
+      if (!writeDuringSearch) return;
+      writeDuringSearch = false;
+      await storage.writeMemory("fact", "write committed while retrieval was in flight");
+    });
+    const withBackend = orchestrator as unknown as { qmd: SearchBackend };
+    withBackend.qmd = {
+      ...backend,
+      async updateCollection() { indexUpdates += 1; },
+      async updateCollectionStrict() { indexUpdates += 1; },
+    };
+
+    const response = await new EngramAccessService(orchestrator).recall({
+      query: QUERY,
+      sessionKey: "corpus-version",
+    });
+
+    assert.ok(observed.calls > 0, "recall must consult the backend");
+    assert.ok(storage.getMemoryCorpusVersion() > beforeRetrieval, "the in-flight write advanced storage");
+    assert.deepEqual(response.storageCorpusVersionsAtRecallStart, [{ namespace: "default", version: beforeRetrieval }]);
+    assert.equal(indexUpdates, 0, "a corpus sentinel does not prove QMD index application");
+  });
+});
+
+test("includeRecall X-ray responses report storage versions sampled before retrieval", async () => {
+  await withOrchestrator("remnic-xray-recall-version-", false, async (orchestrator, observed) => {
+    const storage = await orchestrator.getStorage("default");
+    const beforeRetrieval = storage.getMemoryCorpusVersion();
+    let writeDuringSearch = true;
+    const backend = searchBackend(observed, false, async () => {
+      if (!writeDuringSearch) return;
+      writeDuringSearch = false;
+      await storage.writeMemory("fact", "write committed while X-ray retrieval was in flight");
+    });
+    (orchestrator as unknown as { qmd: SearchBackend }).qmd = backend;
+
+    const response = await new EngramAccessService(orchestrator).recallXray({
+      query: QUERY,
+      sessionKey: "xray-corpus-version",
+      includeRecall: true,
+    });
+
+    assert.ok(observed.calls > 0);
+    assert.ok(response.snapshotFound);
+    assert.ok(storage.getMemoryCorpusVersion() > beforeRetrieval);
+    assert.deepEqual(response.recall?.storageCorpusVersionsAtRecallStart, [
+      { namespace: "default", version: beforeRetrieval },
+    ]);
+  });
+});
+test("no-recall X-ray includes corpus versions read by the standing-memory block", async () => {
+  await withOrchestrator(
+    "remnic-xray-standing-version-",
+    false,
+    async (orchestrator) => {
+      const storage = await orchestrator.getStorage("default");
+      const versionBeforeRead = storage.getMemoryCorpusVersion();
+      const response = await new EngramAccessService(orchestrator).recallXray({
+        query: "thanks",
+        sessionKey: "xray-standing-no-recall",
+        mode: "no_recall",
+        includeRecall: true,
+      });
+
+      assert.ok(response.snapshotFound);
+      assert.deepEqual(response.recall?.storageCorpusVersionsAtRecallStart, [
+        { namespace: "default", version: versionBeforeRead },
+      ]);
+    },
+    { recallStandingBlock: true },
+  );
+});
+
+test("standing-memory corpus version uses configured default namespace", async () => {
+  await withOrchestrator(
+    "remnic-standing-custom-namespace-",
+    false,
+    async (orchestrator) => {
+      const storage = await orchestrator.getStorage("personal");
+      const version = storage.getMemoryCorpusVersion();
+      const response = await new EngramAccessService(orchestrator).recall({
+        query: "thanks",
+        sessionKey: "standing-custom-namespace",
+        mode: "no_recall",
+      });
+
+      assert.deepEqual(response.storageCorpusVersionsAtRecallStart, [
+        { namespace: "personal", version },
+      ]);
+    },
+    { recallStandingBlock: true, defaultNamespace: "personal" },
+  );
+});
+
+test("recall reports a separate storage corpus version for every searched namespace", async () => {
+  await withOrchestrator(
+    "remnic-recall-multi-version-",
+    false,
+    async (orchestrator) => {
+      const primary = await orchestrator.getStorage("default");
+      const shared = await orchestrator.getStorage("shared");
+      const response = await new EngramAccessService(orchestrator).recall({
+        query: QUERY,
+        sessionKey: "multi-corpus-version",
+        authenticatedPrincipal: "alice",
+      });
+      assert.deepEqual(response.storageCorpusVersionsAtRecallStart, [
+        { namespace: "default", version: primary.getMemoryCorpusVersion() },
+        { namespace: "shared", version: shared.getMemoryCorpusVersion() },
+      ]);
+    },
+    { namespacesEnabled: true, defaultRecallNamespaces: ["self", "shared"] },
+  );
+});
+
+test("cold-start recall samples corpus versions only after initialization", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-recall-cold-start-"));
+  const orchestrator = new Orchestrator(parseConfig({
+    memoryDir,
+    workspaceDir: memoryDir,
+    qmdEnabled: true,
+    embeddingFallbackEnabled: false,
+  }));
+  const observed = { calls: 0 };
+  const withBackend = orchestrator as unknown as { qmd: SearchBackend };
+  withBackend.qmd = searchBackend(observed, false);
+  let releaseInitialization!: () => void;
+  let notifyInitializationStarted!: () => void;
+  const initializationGate = new Promise<void>((resolve) => { releaseInitialization = resolve; });
+  const initializationStarted = new Promise<void>((resolve) => { notifyInitializationStarted = resolve; });
+  const initialize = orchestrator.initialize.bind(orchestrator);
+  orchestrator.initialize = async () => {
+    notifyInitializationStarted();
+    await initializationGate;
+    await initialize();
+  };
+  let initialization: Promise<void> | undefined;
+  try {
+    initialization = orchestrator.initialize();
+    await initializationStarted;
+    const recall = new EngramAccessService(orchestrator).recall({
+      query: QUERY,
+      sessionKey: "cold-start-corpus-version",
+    });
+    releaseInitialization();
+    await initialization;
+    const response = await recall;
+    assert.ok(observed.calls > 0);
+    const storage = await orchestrator.getStorage("default");
+    assert.ok(storage.getMemoryCorpusVersion() > 0);
+    assert.deepEqual(response.storageCorpusVersionsAtRecallStart, [
+      { namespace: "default", version: storage.getMemoryCorpusVersion() },
+    ]);
+  } finally {
+    releaseInitialization();
+    await initialization?.catch(() => undefined);
+    await orchestrator.destroy();
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+test("fail-open recall still reports versions when initialization times out", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-recall-init-timeout-"));
+  const orchestrator = new Orchestrator(parseConfig({
+    memoryDir,
+    workspaceDir: memoryDir,
+    qmdEnabled: true,
+    embeddingFallbackEnabled: false,
+    initGateTimeoutMs: 1,
+  }));
+  const observed = { calls: 0 };
+  (orchestrator as unknown as { qmd: SearchBackend }).qmd = searchBackend(observed, false);
+  let releaseInitialization!: () => void;
+  const initializationGate = new Promise<void>((resolve) => { releaseInitialization = resolve; });
+  const initialize = orchestrator.initialize.bind(orchestrator);
+  orchestrator.initialize = async () => {
+    await initializationGate;
+    await initialize();
+  };
+  let initialization: Promise<void> | undefined;
+  try {
+    initialization = orchestrator.initialize();
+    const response = await new EngramAccessService(orchestrator).recall({
+      query: QUERY,
+      sessionKey: "init-timeout-corpus-version",
+    });
+    assert.ok(observed.calls > 0);
+    assert.deepEqual(response.storageCorpusVersionsAtRecallStart?.map(({ namespace }) => namespace), ["default"]);
+  } finally {
+    releaseInitialization();
+    await initialization?.catch(() => undefined);
+    await orchestrator.destroy();
+    await rm(memoryDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  }
+});
+
+test("no_recall does not sample corpus versions for namespaces it did not search", async () => {
+  await withOrchestrator("remnic-recall-no-recall-version-", false, async (orchestrator) => {
+    const response = await new EngramAccessService(orchestrator).recall({
+      query: "thanks",
+      sessionKey: "no-recall-corpus-version",
+      mode: "no_recall",
+    });
+    assert.deepEqual(response.storageCorpusVersionsAtRecallStart, []);
+    const autoPlanned = await new EngramAccessService(orchestrator).recall({
+      query: "thanks",
+      sessionKey: "auto-no-recall-corpus-version",
+    });
+    assert.deepEqual(autoPlanned.storageCorpusVersionsAtRecallStart, []);
+  }, { recallStandingBlock: false });
+});
+
+test("no_recall reports the corpus version read by the standing-memory block", async () => {
+  await withOrchestrator(
+    "remnic-no-recall-standing-version-",
+    false,
+    async (orchestrator) => {
+      const storage = await orchestrator.getStorage("default");
+      const versionBeforeRead = storage.getMemoryCorpusVersion();
+      const manager = orchestrator.storage;
+      const readAllMemories = manager.readAllMemories.bind(manager);
+      manager.readAllMemories = async (...args) => {
+        await storage.writeMemory("fact", "write during no-recall standing-memory read");
+        return readAllMemories(...args);
+      };
+
+      const response = await new EngramAccessService(orchestrator).recall({
+        query: "thanks",
+        sessionKey: "no-recall-standing-version",
+        mode: "no_recall",
+      });
+
+      assert.ok(storage.getMemoryCorpusVersion() > versionBeforeRead);
+      assert.deepEqual(response.storageCorpusVersionsAtRecallStart, [
+        { namespace: "default", version: versionBeforeRead },
+      ]);
+    },
+    { recallStandingBlock: true },
+  );
+});
+
+test("recall captures corpus versions before reading the standing-memory block", async () => {
+  await withOrchestrator(
+    "remnic-recall-standing-version-",
+    false,
+    async (orchestrator) => {
+      const storage = await orchestrator.getStorage("default");
+      const versionBeforeRecall = storage.getMemoryCorpusVersion();
+      const manager = orchestrator.storage;
+      const readAllMemories = manager.readAllMemories.bind(manager);
+      let standingReadCount = 0;
+      manager.readAllMemories = async (...args) => {
+        standingReadCount += 1;
+        if (standingReadCount === 1) {
+          await storage.writeMemory("fact", "write during standing-memory read");
+        }
+        return readAllMemories(...args);
+      };
+
+      const response = await new EngramAccessService(orchestrator).recall({
+        query: QUERY,
+        sessionKey: "standing-corpus-version",
+      });
+
+      assert.ok(standingReadCount >= 1);
+      assert.ok(storage.getMemoryCorpusVersion() > versionBeforeRecall);
+      assert.deepEqual(response.storageCorpusVersionsAtRecallStart, [
+        { namespace: "default", version: versionBeforeRecall },
+      ]);
+    },
+    { recallStandingBlock: true },
+  );
+});
+test("standing-memory fallback survives recall planning failure with its captured version", async () => {
+  await withOrchestrator(
+    "remnic-standing-planning-failure-",
+    false,
+    async (orchestrator) => {
+      const storage = await orchestrator.getStorage("default");
+      await storage.writeMemory("fact", "standing fallback survives planning failure");
+      const versionBeforeRead = storage.getMemoryCorpusVersion();
+      const manager = orchestrator.storage;
+      const readAllMemories = manager.readAllMemories.bind(manager);
+      manager.readAllMemories = async () => {
+        await storage.writeMemory("fact", "write during failing-recall standing read");
+        return [{
+          id: "standing-fallback",
+          content: "Standing fallback memory survives planning failure.",
+          frontmatter: { id: "standing-fallback", origin: "user", status: "active", pinned: true },
+        }] as unknown as Awaited<ReturnType<typeof readAllMemories>>;
+      };
+      (orchestrator as unknown as { recallInternal: () => Promise<string> }).recallInternal = async () => {
+        throw new Error("synthetic planning failure");
+      };
+
+      const response = await new EngramAccessService(orchestrator).recall({
+        query: QUERY,
+        sessionKey: "standing-planning-failure",
+      });
+
+      assert.match(response.context, /Standing fallback memory survives planning failure/);
+      assert.ok(storage.getMemoryCorpusVersion() > versionBeforeRead);
+      assert.deepEqual(response.storageCorpusVersionsAtRecallStart, [
+        { namespace: "default", version: versionBeforeRead },
+      ]);
+    },
+    { recallStandingBlock: true },
+  );
+});
+
+test("a throwing plan callback cannot escape recall or skip the standing block", async () => {
+  await withOrchestrator(
+    "remnic-throwing-plan-callback-",
+    false,
+    async (orchestrator) => {
+      const manager = orchestrator.storage;
+      manager.readAllMemories = async () => [{
+        id: "callback-standing",
+        content: "Standing memory survives a plan callback failure.",
+        frontmatter: { id: "callback-standing", origin: "user", status: "active", pinned: true },
+      }] as unknown as Awaited<ReturnType<typeof manager.readAllMemories>>;
+      let threw = false;
+      const response = await orchestrator.recall(QUERY, "throwing-plan-callback", {
+        onRecallPlanResolved: async (mode) => {
+          if (mode === "no_recall" && !threw) {
+            threw = true;
+            throw new Error("synthetic pre-plan callback failure");
+          }
+        },
+      });
+
+      assert.ok(threw);
+      assert.ok(response.includes("## Standing Memory (Remnic)"));
+      assert.ok(response.includes("Standing memory survives a plan callback failure."));
+      assert.ok(!response.includes("Memory context unavailable"));
+    },
+    { recallStandingBlock: true },
+  );
+});
+
+test("corpus-version capture is failure-open for unavailable secondary storage", async () => {
+  let captured: Array<{ namespace: string; version: number | null }> = [];
+  const capture = createRecallCorpusVersionCapture(
+    {
+      async getStorage(namespace) {
+        if (namespace === "unavailable") throw new Error("store unavailable");
+        return { getMemoryCorpusVersion: () => 7 };
+      },
+    },
+    undefined,
+    (versions) => { captured = versions; },
+    true,
+  );
+
+  await capture("no_recall", ["default", "unavailable"]);
+  assert.deepEqual(captured, [
+    { namespace: "default", version: 7 },
+    { namespace: "unavailable", version: null },
+  ]);
+});
+test("stalled corpus-version storage cannot block recall planning", async () => {
+  let captured: Array<{ namespace: string; version: number | null }> = [];
+  const capture = createRecallCorpusVersionCapture(
+    { getStorage: () => new Promise<never>(() => {}) },
+    undefined,
+    (versions) => { captured = versions; },
+    true,
+    20,
+  );
+  const result = await Promise.race([
+    Promise.resolve(capture("no_recall", ["stalled"])).then(() => "captured"),
+    new Promise<string>((resolve) => setTimeout(() => resolve("deadline"), 100)),
+  ]);
+  assert.equal(result, "captured");
+  assert.deepEqual(captured, [{ namespace: "stalled", version: null }]);
+});
+
+
+test("recall reports the same coding namespaces selected by its scope plan", async () => {
+  await withOrchestrator(
+    "remnic-recall-coding-version-",
+    false,
+    async (orchestrator) => {
+      const sessionKey = "coding-version";
+      orchestrator.setCodingContextForSession(sessionKey, {
+        projectId: "origin:acme/repo",
+        branch: "main",
+        rootPath: "/workspace/repo",
+        defaultBranch: "main",
+      });
+      const scopePlan = resolveScopePlan({
+        config: orchestrator.config,
+        sessionKey,
+        codingContext: orchestrator.getCodingContextForSession(sessionKey),
+        namespacesEnabled: true,
+      });
+      const originalRecall = orchestrator.recall.bind(orchestrator);
+      orchestrator.recall = async (...args) => {
+        orchestrator.setCodingContextForSession(sessionKey, {
+          projectId: "origin:acme/new-repo",
+          branch: "main",
+          rootPath: "/workspace/new-repo",
+          defaultBranch: "main",
+        });
+        return originalRecall(...args);
+      };
+      const actualScopePlan = resolveScopePlan({
+        config: orchestrator.config,
+        sessionKey,
+        codingContext: {
+          projectId: "origin:acme/new-repo",
+          branch: "main",
+          rootPath: "/workspace/new-repo",
+          defaultBranch: "main",
+        },
+        namespacesEnabled: true,
+      });
+      const response = await new EngramAccessService(orchestrator).recall({
+        query: QUERY,
+        sessionKey,
+        authenticatedPrincipal: "alice",
+      });
+
+      assert.deepEqual(
+        response.storageCorpusVersionsAtRecallStart?.map(({ namespace }) => namespace),
+        actualScopePlan.readNamespaces,
+      );
+    },
+    { namespacesEnabled: true, codingMode: { projectScope: true, branchScope: false, globalFallback: true } },
+  );
+});
+
 test("access recall surfaces retrievalFailure on daemon timeout and omits it on genuine empty", async () => {
   await withOrchestrator("remnic-recall-access-timeout-", true, async (timedOut, observed) => {
     const failed = await new EngramAccessService(timedOut).recall({
@@ -192,6 +638,7 @@ test("MCP recall payload keeps retrievalFailure so a tool caller can branch on i
     fallbackUsed: false,
     sourcesUsed: [],
     disclosure: DEFAULT_RECALL_DISCLOSURE,
+    storageCorpusVersionsAtRecallStart: [{ namespace: "default", version: 1 }],
     retrievalFailure: {
       reason: "backend_unavailable",
       detail: "qmd:daemon_timeout (no response within the deadline)",
@@ -207,6 +654,7 @@ test("MCP recall payload keeps retrievalFailure so a tool caller can branch on i
     fallbackUsed: false,
     sourcesUsed: [],
     disclosure: DEFAULT_RECALL_DISCLOSURE,
+    storageCorpusVersionsAtRecallStart: [{ namespace: "default", version: 1 }],
   };
 
   async function callRecall(response: EngramAccessRecallResponse): Promise<EngramAccessRecallResponse> {
@@ -233,6 +681,10 @@ test("MCP recall payload keeps retrievalFailure so a tool caller can branch on i
     assert.ok(parsed && typeof parsed === "object");
     const parsedFailure = "retrievalFailure" in parsed ? parsed.retrievalFailure : undefined;
     assert.deepEqual(parsedFailure, structured.retrievalFailure);
+    assert.deepEqual(
+      "storageCorpusVersionsAtRecallStart" in parsed ? parsed.storageCorpusVersionsAtRecallStart : undefined,
+      structured.storageCorpusVersionsAtRecallStart,
+    );
     return structured;
   }