@remnic/core 9.69.56 → 9.69.58

package/src/access-recall-diagnostics.ts

@@ -0,0 +1,236 @@
+/**
+ * Surface glue for the two recall-diagnostics reads (issue #3033).
+ *
+ * `access-http.ts` is at its file-size ratchet ceiling, so the route bodies
+ * live here and the router keeps only a thin dispatch line each (AGENTS.md:
+ * extract the addition to a sibling module). The pre-existing X-ray route
+ * body moved here unchanged for the same reason and because the two reads
+ * belong together — `xray` explains the results a recall returned, `why`
+ * explains the ones it did not.
+ *
+ * Both helpers return `{ status, body }` rather than writing to the response
+ * so they stay unit-testable, matching `handleWhoKnowsHttpQuery`.
+ */
+
+import { EngramAccessInputError } from "./access-errors.js";
+import type { McpTool } from "./access-mcp.js";
+import { RecallWhyInputError } from "./recall-why.js";
+import type { RecallWhyRequest, RecallWhyResponse } from "./recall-why-service.js";
+import { isRecallDisclosure, type RecallDisclosure } from "./types.js";
+
+export interface RecallDiagnosticsHttpOutcome {
+  status: number;
+  body: unknown;
+}
+
+/**
+ * `remnic.recall_why` / `engram.recall_why`. `withToolAliases` in
+ * `access-mcp.ts` emits the canonical `remnic.` alias automatically
+ * (dual-naming invariant for every new MCP tool).
+ */
+export const RECALL_WHY_MCP_TOOLS: McpTool[] = [
+  {
+    name: "engram.recall_why",
+    description:
+      "Diagnose why a query did NOT surface an expected memory (issue #3033). Replays the real recall once and attributes the outcome to pipeline stages (retrieval -> policy filters -> rerank -> cap -> format) with per-drop reasons. Pass `expect` with a memory id or substring to get the exact stage that dropped it plus a remediation hint. Read-only; never mutates memory, and never gated behind recallDirectAnswerEnabled. A search-backend outage reports backend_unavailable rather than an empty pipeline.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        query: {
+          type: "string",
+          description: "The prompt that failed to recall what you expected. Required; non-empty.",
+        },
+        expect: {
+          type: "string",
+          description:
+            "Memory id, or a substring of an id or path, to trace through every stage. Omit for a whole-pipeline report.",
+        },
+        sessionKey: {
+          type: "string",
+          description: "Optional session key to scope the diagnosis.",
+        },
+        namespace: {
+          type: "string",
+          description:
+            "Optional namespace. Enforced against the caller's principal; a denial yields reportFound:false.",
+        },
+      },
+      required: ["query"],
+      additionalProperties: false,
+    },
+  },
+];
+
+/**
+ * `GET /engram/v1/recall/why` (and the `/remnic/v1/...` spelling).
+ *
+ * Invalid input is a 400 rather than a silent default (Review Prevention
+ * Checklist #1 / #39). Backend faults are NOT mapped here: an outage is
+ * reported inside the 200 body as `backend_unavailable`, and a genuine
+ * server-side fault rethrows so the router returns 500 and logs it.
+ */
+export async function handleRecallWhyHttpQuery(deps: {
+  getParam: (name: string) => string | null;
+  resolveNamespace: (namespace: string | undefined) => string | undefined;
+  principal?: string;
+  connector?: string;
+  run: (request: RecallWhyRequest) => Promise<RecallWhyResponse>;
+}): Promise<RecallDiagnosticsHttpOutcome> {
+  const queryParam = deps.getParam("q");
+  if (queryParam === null || queryParam.trim().length === 0) {
+    return {
+      status: 400,
+      body: {
+        error: "missing_query",
+        code: "missing_query",
+        message: "q search parameter is required and must be non-empty",
+      },
+    };
+  }
+  // `expect` is optional, but an explicitly-supplied blank value is a caller
+  // mistake, not a request for the whole-pipeline report.
+  const expectParam = deps.getParam("expect");
+  if (expectParam !== null && expectParam.trim().length === 0) {
+    return {
+      status: 400,
+      body: {
+        error: "invalid_expect",
+        code: "invalid_expect",
+        message: "expect must be non-empty when supplied",
+      },
+    };
+  }
+  const sessionParam = deps.getParam("session");
+  const namespaceParam = deps.getParam("namespace");
+  // Always resolve (and allow-list check) the effective namespace, even when
+  // the param is omitted — the implicit default namespace is still a
+  // namespace every other GET route gates the same way.
+  const namespace = deps.resolveNamespace(
+    namespaceParam !== null && namespaceParam.length > 0 ? namespaceParam : undefined
+  );
+  try {
+    return {
+      status: 200,
+      body: await deps.run({
+        query: queryParam,
+        ...(expectParam !== null ? { expect: expectParam } : {}),
+        ...(sessionParam !== null && sessionParam.length > 0 ? { sessionKey: sessionParam } : {}),
+        ...(namespace !== undefined ? { namespace } : {}),
+        ...(deps.principal ? { authenticatedPrincipal: deps.principal } : {}),
+        ...(deps.connector ? { sourceConnector: deps.connector } : {}),
+      }),
+    };
+  } catch (err) {
+    if (err instanceof RecallWhyInputError || err instanceof EngramAccessInputError) {
+      return {
+        status: 400,
+        body: { error: "invalid_request", code: "invalid_request", message: err.message },
+      };
+    }
+    throw err;
+  }
+}
+
+/**
+ * `GET /engram/v1/recall/xray`. Moved here verbatim from `access-http.ts`
+ * (issue #570 PR 4 behaviour is unchanged): bearer auth and namespace scope
+ * are still enforced by the router before this runs; the query comes from
+ * `q` so GET stays cacheable, and `namespace` / `session` / `budget` /
+ * `disclosure` stay optional.
+ */
+export async function handleRecallXrayHttpQuery<TResponse>(deps: {
+  getParam: (name: string) => string | null;
+  resolveNamespace: (namespace: string | undefined) => string | undefined;
+  principal?: string;
+  connector?: string;
+  run: (request: {
+    query: string;
+    sessionKey?: string;
+    namespace?: string;
+    budget?: number;
+    authenticatedPrincipal?: string;
+    sourceConnector?: string;
+    disclosure?: RecallDisclosure;
+  }) => Promise<TResponse>;
+}): Promise<RecallDiagnosticsHttpOutcome> {
+  const queryParam = deps.getParam("q");
+  if (queryParam === null || queryParam.trim().length === 0) {
+    return {
+      status: 400,
+      body: {
+        error: "missing_query",
+        code: "missing_query",
+        message: "q search parameter is required and must be non-empty",
+      },
+    };
+  }
+  const sessionParam = deps.getParam("session");
+  const namespaceParam = deps.getParam("namespace");
+  const namespace = deps.resolveNamespace(
+    namespaceParam !== null && namespaceParam.length > 0 ? namespaceParam : undefined
+  );
+  // Reject an invalid `budget` with 400 rather than silently defaulting.
+  const budgetParam = deps.getParam("budget");
+  let budget: number | undefined;
+  if (budgetParam !== null && budgetParam !== "") {
+    const parsedBudget = Number(budgetParam);
+    if (!Number.isFinite(parsedBudget) || parsedBudget <= 0 || !Number.isInteger(parsedBudget)) {
+      return {
+        status: 400,
+        body: {
+          error: "invalid_budget",
+          code: "invalid_budget",
+          message: "budget expects a positive integer",
+        },
+      };
+    }
+    budget = parsedBudget;
+  }
+  // Disclosure depth (issue #677 PR 3/4 telemetry plumbing). When present it
+  // must match the chunk|section|raw allow-list; an invalid value is a 400,
+  // never a silent fallback that quietly disables the per-disclosure summary.
+  const disclosureParam = deps.getParam("disclosure");
+  let disclosure: RecallDisclosure | undefined;
+  if (disclosureParam !== null && disclosureParam.length > 0) {
+    if (!isRecallDisclosure(disclosureParam)) {
+      return {
+        status: 400,
+        body: {
+          error: "invalid_disclosure",
+          code: "invalid_disclosure",
+          message: "disclosure must be one of: chunk, section, raw",
+        },
+      };
+    }
+    disclosure = disclosureParam;
+  }
+  // Only validation errors become 400s. Backend faults (timeouts, storage
+  // errors, unexpected orchestrator failures) must bubble to the router's
+  // error handler so they return 500 and get logged. `service.recallXray`
+  // prefixes its validation errors with "recallXray:" so we key off that
+  // prefix rather than catching everything, and we surface only a real
+  // `Error.message` — never `String(err)` of an arbitrary throw, which
+  // CodeQL flags as stack-trace exposure (js/stack-trace-exposure).
+  try {
+    return {
+      status: 200,
+      body: await deps.run({
+        query: queryParam,
+        ...(sessionParam !== null && sessionParam.length > 0 ? { sessionKey: sessionParam } : {}),
+        ...(namespace !== undefined ? { namespace } : {}),
+        ...(budget !== undefined ? { budget } : {}),
+        ...(deps.principal ? { authenticatedPrincipal: deps.principal } : {}),
+        ...(deps.connector ? { sourceConnector: deps.connector } : {}),
+        ...(disclosure !== undefined ? { disclosure } : {}),
+      }),
+    };
+  } catch (err) {
+    if (err instanceof Error && err.message.startsWith("recallXray:")) {
+      return {
+        status: 400,
+        body: { error: "invalid_request", code: "invalid_request", message: err.message },
+      };
+    }
+    throw err;
+  }
+}

package/src/access-service.ts

@@ -323,6 +323,7 @@ import { decideDisclosureEscalation } from "./recall-disclosure-escalation.js";
 import { toRecallExplainJson } from "./recall-explain-renderer.js";
 import { type TagMatchMode, applyTagFilter, normalizeTags, parseTagMatch } from "./recall-tag-filter.js";
 import { type RecallXraySnapshot, estimateRecallTokens } from "./recall-xray.js";
+import { runRecallWhy, type RecallWhyRequest, type RecallWhyResponse } from "./recall-why-service.js";
 import {
   computeWhoKnows,
   loadWhoKnowsEntities,
@@ -2839,6 +2840,11 @@ export class EngramAccessService extends SupportPassportAccessServiceBase {
     return runMemoryBrowse({ resolveReadableNamespace: (ns, p) => this.resolveReadableNamespace(ns, p), getStorage: (ns) => this.orchestrator.getStorage(ns) }, request);
   }
 
+  /** Recall-miss diagnosis (issue #3033). Read-only; see recall-why.ts. */
+  async recallWhy(request: RecallWhyRequest): Promise<RecallWhyResponse> {
+    return runRecallWhy({ orchestrator: this.orchestrator, resolveNamespace: (ns) => this.resolveNamespace(ns) }, request);
+  }
+
   async recallXray(request: {
     query: string;
     sessionKey?: string;

package/src/access-surface-catalog.ts

@@ -56,6 +56,7 @@ export const MCP_TOOLS: readonly McpToolEntry[] = [
   { tool: "set_coding_context", operation: "set_coding_context" },
   { tool: "recall_tier_explain", operation: "recall_tier_explain" },
   { tool: "recall_xray", operation: "recall_xray" },
+  { tool: "recall_why", operation: "recall_why" },
   { tool: "who_knows", operation: "who_knows" },
   { tool: "promotion_candidates", operation: "promotion_candidates" },
   { tool: "wearables_status", operation: "wearables_status" },
@@ -226,6 +227,7 @@ export const HTTP_ROUTES: readonly HttpRouteEntry[] = [
   { method: "POST", pathname: "/engram/v1/action-confidence", operation: "action_confidence" },
   { method: "GET", pathname: "/engram/v1/recall/tier-explain", operation: "recall_tier_explain" },
   { method: "GET", pathname: "/engram/v1/recall/xray", operation: "recall_xray" },
+  { method: "GET", pathname: "/engram/v1/recall/why", operation: "recall_why" },
   { method: "GET", pathname: "/engram/v1/who-knows", operation: "who_knows" },
   { method: "GET", pathname: "/engram/v1/namespace/writable", operation: "namespace_writable" },
   { method: "GET", pathname: "/engram/v1/wearables/status", operation: "wearables_status" },

package/src/index.ts

@@ -130,6 +130,8 @@ export {
   type ObserveFingerprintParts,
   type ComposeEnvelopeOptions,
 } from "./write-envelope.js";
+export { composeSalvagedEnvelope } from "./salvage-envelope.js";
+export { sanitizeMemoryContent } from "./sanitize.js";
 export {
   getHostEmbeddingProvider,
   normalizeHostEmbeddingVector,
@@ -490,6 +492,15 @@ export {
 export type {
   RecallXraySnapshot, RecallXrayResult, RecallXrayScoreDecomposition, RecallXrayServedBy, RecallFilterTrace,
 } from "./recall-xray.js";
+// Recall-miss diagnosis (issue #3033). Same rationale as the X-ray exports
+// above: the standalone `@remnic/cli` binary wires `remnic why` through the
+// package entry point, never by relative path (rule 26).
+export { explainRecallMiss, RECALL_WHY_STAGES, RECALL_WHY_DROP_REASONS, RecallWhyInputError } from "./recall-why.js";
+export type { RecallWhyReport, RecallWhyStage, RecallWhyStageRecord, RecallWhyDrop, RecallWhyDropReason, RecallWhyExpectation, RecallWhyDeps, RecallWhyOptions, RecallWhyMemoryRef, RecallWhyRecallOutcome } from "./recall-why.js";
+export { renderRecallWhy, renderRecallWhyJson, renderRecallWhyMarkdown, parseRecallWhyFormat, summarizeRecallWhy, RECALL_WHY_FORMATS, type RecallWhyFormat } from "./recall-why-renderer.js";
+export { parseWhyCliOptions, type ParsedWhyCliOptions } from "./recall-why-cli.js";
+export { runRecallWhy, type RecallWhyRequest, type RecallWhyResponse } from "./recall-why-service.js";
+export { RECALL_WHY_MCP_TOOLS, handleRecallWhyHttpQuery, handleRecallXrayHttpQuery, type RecallDiagnosticsHttpOutcome } from "./access-recall-diagnostics.js";
 export * from "./who-knows.js";
 export * from "./who-knows-cli.js";
 export * from "./memory-subject.js";

package/src/mcp-read-only-tools.ts

@@ -16,6 +16,7 @@ export const MCP_READ_ONLY_TOOL_SUFFIXES: Readonly<Record<string, true>> = {
   recall_explain: true,
   recall_tier_explain: true,
   recall_xray: true,
+  recall_why: true,
   who_knows: true,
   promotion_candidates: true,
   briefing: true,

package/src/orchestration/orchestrator-helpers.ts

@@ -26,6 +26,7 @@ import { log } from "../logger.js";
 import type { GraphRecallRankedResult, GraphRecallShadowComparison } from "./graph-recall-coordinator.js";
 import { parseQmdExplain } from "../qmd.js";
 import type { GraphRecallExpandedEntry } from "../recall-state.js";
+import type { SearchDegradation } from "../search/port.js";
 import type { RecallXrayServedBy } from "../recall-xray.js";
 import type { RecallContextComposition } from "../recall-context-composition.js";
 import { type BufferTurn, type MemoryActionEvent, type MemoryFile, type MemoryFrontmatter, type MemoryIntent, type PluginConfig, type QmdSearchResult, type RecallPlanMode, confidenceTier } from "../types.js";
@@ -209,6 +210,12 @@ export interface RecallInvocationOptions {
    * captured and recall behavior is unchanged (schema-only slice).
    */
   xrayCapture?: boolean;
+  /**
+   * Out-parameter receiving the backend degradations this recall observes
+   * (#3033). Absent by default — behavior and allocations unchanged; the
+   * diagnosis passes a sink to tell an outage from "no matches".
+   */
+  degradationSink?: SearchDegradation[];
   /**
    * Per-invocation override for `recallBudgetChars` (issue #570 PR 3/4).
    * Flows through `getRecallBudgetChars()` for this recall only — no

package/src/orchestration/recall-internal.ts

@@ -134,7 +134,7 @@ export class RecallInternalCoordinator {
     // collected via the execution-options observer and attached to the
     // LastRecallSnapshot after it is recorded, so surfaces can distinguish
     // "no matches" from "backend could not answer" (CLAUDE.md rule 34).
-    const backendDegradations: SearchDegradation[] = [];
+    const backendDegradations: SearchDegradation[] = options.degradationSink ?? [];
     // Issue #680 — historical recall.  Parse `options.asOf` once at the
     // top of the recall so each boost-pass uses identical filter logic.
     // Invalid values are rejected at input boundaries (CLI / HTTP / MCP)

package/src/recall-degradation-sink.test.ts

@@ -0,0 +1,144 @@
+/**
+ * Pipeline instrumentation test for the recall-miss diagnosis (issue #3033).
+ *
+ * `RecallInvocationOptions.degradationSink` is the one change this feature
+ * makes to the recall path: the pipeline already collected the degradations
+ * its searches observed, and now a caller may hand it the array to collect
+ * into. This exercises the REAL orchestrator recall — not a stub — so the
+ * wiring cannot rot silently, because a diagnosis that cannot see a
+ * mid-recall backend failure would report an outage as "no matches"
+ * (Review Prevention Checklist #22).
+ *
+ * Fixture data is synthetic and the store is an empty temp directory.
+ */
+
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { parseConfig } from "./config.js";
+import { Orchestrator } from "./orchestrator.js";
+import type { SearchBackend, SearchDegradation, SearchExecutionOptions } from "./search/port.js";
+
+const DEGRADATION: SearchDegradation = {
+  backend: "qmd",
+  code: "daemon_timeout",
+  detail: "no response within the deadline",
+};
+/**
+ * A backend that answers the availability probe and then degrades on every
+ * search — the failure shape a healthy-looking daemon produces under load.
+ * Typed as `Partial<SearchBackend>` so every method signature is checked
+ * against the real port (Review Prevention Checklist #21: a mock whose
+ * signature drifts from production tests nothing).
+ */
+function degradingBackend(observed: { calls: number }): SearchBackend {
+  const degrade = (execution?: SearchExecutionOptions) => {
+    observed.calls += 1;
+    execution?.onDegradation?.(DEGRADATION);
+    return [];
+  };
+  const backend: Partial<SearchBackend> = {
+    async probe() {
+      return true;
+    },
+    isAvailable() {
+      return true;
+    },
+    debugStatus() {
+      return "backend=degrading-stub";
+    },
+    async search(_query, _collection, _maxResults, _options, execution) {
+      return degrade(execution);
+    },
+    async searchGlobal(_query, _maxResults, execution) {
+      return degrade(execution);
+    },
+    async bm25Search(_query, _collection, _maxResults, execution) {
+      return degrade(execution);
+    },
+    async vectorSearch(_query, _collection, _maxResults, execution) {
+      return degrade(execution);
+    },
+    async hybridSearch(_query, _collection, _maxResults, execution) {
+      return degrade(execution);
+    },
+    async update() {},
+    async updateCollection() {},
+    async embed() {},
+    async embedCollection() {},
+    async ensureCollection() {
+      return "skipped";
+    },
+  };
+  // Unchecked cast with reason: the stub implements only the read surface a
+  // recall touches; the rest of the port is never reached here.
+  return backend as SearchBackend;
+}
+
+/**
+ * Both cases boot a real orchestrator, so both must settle deferred init and
+ * `destroy()` it before the temp store is removed: background QMD/warmup
+ * writes otherwise race the teardown and `rm` fails with ENOTEMPTY under
+ * load (the orchestrator's documented shutdown contract).
+ */
+async function withOrchestrator(
+  prefix: string,
+  run: (orchestrator: Orchestrator, observed: { calls: number }) => Promise<void>,
+): Promise<void> {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), prefix));
+  const orchestrator = new Orchestrator(
+    parseConfig({ memoryDir, workspaceDir: memoryDir, qmdEnabled: true, embeddingFallbackEnabled: false }),
+  );
+  const observed = { calls: 0 };
+  // Unchecked cast with reason: `qmd` is assigned post-construction because the
+  // backend is built by the search factory, which would reach for a real
+  // daemon. Same seam orchestrator-xray-capture.test.ts uses.
+  const withBackend = orchestrator as unknown as { qmd: SearchBackend };
+  withBackend.qmd = degradingBackend(observed);
+  try {
+    await orchestrator.initialize();
+    await orchestrator.deferredReady;
+    await run(orchestrator, observed);
+  } finally {
+    await orchestrator.destroy();
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+}
+
+test("a caller-supplied degradationSink receives the real recall's backend degradations", async () => {
+  await withOrchestrator("remnic-degradation-sink-", async (orchestrator, observed) => {
+    const degradationSink: SearchDegradation[] = [];
+    const context = await orchestrator.recall(
+      "what did we decide about the retention window?",
+      "sink-session",
+      { degradationSink },
+    );
+
+    assert.ok(observed.calls > 0, "the recall must actually have consulted the backend");
+    assert.ok(degradationSink.length > 0, "the sink must receive the pipeline's degradations");
+    assert.deepEqual(
+      degradationSink[0],
+      DEGRADATION,
+      "entries must be the degradation the backend reported, verbatim",
+    );
+    assert.ok(
+      degradationSink.every((d) => d.code === DEGRADATION.code),
+      "no entry may be invented or rewritten by the pipeline",
+    );
+    // The recall itself still returns an ordinary (here empty) context: the
+    // sink observes, it never alters recall behavior.
+    assert.equal(typeof context, "string");
+  });
+});
+
+test("recall works unchanged when no sink is supplied", async () => {
+  await withOrchestrator("remnic-degradation-nosink-", async (orchestrator, observed) => {
+    // No `degradationSink`: the pipeline allocates its own array as before.
+    const context = await orchestrator.recall("what did we decide about retention?", "no-sink");
+    assert.equal(typeof context, "string");
+    assert.ok(observed.calls > 0, "the recall must still consult the backend");
+  });
+});

package/src/recall-why-cli.ts

@@ -0,0 +1,59 @@
+/**
+ * Input validation for the `remnic why` command (issue #3033).
+ *
+ * Extracted from the CLI handlers so the validation paths are unit-testable
+ * without booting an orchestrator — the same split `recall-xray-cli.ts`
+ * uses. Every flag rejects a missing or unknown value with a listed-options
+ * error instead of silently defaulting (Review Prevention Checklist #1 /
+ * #39).
+ */
+
+import { parseRecallWhyFormat, type RecallWhyFormat } from "./recall-why-renderer.js";
+
+export interface ParsedWhyCliOptions {
+  format: RecallWhyFormat;
+  /** Memory id or substring to trace, or undefined when not specified. */
+  expect?: string;
+  /** Trimmed namespace, or undefined when not specified. */
+  namespace?: string;
+  /** Trimmed session key, or undefined when not specified. */
+  session?: string;
+  /** Trimmed, tilde-unexpanded output path, or undefined when stdout. */
+  outPath?: string;
+}
+
+export function parseWhyCliOptions(
+  rawQuery: unknown,
+  options: Record<string, unknown>
+): { query: string } & ParsedWhyCliOptions {
+  if (typeof rawQuery !== "string" || rawQuery.trim().length === 0) {
+    throw new Error("why expects a non-empty query argument");
+  }
+  const format = parseRecallWhyFormat(Object.hasOwn(options, "format") ? options.format : undefined);
+  return {
+    query: rawQuery,
+    format,
+    ...maybeStringFlag(options, "expect"),
+    ...maybeStringFlag(options, "namespace"),
+    ...maybeStringFlag(options, "session"),
+    ...maybeStringFlag(options, "out", "outPath"),
+  };
+}
+
+/**
+ * Read one optional string flag. `--flag` with no value arrives as `true`
+ * from commander and must fail rather than be reinterpreted as "not
+ * supplied"; a whitespace-only value is likewise a caller mistake, not a
+ * request for the default.
+ */
+function maybeStringFlag(options: Record<string, unknown>, flag: string, key = flag): Record<string, string> {
+  // `Object.hasOwn` + own-property enumeration: a value inherited from a
+  // doctored prototype must never enable a flag (checklist #46).
+  if (!Object.hasOwn(options, flag)) return {};
+  const value = options[flag];
+  if (value === undefined) return {};
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new Error(`--${flag} expects a non-empty value`);
+  }
+  return { [key]: value.trim() };
+}