@remnic/core 9.69.10 → 9.69.11

package/src/deep-recall-http-glue.ts

@@ -0,0 +1,75 @@
+/**
+ * Deep-recall HTTP glue (issue #2332) — route body lives here so
+ * access-http.ts stays at its structural ceiling (same seam as
+ * meetings/http-glue.ts and location/http-glue.ts).
+ *
+ * Input errors map to 400; everything else rethrows so backend faults keep
+ * flowing to the global 500 handler. `deepRecall.enabled` gating lives in
+ * EngramAccessService — this module only translates transport shape.
+ */
+
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { deepRecallRequestSchema } from "./access-schema.js";
+import { EngramAccessInputError } from "./access-errors.js";
+import type { EngramAccessService } from "./access-service.js";
+
+type RespondJson = (res: ServerResponse, status: number, payload: unknown) => void;
+type ReadJsonBody = (req: IncomingMessage) => Promise<unknown>;
+
+function coerceMaxSteps(raw: unknown): number | undefined {
+  if (raw === undefined || raw === null) return undefined;
+  const value = typeof raw === "string" ? Number(raw) : raw;
+  if (typeof value !== "number" || !Number.isFinite(value) || !Number.isInteger(value) || value < 0) {
+    throw new EngramAccessInputError("maxSteps must be a non-negative integer");
+  }
+  return value;
+}
+
+export async function respondDeepRecall(
+  req: IncomingMessage,
+  res: ServerResponse,
+  respondJson: RespondJson,
+  readJsonBody: ReadJsonBody,
+  service: EngramAccessService,
+  /**
+   * Resolve the caller scope for a body-supplied namespace/sessionKey. The
+   * body `namespace` is user-controlled, so it MUST flow through the SAME
+   * effective-namespace allow-list gate as the query-string value (issue
+   * #1850 finding 2 / #2332 review): a scoped bearer that passes an allowed
+   * `?namespace=` while setting a different `namespace` in the body would
+   * otherwise read another tenant. access-http supplies the gate and applies
+   * the query-string fallback, so this module never selects a namespace of
+   * its own. Throws 403 when the effective namespace is not permitted.
+   *
+   * `authenticatedPrincipal` is the identity the transport authenticated, and
+   * it MUST reach the service: authorization and the audit trail derive from
+   * the presenting principal, never from the client-supplied `sessionKey`.
+   * Dropping it let a crafted key matching another principal rule read that
+   * principal's namespaces, and made a legitimate namespace-enabled request
+   * with no session key fail as unauthenticated.
+   */
+  scopeFor: (
+    bodyNamespace?: string,
+    bodySessionKey?: string,
+  ) => { namespace?: string; sessionKey?: string; authenticatedPrincipal?: string },
+): Promise<void> {
+  const parsed = deepRecallRequestSchema.safeParse(await readJsonBody(req));
+  if (!parsed.success) {
+    respondJson(
+      res,
+      400,
+      { error: "invalid_request", detail: parsed.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ") },
+    );
+    return;
+  }
+  const maxSteps = coerceMaxSteps(parsed.data.maxSteps);
+  const scope = scopeFor(parsed.data.namespace, parsed.data.sessionKey);
+  const result = await service.deepRecall({
+    query: parsed.data.query,
+    ...(maxSteps !== undefined ? { maxSteps } : {}),
+    namespace: scope.namespace,
+    sessionKey: scope.sessionKey,
+    authenticatedPrincipal: scope.authenticatedPrincipal,
+  });
+  respondJson(res, 200, result);
+}

package/src/deep-recall-mcp-tools.ts

@@ -0,0 +1,33 @@
+/**
+ * Deep-recall MCP tool definition (issue #2332) — extracted from
+ * access-mcp.ts so the surface file stays at its structural ceiling.
+ * Spread back into `this.tools` (`...DEEP_RECALL_MCP_TOOLS`); the
+ * migrated-operation mapping spreads into MCP_MIGRATED_OPERATIONS.
+ * The `remnic.` canonical alias is minted by withToolAliases at
+ * tools/list time (same as meetings).
+ */
+
+import type { McpTool } from "./access-mcp.js";
+
+export const DEEP_RECALL_MCP_TOOLS: McpTool[] = [
+  {
+    name: "engram.deep_recall",
+    description:
+      "Budgeted REFINE/EXPAND/STOP multi-hop retrieval over the cue-anchor graph (issue #2332). Slow and thorough (seconds; several LLM calls) — opt in when a question warrants deep search, not for routine recall. Returns ranked entries plus a per-step trace. Typed error when deepRecall.enabled is off.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        query: { type: "string", description: "The question to retrieve for. Required." },
+        maxSteps: {
+          type: "integer",
+          minimum: 0,
+          description: "Optional policy-step ceiling; cannot exceed the configured deepRecall.maxSteps.",
+        },
+        namespace: { type: "string", description: "Caller namespace to scope this operation to." },
+        sessionKey: { type: "string", description: "Session key for principal/namespace resolution." },
+      },
+      required: ["query"],
+      additionalProperties: false,
+    },
+  },
+];

package/src/deep-recall-policy-llm.ts

@@ -0,0 +1,65 @@
+/**
+ * Policy-call LLM routing for deep recall (issue #2332).
+ *
+ * Mirrors `extraction-judge.ts` `callJudgeLlm` exactly: local client first
+ * (json_object response format) unless modelSource is "gateway", then the
+ * fallback client with the shared background-task chain options. Never a
+ * raw OpenAI client. Returns null when both legs fail — the loop treats
+ * that as a stop, never a crash.
+ */
+
+import { gatewayTaskChainOptions, type FallbackLlmClient } from "./fallback-llm.js";
+import type { LocalLlmClient } from "./local-llm.js";
+import type { PluginConfig } from "./types.js";
+
+export const DEEP_RECALL_POLICY_SYSTEM_PROMPT =
+  "You are the retrieval policy for a long-term memory system. Given the question, the current working set of retrieved memories, and the frontier of anchor-linked candidates, choose exactly one action: REFINE (rewrite the query), EXPAND (follow named frontier nodes), or STOP. Respond with JSON only.";
+
+export async function callDeepRecallPolicyLlm(input: {
+  statePrompt: string;
+  config: PluginConfig;
+  localLlm: LocalLlmClient | null;
+  fallbackLlm: FallbackLlmClient | null;
+  timeoutMs: number;
+}): Promise<string | null> {
+  const messages: Array<{ role: "system" | "user"; content: string }> = [
+    { role: "system", content: DEEP_RECALL_POLICY_SYSTEM_PROMPT },
+    { role: "user", content: input.statePrompt },
+  ];
+  const timeout = input.timeoutMs > 0 ? input.timeoutMs : undefined;
+  const skipLocal = input.config.modelSource === "gateway";
+  const gatewayChain = gatewayTaskChainOptions(input.config);
+
+  if (input.localLlm && !skipLocal) {
+    try {
+      const result = await input.localLlm.chatCompletion(messages, {
+        temperature: 0.1,
+        maxTokens: 1024,
+        ...(timeout ? { timeoutMs: timeout } : {}),
+        responseFormat: { type: "json_object" },
+        operation: "deep-recall-policy",
+      });
+      if (result?.content) return result.content;
+    } catch {
+      // fall through to the gateway leg
+    }
+  }
+
+  if (input.fallbackLlm) {
+    try {
+      const result = await input.fallbackLlm.chatCompletion(
+        messages as Array<{ role: "system" | "user" | "assistant"; content: string }>,
+        {
+          temperature: 0.1,
+          maxTokens: 1024,
+          ...(timeout ? { timeoutMs: timeout } : {}),
+          ...gatewayChain,
+        },
+      );
+      if (result?.content) return result.content;
+    } catch {
+      // both legs failed — caller treats null as a stop
+    }
+  }
+  return null;
+}

package/src/deep-recall-renderer.ts

@@ -0,0 +1,41 @@
+/**
+ * Shared markdown renderer for deep recall (issue #2332).
+ *
+ * Rule 22 (AGENTS.md): never fork formatting — CLI text output, MCP text
+ * output, and any later surface all render through this one module.
+ * Deterministic: entries in result order, trace in step order.
+ */
+
+import type { DeepRecallResult } from "./deep-recall.js";
+
+export function renderDeepRecallResult(result: DeepRecallResult): string {
+  if (!result.ok) {
+    return [`# Deep recall`, ``, `- error: ${result.error ?? "unknown"}`, ``].join("\n");
+  }
+  const lines: string[] = ["# Deep recall", ""];
+  if (result.entries.length === 0) {
+    lines.push("No memories retrieved.", "");
+  } else {
+    lines.push("## Memories", "");
+    for (const [index, entry] of result.entries.entries()) {
+      const via = entry.viaAnchor ? ` via anchor "${entry.viaAnchor}"` : "";
+      lines.push(
+        `${index + 1}. **${entry.memoryId}** (score ${entry.score.toFixed(3)}, origin ${entry.origin}${via})`,
+      );
+      lines.push(`   ${entry.content.replace(/\s+/g, " ").trim().slice(0, 400)}`);
+    }
+    lines.push("");
+  }
+  lines.push("## Trace", "");
+  if (result.trace.length === 0) {
+    lines.push("(empty)");
+  } else {
+    for (const step of result.trace) {
+      lines.push(
+        `- step ${step.step}: ${step.action} — ${step.detail} (workingSet=${step.workingSetSize}, frontier=${step.frontierSize}, ${step.durationMs}ms)`,
+      );
+    }
+  }
+  lines.push("");
+  return lines.join("\n");
+}

package/src/deep-recall-seeds.ts

@@ -0,0 +1,134 @@
+/**
+ * Deep-recall seed retrieval (issue #2332 review).
+ *
+ * Seeds route through the SAME namespace search router the rest of the read
+ * path uses, so a non-default caller searches its own suffixed collection
+ * instead of the base (default-namespace) collection — read and write resolve
+ * through one resolver, never two (§30).
+ *
+ * Each hit is resolved to a real memory id through the shared QMD result
+ * resolver (`QmdResultResolver.readQmdResultMemory`), per hit and bounded by
+ * the requested limit, instead of pre-scanning the whole namespace corpus to
+ * build a path index. With namespaces disabled the fanout queries the QMD
+ * backend directly, so hits keep the indexer's raw path forms; the resolver
+ * decodes `qmd://<collection>/` URIs and `<collection>/`-prefixed relatives,
+ * and the two forms it cannot decode (`qmd:///`, a leading `/`) are stripped
+ * for one retry. A hit with no path is dropped: its namespace membership
+ * cannot be verified, and a bare docid from a foreign collection must never
+ * enter the working set as a resolved memory.
+ *
+ * An unavailable namespace backend (or a missing collection) contributes an
+ * empty result set and reports itself only through `execution.onDegradation`,
+ * so the observer is REQUIRED here: without it a dead index is indistinguishable
+ * from a healthy empty one and deep recall would answer `ok: true` with zero
+ * entries instead of the advertised `backend_unavailable` failure.
+ */
+
+import type { DeepRecallSeedHit } from "./deep-recall.js";
+import type { SearchDegradation } from "./search/port.js";
+
+/** Namespace-scoped search entrypoint (satisfied by `Orchestrator.searchAcrossNamespaces`). */
+export interface DeepRecallSeedRouter {
+  searchAcrossNamespaces(options: {
+    query: string;
+    namespaces: string[];
+    maxResults?: number;
+    execution?: { onDegradation?: (degradation: SearchDegradation) => void };
+  }): Promise<ReadonlyArray<{ path?: string; docid?: string; score?: number }>>;
+}
+
+/** Namespace-scoped storage read (satisfied by `StorageManager`). */
+export interface DeepRecallSeedStorage {
+  readonly dir: string;
+  readMemoryByPath(filePath: string): Promise<{ frontmatter: { id?: string } } | null>;
+}
+
+/** Full QMD hit -> memory resolver (satisfied by `QmdResultResolver#readQmdResultMemory`). */
+export interface DeepRecallSeedResultResolver {
+  readQmdResultMemory(
+    resultPath: string,
+    fallbackStorage: DeepRecallSeedStorage,
+    recallNamespaces?: readonly string[],
+    preferredNamespace?: string,
+  ): Promise<{ frontmatter: { id?: string } } | null>;
+}
+
+/**
+ * Strip the collection-qualified forms the indexer can emit but
+ * `readQmdResultMemory` cannot decode: the empty-hostname `qmd:///` URI and
+ * the root-relative `/` path both reach its fallback probe still prefixed and
+ * miss. Returns `null` when the path carries no such prefix — including a
+ * decodable `qmd://<collection>/` or `<collection>/` form, which the resolver
+ * owns, and a foreign collection prefix, which must stay a miss.
+ */
+function collectionPrefixStrippedPath(resultPath: string): string | null {
+  const trimmed = resultPath.trim();
+  if (trimmed.startsWith("qmd:///")) {
+    const relative = trimmed.slice("qmd:///".length).replace(/^\/+/, "");
+    return relative.length > 0 ? relative : null;
+  }
+  if (trimmed.startsWith("/")) {
+    const relative = trimmed.replace(/^\/+/, "");
+    return relative.length > 0 ? relative : null;
+  }
+  return null;
+}
+
+async function resolveSeedMemoryId(
+  resolver: DeepRecallSeedResultResolver,
+  storage: DeepRecallSeedStorage,
+  namespace: string,
+  resultPath: string | undefined,
+): Promise<string | null> {
+  if (typeof resultPath !== "string" || resultPath.length === 0) return null;
+  const attempts = [resultPath];
+  const stripped = collectionPrefixStrippedPath(resultPath);
+  if (stripped !== null) attempts.push(stripped);
+  for (const attempt of attempts) {
+    const memory = await resolver.readQmdResultMemory(attempt, storage, [namespace]);
+    const id = memory?.frontmatter?.id;
+    if (typeof id === "string" && id.length > 0) return id;
+  }
+  return null;
+}
+
+export function createDeepRecallSeedSearch(deps: {
+  namespace: string;
+  storage: DeepRecallSeedStorage;
+  router: DeepRecallSeedRouter;
+  resolver: DeepRecallSeedResultResolver;
+}): (query: string, limit: number) => Promise<DeepRecallSeedHit[]> {
+  return async (query, limit) => {
+    if (limit <= 0) return [];
+    const unavailable: SearchDegradation[] = [];
+    const hits = await deps.router.searchAcrossNamespaces({
+      query,
+      namespaces: [deps.namespace],
+      maxResults: limit,
+      execution: {
+        onDegradation: (degradation) => {
+          if (degradation.code === "backend_unavailable") unavailable.push(degradation);
+        },
+      },
+    });
+    // The requested namespace has no usable index: a seed-search failure, which
+    // is the one condition deep recall reports as `backend_unavailable`.
+    if (unavailable.length > 0) {
+      throw new Error(
+        `deep recall seed search unavailable: ${unavailable.map((d) => d.detail ?? d.code).join("; ")}`,
+      );
+    }
+    const seeds: DeepRecallSeedHit[] = [];
+    for (const hit of hits) {
+      if (typeof hit.path !== "string" || hit.path.length === 0) continue;
+      const memoryId =
+        (await resolveSeedMemoryId(deps.resolver, deps.storage, deps.namespace, hit.path)) ?? hit.docid;
+      if (typeof memoryId !== "string" || memoryId.length === 0) continue;
+      seeds.push({
+        memoryId,
+        score: typeof hit.score === "number" && Number.isFinite(hit.score) ? hit.score : 0,
+      });
+    }
+    return seeds;
+  };
+}