@remnic/core 9.69.46 → 9.69.47

package/src/extraction-evidence-class.ts

@@ -0,0 +1,241 @@
+/**
+ * Deterministic evidence provenance classes (issue #2973).
+ *
+ * Every extracted candidate is backed by spans located in buffered turns.
+ * The CLASS of a span — whose words it records — decides what the span can
+ * license (learned from distill-kura's measured self-reinforcing assertion
+ * loop: an agent asserts, the extractor records, the next agent reads it back
+ * as ground truth and repeats it with more confidence):
+ *
+ *   - `user`:   the human's own words. Licenses user attributions.
+ *   - `tool`:   machine output. The ONLY licit source for numbers.
+ *   - `action`: a tool was invoked. Weaker than reading its output.
+ *   - `agent`:  agent prose. Licenses a first-person judgement, never a
+ *               bare fact.
+ *
+ * Derivation is pure and total: turn role plus tool-part flags in, one class
+ * out, no clock, no randomness, no I/O. The same input always yields the
+ * same class. Span matching reuses `locateQuoteOffsets` /
+ * `stripLeadingRolePrefix` from `provenance.ts` so class location and
+ * provenance building can never disagree (issue #1575 pitfall: one matcher,
+ * not two).
+ *
+ * The post-extraction rules (`applyEvidenceClassRules`) are deterministic
+ * string/flag checks — no LLM — designed to run before the judge. Each
+ * returns a stable rule id for shadow-mode telemetry. Not yet wired into the
+ * extraction pipeline; see the issue's rollout plan.
+ */
+
+import { locateQuoteOffsets, stripLeadingRolePrefix } from "./provenance.js";
+import type { MemorySubject } from "./types.js";
+import { collapseWhitespace } from "./whitespace.js";
+
+export type EvidenceClass = "user" | "tool" | "action" | "agent";
+
+/** Canonical class order: strongest license first. Fixed; never reorder. */
+export const EVIDENCE_CLASSES: readonly EvidenceClass[] = ["user", "tool", "action", "agent"];
+
+/**
+ * A buffered turn reduced to what class derivation needs. Callers map
+ * `BufferTurn`s via {@link evidenceTurnInput}; `role` and the tool flags are
+ * the only fields that influence the class.
+ */
+export interface EvidenceClassTurnInput {
+  content: string;
+  role: "user" | "assistant";
+  /** Turn carries machine output (a tool_result part). */
+  toolResult?: boolean;
+  /** Turn records a tool invocation (a tool_call part). */
+  toolCall?: boolean;
+}
+
+/**
+ * Derive a turn's evidence class. Total and deterministic:
+ *   - `role: "user"` → `"user"` (the human's words, regardless of stray
+ *     part flags a connector may attach).
+ *   - `role: "assistant"` with `toolResult` → `"tool"`; machine output
+ *     outranks the invocation that produced it.
+ *   - else with `toolCall` → `"action"`.
+ *   - else → `"agent"` (agent prose — the weakest license).
+ */
+export function deriveTurnEvidenceClass(turnInput: EvidenceClassTurnInput): EvidenceClass {
+  if (turnInput.role === "user") return "user";
+  if (turnInput.toolResult === true) return "tool";
+  if (turnInput.toolCall === true) return "action";
+  return "agent";
+}
+
+/**
+ * Map a buffered turn (structural subset of `BufferTurn`) to
+ * {@link EvidenceClassTurnInput}. Tool flags derive mechanically from part
+ * kinds — no content inspection, no heuristics.
+ */
+export function evidenceTurnInput(bufferTurn: {
+  role: "user" | "assistant";
+  content: string;
+  parts?: ReadonlyArray<{ kind: string }>;
+}): EvidenceClassTurnInput {
+  let toolResult = false;
+  let toolCall = false;
+  if (bufferTurn.parts) {
+    for (const part of bufferTurn.parts) {
+      if (part.kind === "tool_result") toolResult = true;
+      else if (part.kind === "tool_call") toolCall = true;
+    }
+  }
+  return {
+    role: bufferTurn.role,
+    content: bufferTurn.content,
+    ...(toolResult ? { toolResult: true } : {}),
+    ...(toolCall ? { toolCall: true } : {}),
+  };
+}
+
+/**
+ * Resolve which quote `buildFactProvenance` would search for: the raw quote
+ * when it matches at least one turn, else the leading prompt role label
+ * stripped (mirrors the raw-preferred contract in `provenance.ts`).
+ */
+function resolveSearchableQuote(
+  quote: string | null | undefined,
+  turns: ReadonlyArray<EvidenceClassTurnInput>,
+): string {
+  const rawQuote = typeof quote === "string" ? quote.trim() : "";
+  if (rawQuote.length === 0) return "";
+  const stripped = stripLeadingRolePrefix(rawQuote);
+  if (rawQuote === stripped) return rawQuote;
+  const rawMatches = turns.some(
+    (t) => typeof t?.content === "string" && t.content.length > 0 && locateQuoteOffsets(rawQuote, t.content).matched,
+  );
+  return rawMatches ? rawQuote : stripped;
+}
+
+/**
+ * Classify the evidence backing a fact: the classes of every turn in which
+ * its quote was located, in canonical `EVIDENCE_CLASSES` order regardless of
+ * turn order. Deterministic — the same `(quote, turns)` pair always yields
+ * the same array. An absent or unlocatable quote yields `[]` (no evidence
+ * class; the extractor's own assertion is all that backs the fact).
+ */
+export function classifyEvidenceClasses(input: {
+  quote: string | null | undefined;
+  turns: ReadonlyArray<EvidenceClassTurnInput>;
+}): EvidenceClass[] {
+  const quote = resolveSearchableQuote(input.quote, input.turns);
+  if (quote.length === 0) return [];
+  const present = new Set<EvidenceClass>();
+  for (const turn of input.turns) {
+    if (!turn || typeof turn.content !== "string" || turn.content.length === 0) continue;
+    if (locateQuoteOffsets(quote, turn.content).matched) {
+      present.add(deriveTurnEvidenceClass(turn));
+    }
+  }
+  return EVIDENCE_CLASSES.filter((cls) => present.has(cls));
+}
+
+/**
+ * Collapse a fact's evidence classes to the single strongest license present.
+ * No located span (`[]`) collapses to `"agent"`: the fact is agent-asserted.
+ */
+export function deriveFactEvidenceClass(classes: readonly EvidenceClass[]): EvidenceClass {
+  for (const cls of EVIDENCE_CLASSES) {
+    if (classes.includes(cls)) return cls;
+  }
+  return "agent";
+}
+
+// ---------------------------------------------------------------------------
+// Deterministic post-extraction rules (no LLM; run before the judge)
+// ---------------------------------------------------------------------------
+
+export type EvidenceRuleId =
+  | "numeric_without_tool_evidence"
+  | "user_attribution_without_user_span"
+  | "echo_of_existing_content";
+
+export interface EvidenceRuleFinding {
+  rule: EvidenceRuleId;
+  /** `demote` routes the fact to `pending_review`; `refuse` drops it. */
+  action: "demote" | "refuse";
+  reason: string;
+}
+
+/** Categories that attribute a choice or commitment to the modeled subject. */
+const DECISION_LIKE_CATEGORIES: Record<string, true> = {
+  decision: true,
+  preference: true,
+  commitment: true,
+};
+
+/**
+ * Normalize a quote for echo comparison. Casefold plus whitespace collapse —
+ * the same normalization `locateQuoteOffsets` matches with, via the single
+ * shared normalizer in `whitespace.ts`.
+ */
+export function echoQuoteKey(quote: string): string {
+  return collapseWhitespace(quote.toLowerCase());
+}
+
+/**
+ * Apply the three deterministic evidence rules to one extracted fact.
+ * Pure; findings come back in canonical rule-id order. A fact triggers:
+ *
+ *   1. `numeric_without_tool_evidence` — content carries a digit but no
+ *      `tool`/`action` span backs it. The human and the agent are both
+ *      illicit sources for numbers; demote to review rather than refuse
+ *      (the number may be real — it is just unproven).
+ *   2. `user_attribution_without_user_span` — a decision-like fact about the
+ *      user (absent `subject` reads as `"user"`, mirroring the promotion
+ *      guard's fail-closed default) with no `user` span. This is the
+ *      assertion-loop gate: an agent-asserted "the user decided X" is
+ *      refused, never promoted to active.
+ *   3. `echo_of_existing_content` — every quote backing the fact already
+ *      exists in the store index, so the "evidence" is the store reading
+ *      itself back through a tool result. Refused: not new material.
+ */
+export function applyEvidenceClassRules(input: {
+  content: string;
+  category?: string;
+  subject?: MemorySubject;
+  evidenceClasses?: readonly EvidenceClass[];
+  /** Normalized keys (`echoQuoteKey`) of the fact's surviving quotes. */
+  quoteKeys?: readonly string[];
+  /** Normalized keys of quotes already present in the content-hash index. */
+  existingQuoteKeys?: ReadonlySet<string>;
+}): EvidenceRuleFinding[] {
+  const classes = input.evidenceClasses ?? [];
+  const findings: EvidenceRuleFinding[] = [];
+
+  if (/\d/.test(input.content) && !classes.some((c) => c === "tool" || c === "action")) {
+    findings.push({
+      rule: "numeric_without_tool_evidence",
+      action: "demote",
+      reason: "content carries a number but no tool or action evidence span backs it",
+    });
+  }
+
+  const decisionLike = input.category !== undefined && DECISION_LIKE_CATEGORIES[input.category] === true;
+  const subject = input.subject ?? "user";
+  if (decisionLike && subject === "user" && !classes.includes("user")) {
+    findings.push({
+      rule: "user_attribution_without_user_span",
+      action: "refuse",
+      reason: "decision-like fact about the user with no user-role evidence span",
+    });
+  }
+
+  const quoteKeys = input.quoteKeys ?? [];
+  if (
+    quoteKeys.length > 0
+    && input.existingQuoteKeys !== undefined
+    && quoteKeys.every((key) => input.existingQuoteKeys!.has(key))
+  ) {
+    findings.push({
+      rule: "echo_of_existing_content",
+      action: "refuse",
+      reason: "every evidence quote already exists in the store (echo, not new material)",
+    });
+  }
+
+  return findings;
+}

package/src/index.ts

@@ -49,6 +49,7 @@ export { searchWithGenericExclusion } from "./access-memory-search-fanout.js";
 export * from "./model-registry.js";
 export * from "./contradiction/index.js";
 export * from "./recall-state-view.js";
+export * from "./standing-memory-block.js";
 
 export {
   buildEvidencePack,

package/src/mcp-read-only-tools.ts

@@ -30,6 +30,9 @@ export const MCP_READ_ONLY_TOOL_SUFFIXES: Readonly<Record<string, true>> = {
   deep_recall: true,
   memory_expand: true,
   memory_traverse: true,
+  memory_ls: true,
+  memory_tree: true,
+  memory_find: true,
   standup: true,
   action_confidence: true,
   capsule_list: true,

package/src/memory-browse-http-glue.ts

@@ -0,0 +1,93 @@
+/**
+ * Memory-store browse HTTP glue (issue #2978) — route bodies live here so
+ * access-http.ts stays at its structural ceiling (same seam as
+ * recall-navigation-http-glue.ts).
+ *
+ * Each route carries its literal `enforceTokenOp("<op>")` boundary-dispatch
+ * marker so `access-surface-catalog.test.ts` can prove dispatch statically.
+ * Input errors map to 400; everything else rethrows to the global 500
+ * handler. Namespace resolution flows through the caller-supplied
+ * `scopeFor` gate so a body-supplied namespace passes the SAME
+ * effective-namespace allow-list as the query string (issue #1850 finding
+ * 2), and `authenticatedPrincipal` reaches the service so authorization
+ * derives from the presenting principal, never client-supplied input.
+ */
+
+import type { IncomingMessage, ServerResponse } from "node:http";
+
+import { EngramAccessInputError } from "./access-errors.js";
+import type { EngramAccessService } from "./access-service.js";
+import type { BrowseVerb } from "./memory-browse.js";
+
+type RespondJson = (res: ServerResponse, status: number, payload: unknown) => void;
+type ReadJsonBody = (req: IncomingMessage) => Promise<unknown>;
+
+type ScopeFor = (
+  bodyNamespace?: string,
+) => { namespace?: string; authenticatedPrincipal?: string };
+
+function optStr(value: unknown): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
+function optInt(value: unknown): number | undefined {
+  if (value === undefined || value === null || value === "") return undefined;
+  const parsed = typeof value === "string" ? Number(value) : value;
+  if (typeof parsed !== "number" || !Number.isInteger(parsed)) {
+    throw new EngramAccessInputError("depth must be an integer");
+  }
+  return parsed;
+}
+
+async function respondBrowse(
+  req: IncomingMessage,
+  res: ServerResponse,
+  respondJson: RespondJson,
+  readJsonBody: ReadJsonBody,
+  service: EngramAccessService,
+  scopeFor: ScopeFor,
+  verb: BrowseVerb,
+): Promise<void> {
+  const body = (await readJsonBody(req)) as Record<string, unknown>;
+  const scope = scopeFor(optStr(body.namespace));
+  const depth = verb === "tree" ? optInt(body.depth) : undefined;
+  const result = await service.memoryStoreBrowse({
+    verb,
+    ...(optStr(body.path) !== undefined ? { path: optStr(body.path) } : {}),
+    ...(depth !== undefined ? { depth } : {}),
+    ...(verb === "find" && optStr(body.pattern) !== undefined ? { pattern: optStr(body.pattern) } : {}),
+    namespace: scope.namespace,
+    authenticatedPrincipal: scope.authenticatedPrincipal,
+  });
+  const { rendered: _rendered, ...payload } = result;
+  respondJson(res, 200, payload);
+}
+
+export async function maybeRespondMemoryBrowse(
+  req: IncomingMessage,
+  res: ServerResponse,
+  pathname: string,
+  method: string | undefined,
+  enforceTokenOp: (op: "memory_ls" | "memory_tree" | "memory_find") => void,
+  respondJson: RespondJson,
+  readJsonBody: ReadJsonBody,
+  service: EngramAccessService,
+  scopeFor: ScopeFor,
+): Promise<boolean> {
+  if (method === "POST" && (pathname === "/engram/v1/memory/ls" || pathname === "/remnic/v1/memory/ls")) {
+    enforceTokenOp("memory_ls");
+    await respondBrowse(req, res, respondJson, readJsonBody, service, scopeFor, "ls");
+    return true;
+  }
+  if (method === "POST" && (pathname === "/engram/v1/memory/tree" || pathname === "/remnic/v1/memory/tree")) {
+    enforceTokenOp("memory_tree");
+    await respondBrowse(req, res, respondJson, readJsonBody, service, scopeFor, "tree");
+    return true;
+  }
+  if (method === "POST" && (pathname === "/engram/v1/memory/find" || pathname === "/remnic/v1/memory/find")) {
+    enforceTokenOp("memory_find");
+    await respondBrowse(req, res, respondJson, readJsonBody, service, scopeFor, "find");
+    return true;
+  }
+  return false;
+}

package/src/memory-browse-mcp-tools.ts

@@ -0,0 +1,57 @@
+/**
+ * Memory-store browse MCP tool definitions (issue #2978) — extracted from
+ * access-mcp.ts so the surface file stays at its structural ceiling (same
+ * seam as recall-navigation-mcp-tools.ts). Spread back into `this.tools`
+ * (`...MEMORY_BROWSE_MCP_TOOLS`); the migrated-operation mapping lives in
+ * access-mcp.ts's MCP_MIGRATED_OPERATIONS. The `remnic.` canonical alias
+ * is minted by withToolAliases at tools/list time.
+ *
+ * Always listed: browse is a pure read over the caller's resolved
+ * namespace, so there is no gating flag (unlike recallNavigation).
+ */
+
+import type { McpTool } from "./access-mcp.js";
+
+export const MEMORY_BROWSE_MCP_TOOLS: McpTool[] = [
+  {
+    name: "engram.memory_ls",
+    description:
+      "List what exists at one store path in your namespace: category directories with memory counts, memory files with a one-line description. Answers \"is there anything here at all?\" without a search query. Derived stores (artifacts/, meetings records, wearables, state/) are invisible by design.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        path: { type: "string", description: "Store path to list, relative to the namespace root (\"\" or \"/\" = root). Default root." },
+        namespace: { type: "string", description: "Caller namespace to scope this operation to." },
+      },
+      additionalProperties: false,
+    },
+  },
+  {
+    name: "engram.memory_tree",
+    description:
+      "Depth-limited tree under one store path (depth 1 acts like ls). Deterministic sorted output, entry-capped so a large store cannot flood the response.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        path: { type: "string", description: "Store path to expand, relative to the namespace root (default root)." },
+        depth: { type: "integer", minimum: 1, maximum: 4, description: "Expansion depth, 1-4. Default 1." },
+        namespace: { type: "string", description: "Caller namespace to scope this operation to." },
+      },
+      additionalProperties: false,
+    },
+  },
+  {
+    name: "engram.memory_find",
+    description:
+      "Deterministic name/path lookup over your namespace's store: `*` glob (e.g. \"facts/2026-08-*/*.md\") or substring. Use memory_search for semantic matching; use this when you know (part of) the name.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        pattern: { type: "string", description: "Glob with `*` wildcards or substring to match against path and filename. Required." },
+        namespace: { type: "string", description: "Caller namespace to scope this operation to." },
+      },
+      required: ["pattern"],
+      additionalProperties: false,
+    },
+  },
+];

package/src/memory-browse-operations.ts

@@ -0,0 +1,88 @@
+/**
+ * Memory-store browse boundary operations (issue #2978).
+ *
+ * `memory_ls` / `memory_tree` / `memory_find` route MCP, HTTP, and CLI
+ * dispatch through the shared validation + error-mapping layer
+ * (access-boundary.ts) to the single `EngramAccessService.memoryStoreBrowse`
+ * implementation. Sibling module so access-operations-batch.ts stays at its
+ * ceiling seam (same layout as recall-navigation-operations.ts).
+ */
+
+import { z } from "zod";
+
+import { defineOperation } from "./access-boundary.js";
+import { EngramAccessInputError } from "./access-errors.js";
+import type { BrowseVerb, MemoryStoreBrowseRequest } from "./memory-browse.js";
+
+function strictSchema<T extends z.ZodRawShape>(shape: T): z.ZodType<Record<string, unknown>> {
+  return z.preprocess(
+    (data) => {
+      if (data !== null && typeof data === "object" && !Array.isArray(data)) {
+        return Object.fromEntries(Object.entries(data as Record<string, unknown>).filter(([, value]) => value !== null));
+      }
+      return data;
+    },
+    z.object(shape).passthrough(),
+  ) as unknown as z.ZodType<Record<string, unknown>>;
+}
+
+function optStr(value: unknown): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
+function optInt(value: unknown): number | undefined {
+  if (typeof value === "number") return value;
+  if (typeof value === "string" && Number.isFinite(Number(value))) return Number(value);
+  return undefined;
+}
+
+function reqStr(value: unknown, field: string): string {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new EngramAccessInputError(`${field} is required`);
+  }
+  return value;
+}
+
+const verbShape = {
+  namespace: z.string().optional(),
+} as const;
+
+function browseRequest(verb: BrowseVerb, input: Record<string, unknown>): MemoryStoreBrowseRequest {
+  return {
+    verb,
+    ...(optStr(input.path) !== undefined ? { path: optStr(input.path) } : {}),
+    ...(optInt(input.depth) !== undefined ? { depth: optInt(input.depth) } : {}),
+    ...(optStr(input.pattern) !== undefined ? { pattern: optStr(input.pattern) } : {}),
+    namespace: optStr(input.namespace),
+  };
+}
+
+defineOperation({
+  name: "memory_ls",
+  description: "List the children of one store path in the caller's namespace: category dirs with counts, memory files with a one-line description.",
+  schema: strictSchema({ ...verbShape, path: z.string().optional() }),
+  handler: async (input, ctx) => ({
+    result: await ctx.service.memoryStoreBrowse({ ...browseRequest("ls", input), authenticatedPrincipal: ctx.authenticatedPrincipal }),
+  }),
+});
+
+defineOperation({
+  name: "memory_tree",
+  description: "Depth-limited tree of store paths under one directory, one-line descriptions, entry-capped.",
+  schema: strictSchema({ ...verbShape, path: z.string().optional(), depth: z.union([z.number(), z.string()]).optional() }),
+  handler: async (input, ctx) => ({
+    result: await ctx.service.memoryStoreBrowse({ ...browseRequest("tree", input), authenticatedPrincipal: ctx.authenticatedPrincipal }),
+  }),
+});
+
+defineOperation({
+  name: "memory_find",
+  description: "Deterministic name/path match (`*` glob or substring) over the caller's namespace store; semantic search stays memory_search.",
+  schema: strictSchema({ ...verbShape, pattern: z.string() }),
+  handler: async (input, ctx) => ({
+    result: await ctx.service.memoryStoreBrowse({
+      ...browseRequest("find", { ...input, pattern: reqStr(input.pattern, "pattern") }),
+      authenticatedPrincipal: ctx.authenticatedPrincipal,
+    }),
+  }),
+});