@remnic/core 9.69.49 → 9.69.51

package/src/recall-recognition-tier.ts

@@ -0,0 +1,375 @@
+/**
+ * Full-index recognition tier (issue #2975, foundation slice).
+ *
+ * distill-kura's recall does not search — it recognizes: the entire
+ * namespace index (one line per memory: id + description) goes into a
+ * single prompt and a small model names which entries bear on the
+ * question. Keyword/vector search misses same-subject-different-words
+ * cases; a recognizer that reads the whole index cannot miss what it
+ * can see (~500 entries ≈ 6k tokens, sub-second locally).
+ *
+ * This slice ships the deterministic tier machinery only — nothing on
+ * the live recall path reads it yet, so recall behavior is unchanged:
+ *  - a compact per-namespace index persisted at
+ *    `<namespace memoryDir>/state/index_recognition.json` with
+ *    build/save/load; loading is total: missing, corrupt, wrong-shape,
+ *    or future-version indexes all yield `null`, never an error;
+ *  - the tier decision, pure on (index, maxEntries): index present and
+ *    at or under the entry threshold → recognition; absent or above →
+ *    the vector tier, no error thrown;
+ *  - the runner: renders the FULL index into one prompt, asks the
+ *    recognizer for ids, keeps only ids the index actually carries, and
+ *    returns them in index order — never recognizer order;
+ *  - loud degradation: a recognizer that fails falls back to vector
+ *    search labeled `recognizer_unavailable` (quiet degradation is
+ *    worse than degradation).
+ *
+ * Determinism rules:
+ *  - the decision is a pure function of (index, maxEntries) — no clock,
+ *    no I/O, no locale;
+ *  - an empty index short-circuits: no model call, zero candidates;
+ *  - the prompt and the returned ids are byte-identical across runs
+ *    with unchanged inputs.
+ *
+ * Host contract for the wiring slices:
+ *  - maintenance slice: build/save the index at memory write time from
+ *    each memory's id + description line (a recognition trigger — proper
+ *    nouns, numbers, landmines, the conclusion reached — not a summary),
+ *    serializing writes the way temporal-index does (op chain + lock);
+ *    saveRecognitionIndex here is single-writer and does not lock;
+ *  - recall slice: for each recalled namespace call
+ *    recallViaRecognitionTier with the recognizer (one small-model call)
+ *    and the existing vector search as the fallback; expand picks
+ *    through the graph neighborhood, merge through the normal pipeline
+ *    (filters → rerank → budget), and record tier = recognition plus any
+ *    degraded fallback in xray;
+ *  - config keys recallRecognitionTier (default false) and
+ *    recognitionIndexMaxEntries (default 500) gate the wiring.
+ */
+import { promises as fsp } from "node:fs";
+import path from "node:path";
+import { coerceBooleanLike, coerceNumber } from "./connectors/coerce.js";
+import { log } from "./logger.js";
+
+export const DEFAULT_RECALL_RECOGNITION_TIER = false;
+export const DEFAULT_RECOGNITION_INDEX_MAX_ENTRIES = 500;
+export const RECOGNITION_INDEX_VERSION = 1;
+const RECOGNITION_INDEX_FILE = "index_recognition.json";
+
+export const RECOGNITION_PROMPT_HEADER =
+  "Memory index — one line per memory. Name the ids that bear on the question.";
+export const RECOGNITION_PROMPT_FOOTER =
+  "Answer with the relevant ids only, comma-separated. If none bear on the question, answer with nothing.";
+
+export interface RecognitionIndexEntry {
+  /** Stable memory id. Candidates always come back in index order. */
+  id: string;
+  /** One-line recognition trigger; not a summary (issue #2975 swap test). */
+  description: string;
+}
+
+export interface RecognitionIndex {
+  version: number;
+  entries: RecognitionIndexEntry[];
+}
+
+/**
+ * Config fields this tier owns, mixed into `PluginConfig` (issue #2975) so
+ * the keys live beside their parsers instead of growing `types.ts`.
+ */
+export interface RecognitionTierSettings {
+  /** Small namespaces recall by recognizing against the whole compact index instead of vector search; parsed now, wiring lands in a later slice. Default false. */
+  recallRecognitionTier: boolean;
+  /** Max entries a namespace's recognition index may carry for the tier to engage (inclusive). Default 500. */
+  recognitionIndexMaxEntries: number;
+}
+
+export function parseRecallRecognitionTier(raw: unknown): boolean {
+  return coerceBooleanLike(raw, "recallRecognitionTier") === true;
+}
+
+export function parseRecognitionIndexMaxEntries(raw: unknown): number {
+  if (raw === undefined || raw === null) return DEFAULT_RECOGNITION_INDEX_MAX_ENTRIES;
+  const n = coerceNumber(raw, "recognitionIndexMaxEntries");
+  if (n === undefined || !Number.isFinite(n) || n < 1) {
+    throw new Error(
+      `Invalid recognitionIndexMaxEntries: expected an integer >= 1, got ${JSON.stringify(raw)}`,
+    );
+  }
+  return Math.floor(n);
+}
+
+/** On-disk location of a namespace's recognition index. */
+export function recognitionIndexPath(memoryDir: string): string {
+  return path.join(memoryDir, "state", RECOGNITION_INDEX_FILE);
+}
+
+/**
+ * Normalize raw records into a recognition index: blank/whitespace ids
+ * are dropped, descriptions are trimmed, and the first record for an id
+ * wins. Input order is preserved — it is the recognition order.
+ */
+export function buildRecognitionIndex(
+  records: ReadonlyArray<{ id?: string | null; description?: string | null }>,
+): RecognitionIndex {
+  const seen = new Set<string>();
+  const out: RecognitionIndexEntry[] = [];
+  for (const record of records) {
+    const id = typeof record.id === "string" ? record.id.trim() : "";
+    if (id.length === 0 || seen.has(id)) continue;
+    seen.add(id);
+    out.push({
+      id,
+      description: typeof record.description === "string" ? record.description.trim() : "",
+    });
+  }
+  return { version: RECOGNITION_INDEX_VERSION, entries: out };
+}
+
+/**
+ * Persist a recognition index. Single-writer: the maintenance slice that
+ * owns index writes must serialize calls the way temporal-index does.
+ */
+export async function saveRecognitionIndex(
+  memoryDir: string,
+  index: RecognitionIndex,
+): Promise<void> {
+  const file = recognitionIndexPath(memoryDir);
+  await fsp.mkdir(path.dirname(file), { recursive: true });
+  await fsp.writeFile(file, `${JSON.stringify(index, null, 2)}\n`, "utf8");
+}
+
+function isRecognitionIndex(raw: unknown): raw is RecognitionIndex {
+  if (typeof raw !== "object" || raw === null) return false;
+  const candidate = raw as { version?: unknown; entries?: unknown };
+  if (candidate.version !== RECOGNITION_INDEX_VERSION) return false;
+  if (!Array.isArray(candidate.entries)) return false;
+  return candidate.entries.every(
+    (entry) =>
+      typeof entry === "object" &&
+      entry !== null &&
+      typeof (entry as RecognitionIndexEntry).id === "string" &&
+      typeof (entry as RecognitionIndexEntry).description === "string",
+  );
+}
+
+/**
+ * Load a namespace's recognition index. Total: a missing, corrupt,
+ * wrong-shape, or future-version file all return `null` so the caller
+ * falls back to vector search without an error.
+ */
+export async function loadRecognitionIndex(memoryDir: string): Promise<RecognitionIndex | null> {
+  try {
+    const raw = await fsp.readFile(recognitionIndexPath(memoryDir), "utf8");
+    const parsed: unknown = JSON.parse(raw);
+    if (isRecognitionIndex(parsed)) return parsed;
+    log.warn(
+      `recognition index ignored at ${RECOGNITION_INDEX_FILE}: unexpected shape; falling back to vector search`,
+    );
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+export type RecognitionRecallTier = "recognition" | "vector";
+export type RecognitionTierReason =
+  | "index_absent"
+  | "index_above_threshold"
+  | "index_within_threshold";
+
+export interface RecognitionTierDecision {
+  tier: RecognitionRecallTier;
+  reason: RecognitionTierReason;
+  /** Entries the decision considered; 0 when the index is absent. */
+  entriesConsidered: number;
+  /** Threshold in force. */
+  maxEntries: number;
+}
+
+/**
+ * Deterministic tier choice, pure on (index, maxEntries). The threshold
+ * is inclusive: an index of exactly maxEntries entries still recognizes
+ * (a threshold-sized index is the designed working set, not an overrun).
+ */
+export function decideRecognitionTier(
+  index: RecognitionIndex | null,
+  opts: { maxEntries: number },
+): RecognitionTierDecision {
+  const maxEntries = Math.max(1, Math.floor(opts.maxEntries));
+  if (index === null) {
+    return { tier: "vector", reason: "index_absent", entriesConsidered: 0, maxEntries };
+  }
+  if (index.entries.length > maxEntries) {
+    return {
+      tier: "vector",
+      reason: "index_above_threshold",
+      entriesConsidered: index.entries.length,
+      maxEntries,
+    };
+  }
+  return {
+    tier: "recognition",
+    reason: "index_within_threshold",
+    entriesConsidered: index.entries.length,
+    maxEntries,
+  };
+}
+
+/** Render the full index, one line per entry, in index order. */
+export function renderRecognitionIndex(entries: ReadonlyArray<RecognitionIndexEntry>): string {
+  return entries.map((entry) => `${entry.id}: ${entry.description}`).join("\n");
+}
+
+/** The single recognition prompt: header, the FULL index, the question, the answer shape. */
+export function buildRecognitionPrompt(
+  query: string,
+  entries: ReadonlyArray<RecognitionIndexEntry>,
+): string {
+  return [
+    RECOGNITION_PROMPT_HEADER,
+    renderRecognitionIndex(entries),
+    `Question: ${query}`,
+    RECOGNITION_PROMPT_FOOTER,
+  ].join("\n\n");
+}
+
+function dedupeOrdered(tokens: string[]): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const token of tokens) {
+    if (token.length > 0 && !seen.has(token)) {
+      seen.add(token);
+      out.push(token);
+    }
+  }
+  return out;
+}
+
+const TRAILING_PUNCTUATION: Record<string, true> = { ".": true, ",": true, ";": true, ":": true };
+
+/**
+ * Strip trailing sentence punctuation in linear time. A quantified anchored
+ * regex (`/[.,;:]+$/`) backtracks polynomially on recognizer output made of
+ * many separators, and that output is uncontrolled model text.
+ */
+function stripTrailingPunctuation(value: string): string {
+  let end = value.length;
+  while (end > 0 && TRAILING_PUNCTUATION[value.charAt(end - 1)]) end -= 1;
+  return value.slice(0, end);
+}
+
+/**
+ * Parse recognizer output into ordered unique id tokens. Accepts a JSON
+ * string array or comma/whitespace-separated ids, and strips sentence
+ * punctuation so prose wrappers do not corrupt ids. Tokens that are not
+ * real ids are dropped later by validation against the index.
+ */
+export function parseRecognizerIds(output: string | null | undefined): string[] {
+  if (typeof output !== "string") return [];
+  const text = output.trim();
+  if (text.length === 0) return [];
+  if (text.startsWith("[")) {
+    try {
+      const parsed: unknown = JSON.parse(text);
+      if (Array.isArray(parsed)) {
+        return dedupeOrdered(
+          parsed
+            .filter((item): item is string => typeof item === "string")
+            .map((item) => stripTrailingPunctuation(item.trim()).trim()),
+        );
+      }
+    } catch {
+      // Fall through to separator parsing.
+    }
+  }
+  return dedupeOrdered(
+    text
+      .split(/[\s,]+/)
+      .map((token) => stripTrailingPunctuation(token.replace(/^[•\-*\d.)\]]+/, "")).trim()),
+  );
+}
+
+/** One model call over the full index; supplied by the wiring slice. */
+export type Recognizer = (prompt: string) => Promise<string | null | undefined>;
+
+export interface RecognitionRunResult {
+  /** Recognized ids, validated against the index, in index order. */
+  ids: string[];
+  /** Recognizer tokens the index does not carry (hallucinated ids). */
+  dropped: string[];
+  /** True when no model call was made (empty index). */
+  skipped: boolean;
+}
+
+/**
+ * Run the recognition pass: the FULL index goes into one prompt, the
+ * recognizer names ids, unknown ids are dropped, and the kept ids come
+ * back in index order so downstream merging stays deterministic.
+ */
+export async function runRecognitionTier(
+  query: string,
+  entries: ReadonlyArray<RecognitionIndexEntry>,
+  recognize: Recognizer,
+): Promise<RecognitionRunResult> {
+  if (entries.length === 0) {
+    return { ids: [], dropped: [], skipped: true };
+  }
+  const raw = parseRecognizerIds(await recognize(buildRecognitionPrompt(query, entries)));
+  const byId = new Map(entries.map((entry) => [entry.id, entry]));
+  const picked = new Set(raw.filter((token) => byId.has(token)));
+  return {
+    ids: entries.filter((entry) => picked.has(entry.id)).map((entry) => entry.id),
+    dropped: dedupeOrdered(raw.filter((token) => !byId.has(token))),
+    skipped: false,
+  };
+}
+
+export interface RecognitionTierOutcome<TVector> {
+  decision: RecognitionTierDecision;
+  /** Recognition picks, index order; empty on the vector tier. */
+  ids: string[];
+  /** Vector-tier results from the fallback; empty on the recognition tier. */
+  vectorResults: TVector[];
+  /**
+   * Set when recognition was routed but the recognizer failed and the
+   * vector tier served the recall instead — loud degradation.
+   */
+  degraded?: "recognizer_unavailable";
+}
+
+/**
+ * The wiring-seam composition for one namespace: load the index, decide
+ * the tier, then either recognize against the full index or run the
+ * caller's vector search. Never throws for index problems; a failing
+ * recognizer degrades to vector search labeled `recognizer_unavailable`.
+ */
+export async function recallViaRecognitionTier<TVector>(args: {
+  memoryDir: string;
+  query: string;
+  maxEntries: number;
+  recognize: Recognizer;
+  vectorSearch: () => Promise<TVector[]>;
+}): Promise<RecognitionTierOutcome<TVector>> {
+  const index = await loadRecognitionIndex(args.memoryDir);
+  const decision = decideRecognitionTier(index, { maxEntries: args.maxEntries });
+  if (index === null || decision.tier === "vector") {
+    return { decision, ids: [], vectorResults: await args.vectorSearch() };
+  }
+  try {
+    const run = await runRecognitionTier(args.query, index.entries, args.recognize);
+    return { decision, ids: run.ids, vectorResults: [] };
+  } catch (err) {
+    log.warn(
+      `recognition tier degraded to vector search: recognizer failed${
+        err instanceof Error ? ` (${err.message})` : ""
+      }`,
+    );
+    return {
+      decision,
+      ids: [],
+      vectorResults: await args.vectorSearch(),
+      degraded: "recognizer_unavailable",
+    };
+  }
+}

package/src/recall-xray-renderer.test.ts

@@ -615,3 +615,20 @@ test("renderXrayText tier-explain block matches shared helper output", async ()
     assert.ok(nullText.includes(line));
   }
 });
+
+test("#2972 xray text and markdown surface degradation only when present", () => {
+  const healthy = minimalSnapshot();
+  const degraded = {
+    ...minimalSnapshot(),
+    degradation: {
+      state: "degraded" as const,
+      reason: "budget-clipped" as const,
+    },
+  };
+  const healthyText = renderXrayText(healthy);
+  assert.equal(healthyText.includes("degradation:"), false);
+  assert.match(renderXrayText(degraded), /degradation: degraded\/budget-clipped/);
+  assert.equal(renderXrayMarkdown(healthy).includes("| Degradation |"), false);
+  assert.match(renderXrayMarkdown(degraded), /\| Degradation \| degraded\/budget-clipped \|/);
+});
+

package/src/recall-xray-renderer.ts

@@ -108,6 +108,11 @@ export function renderXrayText(snapshot: RecallXraySnapshot | null): string {
   lines.push(
     `budget: ${snapshot.budget.used} / ${snapshot.budget.chars} chars`,
   );
+  if (snapshot.degradation) {
+    lines.push(
+      `degradation: ${snapshot.degradation.state}/${snapshot.degradation.reason}`,
+    );
+  }
 
   lines.push("");
   lines.push("--- filters ---");
@@ -247,6 +252,11 @@ export function renderXrayMarkdown(
   lines.push(
     `| Budget | ${snapshot.budget.used} / ${snapshot.budget.chars} chars |`,
   );
+  if (snapshot.degradation) {
+    lines.push(
+      `| Degradation | ${snapshot.degradation.state}/${snapshot.degradation.reason} |`,
+    );
+  }
 
   lines.push("");
   lines.push("## Filters");

package/src/recall-xray.test.ts

@@ -650,3 +650,31 @@ test("RecallXrayBuilder clones the trust projection so the snapshot surfaces it
     "quarantine reason preserved so exclusion never looks like 'no result' (rule 34)",
   );
 });
+
+test("#2972 healthy xray snapshot omits the degradation marker", () => {
+  const snapshot = buildXraySnapshot({
+    query: "q",
+    now: fixedNow,
+    snapshotIdGenerator: idGen(),
+  });
+  assert.equal("degradation" in snapshot, false);
+  assert.ok(!JSON.stringify(snapshot).includes("degradation"));
+});
+
+test("#2972 xray snapshot copies degradation and isolates caller mutation", () => {
+  const degradation = {
+    state: "degraded" as const,
+    reason: "budget-clipped" as const,
+    budget: { contextBudget: 20, fullChars: 100, deliveredChars: 20 },
+  };
+  const snapshot = buildXraySnapshot({
+    query: "q",
+    degradation,
+    now: fixedNow,
+    snapshotIdGenerator: idGen(),
+  });
+  assert.deepEqual(snapshot.degradation, degradation);
+  degradation.budget.deliveredChars = 0;
+  assert.equal(snapshot.degradation?.budget?.deliveredChars, 20);
+});
+

package/src/recall-xray.ts

@@ -24,6 +24,7 @@ import { estimateTokenCount } from "./token-estimate.js";
 import type { RecallDisclosure, RecallTierExplain } from "./types.js";
 import type { StateLabel } from "./recall-state-view.js";
 import { isRecallDisclosure } from "./types.js";
+import type { RecallContextDegradation } from "./recall-context-composition.js";
 
 /**
  * Estimate token cost of a payload for X-ray disclosure accounting.
@@ -282,6 +283,12 @@ export interface RecallXraySnapshot {
    * absent) means no peer profile was injected.
    */
   peerProfileInjection?: RecallXrayPeerProfileInjection | null;
+  /**
+   * Injected-context degradation (#2972). Absent on a healthy recall so
+   * the snapshot stays byte-stable. Budget warnings ride here, never
+   * in the injected text.
+   */
+  degradation?: RecallContextDegradation;
 }
 
 // ─── Builder ──────────────────────────────────────────────────────────────
@@ -300,6 +307,8 @@ export interface BuildXraySnapshotInput {
   traceId?: string;
   /** Peer-profile injection metadata (issue #679 completion). */
   peerProfileInjection?: RecallXrayPeerProfileInjection | null;
+  /** Injected-context degradation (#2972). Omit on a healthy recall. */
+  degradation?: RecallContextDegradation;
   /** Optional injected timestamp for deterministic tests. */
   now?: () => number;
   /** Optional injected id generator for deterministic tests. */
@@ -375,6 +384,12 @@ export function buildXraySnapshot(
   if (peerProfileInjection !== undefined) {
     out.peerProfileInjection = peerProfileInjection;
   }
+  if (input.degradation && typeof input.degradation === "object") {
+    out.degradation = { ...input.degradation };
+    if (input.degradation.budget) {
+      out.degradation.budget = { ...input.degradation.budget };
+    }
+  }
   return out;
 }
 

package/src/testing/subjects/recall-recognition-tier.test.ts

@@ -0,0 +1,106 @@
+/**
+ * Recognition-tier swap subject for the scenario-matrix harness (issue #2975).
+ *
+ * Every canonical row runs the real fetchQmdMemoryResultsWithRecognitionSwap
+ * seam twice: gate off (original fetch only, zero index I/O, zero recognizer
+ * calls) and gate on with an under-threshold index (one recognizer call, the
+ * namespace never reaches vector search). The scenario dimensions are
+ * orthogonal to this routing decision, so the same invariant holds per row.
+ */
+import assert from "node:assert/strict";
+import os from "node:os";
+import path from "node:path";
+import { mkdtemp, rm } from "node:fs/promises";
+
+import {
+  fetchQmdMemoryResultsWithRecognitionSwap,
+  type RecognitionSearchDeps,
+} from "../../orchestration/recall-recognition-search.js";
+import { saveRecognitionIndex } from "../../recall-recognition-tier.js";
+import type { QmdSearchResult } from "../../types.js";
+import { type LifecycleSubject, type MatrixRow, runLifecycleMatrix } from "../lifecycle-matrix.js";
+interface SwapState {
+  rowId: string;
+  dir: string;
+  deps: (enabled: boolean) => RecognitionSearchDeps;
+  fetchCalls: number;
+  recognizeCalls: number;
+  storageForCalls: number;
+}
+
+const subject: LifecycleSubject<SwapState> = {
+  async setup(row: MatrixRow): Promise<SwapState> {
+    const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-recognition-subject-"));
+    await saveRecognitionIndex(dir, {
+      version: 1,
+      entries: [
+        { id: `m-${row.id}-a`, description: `trigger for ${row.id}-a` },
+        { id: `m-${row.id}-b`, description: `trigger for ${row.id}-b` },
+      ],
+    });
+    const state: SwapState = { rowId: String(row.id), dir, deps: undefined as never, fetchCalls: 0, recognizeCalls: 0, storageForCalls: 0 };
+    state.deps = (enabled) =>
+      ({
+        config: { recallRecognitionTier: enabled, recognitionIndexMaxEntries: 500 },
+        storageRouter: {
+          storageFor: async () => {
+            state.storageForCalls += 1;
+            return {
+              dir,
+              findExistingMemoryPaths: async (ids: string[]) =>
+                new Map(ids.map((id) => [id, [path.join(dir, `facts/${id}.md`)]])),
+            };
+          },
+        },
+        fastLlmForRerank: {
+          chatCompletion: async () => {
+            state.recognizeCalls += 1;
+            return { content: JSON.stringify([`m-${row.id}-a`, `m-${row.id}-b`]) };
+          },
+        },
+        fetchQmdMemoryResultsWithArtifactTopUp: async (): Promise<QmdSearchResult[]> => {
+          state.fetchCalls += 1;
+          return [{ docid: `default:vector.md`, namespace: "default", path: "vector.md", score: 1, snippet: "vector" }];
+        },
+      }) as unknown as RecognitionSearchDeps;
+    return state;
+  },
+
+  async exercise(state): Promise<void> {
+    const options = {
+      namespacesEnabled: true,
+      recallNamespaces: ["default"],
+      resolveNamespace: () => "default",
+    };
+
+    const off = await fetchQmdMemoryResultsWithRecognitionSwap(state.deps(false), "q", 5, 7, options);
+    assert.equal(off.length, 1);
+    assert.equal(state.fetchCalls, 1);
+    assert.equal(state.recognizeCalls, 0);
+    assert.equal(state.storageForCalls, 0);
+
+    const on = await fetchQmdMemoryResultsWithRecognitionSwap(state.deps(true), "q", 5, 7, options);
+    assert.equal(state.recognizeCalls, 1);
+    assert.equal(state.fetchCalls, 1, "recognition namespace must not reach vector fetch");
+    assert.deepEqual(on.map((r) => r.path).sort(), [`facts/m-${state.rowId}-a.md`, `facts/m-${state.rowId}-b.md`].sort());
+  },
+
+  async invariants(state): Promise<void> {
+    // Off-path again after the on-path run: still a pure passthrough.
+    const before = { fetch: state.fetchCalls, recognize: state.recognizeCalls, storage: state.storageForCalls };
+    await fetchQmdMemoryResultsWithRecognitionSwap(state.deps(false), "q", 5, 7, {
+      namespacesEnabled: true,
+      recallNamespaces: ["default"],
+      resolveNamespace: () => "default",
+    });
+    assert.equal(state.fetchCalls, before.fetch + 1);
+    assert.equal(state.recognizeCalls, before.recognize);
+    assert.equal(state.storageForCalls, before.storage);
+  },
+
+  async teardown(state): Promise<void> {
+    await rm(state.dir, { recursive: true, force: true });
+  },
+};
+
+runLifecycleMatrix("recall-recognition-tier", subject);