@remnic/core 9.65.0 → 9.65.2

package/src/cross-lingual-recall.ts

@@ -0,0 +1,186 @@
+/**
+ * Cross-lingual recall planner (issue #2197).
+ *
+ * Lexical tiers (BM25/FTS) match on shared surface tokens, so a query whose
+ * dominant script differs from the corpus's cannot retrieve lexically — an
+ * English query never token-overlaps a Japanese fact. When the planner
+ * detects that mismatch it leans on the vector tier (the embedding-fallback
+ * supplement in the recall search pipeline), and when NO vector tier exists
+ * it emits a {@link SearchDegradation} so "no results" stays distinguishable
+ * from "cross-script query, vectors unavailable" (rule 34).
+ *
+ * The write-time half is a dominant-script hint stamped into frontmatter
+ * `language` by `StorageManager.writeMemory` (ISO 15924 script codes);
+ * legacy files without the field simply do not vote for the corpus script.
+ */
+
+import type { MemoryFile } from "./types.js";
+import type { SearchDegradation } from "./search/port.js";
+
+type CorpusLanguageStorage = {
+  dir: string;
+  readAllMemories: () => Promise<MemoryFile[]>;
+  getCorpusScanVersion: () => string;
+};
+
+/** Script classes the cheap detector can distinguish. ISO 15924 codes. */
+export type LanguageHint =
+  | "latn"
+  | "jpan"
+  | "kore"
+  | "hani"
+  | "cyrl"
+  | "grek"
+  | "arab"
+  | "hebr"
+  | "thai"
+  | "deva";
+
+/** Inclusive codepoint ranges per script class. `kana` folds into `jpan` at detection. */
+const SCRIPT_RANGES: ReadonlyArray<readonly [LanguageHint | "kana", number, number]> = [
+  ["latn", 0x41, 0x5a],
+  ["latn", 0x61, 0x7a],
+  ["latn", 0xc0, 0x24f],
+  ["grek", 0x370, 0x3ff],
+  ["cyrl", 0x400, 0x4ff],
+  ["cyrl", 0x500, 0x52f],
+  ["hebr", 0x590, 0x5ff],
+  ["arab", 0x600, 0x6ff],
+  ["arab", 0x750, 0x77f],
+  ["deva", 0x900, 0x97f],
+  ["thai", 0xe00, 0xe7f],
+  ["kore", 0x1100, 0x11ff],
+  ["latn", 0x1e00, 0x1eff],
+  ["kana", 0x3040, 0x309f],
+  ["kana", 0x30a0, 0x30ff],
+  ["kana", 0x31f0, 0x31ff],
+  ["hani", 0x3400, 0x4dbf],
+  ["hani", 0x4e00, 0x9fff],
+  ["kore", 0xac00, 0xd7a3],
+  ["kana", 0xff66, 0xff9d],
+];
+
+/**
+ * Dominant-script hint for a text. Codepoint-range counting (no regex):
+ * kana anywhere maps to `jpan` — kana is unique to Japanese and disambiguates
+ * it from `hani` (Chinese, or kanji-only Japanese). Returns `undefined` when
+ * the text contains no lettered script class.
+ */
+export function detectLanguageHint(text: string): LanguageHint | undefined {
+  const counts = new Map<LanguageHint | "kana", number>();
+  for (let i = 0; i < text.length; i += 1) {
+    const code = text.charCodeAt(i);
+    for (const [script, low, high] of SCRIPT_RANGES) {
+      if (code < low) break;
+      if (code <= high) {
+        counts.set(script, (counts.get(script) ?? 0) + 1);
+        break;
+      }
+    }
+  }
+  // SCRIPT_RANGES is sorted by start codepoint, so `break` on the first
+  // range starting past the codepoint is exact — one range test per char.
+  if (counts.has("kana")) return "jpan";
+  let best: LanguageHint | undefined;
+  let bestCount = 0;
+  for (const [script, count] of counts) {
+    if (script === "kana") continue;
+    if (count > bestCount) {
+      best = script;
+      bestCount = count;
+    }
+  }
+  return best;
+}
+
+/** Dominant hint over a corpus sample; `undefined` entries (legacy files) never vote. */
+export function corpusDominantLanguage(
+  hints: Iterable<string | undefined>,
+): LanguageHint | undefined {
+  const counts = new Map<string, number>();
+  for (const hint of hints) {
+    if (hint === undefined || hint === "") continue;
+    counts.set(hint, (counts.get(hint) ?? 0) + 1);
+  }
+  let best: string | undefined;
+  let bestCount = 0;
+  for (const [hint, count] of counts) {
+    if (count > bestCount) {
+      best = hint;
+      bestCount = count;
+    }
+  }
+  return best as LanguageHint | undefined;
+}
+
+/** Planner decision for one recall query. */
+export interface CrossScriptPlan {
+  queryLanguage: LanguageHint | undefined;
+  corpusLanguage: LanguageHint | undefined;
+  /** True when both hints resolved and differ — lexical tiers cannot serve this query. */
+  crossScript: boolean;
+  /** Emitted when `crossScript` is true and no vector tier can compensate. */
+  degradation?: SearchDegradation;
+}
+
+/**
+ * Pure planner core: compare query/corpus hints and, on a mismatch without a
+ * vector tier, build the visible degradation signal.
+ */
+export function planCrossScript(input: {
+  queryLanguage: LanguageHint | undefined;
+  corpusLanguage: LanguageHint | undefined;
+  vectorTierAvailable: boolean;
+}): CrossScriptPlan {
+  const crossScript =
+    input.queryLanguage !== undefined &&
+    input.corpusLanguage !== undefined &&
+    input.queryLanguage !== input.corpusLanguage;
+  return {
+    queryLanguage: input.queryLanguage,
+    corpusLanguage: input.corpusLanguage,
+    crossScript,
+    degradation:
+      crossScript && !input.vectorTierAvailable
+        ? {
+            backend: "qmd",
+            code: "vector_tier_unavailable",
+            detail:
+              `cross-script query (${input.queryLanguage}) over a ${input.corpusLanguage} corpus: ` +
+              "lexical recall cannot match across scripts — enable a multilingual embedding model " +
+              "(embedding fallback) so the vector tier can serve it",
+          }
+        : undefined,
+  };
+}
+
+/** Corpus sample cap: dominant script stabilizes well below the full corpus. */
+const CORPUS_SAMPLE_CAP = 500;
+
+/** Memoized dominant corpus language per storage dir, keyed by corpus scan version. */
+const corpusLanguageCache = new Map<string, { version: string; language: LanguageHint | undefined }>();
+
+/**
+ * Dominant corpus language for a storage manager, sampled from the newest
+ * frontmatter `language` hints (capped, memoized on the corpus scan version so
+ * recurring recalls pay one pass per corpus generation, not per query).
+ * Read failures degrade to `undefined` — never break recall.
+ */
+export async function dominantCorpusLanguage(
+  storage: CorpusLanguageStorage,
+): Promise<LanguageHint | undefined> {
+  try {
+    const version = storage.getCorpusScanVersion();
+    const cached = corpusLanguageCache.get(storage.dir);
+    if (cached?.version === version) return cached.language;
+    const memories = await storage.readAllMemories();
+    const language = corpusDominantLanguage(
+      memories.slice(0, CORPUS_SAMPLE_CAP).map((memory: MemoryFile) => memory.frontmatter.language),
+    );
+    if (corpusLanguageCache.size > 8) corpusLanguageCache.clear();
+    corpusLanguageCache.set(storage.dir, { version, language });
+    return language;
+  } catch {
+    return undefined;
+  }
+}

package/src/extraction-prompt.ts

@@ -22,6 +22,7 @@ export const EXTRACTION_RESPONSE_SHAPE = `{
     "promptedByQuestion": "<optional source-grounded question>",
     "quote": "<optional exact contiguous source span>",
     "scope": "<optional project-or-global>",
+    "subject": "<optional user-or-agent>",
     "structuredAttributes": {"<key>": "<value>"},
     "procedureSteps": [{"order": 1, "intent": "<step>"}, {"order": 2, "intent": "<step>"}],
     "reasoningTrace": {

package/src/extraction.ts

@@ -312,6 +312,8 @@ export class ExtractionEngine {
               promptedByQuestion: extractionText(f.promptedByQuestion),
               scope:
                 f.scope === "global" || f.scope === "project" ? f.scope : undefined,
+              subject:
+                f.subject === "user" || f.subject === "agent" ? f.subject : undefined,
               structuredAttributes: extractionAttributes(f.structuredAttributes),
               procedureSteps,
               reasoningTrace,
@@ -1425,7 +1427,8 @@ Rules:
 - ${OUTPUT_LANGUAGE_POLICY}
 ${CUE_ANCHOR_PROMPT_INSTRUCTION}\n- Include at most five durable relationships.${this.config.provenance?.enabled ? `
 - Each fact must include a quote copied verbatim from one contiguous conversation span.` : ""}${lifecycleCaps.extractionScopeClassification ? `
-- Set each fact scope to "global" for cross-project knowledge or "project" for codebase-specific knowledge. Tool, command, or CLI-flag instructions tied to one agent are "project" (the same tool name can mean different things across agents); when keeping one, begin the fact with a leading "In <agent>," clause naming that agent.` : ""}${ambientCapture ? AMBIENT_CAPTURE_PROMPT_SECTION_COMPACT : ""}
+- Set each fact scope to "global" for cross-project knowledge or "project" for codebase-specific knowledge. Tool, command, or CLI-flag instructions tied to one agent are "project" (the same tool name can mean different things across agents); when keeping one, begin the fact with a leading "In <agent>," clause naming that agent.` : ""}${this.config.subjectClassification?.enabled === true ? `
+- Set each fact subject to "user" when it models the person (preferences, relationships, biography, moments, commitments about their life) or "agent" when it is reusable operating knowledge (procedures, principles, tool-usage lessons, debugging strategies, environment facts with no personal content).` : ""}${ambientCapture ? AMBIENT_CAPTURE_PROMPT_SECTION_COMPACT : ""}
 ${eventTimePromptInstruction(this.config)}
 Return only valid JSON matching this shape. Placeholder text describes field shape only and is never source evidence:
 ${EXTRACTION_RESPONSE_SHAPE}

package/src/index.ts

@@ -229,7 +229,7 @@ export type {
   ProcedureStatsConfigSnapshot,
   ProcedureStatsRecent,
 } from "./procedural/procedure-stats.js";
-
+export { runProcedureLibraryMaintenance, type ProcedureLibraryMaintenanceReport, type ProcedureMaintenanceAction, type ProcedureMaintenanceActionKind, type ProcedureMaintenanceReasonCode, type ProcedureMaintenanceEvidence, type ProcedureMaintenanceMarker } from "./procedural/library-maintenance.js";
 // ---------------------------------------------------------------------------
 // Direct-answer retrieval tier (issue #518)
 // ---------------------------------------------------------------------------
@@ -493,14 +493,13 @@ export {
   type RecallXrayFormat,
 } from "./recall-xray-renderer.js";
 export type {
-  RecallXraySnapshot,
-  RecallXrayResult,
-  RecallXrayScoreDecomposition,
-  RecallXrayServedBy,
-  RecallFilterTrace,
+  RecallXraySnapshot, RecallXrayResult, RecallXrayScoreDecomposition, RecallXrayServedBy, RecallFilterTrace,
 } from "./recall-xray.js";
 export * from "./who-knows.js";
 export * from "./who-knows-cli.js";
+export * from "./memory-subject.js";
+export * from "./memory-subject-cli.js";
+export * from "./subject-config.js";
 
 // ChatGPT Apps-compatible memory inspector demo.
 export {

package/src/mcp-read-only-tools.ts

@@ -17,6 +17,7 @@ export const MCP_READ_ONLY_TOOL_SUFFIXES: Readonly<Record<string, true>> = {
   recall_tier_explain: true,
   recall_xray: true,
   who_knows: true,
+  promotion_candidates: true,
   briefing: true,
   wearables_status: true,
   transcript_day: true,

package/src/memory-subject-cli.ts

@@ -0,0 +1,135 @@
+/**
+ * CLI surface for memory-subject operations (issue #2372).
+ *
+ * `remnic promotion-candidates` — read-only surfacing of agent-subject,
+ * reuse-signaled promotion candidates. `remnic subjects backfill` — the
+ * deterministic shadow/apply stamp for unstamped memories. Runners take an
+ * io seam (service/storage callbacks + stdout) so tests drive them without
+ * booting an orchestrator; the remnic-cli entry supplies the real ones.
+ */
+
+import type {
+  PromotionCandidatesResult,
+  SubjectBackfillReport,
+} from "./memory-subject.js";
+
+export interface PromotionCandidatesCliOptions {
+  namespace?: string;
+  targetNamespace?: string;
+  limit: number;
+  json: boolean;
+}
+
+export function parsePromotionCandidatesCliOptions(args: {
+  namespace?: string;
+  targetNamespace?: string;
+  limit?: string;
+  json?: boolean;
+}): PromotionCandidatesCliOptions {
+  let limit = 20;
+  if (args.limit !== undefined) {
+    const parsed = Number(args.limit);
+    if (!Number.isInteger(parsed) || parsed < 1 || parsed > 100) {
+      throw new Error(
+        `promotion-candidates: --limit expects an integer in [1, 100] (got ${JSON.stringify(args.limit)})`,
+      );
+    }
+    limit = parsed;
+  }
+  return {
+    ...(args.namespace !== undefined && args.namespace !== "" ? { namespace: args.namespace } : {}),
+    ...(args.targetNamespace !== undefined && args.targetNamespace !== ""
+      ? { targetNamespace: args.targetNamespace }
+      : {}),
+    limit,
+    json: args.json === true,
+  };
+}
+
+export function renderPromotionCandidates(result: PromotionCandidatesResult, json: boolean): string {
+  if (json) return JSON.stringify(result, null, 2);
+  const lines = [
+    `Namespace: ${result.namespace}  Target layer: ${result.targetNamespace}  (minAccessCount ${result.minAccessCount})`,
+  ];
+  if (result.candidates.length === 0) {
+    lines.push("No promotion candidates.");
+    return lines.join("\n");
+  }
+  for (const c of result.candidates) {
+    lines.push(
+      `- ${c.id} [${c.category}] signal=${c.reuseSignal} access=${c.accessCount} reinforced=${c.reinforcementCount} mw=${c.mwSuccess}/${c.mwFail}`,
+    );
+    lines.push(`  ${c.content.replace(/\s+/g, " ").slice(0, 100)}`);
+  }
+  lines.push("Promote explicitly with the existing promotion commands; nothing is auto-promoted.");
+  return lines.join("\n");
+}
+
+export async function runPromotionCandidatesCommand(
+  rest: string[],
+  io: {
+    promotionCandidates: (request: {
+      namespace?: string;
+      targetNamespace?: string;
+      limit?: number;
+    }) => Promise<PromotionCandidatesResult>;
+    stdout: (line: string) => void;
+  },
+): Promise<void> {
+  const flag = (name: string): string | undefined => {
+    const hit = rest.find((a) => a.startsWith(`--${name}=`));
+    return hit === undefined ? undefined : hit.slice(name.length + 3);
+  };
+  const options = parsePromotionCandidatesCliOptions({
+    namespace: flag("namespace"),
+    targetNamespace: flag("target-namespace"),
+    limit: flag("limit"),
+    json: rest.includes("--json"),
+  });
+  const result = await io.promotionCandidates({
+    ...(options.namespace !== undefined ? { namespace: options.namespace } : {}),
+    ...(options.targetNamespace !== undefined ? { targetNamespace: options.targetNamespace } : {}),
+    limit: options.limit,
+  });
+  io.stdout(renderPromotionCandidates(result, options.json));
+}
+
+export interface SubjectsBackfillCliOptions {
+  mode: "shadow" | "apply";
+  namespace?: string;
+}
+
+export function parseSubjectsBackfillCliOptions(args: {
+  apply?: boolean;
+  namespace?: string;
+}): SubjectsBackfillCliOptions {
+  return {
+    mode: args.apply === true ? "apply" : "shadow",
+    ...(args.namespace !== undefined && args.namespace !== "" ? { namespace: args.namespace } : {}),
+  };
+}
+
+export async function runSubjectsBackfillCommand(
+  rest: string[],
+  io: {
+    backfill: (options: { mode: "shadow" | "apply"; namespace?: string }) => Promise<SubjectBackfillReport>;
+    stdout: (line: string) => void;
+  },
+): Promise<void> {
+  const namespaceFlag = rest.find((a) => a.startsWith("--namespace="))?.slice("--namespace=".length);
+  const options = parseSubjectsBackfillCliOptions({
+    apply: rest.includes("--apply"),
+    namespace: namespaceFlag,
+  });
+  const report = await io.backfill(options);
+  const lines = [
+    `Mode: ${report.mode}  Scanned: ${report.scanned}  Already stamped: ${report.alreadyStamped}  To stamp: ${report.stamped}`,
+  ];
+  for (const stamp of report.stamps) {
+    lines.push(`- ${stamp.id} [${stamp.category}] → subject: ${stamp.subject}`);
+  }
+  if (report.mode === "shadow") {
+    lines.push("Shadow run — nothing written. Re-run with --apply to stamp.");
+  }
+  io.stdout(lines.join("\n"));
+}

package/src/memory-subject.test.ts

@@ -0,0 +1,48 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  evaluateSubjectGuard,
+  resolveWriteSubject,
+  SUBJECT_GUARD_OVERRIDE_FLAG,
+} from "./memory-subject.js";
+import { parseSubjectRuntimeConfig } from "./subject-config.js";
+
+test("category defaults: procedure is agent, preference is user, unknown is user", () => {
+  assert.equal(resolveWriteSubject("procedure", undefined), "agent");
+  assert.equal(resolveWriteSubject("preference", undefined), "user");
+  assert.equal(resolveWriteSubject("fact", undefined), "user");
+  assert.equal(resolveWriteSubject("procedure", "user"), "user");
+});
+
+test("shared-layer guard: missing subject fails closed as user", () => {
+  const warn = evaluateSubjectGuard({
+    subject: undefined,
+    sharedTarget: true,
+    mode: "warn",
+  });
+  assert.equal(warn.action, "warn");
+  assert.equal(warn.effectiveSubject, "user");
+  assert.ok(warn.reason.includes(SUBJECT_GUARD_OVERRIDE_FLAG));
+
+  const reject = evaluateSubjectGuard({
+    subject: "user",
+    sharedTarget: true,
+    mode: "enforce",
+  });
+  assert.equal(reject.action, "reject");
+
+  const allowed = evaluateSubjectGuard({
+    subject: "agent",
+    sharedTarget: true,
+    mode: "enforce",
+  });
+  assert.equal(allowed.action, "allow");
+});
+
+test("subjectClassification omitted is off; invalid values throw", () => {
+  assert.equal(parseSubjectRuntimeConfig({}).subjectClassification.enabled, false);
+  assert.equal(parseSubjectRuntimeConfig({}).subjectGuard, "warn");
+  assert.throws(() => parseSubjectRuntimeConfig({ subjectGuard: "loud" }));
+  assert.throws(() => parseSubjectRuntimeConfig({ subjectClassification: false }));
+});