@remnic/core 9.3.725 → 9.3.726

package/src/extraction-redaction-rules.test.ts

@@ -0,0 +1,213 @@
+/**
+ * extraction-redaction-rules.test.ts — issue #1669 (#1580 follow-up).
+ *
+ * Proves the extraction-layer redaction-rule helper reads persisted
+ * `redaction_rule` patterns and withholds matching content. The orchestrator
+ * consults these rules before the storage write chokepoint so a never_store
+ * correction actually blocks future extraction of matching content.
+ */
+import { strict as assert } from "node:assert";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { tmpdir } from "node:os";
+import { test } from "node:test";
+import {
+  REDACTION_RULES_SUBDIR,
+  compileRedactionPattern,
+  contentMatchesRedactionRules,
+  loadRedactionRules,
+} from "./extraction-redaction-rules.js";
+import { validateRedactionPattern } from "./correction/correction-contract.js";
+
+async function withTempDir<T>(fn: (dir: string) => Promise<T>): Promise<T> {
+  const dir = await mkdtemp(path.join(tmpdir(), "remnic-redact-"));
+  try {
+    return await fn(dir);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}
+
+async function writeRule(stateDir: string, pattern: string): Promise<void> {
+  const dir = path.join(stateDir, REDACTION_RULES_SUBDIR);
+  await mkdir(dir, { recursive: true });
+  const slug = pattern.replace(/[^a-zA-Z0-9]+/g, "-").slice(0, 64) || "rule";
+  await writeFile(
+    path.join(dir, `${slug}.json`),
+    `${JSON.stringify({ pattern, namespace: "default", createdAt: "2026-07-06T00:00:00Z" })}\n`,
+    "utf-8",
+  );
+}
+
+test("compileRedactionPattern: literal pattern matches by substring", () => {
+  const rule = compileRedactionPattern("secret-token-123");
+  assert.ok(rule.matcher("the secret-token-123 leaked here"));
+  assert.ok(!rule.matcher("nothing here"));
+});
+
+test("compileRedactionPattern: regex-like pattern compiles to a RegExp", () => {
+  // Only /.../-wrapped patterns compile as RegExp (review thread P1).
+  const rule = compileRedactionPattern("/secret-token-\\d+/");
+  assert.ok(rule.matcher("found secret-token-999 in logs"));
+  assert.ok(!rule.matcher("found secret-token-abc in logs"));
+});
+
+test("compileRedactionPattern (P1): unwrapped pattern with metacharacters is literal", () => {
+  // abc+def must match the LITERAL string, not the regex abccccdef.
+  const rule = compileRedactionPattern("abc+def");
+  assert.ok(rule.matcher("the value is abc+def here"),
+    "unwrapped metachar pattern must match the literal substring");
+  assert.ok(!rule.matcher("abccccdef"),
+    "unwrapped metachar pattern must NOT be compiled as a regex");
+  // An API-token-shaped literal like foo(bar) also works.
+  const rule2 = compileRedactionPattern("foo(bar)");
+  assert.ok(rule2.matcher("contains foo(bar) text"));
+  assert.ok(!rule2.matcher("contains foobar text"));
+});
+
+test("compileRedactionPattern: /wrapped/ regex strips delimiters", () => {
+  const rule = compileRedactionPattern("/internal-only-[a-z]+/");
+  assert.ok(rule.matcher("this is internal-only-foo data"));
+  assert.ok(!rule.matcher("this is public data"));
+});
+
+test("compileRedactionPattern: malformed /wrapped/ regex falls back to literal", () => {
+  // /.../-wrapped with an unbalanced bracket — RegExp() throws, falls back
+  // to literal substring on the BODY (without delimiters).
+  const rule = compileRedactionPattern("/[unclosed/");
+  assert.ok(rule.matcher("contains [unclosed literally"));
+  assert.ok(!rule.matcher("no match"));
+});
+
+test("loadRedactionRules: empty when the dir does not exist (cold install)", async () => {
+  await withTempDir(async (dir) => {
+    const rules = await loadRedactionRules(dir);
+    assert.equal(rules.length, 0);
+  });
+});
+
+test("loadRedactionRules: reads persisted literal + regex patterns", async () => {
+  await withTempDir(async (dir) => {
+    await writeRule(dir, "secret-token-123");
+    await writeRule(dir, "/api-key-[a-f0-9]{8}/");
+    const rules = await loadRedactionRules(dir);
+    assert.equal(rules.length, 2);
+    assert.ok(contentMatchesRedactionRules("leaked secret-token-123 here", rules));
+    assert.ok(contentMatchesRedactionRules("api-key-deadbeef was used", rules));
+    assert.ok(!contentMatchesRedactionRules("clean content", rules));
+  });
+});
+
+test("loadRedactionRules: skips a corrupt rule file without failing the pass", async () => {
+  await withTempDir(async (dir) => {
+    await writeRule(dir, "good-pattern");
+    const dir2 = path.join(dir, REDACTION_RULES_SUBDIR);
+    // A non-JSON file masquerading as a rule.
+    await writeFile(path.join(dir2, "broken.json"), "{not valid json", "utf-8");
+    // A rule file missing the `pattern` field.
+    await writeFile(path.join(dir2, "nopattern.json"), JSON.stringify({ foo: "bar" }), "utf-8");
+    const rules = await loadRedactionRules(dir);
+    assert.equal(rules.length, 1, "only the well-formed rule compiles");
+    assert.ok(contentMatchesRedactionRules("good-pattern here", rules));
+  });
+});
+
+test("contentMatchesRedactionRules: empty rule set never matches", () => {
+  assert.equal(contentMatchesRedactionRules("anything", []), false);
+});
+
+test("regression (#1669): a persisted redaction rule blocks matching content", async () => {
+  await withTempDir(async (dir) => {
+    // Simulate a `redaction_rule` correction that persisted a never-store pattern.
+    await writeRule(dir, "do-not-store-this-exact-phrase");
+    const rules = await loadRedactionRules(dir);
+    // Content matching the persisted pattern is withheld.
+    assert.ok(
+      contentMatchesRedactionRules("a fact containing do-not-store-this-exact-phrase inside", rules),
+      "matching content must be withheld",
+    );
+    // Non-matching content passes through.
+    assert.ok(
+      !contentMatchesRedactionRules("a perfectly fine durable fact", rules),
+      "non-matching content must pass",
+    );
+  });
+});
+
+test("#1669 thread #3: catastrophic-backtracking regex falls back to literal (ReDoS guard)", () => {
+  // A pattern like (a+)+ is a classic ReDoS shape. It must NOT compile as a
+  // RegExp — it would hang persistExtraction on a near-miss fact. Instead it
+  // falls back to literal substring match.
+  const rule = compileRedactionPattern("(a+)+");
+  // The literal fallback matches when the exact string "(a+)+" appears:
+  assert.equal(rule.matcher("a fact with (a+)+ inside"), true,
+    "literal fallback must still match the pattern as a substring");
+  // A long near-miss string that would cause exponential backtracking on the
+  // regex must return quickly (literal match, no regex evaluation):
+  const nearMiss = "a".repeat(100) + "!";
+  assert.equal(rule.matcher(nearMiss), false,
+    "near-miss must not hang — literal fallback, not regex");
+});
+
+test("#1669 thread #3: safe regex pattern still compiles as RegExp", () => {
+  // A benign anchored pattern compiles normally.
+  const rule = compileRedactionPattern("/^secret-key-[a-f0-9]+$/");
+  assert.equal(rule.matcher("secret-key-deadbeef"), true);
+  assert.equal(rule.matcher("not-a-secret"), false);
+});
+
+test("#1669 cursor thread: overly broad /wrapped/ regex falls back to literal (not compiled)", () => {
+  // A hand-edited rule file with /.*/ must NOT compile — it would match every
+  // fact and withhold all extraction. isSafeRegex now rejects overly-broad
+  // bodies, so the rule falls back to literal substring on the body.
+  const rule = compileRedactionPattern("/.*/");
+  assert.ok(!rule.matcher("any fact content"),
+    "overly broad regex must not match arbitrary content");
+  assert.ok(rule.matcher("contains .* literally"),
+    "literal fallback on the body must still match the substring .*");
+});
+
+test("#1669 thread #3 (first-line guard): validateRedactionPattern rejects catastrophic-backtracking shapes", () => {
+  // Classic ReDoS shapes must be rejected at apply time — not just fall back
+  // to literal at extraction time. Each is a nested quantifier or overlapping
+  // alternation that would hang on a near-miss fact.
+  const pathological = ["/(a+)+/", "/(a*)*/", "/(a?)+/", "/(a|a)+/"];
+  for (const p of pathological) {
+    assert.throws(
+      () => validateRedactionPattern(p),
+      { message: /unsafe/ },
+      `validateRedactionPattern must reject catastrophic pattern: ${p}`,
+    );
+  }
+  // The overly-broad check still fires for /.../-wrapped .* / .+ patterns.
+  assert.throws(() => validateRedactionPattern("/.*/"));
+  assert.throws(() => validateRedactionPattern("/.+/"));
+  // A near-miss string that would cause exponential backtracking on the
+  // regex never reaches extraction — the plan is rejected at apply time.
+  const nearMiss = "a".repeat(100) + "!";
+  assert.ok(nearMiss.length === 101);
+});
+
+test("#1669 thread #3 (first-line guard): valid patterns still pass", () => {
+  // Bounded literal + benign anchored regex must NOT be rejected.
+  assert.equal(validateRedactionPattern("secret-token-\\d+"), "secret-token-\\d+");
+  assert.equal(validateRedactionPattern("(a|b)+"), "(a|b)+");
+  assert.equal(validateRedactionPattern("/^secret-key-[a-f0-9]+$/"), "/^secret-key-[a-f0-9]+$/");
+});
+
+test("#1669 P2: validateRedactionPattern rejects zero-width regex", () => {
+  // Empty body or regex that matches the empty string would withhold every fact.
+  assert.throws(() => validateRedactionPattern("//"));
+  assert.throws(() => validateRedactionPattern("/a?/"));
+  assert.throws(() => validateRedactionPattern("/a*/"));
+  assert.throws(() => validateRedactionPattern("/(a|)/"));
+  // A bounded non-zero-width regex still passes.
+  assert.equal(validateRedactionPattern("/^secret-[a-f]+$/"), "/^secret-[a-f]+$/");
+});
+
+test("#1669 P2: compileRedactionPattern empty body never matches", () => {
+  // A hand-edited // rule file must not withhold all extraction.
+  const rule = compileRedactionPattern("//");
+  assert.equal(rule.matcher("anything"), false);
+  assert.equal(rule.matcher(""), false);
+});

package/src/extraction-redaction-rules.ts

@@ -0,0 +1,208 @@
+/**
+ * extraction-redaction-rules.ts — consults persisted correction redaction
+ * rules during the extraction→persist path so a `never_store` / redaction_rule
+ * correction actually blocks future extraction of matching content (issue
+ * #1669, #1580 follow-up).
+ *
+ * The Correction Contract persists `redaction_rule` patterns to
+ * `<storage>/state/corrections/redaction-rules/<slug>.json` (one file per
+ * rule, idempotent on pattern). This module reads those patterns at the start
+ * of each extraction persist pass and exposes a pure matcher the orchestrator
+ * consults BEFORE a fact reaches the storage write chokepoint — mirroring how
+ * tombstones are consulted at the write chokepoint (#1579), but one stage
+ * earlier so the content never even lands as `pending_review`.
+ *
+ * Scope (issue #1669): the extraction layer consults the rules. Persistence
+ * stays owned by the correction module. This helper is the read surface.
+ *
+ * Safety: patterns are validated at correction-apply time
+ * (`validateRedactionPattern` — bounded literal/safe-regex, no catastrophic
+ * shapes). We compile defensively here too: a pattern that fails to compile
+ * as a regex is treated as a literal substring, and any matcher error is
+ * swallowed (fail-open to "no match") so a malformed rule can never block the
+ * entire extraction pipeline.
+ */
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Directory (relative to a namespace storage dir) where redaction rules live.
+ * Kept in sync with `registerRedactionRuleFn` in correction-access-wiring.ts.
+ */
+export const REDACTION_RULES_SUBDIR = path.join("state", "corrections", "redaction-rules");
+
+/** Maximum number of rule files read per pass (defense in depth). */
+const MAX_RULE_FILES = 256;
+
+/**
+ * A compiled redaction rule. `pattern` is the original (validated) pattern;
+ * `matcher` returns true when content should be withheld.
+ */
+export interface CompiledRedactionRule {
+  pattern: string;
+  matcher: (content: string) => boolean;
+}
+
+/** Shape persisted by `registerRedactionRuleFn`. */
+interface RedactionRuleFile {
+  pattern: string;
+  namespace?: string;
+  createdAt?: string;
+}
+
+/**
+ * Only `/…/`-wrapped patterns compile as RegExp; everything else is a literal
+ * substring (review thread P1). An unwrapped pattern like `abc+def` must match
+ * the literal string `abc+def`, NOT the regex `abccccdef`.
+ */
+function isRegexLike(pattern: string): boolean {
+  return pattern.startsWith("/") && pattern.endsWith("/") && pattern.length >= 2;
+}
+
+/**
+ * Reject patterns prone to catastrophic backtracking (ReDoS, review thread #3).
+ * Mirrors the safe-regex heuristic: a quantifier inside a group that is itself
+ * quantified creates exponential blowup on near-miss inputs. Also rejects
+ * patterns with overlapping alternation under repetition
+ * (e.g. (a|a)*). Returns true when the pattern is safe to compile.
+ *
+ * This is a second line of defense — validateRedactionPattern runs first at
+ * apply time, but a hand-edited rule file or an edge case can still slip past.
+ */
+function isSafeRegex(source: string): boolean {
+  if (source.length === 0 || source.length > 512) return false;
+  // Reject overly-broad patterns (.*) that would match every fact and
+  // withhold all extraction (cursor Bugbot thread — mirrors
+  // isUnsafeRedactionRegex in correction-contract.ts).
+  if (/(?:^|[^\\])\(\.\*\)|(?:^|[^\\])\.\*|(?:^|[^\\])\.\+|^\.([^*+]?)$/.test(source)) {
+    return false;
+  }
+  // group body itself ends with a quantifier. This catches (a+)+, (a*)*, etc.
+  // Also catches overlapping alternation like (a|a)+.
+  for (let i = 0; i < source.length; i++) {
+    if (source[i] !== "(") continue;
+    // Find the matching close paren
+    let depth = 1;
+    let j = i + 1;
+    while (j < source.length && depth > 0) {
+      if (source[j] === "\\") { j += 2; continue; }
+      if (source[j] === "(") depth++;
+      else if (source[j] === ")") depth--;
+      j++;
+    }
+    if (depth !== 0) continue; // unbalanced — will fail RegExp compile anyway
+    // Check if the char after the closing paren is a quantifier
+    const afterGroup = source[j];
+    if (afterGroup !== "+" && afterGroup !== "*" && afterGroup !== "{") continue;
+    const groupBody = source.slice(i + 1, j - 1);
+    // The group body ends with a quantifier on a non-escape char
+    if (/[+*?]$/.test(groupBody) || /\{\d+,?\d*\}[+*?]?$/.test(groupBody)) {
+      // Nested quantifier → potential ReDoS
+      return false;
+    }
+    // Overlapping alternation: (a|a)+ where branches share a common prefix
+    if (groupBody.includes("|")) {
+      const branches = groupBody.split("|");
+      if (branches.length >= 2) {
+        const firstChars = new Set(branches.map((b) => b[0]).filter(Boolean));
+        if (firstChars.size < branches.filter((b) => b.length > 0).length) {
+          return false;
+        }
+      }
+    }
+    i = j;
+  }
+  return true;
+}
+
+/**
+ * Compile a single pattern into a matcher. A literal pattern matches by
+ * case-sensitive substring; a `/…/`-wrapped pattern compiles into a RegExp
+ * anchored to search (global flag off — we only need a boolean). Compilation
+ * failures fall back to literal substring so a bad pattern never throws here.
+ */
+export function compileRedactionPattern(pattern: string): CompiledRedactionRule {
+  const trimmed = pattern.trim();
+  if (!isRegexLike(trimmed)) {
+    return { pattern: trimmed, matcher: (content) => content.includes(trimmed) };
+  }
+  const body = trimmed.slice(1, -1);
+  try {
+    if (!isSafeRegex(body)) {
+      if (body.length === 0) return { pattern: trimmed, matcher: () => false };
+      return { pattern: trimmed, matcher: (content) => content.includes(body) };
+    }
+    const re = new RegExp(body);
+    if (re.test("")) {
+      // Zero-width regex matches every string — would withhold all extraction.
+      return { pattern: trimmed, matcher: (content) => content.includes(body) };
+    }
+    return { pattern: trimmed, matcher: (content) => re.test(content) };
+  } catch {
+    // Malformed regex despite validation (e.g. rule file hand-edited) —
+    // treat as a literal on the BODY so the rule still does something useful.
+    return { pattern: trimmed, matcher: (content) => content.includes(body) };
+  }
+}
+
+/**
+ * Load + compile every redaction rule persisted under a namespace's storage
+ * dir. Returns an empty array when the directory is absent or unreadable
+ * (fail-open — a missing or corrupt rules dir must never block extraction).
+ */
+export async function loadRedactionRules(stateDir: string): Promise<CompiledRedactionRule[]> {
+  const dir = path.join(stateDir, REDACTION_RULES_SUBDIR);
+  let names: string[];
+  try {
+    names = await readdir(dir);
+  } catch {
+    // ENOENT (cold install, no rules yet) or permission error — no rules.
+    return [];
+  }
+  // Sort deterministically (alphabetical by filename) so the set of rules
+  // that survives the MAX_RULE_FILES cap is NOT filesystem-dependent (review
+  // thread P2). Without this, readdir order varies by OS/FS and the silently
+  // truncated rules are arbitrary — a never-store pattern could be dropped.
+  const jsonFiles = names.filter((n) => n.endsWith(".json")).sort();
+  if (jsonFiles.length > MAX_RULE_FILES) {
+    // Visible failure mode: log a warning so the operator knows rules were
+    // truncated, rather than silently dropping enforcement.
+    console.warn(
+      `extraction-redaction: ${jsonFiles.length} redaction rules in ${dir} exceed the ${MAX_RULE_FILES} cap; ` +
+      `${jsonFiles.length - MAX_RULE_FILES} rules will NOT be enforced. Remove unused rules or raise the cap.`,
+    );
+  }
+  const rules: CompiledRedactionRule[] = [];
+  for (const name of jsonFiles.slice(0, MAX_RULE_FILES)) {
+    try {
+      const raw = await readFile(path.join(dir, name), "utf-8");
+      const parsed = JSON.parse(raw) as RedactionRuleFile;
+      if (typeof parsed?.pattern !== "string" || parsed.pattern.trim().length === 0) continue;
+      rules.push(compileRedactionPattern(parsed.pattern));
+    } catch {
+      // Skip a corrupt rule file rather than failing the pass (rule 34 —
+      // visible skip, never a silent pipeline block).
+    }
+  }
+  return rules;
+}
+
+/**
+ * Test content against a set of compiled rules. Returns true if ANY rule
+ * matches (the content should be withheld). An empty rule set never matches.
+ */
+export function contentMatchesRedactionRules(
+  content: string,
+  rules: readonly CompiledRedactionRule[],
+): boolean {
+  for (const rule of rules) {
+    try {
+      if (rule.matcher(content)) return true;
+    } catch {
+      // A matcher should not throw (compilation already fell back), but if a
+      // RegExp blows up on pathological input we skip that rule rather than
+      // failing the whole gate.
+    }
+  }
+  return false;
+}

package/src/orchestrator.ts

@@ -44,6 +44,11 @@ import {
   createFaithfulnessCounters,
   runFaithfulnessGateBatch,
 } from "./extraction-faithfulness.js";
+import {
+  contentMatchesRedactionRules,
+  loadRedactionRules,
+  type CompiledRedactionRule,
+} from "./extraction-redaction-rules.js";
 import type { FaithfulnessGateCounters } from "./extraction-faithfulness.js";
 import {
   EXTRACTION_JUDGE_VERDICT_CATEGORY,
@@ -14119,6 +14124,23 @@ export class Orchestrator {
     // (declared here, inside persistExtraction) so a transient hiccup in one
     // batch does not permanently disable dedup in future batches.
     let batchBackendUnavailable = false;
+    // #1669: per-namespace redaction-rule cache for this persist pass. A
+    // `never_store` / redaction_rule correction persists patterns under each
+    // namespace's state/corrections/redaction-rules/ dir; we consult them
+    // before a fact reaches the storage write chokepoint so matching content
+    // is withheld entirely rather than landing as pending_review. Cached per
+    // dir so a multi-fact batch over one namespace reads the dir once.
+    let redactionGatedCount = 0;
+    const redactionRulesByDir = new Map<string, CompiledRedactionRule[]>();
+    const redactionRulesFor = async (...dirs: string[]): Promise<CompiledRedactionRule[]> => {
+      const out: CompiledRedactionRule[] = [];
+      for (const d of dirs) {
+        let r = redactionRulesByDir.get(d);
+        if (!r) { r = await loadRedactionRules(d); redactionRulesByDir.set(d, r); }
+        out.push(...r);
+      }
+      return out;
+    };
     const behaviorSignalsByStorage = new Map<
       string,
       { storage: StorageManager; events: BehaviorSignalEvent[] }
@@ -14846,6 +14868,10 @@ export class Orchestrator {
     // into the caller's buffer-retention decision.
     this.lastPersistExtractionDeferredCount = 0;
     if (this.config.extractionJudgeEnabled) {
+      // #1669 P1: pre-filter redacted facts from judge candidates so never-store
+      // content is not persisted as judge training data.
+      let preJudgeRedactionRules: CompiledRedactionRule[] = [];
+      try { preJudgeRedactionRules = await redactionRulesFor(storage.dir); } catch { /* fail open */ }
       try {
         const judgeCandidates: JudgeCandidate[] = [];
         const candidateToFactIndex: number[] = [];
@@ -14886,6 +14912,11 @@ export class Orchestrator {
           ) {
             continue;
           }
+          if (preJudgeRedactionRules.length > 0) {
+            const rc = f.content + (f.structuredAttributes ? " " + JSON.stringify(f.structuredAttributes) : "")
+              + (f.procedureSteps ? " " + f.procedureSteps.map((s) => `${s.intent} ${s.expectedOutcome ?? ""} ${s.toolCall ? `${s.toolCall.kind} ${s.toolCall.signature}` : ""}`.trim()).join(" ") : "");
+            if (contentMatchesRedactionRules(rc, preJudgeRedactionRules)) continue;
+          }
           judgeCandidates.push({
             text: f.content,
             category: judgeCategory,
@@ -15064,6 +15095,7 @@ export class Orchestrator {
       // affect both the dedup fingerprint and importance (issue #519 procedure routing).
       let writeCategory = fact.category;
       let targetStorage = storage;
+      const sourceStorageDir = storage.dir; // #1669 thread #2: pre-routing source ns for redaction gate
       // Track the KNOWN target namespace NAME alongside targetStorage (round 6,
       // codex P2 — NCQI0). Re-deriving it from `targetStorage.dir` mangles a raw
       // namespace literally named like a canonical token (e.g. `ns-616c706861`
@@ -15138,6 +15170,23 @@ export class Orchestrator {
           );
         }
       }
+      // #1669 redaction-rule gate: consult BOTH source and target namespace
+      // rules before any write. A never-store pattern registered under the
+      // source namespace must survive scope-routing to a different target
+      // (review thread #2). Fails open on read error.
+      try {
+        const redactionRules = await redactionRulesFor(sourceStorageDir, targetStorage.dir);
+        const redactionCandidate = fact.content
+          + (fact.structuredAttributes ? " " + JSON.stringify(fact.structuredAttributes) : "")
+          + (fact.procedureSteps ? " " + fact.procedureSteps.map((s) => `${s.intent} ${s.expectedOutcome ?? ""} ${s.toolCall ? `${s.toolCall.kind} ${s.toolCall.signature}` : ""}`.trim()).join(" ") : "");
+        if (redactionRules.length > 0 && contentMatchesRedactionRules(redactionCandidate, redactionRules)) {
+          redactionGatedCount++;
+          log.debug(`extraction: redaction-rule withheld fact #${redactionGatedCount} in ${targetStorage.dir}`);
+          continue;
+        }
+      } catch (redactionErr) {
+        log.warn(`extraction: redaction-rule gate failed open: ${redactionErr}`);
+      }
 
       // Procedures: fingerprint the full serialized body (title + steps), not
       // the title alone, so distinct step lists are not collapsed (issue #519).
@@ -16263,8 +16312,10 @@ export class Orchestrator {
       importanceGatedCount > 0 ? ` (${importanceGatedCount} gated)` : "";
     const judgeSuffix =
       judgeGatedCount > 0 ? ` (${judgeGatedCount} judge-rejected)` : "";
+    const redactionSuffix =
+      redactionGatedCount > 0 ? ` (${redactionGatedCount} redacted)` : "";
     log.info(
-      `persisted: ${facts.length - dedupedCount - importanceGatedCount - judgeGatedCount} facts${dedupSuffix}${gatedSuffix}${judgeSuffix}, ${entities.length} entities, ${questions.length} questions, ${profileUpdates.length} profile updates`,
+      `persisted: ${facts.length - dedupedCount - importanceGatedCount - judgeGatedCount - redactionGatedCount} facts${dedupSuffix}${gatedSuffix}${judgeSuffix}${redactionSuffix}, ${entities.length} entities, ${questions.length} questions, ${profileUpdates.length} profile updates`,
     );
 
     // Update temporal + tag indexes (v8.1) — fire-and-forget, fail-open

package/src/storage.ts

@@ -1,6 +1,7 @@
 import { access, lstat, readdir, readFile, realpath, stat, writeFile, mkdir, unlink, rename, appendFile, open } from "node:fs/promises";
 import { appendFileSync, createReadStream, mkdirSync, readFileSync, statSync, type Dirent } from "node:fs";
-import { createHash } from "node:crypto";
+import { createHash } from "node:crypto"
+import { normalizeContent, computeContentHash } from "./content-hash.js";;
 import path from "node:path";
 import { log } from "./logger.js";
 import { isErrnoCode } from "./utils/errno.js";
@@ -1189,19 +1190,14 @@ export class ContentHashIndex {
     }
   }
 
-  /** Normalize content and compute SHA-256 hash. */
+  /** Normalize content (delegates to content-hash.ts for a single source of truth). */
   static normalizeContent(content: string): string {
-    return content
-      .toLowerCase()
-      .replace(/[^a-z0-9\s]/g, " ")
-      .replace(/\s+/g, " ")
-      .trim();
+    return normalizeContent(content);
   }
 
-  /** Normalize content and compute SHA-256 hash. */
+  /** Compute SHA-256 hash (delegates to content-hash.ts for a single source of truth). */
   static computeHash(content: string): string {
-    const normalized = ContentHashIndex.normalizeContent(content);
-    return createHash("sha256").update(normalized).digest("hex");
+    return computeContentHash(content);
   }
 }