@remnic/core 9.3.725 → 9.3.726

package/dist/chunk-LTHVM4XN.js

@@ -0,0 +1,109 @@
+// src/extraction-redaction-rules.ts
+import { readdir, readFile } from "fs/promises";
+import path from "path";
+var REDACTION_RULES_SUBDIR = path.join("state", "corrections", "redaction-rules");
+var MAX_RULE_FILES = 256;
+function isRegexLike(pattern) {
+  return pattern.startsWith("/") && pattern.endsWith("/") && pattern.length >= 2;
+}
+function isSafeRegex(source) {
+  if (source.length === 0 || source.length > 512) return false;
+  if (/(?:^|[^\\])\(\.\*\)|(?:^|[^\\])\.\*|(?:^|[^\\])\.\+|^\.([^*+]?)$/.test(source)) {
+    return false;
+  }
+  for (let i = 0; i < source.length; i++) {
+    if (source[i] !== "(") continue;
+    let depth = 1;
+    let j = i + 1;
+    while (j < source.length && depth > 0) {
+      if (source[j] === "\\") {
+        j += 2;
+        continue;
+      }
+      if (source[j] === "(") depth++;
+      else if (source[j] === ")") depth--;
+      j++;
+    }
+    if (depth !== 0) continue;
+    const afterGroup = source[j];
+    if (afterGroup !== "+" && afterGroup !== "*" && afterGroup !== "{") continue;
+    const groupBody = source.slice(i + 1, j - 1);
+    if (/[+*?]$/.test(groupBody) || /\{\d+,?\d*\}[+*?]?$/.test(groupBody)) {
+      return false;
+    }
+    if (groupBody.includes("|")) {
+      const branches = groupBody.split("|");
+      if (branches.length >= 2) {
+        const firstChars = new Set(branches.map((b) => b[0]).filter(Boolean));
+        if (firstChars.size < branches.filter((b) => b.length > 0).length) {
+          return false;
+        }
+      }
+    }
+    i = j;
+  }
+  return true;
+}
+function compileRedactionPattern(pattern) {
+  const trimmed = pattern.trim();
+  if (!isRegexLike(trimmed)) {
+    return { pattern: trimmed, matcher: (content) => content.includes(trimmed) };
+  }
+  const body = trimmed.slice(1, -1);
+  try {
+    if (!isSafeRegex(body)) {
+      if (body.length === 0) return { pattern: trimmed, matcher: () => false };
+      return { pattern: trimmed, matcher: (content) => content.includes(body) };
+    }
+    const re = new RegExp(body);
+    if (re.test("")) {
+      return { pattern: trimmed, matcher: (content) => content.includes(body) };
+    }
+    return { pattern: trimmed, matcher: (content) => re.test(content) };
+  } catch {
+    return { pattern: trimmed, matcher: (content) => content.includes(body) };
+  }
+}
+async function loadRedactionRules(stateDir) {
+  const dir = path.join(stateDir, REDACTION_RULES_SUBDIR);
+  let names;
+  try {
+    names = await readdir(dir);
+  } catch {
+    return [];
+  }
+  const jsonFiles = names.filter((n) => n.endsWith(".json")).sort();
+  if (jsonFiles.length > MAX_RULE_FILES) {
+    console.warn(
+      `extraction-redaction: ${jsonFiles.length} redaction rules in ${dir} exceed the ${MAX_RULE_FILES} cap; ${jsonFiles.length - MAX_RULE_FILES} rules will NOT be enforced. Remove unused rules or raise the cap.`
+    );
+  }
+  const rules = [];
+  for (const name of jsonFiles.slice(0, MAX_RULE_FILES)) {
+    try {
+      const raw = await readFile(path.join(dir, name), "utf-8");
+      const parsed = JSON.parse(raw);
+      if (typeof parsed?.pattern !== "string" || parsed.pattern.trim().length === 0) continue;
+      rules.push(compileRedactionPattern(parsed.pattern));
+    } catch {
+    }
+  }
+  return rules;
+}
+function contentMatchesRedactionRules(content, rules) {
+  for (const rule of rules) {
+    try {
+      if (rule.matcher(content)) return true;
+    } catch {
+    }
+  }
+  return false;
+}
+
+export {
+  REDACTION_RULES_SUBDIR,
+  compileRedactionPattern,
+  loadRedactionRules,
+  contentMatchesRedactionRules
+};
+//# sourceMappingURL=chunk-LTHVM4XN.js.map

package/dist/chunk-LTHVM4XN.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/extraction-redaction-rules.ts"],"sourcesContent":["/**\n * extraction-redaction-rules.ts — consults persisted correction redaction\n * rules during the extraction→persist path so a `never_store` / redaction_rule\n * correction actually blocks future extraction of matching content (issue\n * #1669, #1580 follow-up).\n *\n * The Correction Contract persists `redaction_rule` patterns to\n * `<storage>/state/corrections/redaction-rules/<slug>.json` (one file per\n * rule, idempotent on pattern). This module reads those patterns at the start\n * of each extraction persist pass and exposes a pure matcher the orchestrator\n * consults BEFORE a fact reaches the storage write chokepoint — mirroring how\n * tombstones are consulted at the write chokepoint (#1579), but one stage\n * earlier so the content never even lands as `pending_review`.\n *\n * Scope (issue #1669): the extraction layer consults the rules. Persistence\n * stays owned by the correction module. This helper is the read surface.\n *\n * Safety: patterns are validated at correction-apply time\n * (`validateRedactionPattern` — bounded literal/safe-regex, no catastrophic\n * shapes). We compile defensively here too: a pattern that fails to compile\n * as a regex is treated as a literal substring, and any matcher error is\n * swallowed (fail-open to \"no match\") so a malformed rule can never block the\n * entire extraction pipeline.\n */\nimport { readdir, readFile } from \"node:fs/promises\";\nimport path from \"node:path\";\n\n/**\n * Directory (relative to a namespace storage dir) where redaction rules live.\n * Kept in sync with `registerRedactionRuleFn` in correction-access-wiring.ts.\n */\nexport const REDACTION_RULES_SUBDIR = path.join(\"state\", \"corrections\", \"redaction-rules\");\n\n/** Maximum number of rule files read per pass (defense in depth). */\nconst MAX_RULE_FILES = 256;\n\n/**\n * A compiled redaction rule. `pattern` is the original (validated) pattern;\n * `matcher` returns true when content should be withheld.\n */\nexport interface CompiledRedactionRule {\n  pattern: string;\n  matcher: (content: string) => boolean;\n}\n\n/** Shape persisted by `registerRedactionRuleFn`. */\ninterface RedactionRuleFile {\n  pattern: string;\n  namespace?: string;\n  createdAt?: string;\n}\n\n/**\n * Only `/…/`-wrapped patterns compile as RegExp; everything else is a literal\n * substring (review thread P1). An unwrapped pattern like `abc+def` must match\n * the literal string `abc+def`, NOT the regex `abccccdef`.\n */\nfunction isRegexLike(pattern: string): boolean {\n  return pattern.startsWith(\"/\") && pattern.endsWith(\"/\") && pattern.length >= 2;\n}\n\n/**\n * Reject patterns prone to catastrophic backtracking (ReDoS, review thread #3).\n * Mirrors the safe-regex heuristic: a quantifier inside a group that is itself\n * quantified creates exponential blowup on near-miss inputs. Also rejects\n * patterns with overlapping alternation under repetition\n * (e.g. (a|a)*). Returns true when the pattern is safe to compile.\n *\n * This is a second line of defense — validateRedactionPattern runs first at\n * apply time, but a hand-edited rule file or an edge case can still slip past.\n */\nfunction isSafeRegex(source: string): boolean {\n  if (source.length === 0 || source.length > 512) return false;\n  // Reject overly-broad patterns (.*) that would match every fact and\n  // withhold all extraction (cursor Bugbot thread — mirrors\n  // isUnsafeRedactionRegex in correction-contract.ts).\n  if (/(?:^|[^\\\\])\\(\\.\\*\\)|(?:^|[^\\\\])\\.\\*|(?:^|[^\\\\])\\.\\+|^\\.([^*+]?)$/.test(source)) {\n    return false;\n  }\n  // group body itself ends with a quantifier. This catches (a+)+, (a*)*, etc.\n  // Also catches overlapping alternation like (a|a)+.\n  for (let i = 0; i < source.length; i++) {\n    if (source[i] !== \"(\") continue;\n    // Find the matching close paren\n    let depth = 1;\n    let j = i + 1;\n    while (j < source.length && depth > 0) {\n      if (source[j] === \"\\\\\") { j += 2; continue; }\n      if (source[j] === \"(\") depth++;\n      else if (source[j] === \")\") depth--;\n      j++;\n    }\n    if (depth !== 0) continue; // unbalanced — will fail RegExp compile anyway\n    // Check if the char after the closing paren is a quantifier\n    const afterGroup = source[j];\n    if (afterGroup !== \"+\" && afterGroup !== \"*\" && afterGroup !== \"{\") continue;\n    const groupBody = source.slice(i + 1, j - 1);\n    // The group body ends with a quantifier on a non-escape char\n    if (/[+*?]$/.test(groupBody) || /\\{\\d+,?\\d*\\}[+*?]?$/.test(groupBody)) {\n      // Nested quantifier → potential ReDoS\n      return false;\n    }\n    // Overlapping alternation: (a|a)+ where branches share a common prefix\n    if (groupBody.includes(\"|\")) {\n      const branches = groupBody.split(\"|\");\n      if (branches.length >= 2) {\n        const firstChars = new Set(branches.map((b) => b[0]).filter(Boolean));\n        if (firstChars.size < branches.filter((b) => b.length > 0).length) {\n          return false;\n        }\n      }\n    }\n    i = j;\n  }\n  return true;\n}\n\n/**\n * Compile a single pattern into a matcher. A literal pattern matches by\n * case-sensitive substring; a `/…/`-wrapped pattern compiles into a RegExp\n * anchored to search (global flag off — we only need a boolean). Compilation\n * failures fall back to literal substring so a bad pattern never throws here.\n */\nexport function compileRedactionPattern(pattern: string): CompiledRedactionRule {\n  const trimmed = pattern.trim();\n  if (!isRegexLike(trimmed)) {\n    return { pattern: trimmed, matcher: (content) => content.includes(trimmed) };\n  }\n  const body = trimmed.slice(1, -1);\n  try {\n    if (!isSafeRegex(body)) {\n      if (body.length === 0) return { pattern: trimmed, matcher: () => false };\n      return { pattern: trimmed, matcher: (content) => content.includes(body) };\n    }\n    const re = new RegExp(body);\n    if (re.test(\"\")) {\n      // Zero-width regex matches every string — would withhold all extraction.\n      return { pattern: trimmed, matcher: (content) => content.includes(body) };\n    }\n    return { pattern: trimmed, matcher: (content) => re.test(content) };\n  } catch {\n    // Malformed regex despite validation (e.g. rule file hand-edited) —\n    // treat as a literal on the BODY so the rule still does something useful.\n    return { pattern: trimmed, matcher: (content) => content.includes(body) };\n  }\n}\n\n/**\n * Load + compile every redaction rule persisted under a namespace's storage\n * dir. Returns an empty array when the directory is absent or unreadable\n * (fail-open — a missing or corrupt rules dir must never block extraction).\n */\nexport async function loadRedactionRules(stateDir: string): Promise<CompiledRedactionRule[]> {\n  const dir = path.join(stateDir, REDACTION_RULES_SUBDIR);\n  let names: string[];\n  try {\n    names = await readdir(dir);\n  } catch {\n    // ENOENT (cold install, no rules yet) or permission error — no rules.\n    return [];\n  }\n  // Sort deterministically (alphabetical by filename) so the set of rules\n  // that survives the MAX_RULE_FILES cap is NOT filesystem-dependent (review\n  // thread P2). Without this, readdir order varies by OS/FS and the silently\n  // truncated rules are arbitrary — a never-store pattern could be dropped.\n  const jsonFiles = names.filter((n) => n.endsWith(\".json\")).sort();\n  if (jsonFiles.length > MAX_RULE_FILES) {\n    // Visible failure mode: log a warning so the operator knows rules were\n    // truncated, rather than silently dropping enforcement.\n    console.warn(\n      `extraction-redaction: ${jsonFiles.length} redaction rules in ${dir} exceed the ${MAX_RULE_FILES} cap; ` +\n      `${jsonFiles.length - MAX_RULE_FILES} rules will NOT be enforced. Remove unused rules or raise the cap.`,\n    );\n  }\n  const rules: CompiledRedactionRule[] = [];\n  for (const name of jsonFiles.slice(0, MAX_RULE_FILES)) {\n    try {\n      const raw = await readFile(path.join(dir, name), \"utf-8\");\n      const parsed = JSON.parse(raw) as RedactionRuleFile;\n      if (typeof parsed?.pattern !== \"string\" || parsed.pattern.trim().length === 0) continue;\n      rules.push(compileRedactionPattern(parsed.pattern));\n    } catch {\n      // Skip a corrupt rule file rather than failing the pass (rule 34 —\n      // visible skip, never a silent pipeline block).\n    }\n  }\n  return rules;\n}\n\n/**\n * Test content against a set of compiled rules. Returns true if ANY rule\n * matches (the content should be withheld). An empty rule set never matches.\n */\nexport function contentMatchesRedactionRules(\n  content: string,\n  rules: readonly CompiledRedactionRule[],\n): boolean {\n  for (const rule of rules) {\n    try {\n      if (rule.matcher(content)) return true;\n    } catch {\n      // A matcher should not throw (compilation already fell back), but if a\n      // RegExp blows up on pathological input we skip that rule rather than\n      // failing the whole gate.\n    }\n  }\n  return false;\n}\n"],"mappings":";AAwBA,SAAS,SAAS,gBAAgB;AAClC,OAAO,UAAU;AAMV,IAAM,yBAAyB,KAAK,KAAK,SAAS,eAAe,iBAAiB;AAGzF,IAAM,iBAAiB;AAuBvB,SAAS,YAAY,SAA0B;AAC7C,SAAO,QAAQ,WAAW,GAAG,KAAK,QAAQ,SAAS,GAAG,KAAK,QAAQ,UAAU;AAC/E;AAYA,SAAS,YAAY,QAAyB;AAC5C,MAAI,OAAO,WAAW,KAAK,OAAO,SAAS,IAAK,QAAO;AAIvD,MAAI,mEAAmE,KAAK,MAAM,GAAG;AACnF,WAAO;AAAA,EACT;AAGA,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,QAAI,OAAO,CAAC,MAAM,IAAK;AAEvB,QAAI,QAAQ;AACZ,QAAI,IAAI,IAAI;AACZ,WAAO,IAAI,OAAO,UAAU,QAAQ,GAAG;AACrC,UAAI,OAAO,CAAC,MAAM,MAAM;AAAE,aAAK;AAAG;AAAA,MAAU;AAC5C,UAAI,OAAO,CAAC,MAAM,IAAK;AAAA,eACd,OAAO,CAAC,MAAM,IAAK;AAC5B;AAAA,IACF;AACA,QAAI,UAAU,EAAG;AAEjB,UAAM,aAAa,OAAO,CAAC;AAC3B,QAAI,eAAe,OAAO,eAAe,OAAO,eAAe,IAAK;AACpE,UAAM,YAAY,OAAO,MAAM,IAAI,GAAG,IAAI,CAAC;AAE3C,QAAI,SAAS,KAAK,SAAS,KAAK,sBAAsB,KAAK,SAAS,GAAG;AAErE,aAAO;AAAA,IACT;AAEA,QAAI,UAAU,SAAS,GAAG,GAAG;AAC3B,YAAM,WAAW,UAAU,MAAM,GAAG;AACpC,UAAI,SAAS,UAAU,GAAG;AACxB,cAAM,aAAa,IAAI,IAAI,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,OAAO,OAAO,CAAC;AACpE,YAAI,WAAW,OAAO,SAAS,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,QAAQ;AACjE,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AACA,QAAI;AAAA,EACN;AACA,SAAO;AACT;AAQO,SAAS,wBAAwB,SAAwC;AAC9E,QAAM,UAAU,QAAQ,KAAK;AAC7B,MAAI,CAAC,YAAY,OAAO,GAAG;AACzB,WAAO,EAAE,SAAS,SAAS,SAAS,CAAC,YAAY,QAAQ,SAAS,OAAO,EAAE;AAAA,EAC7E;AACA,QAAM,OAAO,QAAQ,MAAM,GAAG,EAAE;AAChC,MAAI;AACF,QAAI,CAAC,YAAY,IAAI,GAAG;AACtB,UAAI,KAAK,WAAW,EAAG,QAAO,EAAE,SAAS,SAAS,SAAS,MAAM,MAAM;AACvE,aAAO,EAAE,SAAS,SAAS,SAAS,CAAC,YAAY,QAAQ,SAAS,IAAI,EAAE;AAAA,IAC1E;AACA,UAAM,KAAK,IAAI,OAAO,IAAI;AAC1B,QAAI,GAAG,KAAK,EAAE,GAAG;AAEf,aAAO,EAAE,SAAS,SAAS,SAAS,CAAC,YAAY,QAAQ,SAAS,IAAI,EAAE;AAAA,IAC1E;AACA,WAAO,EAAE,SAAS,SAAS,SAAS,CAAC,YAAY,GAAG,KAAK,OAAO,EAAE;AAAA,EACpE,QAAQ;AAGN,WAAO,EAAE,SAAS,SAAS,SAAS,CAAC,YAAY,QAAQ,SAAS,IAAI,EAAE;AAAA,EAC1E;AACF;AAOA,eAAsB,mBAAmB,UAAoD;AAC3F,QAAM,MAAM,KAAK,KAAK,UAAU,sBAAsB;AACtD,MAAI;AACJ,MAAI;AACF,YAAQ,MAAM,QAAQ,GAAG;AAAA,EAC3B,QAAQ;AAEN,WAAO,CAAC;AAAA,EACV;AAKA,QAAM,YAAY,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,CAAC,EAAE,KAAK;AAChE,MAAI,UAAU,SAAS,gBAAgB;AAGrC,YAAQ;AAAA,MACN,yBAAyB,UAAU,MAAM,uBAAuB,GAAG,eAAe,cAAc,SAC7F,UAAU,SAAS,cAAc;AAAA,IACtC;AAAA,EACF;AACA,QAAM,QAAiC,CAAC;AACxC,aAAW,QAAQ,UAAU,MAAM,GAAG,cAAc,GAAG;AACrD,QAAI;AACF,YAAM,MAAM,MAAM,SAAS,KAAK,KAAK,KAAK,IAAI,GAAG,OAAO;AACxD,YAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,UAAI,OAAO,QAAQ,YAAY,YAAY,OAAO,QAAQ,KAAK,EAAE,WAAW,EAAG;AAC/E,YAAM,KAAK,wBAAwB,OAAO,OAAO,CAAC;AAAA,IACpD,QAAQ;AAAA,IAGR;AAAA,EACF;AACA,SAAO;AACT;AAMO,SAAS,6BACd,SACA,OACS;AACT,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,UAAI,KAAK,QAAQ,OAAO,EAAG,QAAO;AAAA,IACpC,QAAQ;AAAA,IAIR;AAAA,EACF;AACA,SAAO;AACT;","names":[]}

package/dist/{chunk-4R3CGCI5.js → chunk-MQVKHFIU.js}

@@ -1,18 +1,21 @@
+import {
+  WEARABLES_DIR_NAME,
+  isValidTranscriptDate
+} from "./chunk-M7XQSUBB.js";
 import {
   TombstoneStore,
   buildRetiredFactTombstoneInputs,
   collectRetiredMemoriesForRebuild
 } from "./chunk-L2B3EE7B.js";
-import {
-  WEARABLES_DIR_NAME,
-  isValidTranscriptDate
-} from "./chunk-M7XQSUBB.js";
 import {
   isErrnoCode
 } from "./chunk-5UZXUTVO.js";
 import {
   assertPathInsideRoot
 } from "./chunk-5GPPACXK.js";
+import {
+  stripAttributesSuffix
+} from "./chunk-SVOZFLIQ.js";
 import {
   supersessionKeysForFact
 } from "./chunk-TKZ3AL5V.js";
@@ -21,9 +24,6 @@ import {
   hasCitation,
   stripCitationForTemplate
 } from "./chunk-J6A3CX5N.js";
-import {
-  stripAttributesSuffix
-} from "./chunk-SVOZFLIQ.js";
 import {
   SPECULATIVE_TTL_DAYS,
   confidenceTier
@@ -72,6 +72,10 @@ import {
 import {
   createVersion
 } from "./chunk-VF4XKTX3.js";
+import {
+  computeContentHash,
+  normalizeContent
+} from "./chunk-W5H6ZRPT.js";
 import {
   MAGIC_HEADER_SIZE,
   SecureStoreLockedError,
@@ -886,14 +890,13 @@ var ContentHashIndex = class _ContentHashIndex {
       this.dirty = true;
     }
   }
-  /** Normalize content and compute SHA-256 hash. */
+  /** Normalize content (delegates to content-hash.ts for a single source of truth). */
   static normalizeContent(content) {
-    return content.toLowerCase().replace(/[^a-z0-9\s]/g, " ").replace(/\s+/g, " ").trim();
+    return normalizeContent(content);
   }
-  /** Normalize content and compute SHA-256 hash. */
+  /** Compute SHA-256 hash (delegates to content-hash.ts for a single source of truth). */
   static computeHash(content) {
-    const normalized = _ContentHashIndex.normalizeContent(content);
-    return createHash("sha256").update(normalized).digest("hex");
+    return computeContentHash(content);
   }
 };
 function normalizeAttributePairs(pairs) {
@@ -5863,4 +5866,4 @@ export {
   serializeEntityFile,
   StorageManager
 };
-//# sourceMappingURL=chunk-4R3CGCI5.js.map
+//# sourceMappingURL=chunk-MQVKHFIU.js.map