@ijfw/memory-server 1.4.3 → 1.5.0

package/src/recovery/truncation.js

@@ -0,0 +1,317 @@
+/**
+ * recovery/truncation.js — v1.5.0 T20: subagent truncation detection +
+ * recovery against the T4 intent-journal + T5 per-subagent event stream.
+ *
+ * ROLES:
+ *   * `detectTruncation({events, journal, expectedTerminalVerb})` — pure
+ *     classification of a subagent's event stream + journal state. Returns
+ *     `{ truncated:boolean, reason:string }`. The "tell" signals (in
+ *     priority order, per T19's reporter contract):
+ *       1. `events` ends with `outcome:'error'`            → truncated:'error-terminated'
+ *       2. `events` is empty AND `journal` has an open
+ *          begin (begin without commit)                    → truncated:'no-events-open-begin'
+ *       3. last event predates the wave's expected
+ *          terminal verb (e.g. `subagent.post-done`)       → truncated:'missing-terminal'
+ *       4. journal has any begin-without-commit
+ *          regardless of event stream end                  → truncated:'open-partial'
+ *       5. else                                            → truncated:false (clean)
+ *
+ *   * `recoverSubagent({projectRoot, waveId, subId, sinceVerbId})` — calls
+ *     `query('state.replay', { sinceVerbId })` which (per T4) snapshot-rolls-
+ *     back overwrite-verb partials + seals append-verb partials. Returns
+ *     `{ recovered:boolean, replayResult, verdict, reason }`.
+ *
+ *   * `measureTruncationRate({ fixtures, runOne })` — iterates a corpus of
+ *     fixtures, runs the recovery routine against each via `runOne`, and
+ *     returns `{ corpusSize, truncatedCount, recoveredCount, unrecoveredCount,
+ *     ratePostRecovery, baselineRate, byCategory[] }`. The "rate" we publish
+ *     is the FRACTION OF FIXTURES WHERE A TRUNCATION OCCURRED AND RECOVERY
+ *     COULD NOT RESTORE THE EXPECTED FINAL STATE — i.e. the truncation rate
+ *     that SURVIVES recovery. Clean fixtures (no truncation injected) are
+ *     part of the denominator because they prove recovery does not corrupt
+ *     non-truncated runs.
+ *
+ *   * `writeRateArtifact(projectRoot, result)` — atomically persists the
+ *     measurement to `<projectRoot>/.ijfw/telemetry/truncation-rate.json`,
+ *     fitting the T21 convergence-telemetry directory convention.
+ *
+ * NO PRODUCTION DEPENDENCIES; ESM; Node >=18.
+ */
+
+import { existsSync, mkdirSync, readFileSync, readdirSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+
+import { writeAtomic } from '../lib/atomic-io.js';
+import { query } from '../orchestrator/state-sdk.js';
+
+// -- Constants ------------------------------------------------------------
+
+/**
+ * Brief-locked threshold (BUILD-PLAN T20): the measured truncation rate that
+ * survives recovery MUST be at or below this fraction to pass. Halves the
+ * documented 62% baseline (see PLAN-CROSS-AUDIT-ADJUDICATION row M6).
+ */
+export const TRUNCATION_RATE_THRESHOLD = 0.31;
+
+/**
+ * Documented v1.4.x truncation-rate baseline (the rate WITHOUT this T20
+ * recovery layer). Published so the artifact is self-describing — anyone
+ * reading `.ijfw/telemetry/truncation-rate.json` can compute the improvement
+ * factor against this number.
+ */
+export const TRUNCATION_BASELINE_RATE = 0.62;
+
+/** Default expected terminal verb for a subagent's event stream. */
+export const DEFAULT_TERMINAL_VERB = 'subagent.post-done';
+
+// -- Detection -----------------------------------------------------------
+
+/**
+ * Classify a subagent's event stream + journal as truncated-or-not.
+ *
+ * @param {object} args
+ * @param {object[]} args.events                 the event stream (T5 envelope shape)
+ * @param {object[]} [args.journal]              intent-journal records visible to recovery
+ * @param {string} [args.expectedTerminalVerb]   wave's expected terminal verb
+ *                                               (default `subagent.post-done`)
+ * @returns {{truncated: false|string, reason: string, lastEvent: object|null}}
+ */
+export function detectTruncation({
+  events, journal, expectedTerminalVerb,
+} = {}) {
+  const ev = Array.isArray(events) ? events : [];
+  const j = Array.isArray(journal) ? journal : [];
+  const terminal = typeof expectedTerminalVerb === 'string' && expectedTerminalVerb
+    ? expectedTerminalVerb
+    : DEFAULT_TERMINAL_VERB;
+
+  const lastEvent = ev.length > 0 ? ev[ev.length - 1] : null;
+
+  // Build a quick view of journal partial-vs-committed.
+  const commits = new Set();
+  const begins = new Map();
+  for (const r of j) {
+    if (!r || typeof r.verbId !== 'string') continue;
+    if (r.phase === 'commit') commits.add(r.verbId);
+    else if (r.phase === 'begin') begins.set(r.verbId, r);
+  }
+  const openPartials = [];
+  for (const [verbId, beginRec] of begins) {
+    if (!commits.has(verbId)) openPartials.push(beginRec);
+  }
+
+  // 1. outcome:'error' tail — T19 contract: the subagent's stream ending
+  //    with an error verb is the canonical truncation tell.
+  if (lastEvent && lastEvent.outcome === 'error') {
+    return { truncated: 'error-terminated', reason: `last event outcome='error' (verb=${lastEvent.verb})`, lastEvent };
+  }
+
+  // 2. No events at all BUT journal carries an open begin → subagent
+  //    started a mutating verb and never emitted (truncated before tap).
+  if (ev.length === 0 && openPartials.length > 0) {
+    return {
+      truncated: 'no-events-open-begin',
+      reason: `no events, ${openPartials.length} open begin(s) in journal`,
+      lastEvent: null,
+    };
+  }
+
+  // 3. Last event is not the expected terminal verb. We accept either an
+  //    exact terminal-verb match OR an outcome marker explicitly indicating
+  //    a clean exit ('ok' with the terminal verb). Anything else with an
+  //    open partial OR with no terminal at all is treated as truncated.
+  const lastIsTerminal = lastEvent && lastEvent.verb === terminal
+    && (lastEvent.outcome === 'ok' || lastEvent.outcome === 'advisory');
+
+  if (!lastIsTerminal && openPartials.length > 0) {
+    return {
+      truncated: 'open-partial',
+      reason: `open begin without commit (${openPartials.length}) and no terminal verb`,
+      lastEvent,
+    };
+  }
+
+  if (!lastIsTerminal && ev.length > 0) {
+    return {
+      truncated: 'missing-terminal',
+      reason: `last event verb='${lastEvent.verb}' (expected terminal '${terminal}')`,
+      lastEvent,
+    };
+  }
+
+  // 4. Clean: last event is the expected terminal AND no open partials.
+  if (openPartials.length > 0) {
+    // Edge: terminal verb fired but a partial mutating verb never committed.
+    // Treat as truncated — recovery should still seal/rollback the partial.
+    return {
+      truncated: 'open-partial',
+      reason: `terminal verb emitted but ${openPartials.length} open begin(s) remain`,
+      lastEvent,
+    };
+  }
+
+  return { truncated: false, reason: 'clean exit', lastEvent };
+}
+
+// -- Recovery ------------------------------------------------------------
+
+/**
+ * Apply T4's `state.replay` to recover a truncated subagent. Snapshot-rolls
+ * back overwrite-verb partials; seals append-verb partials in place; leaves
+ * already-committed verbs untouched.
+ *
+ * @param {object} args
+ * @param {string} args.projectRoot   the wave's project root
+ * @param {string} args.waveId        the truncated subagent's wave
+ * @param {string} args.subId         the truncated subagent's id
+ * @param {string} [args.sinceVerbId] scope replay to verbs at/after this id
+ *                                    (default — full journal)
+ * @returns {Promise<{recovered:boolean, replayResult:object, reason:string}>}
+ */
+export async function recoverSubagent({
+  projectRoot, waveId, subId, sinceVerbId,
+} = {}) {
+  if (typeof projectRoot !== 'string' || !projectRoot) {
+    throw new Error('recovery/truncation: projectRoot required');
+  }
+  const payload = {};
+  if (typeof sinceVerbId === 'string' && sinceVerbId) payload.sinceVerbId = sinceVerbId;
+  const ctx = { projectRoot, subagentId: subId };
+  let replayResult;
+  try {
+    replayResult = await query('state.replay', payload, ctx);
+  } catch (err) {
+    return {
+      recovered: false,
+      replayResult: null,
+      reason: `replay threw: ${err?.message || err}`,
+      waveId,
+      subId,
+    };
+  }
+  const sealed = Array.isArray(replayResult?.sealed) ? replayResult.sealed.length : 0;
+  const rolledBack = Array.isArray(replayResult?.rolledBack) ? replayResult.rolledBack.length : 0;
+  const skipped = Array.isArray(replayResult?.skipped) ? replayResult.skipped.length : 0;
+  return {
+    recovered: replayResult?.ok === true,
+    replayResult,
+    reason: `replay ok=${replayResult?.ok} sealed=${sealed} rolledBack=${rolledBack} skipped=${skipped}`,
+    waveId,
+    subId,
+  };
+}
+
+// -- Corpus harness ------------------------------------------------------
+
+/**
+ * Discover fixture subdirectories under a corpus root. Every subdir must
+ * carry a `meta.json`; subdirs without one are skipped (allows README files /
+ * stray dirs to coexist).
+ *
+ * @param {string} corpusDir
+ * @returns {{id:string, dir:string, meta:object}[]}
+ */
+export function listFixtures(corpusDir) {
+  if (!existsSync(corpusDir)) return [];
+  const out = [];
+  for (const name of readdirSync(corpusDir).sort()) {
+    const dir = join(corpusDir, name);
+    const metaPath = join(dir, 'meta.json');
+    if (!existsSync(metaPath)) continue;
+    let meta;
+    try { meta = JSON.parse(readFileSync(metaPath, 'utf8')); }
+    catch { continue; }
+    out.push({ id: name, dir, meta });
+  }
+  return out;
+}
+
+/**
+ * Run the recovery routine across a corpus + return a measurement.
+ *
+ * The caller supplies `runOne(fixture)` — a function that materialises the
+ * fixture into a real temp project, invokes `recoverSubagent`, and returns
+ * `{ truncated, recovered, expectedFinalStateMatches, category }`. We keep
+ * runOne pluggable so the test can use real `query()` while a future caller
+ * could plug in a different I/O backend without changing this aggregator.
+ *
+ * @param {object} args
+ * @param {{id:string, meta:object}[]} args.fixtures
+ * @param {(fixture:object) => Promise<{truncated:string|false, recovered:boolean, expectedFinalStateMatches:boolean, category:string}>} args.runOne
+ * @returns {Promise<{corpusSize, truncatedCount, recoveredCount,
+ *                    unrecoveredCount, ratePostRecovery, baselineRate,
+ *                    byCategory:Array, fixtures:Array}>}
+ */
+export async function measureTruncationRate({ fixtures, runOne }) {
+  if (!Array.isArray(fixtures)) throw new Error('measureTruncationRate: fixtures[] required');
+  if (typeof runOne !== 'function') throw new Error('measureTruncationRate: runOne fn required');
+
+  const perFixture = [];
+  const byCategoryMap = new Map();
+  for (const fx of fixtures) {
+    const r = await runOne(fx);
+    const row = {
+      id: fx.id,
+      category: r.category || fx.meta?.category || 'unknown',
+      truncated: r.truncated,
+      recovered: r.recovered,
+      expectedFinalStateMatches: r.expectedFinalStateMatches,
+    };
+    perFixture.push(row);
+    const key = row.category;
+    if (!byCategoryMap.has(key)) {
+      byCategoryMap.set(key, {
+        category: key, total: 0, truncated: 0, unrecovered: 0,
+      });
+    }
+    const bucket = byCategoryMap.get(key);
+    bucket.total += 1;
+    if (row.truncated) bucket.truncated += 1;
+    if (!row.expectedFinalStateMatches) bucket.unrecovered += 1;
+  }
+
+  const corpusSize = perFixture.length;
+  const truncatedCount = perFixture.filter((r) => r.truncated).length;
+  // A fixture COUNTS AS UNRECOVERED when its post-recovery state does not
+  // match the expected final state. Clean fixtures contribute 0 (their
+  // expectedFinalStateMatches is required to be true regardless).
+  const unrecoveredCount = perFixture.filter((r) => !r.expectedFinalStateMatches).length;
+  const recoveredCount = corpusSize - unrecoveredCount;
+  const ratePostRecovery = corpusSize === 0 ? 0 : unrecoveredCount / corpusSize;
+
+  return {
+    corpusSize,
+    truncatedCount,
+    recoveredCount,
+    unrecoveredCount,
+    ratePostRecovery,
+    baselineRate: TRUNCATION_BASELINE_RATE,
+    threshold: TRUNCATION_RATE_THRESHOLD,
+    passed: ratePostRecovery <= TRUNCATION_RATE_THRESHOLD,
+    byCategory: Array.from(byCategoryMap.values()).sort(
+      (a, b) => a.category.localeCompare(b.category),
+    ),
+    fixtures: perFixture,
+    measuredAt: new Date().toISOString(),
+  };
+}
+
+// -- Artifact emission --------------------------------------------------
+
+/**
+ * Persist the measurement under `<projectRoot>/.ijfw/telemetry/`. Atomic
+ * write (tmp-rename) via `writeAtomic`. Returns the absolute path written.
+ *
+ * @param {string} projectRoot
+ * @param {object} result   value returned by `measureTruncationRate`
+ * @returns {string}        absolute path of the artifact
+ */
+export function writeRateArtifact(projectRoot, result) {
+  if (typeof projectRoot !== 'string' || !projectRoot) {
+    throw new Error('recovery/truncation: writeRateArtifact requires projectRoot');
+  }
+  const path = join(projectRoot, '.ijfw', 'telemetry', 'truncation-rate.json');
+  if (!existsSync(dirname(path))) mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  writeAtomic(path, `${JSON.stringify(result, null, 2)}\n`);
+  return path;
+}

package/src/redactor.js

@@ -16,10 +16,31 @@ const PATTERNS = [
   { re: /ghp_[A-Za-z0-9]{20,}/g,                   label: 'github'    },
   { re: /github_pat_[A-Za-z0-9_]{20,}/g,           label: 'github'    },
   { re: /gh[ousr]_[A-Za-z0-9]{30,}/g,              label: 'github'    }, // gho_/ghu_/ghs_/ghr_
+  // GitLab Personal Access Tokens (glpat-) + CI job tokens (glcbt-).
+  // GitLab PAT spec: `glpat-` + 20 base64url chars. We accept 20+ to be future-proof.
+  { re: /glpat-[A-Za-z0-9_-]{20,}/g,               label: 'gitlab'    },
+  { re: /glcbt-[A-Za-z0-9_-]{20,}/g,               label: 'gitlab'    }, // CI build token
+  { re: /gldt-[A-Za-z0-9_-]{20,}/g,                label: 'gitlab'    }, // GitLab deploy token
   // AWS permanent access key ID (AKIA) + temporary (ASIA) key ID.
   { re: /(?:AKIA|ASIA)[0-9A-Z]{16}/g,              label: 'aws'       },
+  // AWS secret access key — contextualized because bare 40-char base64 is
+  // catastrophically false-positive. Match only when paired with a known key
+  // name (AWS_SECRET_ACCESS_KEY=, aws_secret=, etc).
+  // eslint-disable-next-line security/detect-unsafe-regex -- redactor scans bounded tool output; pattern requires contextual prefix and is anchored to AWS secret key naming conventions.
+  { re: /(?:AWS|aws)[_-]?(?:SECRET|secret)[_-]?(?:ACCESS[_-]?)?(?:KEY|key)\s*[:=]\s*['"]?[A-Za-z0-9/+=]{40}['"]?/g, label: 'aws' },
+  // Discord bot tokens — three base64url segments with fixed first-segment width (24).
+  // Must come BEFORE the generic JWT pattern so the structural difference is preserved.
+  // Format: <24 chars>.<6 chars>.<27+ chars>. Exclude any match whose first
+  // segment starts with `eyJ` to avoid stealing JWT matches.
+  // eslint-disable-next-line security/detect-unsafe-regex -- redactor scans bounded tool output; this is an anchored secret pattern, not user-controlled matching logic.
+  { re: /(?<![A-Za-z0-9_])(?!eyJ)[A-Za-z0-9_-]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}(?![A-Za-z0-9_])/g, label: 'discord' },
   // Authorization: Bearer <token>.
   { re: /Bearer\s+[A-Za-z0-9._~+/=-]{10,}/g,       label: 'bearer'    },
+  // Generic JWT (three base64url segments). Catches Supabase service-role keys,
+  // Auth0/Okta/Firebase ID tokens, and any bare JWT in tool output. Anchored on
+  // `eyJ` header (base64url of `{`) which all JWTs share.
+  // eslint-disable-next-line security/detect-unsafe-regex -- redactor scans bounded tool output; the `eyJ` anchor and three-segment shape minimise false positives.
+  { re: /eyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, label: 'jwt' },
   // Slack bot / user / legacy tokens.
   { re: /xox[baprs]-[A-Za-z0-9-]{10,}/g,           label: 'slack'     },
   // Stripe live + test secret keys.
@@ -29,6 +50,32 @@ const PATTERNS = [
   { re: /npm_[A-Za-z0-9]{36}/g,                    label: 'npm'       },
   // HuggingFace user tokens.
   { re: /hf_[A-Za-z0-9]{34,}/g,                    label: 'huggingface' },
+  // OpenAI organization IDs — `org-` followed by 24 alphanumeric chars.
+  // Not a secret per se, but often pasted alongside the API key and a
+  // valuable target identifier; redact in case of cross-tenant leakage.
+  { re: /\borg-[A-Za-z0-9]{24}\b/g,                label: 'openai-org' },
+  // Vercel API tokens — 24 hex/alnum chars after a `vrcl_` style prefix; the
+  // newer (2026) Vercel CLI emits tokens prefixed with `vercel_pat_` or
+  // contextualised env vars. Cover both the explicit prefix and the env-var
+  // contextual form.
+  { re: /vercel_pat_[A-Za-z0-9_]{20,}/gi,          label: 'vercel'    },
+  // eslint-disable-next-line security/detect-unsafe-regex -- redactor scans bounded tool output and requires a Vercel context prefix before matching a token.
+  { re: /(?:VERCEL|vercel)[_-]?(?:API[_-]?)?TOKEN\s*[:=]\s*['"]?[A-Za-z0-9_-]{24,}['"]?/g, label: 'vercel' },
+  // Supabase service-role / anon keys. The JWT pattern above already catches
+  // the typical service-role JWT shape; this rule covers the legacy
+  // `sbp_` (project access) and `sb_` style tokens distributed via the CLI.
+  { re: /sbp_[A-Za-z0-9]{40,}/g,                   label: 'supabase'  },
+  // Notion integration tokens — `secret_` + 43 chars (legacy) and `ntn_` + alnum (2026).
+  { re: /secret_[A-Za-z0-9]{43}/g,                 label: 'notion'    },
+  { re: /ntn_[A-Za-z0-9_]{40,}/g,                  label: 'notion'    },
+  // Linear API keys — `lin_api_` + alnum, `lin_oauth_` + alnum.
+  { re: /lin_api_[A-Za-z0-9]{32,}/g,               label: 'linear'    },
+  { re: /lin_oauth_[A-Za-z0-9]{32,}/g,             label: 'linear'    },
+  // Twilio Account SID + Auth Token. SID is `AC` + 32 hex; auth token is
+  // 32 hex which is high-FP so we require contextual naming.
+  { re: /AC[a-f0-9]{32}/g,                         label: 'twilio'    },
+  // eslint-disable-next-line security/detect-unsafe-regex -- redactor scans bounded tool output; pattern requires a Twilio context prefix before matching.
+  { re: /(?:TWILIO|twilio)[_-]?(?:AUTH[_-]?)?TOKEN\s*[:=]\s*['"]?[a-f0-9]{32}['"]?/g, label: 'twilio' },
   // Azure Storage connection-string AccountKey (base64, 88 chars with padding).
   { re: /AccountKey=[A-Za-z0-9+/]{86,88}={0,2}/g,  label: 'azure'     },
   // GCP service-account private key PEM block.
@@ -47,6 +94,18 @@ const PATTERNS = [
   { re: /https:\/\/hooks\.slack\.com\/services\/T[A-Z0-9]+\/B[A-Z0-9]+\/[A-Za-z0-9]+/g, label: 'webhook' },
   { re: /https:\/\/discord(?:app)?\.com\/api\/webhooks\/\d+\/[A-Za-z0-9_-]+/g,           label: 'webhook' },
   { re: /https:\/\/[\w-]+\.webhook\.office\.com\/webhookb2\/[\w@/-]+/g,                  label: 'webhook' },
+  // v1.5.0 audit-LOW-tok-L1: PII patterns.
+  // Email -- conservative RFC-ish match. Bounds local-part to 1-64 chars,
+  // domain to a label.label pattern with TLD ≥2 chars. Avoids matching
+  // bare `@handle` references in prose.
+  { re: /(?<![A-Za-z0-9._%+-])[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9.-]+\.[A-Za-z]{2,24}(?![A-Za-z0-9])/g, label: 'email' },
+  // Phone -- E.164 international form (+CCNNNNNNNNNN, 10-15 digits total)
+  // plus common US/EU formats with separators. Anchored to word boundaries
+  // and a digit-density threshold to avoid eating commit SHAs / version
+  // strings. Conservative: requires either a leading `+` or one of
+  // `(NNN)`, `NNN-NNN-NNNN`, `NNN.NNN.NNNN` shapes.
+  { re: /(?<![\d+])\+[1-9]\d{1,3}[\s.-]?\d{2,4}[\s.-]?\d{2,4}[\s.-]?\d{2,4}(?![\d])/g, label: 'phone' },
+  { re: /(?<![\d-])(?:\(\d{3}\)\s?|\d{3}[.-])\d{3}[.-]\d{4}(?![\d-])/g, label: 'phone' },
 ];
 
 // INLINE rules match `key=value` style assignments. Value regex excludes
@@ -78,7 +137,7 @@ export function redactSecrets(s) {
   return out;
 }
 
-// classify(value) -> { clean: boolean, redacted_kind: string | null }
+// classifyAnchored(value) -> { clean: boolean, redacted_kind: string | null }
 //
 // D-PILLAR-SPEC section 3 surface used by D2 entity extraction. Passes the
 // value through the same PATTERNS list redactSecrets uses; if any pattern
@@ -90,20 +149,30 @@ export function redactSecrets(s) {
 //
 // Important: PATTERNS are anchored implicitly via length minimums (e.g.
 // `sk-(?:proj-)?[A-Za-z0-9_-]{32,}`), but to avoid classifying a long file
-// path that happens to contain a token-shaped substring, classify() rejects
-// only when the pattern matches the FULL trimmed value. File paths and
-// function/identifier names are always shorter than the secret patterns'
+// path that happens to contain a token-shaped substring, classifyAnchored()
+// rejects only when the pattern matches the FULL trimmed value. File paths
+// and function/identifier names are always shorter than the secret patterns'
 // minimum lengths, so the conservative cut-line is "match must equal the
 // candidate" -- a substring match doesn't trigger classification.
-export function classify(value) {
+//
+// Naming (v1.5.0 audit LOW #13): renamed conceptually from `classify` to
+// `classifyAnchored` to signal the asymmetric contract -- callers must pass
+// the candidate as a discrete value, NOT a free-form text body that contains
+// the value somewhere inside. `classify` is retained as a back-compat alias.
+export function classifyAnchored(value) {
   if (typeof value !== 'string') return { clean: true, redacted_kind: null };
   const v = value.trim();
   if (!v) return { clean: true, redacted_kind: null };
   for (const { re, label } of PATTERNS) {
     // Build a fresh non-global RegExp per check; the source PATTERNS use /g
-    // for redactSecrets but classify needs a single full-value match.
+    // for redactSecrets but classifyAnchored needs a single full-value match.
     const r = new RegExp(`^(?:${re.source})$`, re.flags.replace('g', ''));
     if (r.test(v)) return { clean: false, redacted_kind: label };
   }
   return { clean: true, redacted_kind: null };
 }
+
+// Back-compat alias. New callers should prefer `classifyAnchored` so the
+// "value must be the whole candidate, not embedded in prose" contract is
+// obvious at the call site.
+export const classify = classifyAnchored;

package/src/runtime-mediator.js

@@ -15,6 +15,21 @@
  * Fail-closed invariant: if the file exists but is unparseable / malformed,
  * the caller MUST treat it as a deny. A corrupted state file is not a free
  * pass -- that would defeat the sandbox.
+ *
+ * ## Cross-platform enforcement boundary
+ * This module is the single tier-2 enforcement point for platforms without
+ * a native pre-tool hook lifecycle: Gemini CLI, Cursor, Windsurf, and
+ * Copilot (VS Code). All MCP tool calls from those platforms pass through
+ * `checkPermission()` at `server.js:98` BEFORE any handler executes.
+ * Hook-lifecycle platforms (Claude Code, Codex, Hermes, Wayland) get
+ * parallel enforcement via their own pre-tool-use hook scripts in addition
+ * to this MCP boundary.
+ *
+ * Coverage:
+ *   - `test-runtime-mediator.js`         unit-level primitives
+ *   - `test-mcp-gate-integration.js`     integration through the exported
+ *                                        `gatePermissionAndQuota` (server.js:98)
+ *                                        — locks in the four no-hook platforms.
  */
 
 import { readFile, mkdir, appendFile, rename, stat } from 'node:fs/promises';

package/src/sanitizer.js

@@ -20,6 +20,16 @@ export function sanitizeContent(s) {
   // U+200B-U+200F, U+202A-U+202E, U+2066-U+2069, U+FEFF
   out = out.replace(/[\u200B-\u200F\u202A-\u202E\u2066-\u2069\uFEFF]/g, '');
 
+  // 2b. Strip Unicode tag-block chars (U+E0000-U+E007F) \u2014 the "ASCII Smuggler"
+  // prompt-injection vector. These codepoints are invisible to humans but map
+  // 1:1 to printable ASCII (U+E0041 = "A", U+E0061 = "a", etc.) and many LLMs
+  // interpret them as the corresponding text. An attacker can hide an
+  // instruction like "ignore all prior" inside otherwise-benign memory content.
+  // v1.5.1 H1.4 (audit memory-engine.md F-SEC-3).
+  // Ref: https://embracethered.com/blog/posts/2024/hiding-and-finding-text-with-unicode-tags/
+  // eslint-disable-next-line security/detect-unsafe-regex -- single-char Unicode range class; no quantifier; not backtrack-exploitable
+  out = out.replace(/[\u{E0000}-\u{E007F}]/gu, '');
+
   // 3. Defang ANY heading prefix (1+ hashes, optional whitespace) -- entry must
   // never produce a structural ## section that mimics a journal timestamp.
   out = out.replace(/^[ \t]*#+[ \t]+/gm, '> ');

package/src/search-hybrid.js

@@ -0,0 +1,139 @@
+// r17 (cold-tier wire-up): hybrid BM25+vector rerank step for searchMemory.
+//
+// Pure module, no side effects on import. server.js imports the rerank
+// helper; test-search-hybrid.js imports the same helper without dragging in
+// the MCP server's stdio bootstrap (which would hang the test runner).
+//
+// Behavior:
+//   - When IJFW_VECTORS is off (default) OR no opts.embedder is injected,
+//     this is a pure no-op and returns the input `ranked` array unchanged.
+//   - When IJFW_VECTORS=on AND the embedder is available (either via
+//     @xenova/transformers installed OR via opts.embedder injection), the
+//     BM25 top-K is reranked using cosine similarity over each result's
+//     snippet, blended with the BM25 score via vectors.hybridRerank.
+//   - Any failure during embedding falls back to BM25 with a single
+//     stderr warning per distinct reason. Never throws into the caller.
+
+import { vectorsEnabled, getEmbedder, hybridRerank } from './vectors.js';
+// v1.5.0 wire-W1.C: persistent embedding cache so repeated queries over a
+// stable corpus don't pay the per-snippet embed cost twice. Cache is keyed
+// on sha256(snippet) + model_id; falls back to live re-embed on any miss
+// or when opts.db is absent.
+import {
+  cacheKeyFor,
+  getCachedEmbedding,
+  setCachedEmbedding,
+  hasVectorCache,
+} from './memory/embedding-cache.js';
+
+// Throttle stderr noise — single warning per distinct failure reason.
+let _vectorWarnedReason = null;
+
+/**
+ * Optional hybrid rerank step. Returns the input `ranked` unchanged when
+ * vectors are disabled or the embedder cannot be loaded. On success, returns
+ * the merged ranking from `hybridRerank` (BM25 score + cosine similarity).
+ *
+ * @param {string} query
+ * @param {Array<{id, score, snippet, meta}>} ranked  - BM25 top-K
+ * @param {object} opts
+ * @param {object} [opts.embedder]  - injected for tests; defaults to getEmbedder()
+ * @param {number} [opts.wBm25]     - BM25 weight (default 0.6 via vectors.js)
+ * @param {number} [opts.wVec]      - vector weight (default 0.4 via vectors.js)
+ * @returns {Promise<Array>} reranked list (or `ranked` on no-op)
+ */
+export async function maybeRerankWithVectors(query, ranked, opts = {}) {
+  // Skip the embedder load entirely when vectors are off — env check is the
+  // cheap path. Tests can force the embedder path by passing opts.embedder.
+  if (!opts.embedder && !vectorsEnabled()) return ranked;
+
+  let embedder = opts.embedder;
+  if (!embedder) {
+    try {
+      embedder = await getEmbedder();
+    } catch (err) {
+      // getEmbedder shouldn't throw (it returns {available:false}), but defend.
+      embedder = { available: false, reason: `getEmbedder-threw: ${err.message}` };
+    }
+  }
+  if (!embedder || !embedder.available) {
+    const reason = embedder?.reason || 'unavailable';
+    if (_vectorWarnedReason !== reason) {
+      _vectorWarnedReason = reason;
+      // Stderr only; stdout is the JSON-RPC framing channel.
+      process.stderr.write(
+        `IJFW: cold-tier vectors requested (IJFW_VECTORS=on) but embedder unavailable (${reason}). Falling back to BM25.\n`
+      );
+    }
+    return ranked;
+  }
+
+  try {
+    // v1.5.0 audit MED #6 (memory-engine.md F-SPD-1): batch-embed the
+    // query + all snippets in parallel via Promise.all. The previous
+    // sequential `for (... await embedder.embed)` loop serialised K+1
+    // calls -- when `embedder.embed` is an HTTP round-trip (e.g. a
+    // remote inference server) p99 was ~600ms for the top-K=10 case.
+    // Concurrent dispatch drops that to ~80ms (single-round-trip cost
+    // plus per-call overhead). For the local @xenova/transformers
+    // pipeline the calls still resolve serially under the hood, but
+    // Promise.all is no worse than the sequential await and lets future
+    // batch-aware embedders win without further changes.
+    //
+    // v1.5.0 wire-W1.C: when opts.db + opts.modelId are supplied AND the
+    // memory_entry_vectors table exists, route each embed through the
+    // persistent cache. The cache is keyed on sha256(text), so a second
+    // call with the same query + corpus serves entirely from SQLite —
+    // zero embedder calls, zero re-embed cost. The query embedding is
+    // also cached (queries repeat in long sessions / dashboard polls).
+    //
+    // cacheReady flips to false when the table is missing OR no db is
+    // passed; the rerank then degrades to the existing live-embed path
+    // with no observable behavior change.
+    const snippets = ranked.map((r) => r.snippet || '');
+    const cacheDb = opts.db || null;
+    const modelId = opts.modelId || embedder.modelId || null;
+    const cacheReady = cacheDb && typeof modelId === 'string' && modelId.length > 0 && hasVectorCache(cacheDb);
+
+    const embedWithCache = async (text) => {
+      if (!cacheReady) return embedder.embed(text);
+      const key = cacheKeyFor(text);
+      if (key === null) return embedder.embed(text);
+      const hit = getCachedEmbedding(cacheDb, key, modelId);
+      if (hit) return hit;
+      const vec = await embedder.embed(text);
+      setCachedEmbedding(cacheDb, key, modelId, vec);
+      return vec;
+    };
+
+    const [queryVec, ...docVecs] = await Promise.all([
+      embedWithCache(query),
+      ...snippets.map((s) => embedWithCache(s)),
+    ]);
+    const vectorScores = new Map();
+    for (let i = 0; i < ranked.length; i++) {
+      const docVec = docVecs[i] || [];
+      // Cosine over L2-normalized vectors === dot product.
+      let dot = 0;
+      const n = Math.min(queryVec.length, docVec.length);
+      for (let j = 0; j < n; j++) dot += queryVec[j] * docVec[j];
+      vectorScores.set(ranked[i].id, dot);
+    }
+    return hybridRerank(ranked, vectorScores, {
+      wBm25: opts.wBm25,
+      wVec: opts.wVec,
+    });
+  } catch (err) {
+    const reason = `embed-failed: ${err.message}`;
+    if (_vectorWarnedReason !== reason) {
+      _vectorWarnedReason = reason;
+      process.stderr.write(
+        `IJFW: cold-tier vectors hit an error mid-pipeline (${reason}). Falling back to BM25 for this query.\n`
+      );
+    }
+    return ranked;
+  }
+}
+
+// Test seam — reset the once-per-process warning gate.
+export function _resetVectorWarnGate() { _vectorWarnedReason = null; }