@mmerterden/multi-agent-pipeline 20.8.0 → 20.8.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mmerterden/multi-agent-pipeline",
-  "version": "20.8.0",
+  "version": "20.8.2",
   "description": "6-phase AI development pipeline with full orchestration on Claude Code, Copilot CLI and Codex CLI. Analysis, planning, TDD, CLI-aware parallel review with consensus surfacing + Fable triage, default-FAIL evidence gates, secret + intent guards, per-phase cost ledger, persistent learnings memory, wiki generation, commit automation. Token-preserving uninstall.",
   "type": "module",
   "main": "index.js",
@@ -20,7 +20,7 @@
   },
   "scripts": {
     "start": "node index.js",
-    "test": "npm run format:check && npm run lint && node --test test/*.test.mjs && node pipeline/scripts/run-smokes.mjs && node pipeline/scripts/lint-skills.mjs && node pipeline/scripts/lint-personas.mjs && node pipeline/scripts/lint-mcp-refs.mjs && node pipeline/scripts/eval-triage.mjs && node pipeline/scripts/eval-golden-tasks.mjs && node pipeline/scripts/eval-intent.mjs && node pipeline/scripts/eval-recall.mjs && node pipeline/scripts/eval-guard-robustness.mjs && node pipeline/scripts/validate-schemas.mjs && node pipeline/scripts/validate-prefs.mjs && node pipeline/scripts/scorecard.mjs",
+    "test": "npm run format:check && npm run lint && node --test test/*.test.mjs && node pipeline/scripts/run-smokes.mjs && node pipeline/scripts/lint-skills.mjs && node pipeline/scripts/gen-skills-index.mjs --check && node pipeline/scripts/lint-personas.mjs && node pipeline/scripts/lint-mcp-refs.mjs && node pipeline/scripts/eval-triage.mjs && node pipeline/scripts/eval-golden-tasks.mjs && node pipeline/scripts/eval-intent.mjs && node pipeline/scripts/eval-recall.mjs && node pipeline/scripts/eval-guard-robustness.mjs && node pipeline/scripts/validate-schemas.mjs && node pipeline/scripts/validate-prefs.mjs && node pipeline/scripts/scorecard.mjs",
     "test:unit": "node --test test/*.test.mjs",
     "test:smoke": "node pipeline/scripts/run-smokes.mjs",
     "lint:skills": "node pipeline/scripts/lint-skills.mjs",
@@ -94,6 +94,9 @@
     "!pipeline/scripts/run-smokes.mjs",
     "!pipeline/scripts/eval-*.mjs",
     "!pipeline/scripts/check-rewrite-overlap.mjs",
+    "!pipeline/scripts/gen-ref-toc.mjs",
+    "!pipeline/scripts/make-manifest.mjs",
+    "!pipeline/scripts/scorecard-snapshot.mjs",
     "!pipeline/scripts/scorecard.mjs",
     "!pipeline/scripts/lint-skills.mjs",
     "!pipeline/scripts/lint-personas.mjs",

package/pipeline/commands/multi-agent/update/SKILL.md

@@ -28,10 +28,16 @@ A git clone of the pipeline repo is a maintainer workspace, kept in sync by `/mu
 
 ## Steps
 
-1. **Resolve the package and both versions** (run steps 1-3 in ONE shell block so the variables survive):
+Every fenced block below runs in a fresh shell, so no variable survives from one
+block to the next. Step 1 writes the values the later steps need to a state file,
+and every later block that needs them starts by sourcing it.
+
+1. **Resolve the package and both versions**:
    ```bash
+   STATE="${TMPDIR:-/tmp}/multi-agent-update.env"
    PKG="@{npm-scope}/multi-agent-pipeline"
    REG="https://registry.npmjs.org"
+   UPD_DIR="${TMPDIR:-/tmp}/multi-agent-update"
    CUR=$(tr -d '[:space:]' < "$HOME/.claude/.pipeline-version" 2>/dev/null)
    [ -n "$CUR" ] || CUR="unknown"
    # Read the registry directly - `npm view` can answer from a stale local cache.
@@ -45,23 +51,25 @@ A git clone of the pipeline repo is a maintainer workspace, kept in sync by `/mu
    REQUIRED=$(curl -fsS -H 'Accept: application/vnd.npm.install-v1+json' \
      "$REG/$(printf '%s' "$PKG" | sed 's|/|%2F|')" \
      | node -pe 'JSON.parse(require("fs").readFileSync(0,"utf8"))["dist-tags"].required || ""' 2>/dev/null)
+   printf 'PKG=%q\nREG=%q\nUPD_DIR=%q\nCUR=%q\nLATEST=%q\n' \
+     "$PKG" "$REG" "$UPD_DIR" "$CUR" "$LATEST" > "$STATE"
    echo "Current: v$CUR   Latest: v$LATEST${REQUIRED:+   Required floor: v$REQUIRED}"
    ```
 
    This command is never blocked by the floor  -  it is the remedy for it. The
-   line above is where the floor is reported, so the user sees why a run halted;
-   `$REQUIRED` does not survive into the later shell blocks.
+   line above is where the floor is reported, so the user sees why a run halted.
 
 2. **Stop early when already current.** When the line below prints, skip
    step 3 (no download, no reinstall) and continue at step 4b, so the stack
    plugins are still refreshed:
    ```bash
+   . "${TMPDIR:-/tmp}/multi-agent-update.env" || { echo "Run step 1 first."; exit 1; }
    [ "$CUR" = "$LATEST" ] && echo "✓ Already up to date: v$LATEST"
    ```
 
 3. **Download the release and install**:
    ```bash
-   UPD_DIR="${TMPDIR:-/tmp}/multi-agent-update"
+   . "${TMPDIR:-/tmp}/multi-agent-update.env" || { echo "Run step 1 first."; exit 1; }
    rm -rf "$UPD_DIR" && mkdir -p "$UPD_DIR"
    # `--registry` does NOT override a scope mapping. A user-level .npmrc line
    # like `@scope:registry=https://npm.pkg.github.com` wins over it, so the
@@ -80,10 +88,15 @@ A git clone of the pipeline repo is a maintainer workspace, kept in sync by `/mu
    TGZ="$UPD_DIR/$(printf '%s' "$PACK_OUT" | tail -1 | tr -d '[:space:]')"
    [ -f "$TGZ" ] || { echo "npm pack reported no tarball; aborting."; exit 1; }
    tar -xzf "$TGZ" -C "$UPD_DIR"
-   node "$UPD_DIR/package/install.js" --all
+   node "$UPD_DIR/package/install.js" --installed
    ```
 
-   The installer refreshes every configured CLI target (Claude Code, Copilot CLI, Codex CLI) and writes the new version to `$HOME/.claude/.pipeline-version`.
+   `--installed` refreshes exactly the CLI targets that already carry a version
+   stamp (`$HOME/.claude/.pipeline-version`, `$HOME/.copilot/.pipeline-version`,
+   `$HOME/.codex/.pipeline-version`) and adds none: an update never installs
+   Copilot CLI or Codex CLI on a machine that only had Claude Code. With no stamp
+   at all it installs Claude Code. Each refreshed target gets the new version
+   stamp.
 
 4b. **Update the stack plugins** (catalog refresh, then every installed plugin in its own scope):
    ```bash
@@ -98,10 +111,8 @@ A git clone of the pipeline repo is a maintainer workspace, kept in sync by `/mu
    echo "  -> no global adapter files to prune"
    ```
 
-   > **Do NOT delete `$HOME/.codex/prompts/multi-agent.md`.** Releases up to
-   > v12.11.0 removed it here as a retired v9.7.0 adapter leftover. Codex CLI is a
-   > supported target again as of v13.0.0 and the installer writes that file, so
-   > deleting it silently breaks the `/multi-agent` slash command on Codex.
+   > **Do NOT delete `$HOME/.codex/prompts/multi-agent.md`.** The Codex installer
+   > writes it and it backs the `/multi-agent` slash command on Codex CLI.
 
 5. **Migrate preferences** (if there is an old schema):
    ```bash
@@ -110,11 +121,9 @@ A git clone of the pipeline repo is a maintainer workspace, kept in sync by `/mu
    fi
    ```
 
-5b. **Auto-configure operational reporting.** One call, and it is the same call
-   `/multi-agent:setup` and a first run make - the registration used to live here
-   as forty lines of shell, so a user who installed, ran setup and never ran
-   update was never registered and never reported, which reads in the panel
-   exactly like nobody using the pipeline.
+5b. **Auto-configure operational reporting.** One call, the same one
+   `/multi-agent:setup` and a first run make, so a machine is registered by
+   whichever of the three runs first.
 
    ```bash
    node "$HOME/.claude/scripts/usage-register.mjs"
@@ -128,6 +137,7 @@ A git clone of the pipeline repo is a maintainer workspace, kept in sync by `/mu
 
 6. **Show the new version and its changes** (from the packaged CHANGELOG - there is no git history on this channel):
    ```bash
+   . "${TMPDIR:-/tmp}/multi-agent-update.env" || { echo "Run step 1 first."; exit 1; }
    NEW=$(tr -d '[:space:]' < "$HOME/.claude/.pipeline-version" 2>/dev/null)
    echo ""
    echo "✓ Updated: v$CUR → v$NEW"
@@ -140,11 +150,12 @@ A git clone of the pipeline repo is a maintainer workspace, kept in sync by `/mu
 
 7. **Smoke test (optional)** - the release tarball ships exactly two smokes for this purpose:
    ```bash
+   . "${TMPDIR:-/tmp}/multi-agent-update.env" || { echo "Run step 1 first."; exit 1; }
    echo ""
    echo "Verification:"
    bash "$UPD_DIR/package/pipeline/scripts/smoke-schema-validation.sh" 2>&1 | tail -1
    bash "$UPD_DIR/package/pipeline/scripts/smoke-cross-cli-behavior.sh" 2>&1 | tail -1
-   rm -rf "$UPD_DIR"
+   rm -rf "$UPD_DIR" "${TMPDIR:-/tmp}/multi-agent-update.env"
    ```
 
 ## Output
@@ -152,7 +163,7 @@ A git clone of the pipeline repo is a maintainer workspace, kept in sync by `/mu
 ```
 Current: v15.6.0   Latest: v15.6.1
   -> npm pack @{npm-scope}/multi-agent-pipeline@15.6.1
-  -> node install.js --all (56 commands, 263 scripts, 210 skills)
+  -> node install.js --installed (Claude Code; no Copilot CLI or Codex CLI stamp, not installed)
   -> migrate-prefs.mjs (0 changes  -  already v2.6.0)
 
 ✓ Updated: v15.6.0 → v15.6.1

package/pipeline/lib/confusables.json

@@ -1,36 +1,23 @@
 {
-  "_readme": "Characters a guard folds before matching, applied after NFKC. `strip` lists invisible characters that are removed. `map` sends a character that renders like an ASCII one to that ASCII character: the dash family, and the Cyrillic and Greek letters whose glyphs match Latin ones. Chosen from the Unicode confusables data for the letters and punctuation the guard rules use. Read by lib/normalize-text.mjs and scripts/agent-guard.py.",
-  "strip": [
-    "\u00ad",
-    "\u034f",
-    "\u061c",
-    "\u115f",
-    "\u1160",
-    "\u17b4",
-    "\u17b5",
-    "\u180e",
-    "\u200b",
-    "\u200c",
-    "\u200d",
-    "\u200e",
-    "\u200f",
-    "\u202a",
-    "\u202b",
-    "\u202c",
-    "\u202d",
-    "\u202e",
-    "\u2060",
-    "\u2061",
-    "\u2062",
-    "\u2063",
-    "\u2064",
-    "\u2066",
-    "\u2067",
-    "\u2068",
-    "\u2069",
-    "\u3164",
-    "\ufeff",
-    "\uffa0"
+  "_readme": "Characters a guard folds before matching, applied after NFKC. `stripRanges` lists the Default_Ignorable_Code_Point set as hex code point ranges (a single value or `FIRST-LAST`); every character in them is removed. `map` sends a character that renders like an ASCII one to that ASCII character: the dash family, the Cyrillic and Greek letters whose glyphs match Latin ones, and the compatibility letters and digits added after Unicode 13 whose NFKC form is ASCII, so a Python whose unicodedata predates them folds them the way Node does. Chosen from the Unicode confusables data for the letters and punctuation the guard rules use. Read by lib/normalize-text.mjs and scripts/agent-guard.py.",
+  "stripRanges": [
+    "00AD",
+    "034F",
+    "061C",
+    "115F-1160",
+    "17B4-17B5",
+    "180B-180F",
+    "200B-200F",
+    "202A-202E",
+    "2060-206F",
+    "3164",
+    "FE00-FE0F",
+    "FEFF",
+    "FFA0",
+    "FFF0-FFF8",
+    "1BCA0-1BCA3",
+    "1D173-1D17A",
+    "E0000-E0FFF"
   ],
   "map": {
     "\u2010": "-",
@@ -95,6 +82,65 @@
     "\u03b1": "a",
     "\u03bd": "v",
     "\u03c1": "p",
-    "\u0131": "i"
+    "\u0131": "i",
+    "\ua7f1": "S",
+    "\ua7f2": "C",
+    "\ua7f3": "F",
+    "\ua7f4": "Q",
+    "\ud801\udf81": ":",
+    "\ud801\udfa5": "q",
+    "\ud833\udcd6": "A",
+    "\ud833\udcd7": "B",
+    "\ud833\udcd8": "C",
+    "\ud833\udcd9": "D",
+    "\ud833\udcda": "E",
+    "\ud833\udcdb": "F",
+    "\ud833\udcdc": "G",
+    "\ud833\udcdd": "H",
+    "\ud833\udcde": "I",
+    "\ud833\udcdf": "J",
+    "\ud833\udce0": "K",
+    "\ud833\udce1": "L",
+    "\ud833\udce2": "M",
+    "\ud833\udce3": "N",
+    "\ud833\udce4": "O",
+    "\ud833\udce5": "P",
+    "\ud833\udce6": "Q",
+    "\ud833\udce7": "R",
+    "\ud833\udce8": "S",
+    "\ud833\udce9": "T",
+    "\ud833\udcea": "U",
+    "\ud833\udceb": "V",
+    "\ud833\udcec": "W",
+    "\ud833\udced": "X",
+    "\ud833\udcee": "Y",
+    "\ud833\udcef": "Z",
+    "\ud833\udcf0": "0",
+    "\ud833\udcf1": "1",
+    "\ud833\udcf2": "2",
+    "\ud833\udcf3": "3",
+    "\ud833\udcf4": "4",
+    "\ud833\udcf5": "5",
+    "\ud833\udcf6": "6",
+    "\ud833\udcf7": "7",
+    "\ud833\udcf8": "8",
+    "\ud833\udcf9": "9",
+    "\ud838\udc30": "a",
+    "\ud838\udc35": "e",
+    "\ud838\udc3c": "o",
+    "\ud838\udc3e": "p",
+    "\ud838\udc3f": "c",
+    "\ud838\udc41": "y",
+    "\ud838\udc43": "x",
+    "\ud838\udc4c": "i",
+    "\ud838\udc4d": "j",
+    "\ud838\udc51": "a",
+    "\ud838\udc56": "e",
+    "\ud838\udc5c": "o",
+    "\ud838\udc5e": "c",
+    "\ud838\udc5f": "y",
+    "\ud838\udc61": "x",
+    "\ud838\udc68": "i",
+    "\ud838\udc69": "s"
   }
 }

package/pipeline/lib/extract-conventions.sh

@@ -927,11 +927,11 @@ bucket_di_registration() {
   case "$PLATFORM" in
     ios)
       local configurators
-      configurators=$(run_find -type f \( -name '*DependencyConfigurator.swift' -o -name '*Assembly.swift' -o -name '*Module.swift' \) 2>/dev/null | head -20 || true)
+      configurators=$(run_find -type f \( -name '*DependencyConfigurator.swift' -o -name '*DependencyRegistrar.swift' -o -name '*Assembly.swift' -o -name '*Module.swift' \) 2>/dev/null | head -20 || true)
       local resolver_count factory_count manual_count swinject_count
       manual_count=0; resolver_count=0; factory_count=0; swinject_count=0
       if [ -n "$configurators" ]; then
-        manual_count=$(printf '%s\n' "$configurators" | count_matches 'DependencyConfigurator')
+        manual_count=$(printf '%s\n' "$configurators" | count_matches -E 'Dependency(Configurator|Registrar)')
       fi
       files=$(run_find -type f -name '*.swift' 2>/dev/null || true)
       if [ -n "$files" ]; then
@@ -943,7 +943,7 @@ bucket_di_registration() {
 
       sample=$(printf '%s\n' "$configurators" | head -3 | sed "s|^$REPO_PATH/||")
       if [ "$manual_count" -ge "$resolver_count" ] && [ "$manual_count" -ge "$factory_count" ] && [ "$manual_count" -ge "$swinject_count" ] && [ "$manual_count" -gt 0 ]; then
-        emit_bucket "manual *DependencyConfigurator" "FooDependencyConfigurator.swift" "$(confidence_for "$manual_count")" "$(files_to_json "$sample")" '["Resolver","Factory","Swinject"]'
+        emit_bucket "manual per-module registrar" "FooDependencyRegistrar.swift" "$(confidence_for "$manual_count")" "$(files_to_json "$sample")" '["Resolver","Factory","Swinject"]'
       elif [ "$resolver_count" -ge "$factory_count" ] && [ "$resolver_count" -ge "$swinject_count" ] && [ "$resolver_count" -gt 0 ]; then
         emit_bucket "Resolver" "Resolver.register { FooViewModel() }" "$(confidence_for "$resolver_count")" "[]" '["manual-configurator","Factory","Swinject"]'
       elif [ "$factory_count" -ge "$swinject_count" ] && [ "$factory_count" -gt 0 ]; then

package/pipeline/lib/json-file-lock.mjs

@@ -19,7 +19,9 @@
 import {
   chmodSync,
   linkSync,
+  lstatSync,
   readFileSync,
+  realpathSync,
   renameSync,
   rmSync,
   statSync,
@@ -125,17 +127,31 @@ export function withFileLockSync(
 /**
  * Replace `path` with `data` as JSON, atomically, at `mode`.
  *
+ * `followSymlink` is for a file the user owns and may keep in a dotfiles repo
+ * (the preferences file): when `path` is a symlink, the rename lands on its
+ * target, so the link and the dotfiles copy stay in step. Without it a
+ * symlink at `path` is replaced by a regular file, which is the safe default
+ * for a path an unattended run could plant a link at.
+ *
  * @param {string} path
  * @param {unknown} data
- * @param {{mode?: number}} [opts]
+ * @param {{mode?: number, followSymlink?: boolean}} [opts]
  */
-export function writeJsonAtomicSync(path, data, { mode = 0o600 } = {}) {
-  const tmp = join(dirname(path), `.${basename(path)}.${process.pid}.tmp`);
+export function writeJsonAtomicSync(path, data, { mode = 0o600, followSymlink = false } = {}) {
+  let target = path;
+  if (followSymlink) {
+    try {
+      if (lstatSync(path).isSymbolicLink()) target = realpathSync(path);
+    } catch {
+      // Missing file: write the path as given.
+    }
+  }
+  const tmp = join(dirname(target), `.${basename(target)}.${process.pid}.tmp`);
   rmSync(tmp, { force: true });
   try {
     writeFileSync(tmp, `${JSON.stringify(data, null, 2)}\n`, { mode, flag: "wx" });
     chmodSync(tmp, mode);
-    renameSync(tmp, path);
+    renameSync(tmp, target);
   } catch (err) {
     rmSync(tmp, { force: true });
     throw err;
@@ -149,16 +165,20 @@ export function writeJsonAtomicSync(path, data, { mode = 0o600 } = {}) {
  *
  * @param {string} path
  * @param {(current: any) => any} mutate
- * @param {{fallback?: any, mode?: number, timeoutMs?: number, staleMs?: number}} [opts]
+ * @param {{fallback?: any, mode?: number, followSymlink?: boolean, timeoutMs?: number, staleMs?: number}} [opts]
  * @returns {any} the document written, or undefined when nothing was
  */
-export function updateJsonFileSync(path, mutate, { fallback = null, mode = 0o600, ...lock } = {}) {
+export function updateJsonFileSync(
+  path,
+  mutate,
+  { fallback = null, mode = 0o600, followSymlink = false, ...lock } = {},
+) {
   return withFileLockSync(
     path,
     () => {
       const next = mutate(readRegularJson(path, fallback));
       if (next === undefined) return undefined;
-      writeJsonAtomicSync(path, next, { mode });
+      writeJsonAtomicSync(path, next, { mode, followSymlink });
       return next;
     },
     lock,

package/pipeline/lib/normalize-text.mjs

@@ -2,13 +2,20 @@
 /**
  * normalize-text.mjs  -  fold look-alike text to the form a guard matches on.
  *
- * NFKC, then the invisible characters in confusables.json removed and its
- * look-alike letters and punctuation mapped to ASCII. A guard matches on the
- * raw text and on this folded copy, so a trailer or a token written with a
- * zero-width space or a Cyrillic letter is still recognised. The folded copy is
- * only ever matched against; nothing written or shown uses it.
+ * NFKC, then the Default_Ignorable_Code_Point ranges in confusables.json
+ * removed and its look-alike letters and punctuation mapped to ASCII. A guard
+ * matches on the raw text and on this folded copy, so a trailer or a token
+ * written with a zero-width space or a Cyrillic letter is still recognised. The
+ * folded copy is only ever matched against; nothing written or shown uses it.
  *
- * As a CLI it reads stdin and writes the folded text to stdout.
+ * The table is loaded fail-soft: when confusables.json cannot be read the fold
+ * is NFKC alone, `TABLE_LOADED` is false and one notice goes to stderr. The log
+ * redactor keeps working on that reduced fold; the outbound gate checks
+ * `TABLE_LOADED` and refuses to scan.
+ *
+ * As a CLI it reads stdin and writes the folded text to stdout. With `--nul`
+ * it reads NUL-separated records and writes each folded record followed by a
+ * NUL, so a caller folds many texts in one process.
  *
  * @module pipeline/lib/normalize-text
  */
@@ -17,32 +24,94 @@ import { readFileSync } from "node:fs";
 import { join } from "node:path";
 import { invokedDirectly } from "./invoked-directly.mjs";
 
-const TABLE = JSON.parse(readFileSync(join(import.meta.dirname, "confusables.json"), "utf8"));
-const STRIP = new Set(TABLE.strip);
-const MAP = new Map(Object.entries(TABLE.map));
+function loadTable() {
+  try {
+    const table = JSON.parse(readFileSync(join(import.meta.dirname, "confusables.json"), "utf8"));
+    return { table, loaded: true };
+  } catch (err) {
+    process.stderr.write(
+      `normalize-text: confusables.json unreadable (${err.code || err.name}) - folding with NFKC only\n`,
+    );
+    return { table: {}, loaded: false };
+  }
+}
+
+const { table: TABLE, loaded } = loadTable();
+
+/** False when confusables.json could not be read and the fold is NFKC only. */
+export const TABLE_LOADED = loaded;
+
+function stripSet(table) {
+  const set = new Set();
+  for (const range of table.stripRanges || []) {
+    const [first, last = first] = String(range).split("-");
+    const lo = parseInt(first, 16);
+    const hi = parseInt(last, 16);
+    if (Number.isNaN(lo) || Number.isNaN(hi)) continue;
+    for (let cp = lo; cp <= hi; cp++) set.add(cp);
+  }
+  return set;
+}
+
+const STRIP = stripSet(TABLE);
+const MAP = new Map(Object.entries(TABLE.map || {}));
 
 /** @param {string} text */
 export function normalizeForMatch(text) {
   if (typeof text !== "string" || !text) return text;
   let out = "";
   for (const ch of text.normalize("NFKC")) {
-    if (STRIP.has(ch)) continue;
+    if (STRIP.has(ch.codePointAt(0))) continue;
     out += MAP.get(ch) ?? ch;
   }
   return out;
 }
 
-/** The text with its invisible characters removed and nothing else changed. */
-export function stripInvisible(text) {
-  if (typeof text !== "string" || !text) return text;
-  let out = "";
-  for (const ch of text) if (!STRIP.has(ch)) out += ch;
-  return out;
+/**
+ * The fold of `text` one code point at a time, with the span of the original
+ * each folded UTF-16 unit came from: folded unit `j` came from
+ * `text.slice(start[j], end[j])`. A match on `folded` from `a` to `b` covers
+ * `text.slice(start[a], end[b - 1])`. Folding per code point rather than over
+ * the whole string differs from normalizeForMatch only where NFKC would
+ * compose a base letter with a following combining mark, which no credential
+ * shape contains.
+ *
+ * @param {string} text
+ * @returns {{ folded: string, start: number[], end: number[] }}
+ */
+export function foldWithIndex(text) {
+  const start = [];
+  const end = [];
+  let folded = "";
+  let i = 0;
+  for (const ch of text) {
+    const next = i + ch.length;
+    for (const part of ch.normalize("NFKC")) {
+      if (STRIP.has(part.codePointAt(0))) continue;
+      const mapped = MAP.get(part) ?? part;
+      for (let k = 0; k < mapped.length; k++) {
+        start.push(i);
+        end.push(next);
+      }
+      folded += mapped;
+    }
+    i = next;
+  }
+  return { folded, start, end };
 }
 
 if (invokedDirectly(import.meta.url)) {
+  const nul = process.argv.includes("--nul");
   let input = "";
   process.stdin.setEncoding("utf8");
   process.stdin.on("data", (d) => (input += d));
-  process.stdin.on("end", () => process.stdout.write(normalizeForMatch(input)));
+  process.stdin.on("end", () => {
+    if (!nul) {
+      process.stdout.write(normalizeForMatch(input));
+      return;
+    }
+    const records = input.split("\0");
+    if (records.at(-1) === "") records.pop();
+    process.stdout.write(records.map((r) => `${normalizeForMatch(r)}\0`).join(""));
+  });
 }

package/pipeline/lib/outbound-gate.mjs

@@ -42,14 +42,15 @@
  *   cat body.md | outbound-gate.mjs --warn          # report, never block
  *
  * Exit codes:
- *   0  -  nothing found
+ *   0  -  nothing found, or an empty body
  *   7  -  at least one finding; the caller must not publish
- *   2  -  usage error (no input)
+ *   2  -  usage error (--file without a path)
+ *         or a stdin that cannot be read at all
  */
 
 import { readFileSync } from "node:fs";
 import { invokedDirectly } from "./invoked-directly.mjs";
-import { normalizeForMatch } from "./normalize-text.mjs";
+import { TABLE_LOADED, normalizeForMatch } from "./normalize-text.mjs";
 
 /**
  * Each rule is a SHAPE, not a dictionary. A list of known token prefixes goes
@@ -148,17 +149,25 @@ export const SURFACES = [
   },
 ];
 
+// Printable ASCII and tab fold to themselves; any other line is folded.
+const NEEDS_FOLD = /[^\t -~]/;
+
 /**
  * @param {string} text
  * @returns {{rule: string, certain: boolean, line: number}[]}
  */
 export function scan(text) {
+  // Without the fold table a disguised credential would pass as clean text,
+  // so the gate refuses to scan rather than report nothing.
+  if (!TABLE_LOADED) {
+    throw new Error("outbound-gate: confusables.json unreadable - refusing to scan");
+  }
   const findings = [];
   const lines = String(text).split("\n");
   for (let i = 0; i < lines.length; i++) {
     // A credential written with an invisible character or a look-alike letter
     // still matches: the certain rules also run on the folded line.
-    const folded = normalizeForMatch(lines[i]);
+    const folded = NEEDS_FOLD.test(lines[i]) ? normalizeForMatch(lines[i]) : lines[i];
     for (const rule of RULES) {
       // `lastIndex` survives between calls on a /g regex and would make every
       // second line skip its start. Reset per line.