@codyswann/lisa 4.17.5 → 4.17.6

package/scripts/check-shell-guard-refusal-coverage.mjs

@@ -0,0 +1,233 @@
+#!/usr/bin/env node
+/**
+ * check-shell-guard-refusal-coverage — refuse a shell guard that tests EXECUTE
+ * but never drive onto a refusal path (CodySwannGT/lisa#3190).
+ *
+ * ## What it does
+ *
+ * Runs the suites that can execute a shipped `.sh` guard with
+ * {@link module:scripts/lib/shell-guard-trace} loaded, so every guard execution
+ * and its exit status is recorded, then judges the trace with
+ * {@link module:scripts/lib/shell-guard-refusal-coverage}. A guard the run
+ * observed only ever exiting 0 is reported by name.
+ *
+ * The population is what RAN. That is the whole point: the roster this check
+ * governs cannot be written down, because a written roster is what went stale
+ * in CodySwannGT/lisa#2847 and what reads as covered in CodySwannGT/lisa#3190.
+ * A new guard with a new driving test joins the population the moment its
+ * suite runs it, with nothing edited here.
+ *
+ * ## Why it fails at zero observed
+ *
+ * A run that observed nothing and a clean tree would otherwise print the same
+ * tick. A missing tracer, an unwritable trace file, a candidate filter that
+ * matched no suite, and a vitest invocation that never started all reach that
+ * branch — and every one of them is a control reporting success while inert,
+ * which is the family this whole campaign has been closing. Zero observed is
+ * exit 2, never exit 0.
+ *
+ * CLI:
+ *   node scripts/check-shell-guard-refusal-coverage.mjs [--json] [--trace FILE]
+ *
+ *   --trace FILE   judge an existing trace instead of producing one. Use it in
+ *                  CI when the full suite already ran with the tracer imported.
+ *
+ * Exit codes (mirroring the sibling check-* scripts):
+ *   0 — every guard the run drove was observed refusing and allowing.
+ *   1 — >=1 finding.
+ *   2 — operational error: unknown flag, vitest could not run, or ZERO guards
+ *       observed.
+ *
+ * @module scripts/check-shell-guard-refusal-coverage
+ */
+import { execFileSync, spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+
+import { invokedAsScript } from "./lib/invoked-as-script.mjs";
+import {
+  formatReport,
+  guardPopulation,
+  judge,
+  parseTrace,
+  tracerIndex,
+} from "./lib/shell-guard-refusal-coverage.mjs";
+
+/** Wall-clock ceiling for the traced vitest run. */
+const RUN_TIMEOUT_MS = 45 * 60_000;
+
+/** Wall-clock ceiling for the one `git` call this script makes. */
+const GIT_TIMEOUT_MS = 60_000;
+
+/** Substrings whose presence means a module starts a child process. */
+const CHILD_STARTS = Object.freeze([
+  "spawnSync",
+  "execFileSync",
+  "execSync",
+  "boundedSpawnSync",
+  "boundedExecFileSync",
+  "boundedBash",
+]);
+
+/**
+ * Read one tracked file, tolerating a path git lists but the tree lacks.
+ * @param {string} root - Repository root.
+ * @param {string} file - Repository-relative path.
+ * @returns {string} File text, or an empty string.
+ */
+function readOrEmpty(root, file) {
+  try {
+    return readFileSync(path.join(root, file), "utf8");
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * The test files that could execute a shipped guard.
+ *
+ * A deliberate SUPERSET: a file is a candidate when its unit — itself plus the
+ * `tests/` modules it transitively imports — both names something ending in
+ * `.sh` and starts a child process. Over-inclusion costs runtime;
+ * under-inclusion costs a guard silently leaving the population, which is the
+ * failure this check exists to prevent.
+ * @param {string} root - Repository root.
+ * @returns {string[]} Repository-relative test file paths, sorted.
+ */
+export function candidateSuites(root) {
+  const tracked = execFileSync("git", ["ls-files", "-z", "tests"], {
+    cwd: root,
+    encoding: "utf8",
+    maxBuffer: 64 * 1024 * 1024,
+    timeout: GIT_TIMEOUT_MS,
+  })
+    .split("\0")
+    .filter(file => file.endsWith(".ts"));
+  const text = new Map(tracked.map(file => [file, readOrEmpty(root, file)]));
+
+  /**
+   * Every `tests/` module one file reaches, itself included.
+   * @param {string} file - Repository-relative path.
+   * @param {Set<string>} seen - Modules already walked.
+   * @returns {string[]} Repository-relative module paths.
+   */
+  const unit = (file, seen) => {
+    if (seen.has(file)) return [];
+    seen.add(file);
+    const source = text.get(file) ?? "";
+    const imports = [...source.matchAll(/from\s+"(\.[^"]+)"/gu)]
+      .map(hit =>
+        path.join(path.dirname(file), hit[1] ?? "").replace(/\.js$/u, ".ts")
+      )
+      .filter(candidate => text.has(candidate));
+    return [file, ...imports.flatMap(next => unit(next, seen))];
+  };
+
+  return tracked
+    .filter(file => file.endsWith(".test.ts"))
+    .filter(file => {
+      const sources = unit(file, new Set()).map(part => text.get(part) ?? "");
+      return (
+        sources.some(source => source.includes(".sh")) &&
+        sources.some(source => CHILD_STARTS.some(call => source.includes(call)))
+      );
+    })
+    .sort((left, right) => left.localeCompare(right));
+}
+
+/**
+ * Run the candidate suites with the tracer imported and return the raw trace.
+ * @param {string} root - Repository root.
+ * @param {ReturnType<typeof guardPopulation>} population - Tracked guards.
+ * @returns {{trace: string, suites: number, ok: boolean, detail: string}} Result.
+ */
+function produceTrace(root, population) {
+  const suites = candidateSuites(root);
+  const workspace = mkdtempSync(path.join(tmpdir(), "lisa-guard-trace-"));
+  const tracePath = path.join(workspace, "trace.jsonl");
+  const indexPath = path.join(workspace, "index.json");
+  writeFileSync(tracePath, "");
+  writeFileSync(indexPath, JSON.stringify(tracerIndex(population)));
+  const tracer = path.join(
+    path.dirname(fileURLToPath(import.meta.url)),
+    "lib",
+    "shell-guard-trace.mjs"
+  );
+  const result = spawnSync(
+    path.join(root, "node_modules", ".bin", "vitest"),
+    ["run", "--reporter=dot", ...suites],
+    {
+      cwd: root,
+      encoding: "utf8",
+      timeout: RUN_TIMEOUT_MS,
+      stdio: ["ignore", "pipe", "pipe"],
+      env: {
+        ...process.env,
+        LISA_SHELL_GUARD_TRACE: tracePath,
+        LISA_SHELL_GUARD_INDEX: indexPath,
+        NODE_OPTIONS:
+          `${process.env["NODE_OPTIONS"] ?? ""} --import=${tracer}`.trim(),
+      },
+    }
+  );
+  const detail =
+    result.signal !== null
+      ? `vitest was KILLED (${result.signal}) rather than completing`
+      : `vitest exited ${String(result.status)}`;
+  return {
+    trace: readOrEmpty(workspace, "trace.jsonl"),
+    suites: suites.length,
+    ok: result.signal === null && result.status !== null,
+    detail,
+  };
+}
+
+/**
+ * CLI entry point.
+ * @returns {void}
+ */
+export function main() {
+  const args = process.argv.slice(2);
+  const known = new Set(["--json", "--trace"]);
+  const unknown = args.find(
+    (arg, at) =>
+      arg.startsWith("--") && !known.has(arg) && args[at - 1] !== "--trace"
+  );
+  if (unknown) {
+    console.error(
+      `check:shell-guard-refusal-coverage: unknown flag ${unknown}`
+    );
+    process.exitCode = 2;
+    return;
+  }
+  const root = process.cwd();
+  const population = guardPopulation(root);
+  const at = args.indexOf("--trace");
+  const supplied = at === -1 ? undefined : args[at + 1];
+  const produced =
+    supplied === undefined
+      ? produceTrace(root, population)
+      : { trace: readOrEmpty(root, supplied), suites: 0, ok: true, detail: "" };
+  if (!produced.ok) {
+    console.error(`check:shell-guard-refusal-coverage: ${produced.detail}`);
+    process.exitCode = 2;
+    return;
+  }
+  const report = judge({ population, observed: parseTrace(produced.trace) });
+  console.log(
+    args.includes("--json")
+      ? JSON.stringify(report, (_key, value) =>
+          value instanceof Set ? [...value] : value
+        )
+      : formatReport(report)
+  );
+  if (report.driven === 0) process.exitCode = 2;
+  else if (report.findings.length > 0) process.exitCode = 1;
+}
+
+if (invokedAsScript(import.meta.url)) {
+  main();
+}

package/scripts/lib/shell-guard-refusal-coverage.mjs

@@ -0,0 +1,315 @@
+/**
+ * shell-guard-refusal-coverage — the population, the rule, and the verdict for
+ * "a shell guard that tests run must be seen REFUSING, not only allowing"
+ * (CodySwannGT/lisa#3190).
+ *
+ * @remarks
+ * ## The defect this exists to keep closed
+ *
+ * CodySwannGT/lisa#3111 established that Stryker cannot instrument shell at
+ * all, so a driving test — a payload table asserting blocked and allowed with a
+ * control on both sides — is the only bite evidence a shell guard can have.
+ * That survey produced two rosters. The one this module governs is the
+ * deceptive half: nine guards that WERE executed by passing tests where every
+ * case exercised the allow path. Replace any of them with `exit 0` and its
+ * suite still passed. A guard with no test is visibly unproven; a guard with a
+ * green allows-only suite reads as covered.
+ *
+ * CodySwannGT/lisa#3054 is why the class is not hypothetical: every verdict
+ * site in `parity-safety-net.sh` reached its answer through `printf … | grep -q
+ * …`, and a `grep` exiting 2 made the shipped hook ALLOW catastrophic deletes,
+ * silently. An allows-only suite cannot detect that by construction.
+ *
+ * ## Why this reports on observations rather than on assertions
+ *
+ * The population comes from {@link module:scripts/lib/shell-guard-trace} — what
+ * the suites actually EXECUTED — and the coverage question is asked of the exit
+ * statuses observed. This module therefore proves "the guard was seen refusing
+ * under test", which is one step weaker than "an `expect` named the refusal".
+ * Stated plainly rather than papered over: the discriminating evidence for the
+ * latter is neutering the guard to an unconditional `exit 0` and watching the
+ * named case fail, which is a thing a human or a mutation gate does, not a
+ * thing a scan can assert. What this control buys is that the *class* cannot
+ * come back silently — a new driven guard with no refusal observation is named
+ * on the next run.
+ *
+ * ## Byte-identical copies are one guard, and drift creates a new one
+ *
+ * Matching is by content hash, so the thirteen tracked copies of
+ * `lisa-edit-gate.sh` are the same program and one execution covers all of
+ * them. That is not a shortcut around the copy problem, it is the correct
+ * answer to it: the copies are proven together precisely while they are
+ * identical, and the moment one DRIFTS it hashes differently, becomes its own
+ * unobserved guard, and this check names it.
+ *
+ * ## Two rules, and why `set -e` is not one of them
+ *
+ * A guard is required to be seen refusing only if its source contains a literal
+ * non-zero `exit`. A script that merely runs under `set -e` is NOT counted:
+ * dying because a tool it called died is a crash path, not a verdict, and
+ * requiring a "refusal case" for it would turn this control into a demand for
+ * fault injection on every setup script in the tree — which is how a guard
+ * earns an exemption list, and an exemption list added to harden a guard has
+ * already become the way around one here.
+ *
+ * A guard at the agent TOOL BOUNDARY — one installed as a hook or reading a
+ * tool payload, WHOSE OWN SOURCE contains a literal `exit 2` — is held to the
+ * sharper rule: it must be seen exiting exactly 2. Both halves are needed. A
+ * `SessionStart` hook that only ever exits 1 is not making a deny decision, and
+ * demanding a 2 from it would be this check inventing a contract; a hook that
+ * DOES spell `exit 2` has declared the deny contract itself.
+ *
+ * Why the sharper rule is worth the extra clause: exit 2 is the `PreToolUse`
+ * deny contract and exit 1 is a guard that crashed, so evidence that accepts
+ * either cannot tell a working guard from a broken one — and
+ * CodySwannGT/lisa#3188 was precisely a guard that, when it could not run,
+ * permitted everything.
+ *
+ * @module scripts/lib/shell-guard-refusal-coverage
+ */
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+
+/** Wall-clock ceiling for the one `git` call this module makes. */
+const GIT_TIMEOUT_MS = 60_000;
+
+/** The exit status an agent tool-boundary guard uses to refuse. */
+export const REFUSED = 2;
+
+/**
+ * Path prefixes whose `.sh` files are not shipped guards.
+ *
+ * `tests/` holds pinned pre-change snapshots and fixture scripts, which are
+ * executed constantly and are not the subject; `dist/` is this repository's own
+ * source a second time, from its build output.
+ */
+const NOT_GUARDS = Object.freeze(["tests/", "dist/", ".husky/_/"]);
+
+/**
+ * Guards that cannot be driven to refuse hermetically, with the reason.
+ *
+ * Deliberately checked in and deliberately EMPTY. An omission recorded here is
+ * visible; an omission left out of the roster is absent, and CodySwannGT/lisa#3190
+ * is a ticket about the difference. An entry must name a guard the trace
+ * actually observed — an exclusion for a guard nothing runs is reported as
+ * stale rather than quietly kept.
+ *
+ * @type {readonly {script: string, reason: string}[]}
+ */
+export const HERMETIC_EXCLUSIONS = Object.freeze([]);
+
+/**
+ * Every tracked shell guard in a checkout, with what its source can do.
+ * @param {string} root - Repository root.
+ * @returns {{path: string, sha256: string, size: number, canRefuse: boolean, toolBoundary: boolean}[]}
+ *   One entry per tracked guard, sorted by path.
+ */
+export function guardPopulation(root) {
+  const tracked = execFileSync("git", ["ls-files", "-z", "*.sh"], {
+    cwd: root,
+    encoding: "utf8",
+    maxBuffer: 64 * 1024 * 1024,
+    timeout: GIT_TIMEOUT_MS,
+  })
+    .split("\0")
+    .filter(file => file.length > 0)
+    .filter(file => !NOT_GUARDS.some(prefix => file.startsWith(prefix)))
+    .sort((left, right) => left.localeCompare(right));
+  return tracked.flatMap(file => {
+    let bytes;
+    try {
+      bytes = readFileSync(path.join(root, file));
+    } catch {
+      return [];
+    }
+    const source = bytes.toString("utf8");
+    return [
+      {
+        path: file,
+        sha256: createHash("sha256").update(bytes).digest("hex"),
+        size: bytes.length,
+        canRefuse: /\bexit[ \t]+[1-9]\d*\b/u.test(source),
+        toolBoundary:
+          (file.includes("/hooks/") ||
+            /tool_input|"file_path"/u.test(source)) &&
+          /\bexit[ \t]+2\b/u.test(source),
+      },
+    ];
+  });
+}
+
+/**
+ * The index the tracer consults, derived from the population.
+ * @param {ReturnType<typeof guardPopulation>} population - Tracked guards.
+ * @returns {{byHash: Record<string, string[]>, sizes: number[]}} Tracer index.
+ */
+export function tracerIndex(population) {
+  /** @type {Record<string, string[]>} */
+  const byHash = {};
+  for (const guard of population) {
+    byHash[guard.sha256] = [...(byHash[guard.sha256] ?? []), guard.path];
+  }
+  return { byHash, sizes: [...new Set(population.map(g => g.size))] };
+}
+
+/**
+ * Parse a JSONL trace into the statuses observed per guard.
+ * @param {string} jsonl - Raw trace content.
+ * @returns {Map<string, {statuses: Set<number>, origins: Set<string>}>} Observations.
+ */
+export function parseTrace(jsonl) {
+  /** @type {Map<string, {statuses: Set<number>, origins: Set<string>}>} */
+  const observed = new Map();
+  for (const line of jsonl.split("\n")) {
+    if (line.trim().length === 0) continue;
+    /** @type {{script?: unknown, status?: unknown, origin?: unknown}} */
+    let record;
+    try {
+      record = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (typeof record.script !== "string") continue;
+    const entry = observed.get(record.script) ?? {
+      statuses: new Set(),
+      origins: new Set(),
+    };
+    if (typeof record.status === "number") entry.statuses.add(record.status);
+    if (typeof record.origin === "string" && record.origin.length > 0) {
+      entry.origins.add(record.origin);
+    }
+    observed.set(record.script, entry);
+  }
+  return observed;
+}
+
+/**
+ * Judge one run's observations against the population.
+ * @param {object} input - Everything the verdict depends on.
+ * @param {ReturnType<typeof guardPopulation>} input.population - Tracked guards.
+ * @param {ReturnType<typeof parseTrace>} input.observed - Trace observations.
+ * @param {readonly {script: string, reason: string}[]} [input.exclusions] - Recorded omissions.
+ * @returns {{driven: number, findings: {script: string, kind: string, detail: string}[], excluded: readonly {script: string, reason: string}[]}}
+ *   The report.
+ */
+export function judge({
+  population,
+  observed,
+  exclusions = HERMETIC_EXCLUSIONS,
+}) {
+  const excludedScripts = new Set(exclusions.map(entry => entry.script));
+  const byPath = new Map(population.map(guard => [guard.path, guard]));
+  /** @type {{script: string, kind: string, detail: string}[]} */
+  const findings = [];
+  const driven = [...observed.keys()]
+    .filter(script => byPath.has(script))
+    .sort();
+
+  for (const script of driven) {
+    const guard = /** @type {NonNullable<ReturnType<typeof byPath.get>>} */ (
+      byPath.get(script)
+    );
+    const statuses = /** @type {{statuses: Set<number>}} */ (
+      observed.get(script)
+    ).statuses;
+    const seen = [...statuses].sort((a, b) => a - b).join(", ");
+    if (!statuses.has(0)) {
+      findings.push({
+        script,
+        kind: "no-allow-control",
+        detail: `never observed allowing ordinary work (statuses seen: ${seen || "none"})`,
+      });
+    }
+    if (excludedScripts.has(script) || !guard.canRefuse) continue;
+    const refusals = [...statuses].filter(status => status !== 0);
+    if (refusals.length === 0) {
+      findings.push({
+        script,
+        kind: "no-refusal-case",
+        detail: `driven ${statuses.size} time(s), only ever exit 0 — nothing proves it can refuse`,
+      });
+      continue;
+    }
+    if (guard.toolBoundary && !statuses.has(REFUSED)) {
+      findings.push({
+        script,
+        kind: "no-refusal-case",
+        detail: `refuses only with exit ${refusals.join("/")}; the tool boundary contract is exit ${REFUSED}, and exit 1 is a guard that crashed`,
+      });
+    }
+  }
+
+  for (const entry of exclusions) {
+    if (!byPath.has(entry.script)) {
+      findings.push({
+        script: entry.script,
+        kind: "unknown-exclusion",
+        detail: "recorded as un-drivable but is not a tracked guard",
+      });
+      continue;
+    }
+    if (!observed.has(entry.script)) {
+      findings.push({
+        script: entry.script,
+        kind: "stale-exclusion",
+        detail: "recorded as un-drivable but no test drives it at all",
+      });
+      continue;
+    }
+    const statuses = /** @type {{statuses: Set<number>}} */ (
+      observed.get(entry.script)
+    ).statuses;
+    if ([...statuses].some(status => status !== 0)) {
+      findings.push({
+        script: entry.script,
+        kind: "stale-exclusion",
+        detail: "recorded as un-drivable, yet it was observed refusing",
+      });
+    }
+  }
+
+  return { driven: driven.length, findings, excluded: exclusions };
+}
+
+/**
+ * Render one report for a terminal.
+ * @param {ReturnType<typeof judge>} report - The verdict.
+ * @returns {string} Human-readable text.
+ */
+export function formatReport(report) {
+  const lines = [
+    `shell-guard refusal coverage: ${report.driven} guard(s) observed running under test`,
+  ];
+  if (report.driven === 0) {
+    lines.push(
+      "",
+      "  ✖ NOTHING was observed. No guard ran, so no guard was measured, and",
+      "    nothing passed — this is the inert-control shape the check exists to",
+      "    refuse. Check that the trace file and the tracer import both reached",
+      "    the vitest workers."
+    );
+    return lines.join("\n");
+  }
+  for (const entry of report.excluded) {
+    lines.push(`  ⚪ ${entry.script} — omitted on purpose: ${entry.reason}`);
+  }
+  if (report.findings.length === 0) {
+    lines.push("  ✔ Every driven guard was observed refusing AND allowing.");
+    return lines.join("\n");
+  }
+  for (const finding of report.findings) {
+    lines.push(`  ✖ ${finding.script} — ${finding.detail}`);
+  }
+  lines.push(
+    "",
+    "Fix: add a case that drives the guard onto a refusal path and asserts the",
+    "exit status EXACTLY — `expect(result.status).toBe(2)` for a tool-boundary",
+    "guard. `not.toBe(0)` is not enough: it cannot tell a guard that refused",
+    "from one that crashed. Keep the allow control beside it.",
+    "A guard that genuinely cannot be driven hermetically belongs in",
+    "HERMETIC_EXCLUSIONS with its reason, so the omission is visible."
+  );
+  return lines.join("\n");
+}