@codyswann/lisa 4.17.4 → 4.17.6

package/scripts/lib/shell-guard-trace.mjs

@@ -0,0 +1,256 @@
+/**
+ * shell-guard-trace — record which shipped `.sh` guards a test run actually
+ * EXECUTES, and with what exit status (CodySwannGT/lisa#3190).
+ *
+ * @remarks
+ * ## Why a runtime trace rather than a scan of the tests
+ *
+ * The question this module answers is "which guard did the suite run", and
+ * every static spelling of that question has been tried here and is wrong in
+ * the direction that produces silence:
+ *
+ *  - **By filename.** `foo.test.ts` next to `foo.sh` proves nothing. A test
+ *    named after a guard that never runs it is the deceptive case
+ *    CodySwannGT/lisa#3190 was filed about, and a scan keyed on names reports
+ *    it as covered.
+ *  - **By the path appearing in the test source.** Measured on this tree:
+ *    258 of 259 tracked `.sh` files are named somewhere inside a test file that
+ *    also spawns something — inventories, manifests, and parity tables mention
+ *    every shipped artifact. The signal is noise.
+ *  - **By resolving the spawn argument statically.** Precise where it works
+ *    (26 guards found) and blind exactly where this repository's suites are
+ *    most interesting: the dominant idiom READS a guard's bytes, writes them
+ *    into a fixture project under a different name, and executes THAT. The
+ *    shipped path never reaches a spawn argument at all.
+ *
+ * The trace has none of those failure modes because it observes the process
+ * table instead of arguing about it, and it handles the write-a-copy idiom by
+ * matching on **content hash**: the copy is byte-identical to the guard, so it
+ * is the guard.
+ *
+ * ## What it does not see, stated rather than hidden
+ *
+ *  - A helper reached by `source` rather than by `bash` is not a child process
+ *    and never appears here. `lisa-edit-gate.sh` is the live example — every
+ *    on-edit hook sources it. Such a helper enters the population only when a
+ *    suite executes it directly, which is how CodySwannGT/lisa#3190's roster
+ *    covers it.
+ *  - Asynchronous child starts are not patched. Every child start in this
+ *    test tree is synchronous by rule — see `tests/helpers/unbounded-spawn-scan`
+ *    — so patching the three synchronous entry points covers the population,
+ *    and a suite that started a guard asynchronously would be a rule violation
+ *    caught by that scan first.
+ *  - A suite that mocks `node:child_process` wholesale never reaches this
+ *    module, and nothing real ran, so there is nothing to record.
+ *
+ * ## Loaded with `--import`, not as a vitest setup file
+ *
+ * `--import` runs before any user module links, so the patch lands on the CJS
+ * `child_process` exports that node's ESM facade for the builtin is later
+ * generated from. Measured: a vitest case using `import { spawnSync } from
+ * "node:child_process"` is intercepted. A vitest `setupFiles` entry runs after
+ * linking and would miss every already-bound import, and it would also collide
+ * with the suites that `vi.mock` the module.
+ *
+ * It NEVER throws and never changes a return value: a tracer that can break the
+ * suite it observes would be turned off, and then this whole control is prose.
+ *
+ * @module scripts/lib/shell-guard-trace
+ */
+import { appendFileSync, readFileSync, statSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { createRequire } from "node:module";
+import path from "node:path";
+import process from "node:process";
+
+/** Largest file the tracer will hash, in bytes. */
+const MAX_HASHED_BYTES = 2_000_000;
+
+/** Longest argv element the tracer will treat as a possible path. */
+const MAX_PATH_LENGTH = 4096;
+
+const traceFile = process.env["LISA_SHELL_GUARD_TRACE"];
+const indexFile = process.env["LISA_SHELL_GUARD_INDEX"];
+
+if (traceFile && indexFile) install(traceFile, indexFile);
+
+/**
+ * Patch the synchronous child starts so every guard execution is recorded.
+ * @param {string} trace - Path of the JSONL trace to append to.
+ * @param {string} index - Path of the guard index JSON to read.
+ * @returns {void}
+ */
+function install(trace, index) {
+  /** @type {{ byHash: Record<string, string[]>, sizes: number[] }} */
+  let inventory;
+  try {
+    inventory = JSON.parse(readFileSync(index, "utf8"));
+  } catch {
+    return;
+  }
+  const sizes = new Set(inventory.sizes);
+  const seen = new Set();
+
+  /**
+   * Append one observation, at most once per guard/status pair per process.
+   * @param {string} script - Repository-relative guard path.
+   * @param {number | null} status - The guard's exit status.
+   * @param {string} origin - Test frame that started it.
+   * @returns {void}
+   */
+  const record = (script, status, origin) => {
+    const key = `${script}:${String(status)}`;
+    if (seen.has(key)) return;
+    seen.add(key);
+    try {
+      appendFileSync(trace, `${JSON.stringify({ script, status, origin })}\n`);
+    } catch {
+      /* A trace that cannot be written must not fail the suite. */
+    }
+  };
+
+  /**
+   * Whether one child start RUNS its shell operand rather than inspecting it.
+   *
+   * This distinction is load-bearing and was found by measurement, not by
+   * reasoning. Without it, `bash -n script.sh` — a syntax check that never
+   * executes a line — reported fourteen `scripts/*.sh` as "driven, only ever
+   * exit 0", which is a false accusation of exactly the defect this control
+   * looks for. `shellcheck`, `cat` and `cp` name a guard in argv for the same
+   * reason and run none of it.
+   *
+   * So the operand counts only under an interpreter that will execute it, with
+   * POSIX `-n` (noexec) absent.
+   * @param {string} command - The command the child start was given.
+   * @param {string[]} argv - Its argument vector.
+   * @returns {boolean} Whether the operand is being executed.
+   */
+  const executes = (command, argv) => {
+    const runner = path.basename(command.split(/\s+/u)[0] ?? "");
+    if (!["bash", "sh", "dash", "ksh", "zsh", "env"].includes(runner)) {
+      return false;
+    }
+    return !argv.includes("-n");
+  };
+
+  /**
+   * Every tracked guard whose bytes appear in one child start's argv.
+   * @param {unknown} command - First argument of the child start.
+   * @param {unknown} args - Second argument: an argv array, or the options.
+   * @param {unknown} third - Third argument, when the second was an argv array.
+   * @returns {string[]} Repository-relative guard paths.
+   */
+  const matched = (command, args, third) => {
+    if (typeof command !== "string") return [];
+    const argv = Array.isArray(args)
+      ? args.filter(entry => typeof entry === "string")
+      : [];
+    // A guard invoked as the command itself is executing itself; anything else
+    // is executing only when the interpreter says so.
+    const tokens = executes(command, argv)
+      ? [...command.split(/\s+/u), ...argv]
+      : [command];
+    const options = Array.isArray(args) ? third : args;
+    const cwd =
+      options !== null &&
+      typeof options === "object" &&
+      typeof (/** @type {{cwd?: unknown}} */ (options).cwd) === "string"
+        ? /** @type {{cwd: string}} */ (options).cwd
+        : process.cwd();
+    const hits = new Set();
+    for (const token of tokens) {
+      if (!token || token.length > MAX_PATH_LENGTH) continue;
+      const absolute = path.isAbsolute(token)
+        ? token
+        : path.resolve(cwd, token);
+      let size = -1;
+      try {
+        const stats = statSync(absolute);
+        if (!stats.isFile()) continue;
+        size = stats.size;
+      } catch {
+        continue;
+      }
+      // Hash only what could be a guard. Every tracked guard's size is known,
+      // so a `git`, a `node`, or a fixture source is rejected by an integer
+      // comparison rather than by reading it.
+      if (!sizes.has(size) || size > MAX_HASHED_BYTES) continue;
+      try {
+        const digest = createHash("sha256")
+          .update(readFileSync(absolute))
+          .digest("hex");
+        for (const guard of inventory.byHash[digest] ?? []) hits.add(guard);
+      } catch {
+        continue;
+      }
+    }
+    return [...hits];
+  };
+
+  /**
+   * The first stack frame inside the test tree, for the failure report.
+   * @returns {string} A `file:line` frame, or an empty string.
+   */
+  const origin = () => {
+    const frames = (new Error("trace").stack ?? "")
+      .split("\n")
+      .filter(line => line.includes("/tests/"));
+    // The suite, not the helper it reached through. Every bounded spawn in this
+    // tree goes through `tests/helpers/io-latency-budget`, so naming the first
+    // `tests/` frame would name that helper for nearly every observation and
+    // the report would say nothing about where to look.
+    const frame = frames.find(line => line.includes(".test.ts")) ?? frames[0];
+    return (frame ?? "").trim().replace(/^at\s+/u, "");
+  };
+
+  const require = createRequire(import.meta.url);
+  const childProcess = require("node:child_process");
+
+  const originalSpawnSync = childProcess.spawnSync;
+  childProcess.spawnSync = function tracedSpawnSync(command, args, options) {
+    const result = originalSpawnSync.call(this, command, args, options);
+    try {
+      const guards = matched(command, args, options);
+      if (guards.length > 0) {
+        const where = origin();
+        for (const guard of guards) record(guard, result.status, where);
+      }
+    } catch {
+      /* Never let observation change the observed. */
+    }
+    return result;
+  };
+
+  for (const name of ["execFileSync", "execSync"]) {
+    const original = childProcess[name];
+    childProcess[name] = function tracedExec(command, args, options) {
+      /**
+       * Record one outcome of the wrapped call.
+       * @param {number | null} status - Its exit status.
+       * @returns {void}
+       */
+      const observe = status => {
+        try {
+          const guards = matched(command, args, options);
+          if (guards.length === 0) return;
+          const where = origin();
+          for (const guard of guards) record(guard, status, where);
+        } catch {
+          /* Never let observation change the observed. */
+        }
+      };
+      try {
+        const result = original.call(this, command, args, options);
+        observe(0);
+        return result;
+      } catch (error) {
+        observe(
+          typeof (/** @type {{status?: unknown}} */ (error).status) === "number"
+            ? /** @type {{status: number}} */ (error).status
+            : null
+        );
+        throw error;
+      }
+    };
+  }
+}

package/typescript/copy-overwrite/scripts/check-skipped-required-checks.mjs

@@ -167,6 +167,16 @@
  *     `no_work` vocabulary, so evaluating on the `pull_request` event without
  *     waiting manufactures a finding on EVERY pull request — and a guard that
  *     fires every time gets deleted rather than read.
+ *
+ *     RE-MEASURED (CodySwannGT/lisa#3221) across the last 40 merged pull
+ *     requests: EVERY one carried multiple `CodeRabbit` statuses on the same
+ *     head SHA — the sequence `Review queued` -> `Review in progress` -> the
+ *     verdict, and SEVEN statuses on one of them, because a re-request restarts
+ *     the sequence on the same commit. Only the LAST settled status is the
+ *     verdict; reading any earlier one reports a hollow review on a pull
+ *     request that may have been genuinely reviewed. Anyone tempted to drop
+ *     `--settle-timeout`/`--settle-interval` for a single immediate read is
+ *     re-introducing exactly that.
  *  3. **It REFUSES rather than reporting all-clear from an empty inspection.**
  *     An unresolvable pull request, a `gh` that could not be read, a roster
  *     with no checks in it, or a declaration naming no evidence-bearing check
@@ -992,13 +1002,17 @@ export function classifyCheckDescription(description, vocabulary = {}) {
  * @param {object} declaration - The per-repo declaration
  * @param {ReadonlyArray<{name: string, state: string, bucket?: string, description?: string}>} checks -
  *   Checks as `gh pr checks --json name,state,bucket,description` returns them
- * @param {{trustRequiredContexts?: boolean}} [options] - Set
- *   `trustRequiredContexts: false` to stop asserting whether a check is required
+ * @param {{trustRequiredContexts?: boolean, headSha?: string}} [options] - Set
+ *   `trustRequiredContexts: false` to stop asserting whether a check is required;
+ *   `headSha` is the commit the checks were read at, cited in every finding
  * @returns {{violations: object[], checked: number}} Violations and how many declared checks were examined
  */
 export function evaluateVacuousChecks(declaration, checks, options = {}) {
   const declared = declaration.evidence_bearing_checks ?? {};
   const trustRequired = options.trustRequiredContexts !== false;
+  // Every finding names the commit it was read at. A description is a property
+  // of a SHA, not of a pull request — see {@link resolveHeadSha}.
+  const at = citeHeadSha(options.headSha);
   const required = new Set(declaration.required_contexts ?? []);
   const violations = [];
   let checked = 0;
@@ -1012,7 +1026,7 @@ export function evaluateVacuousChecks(declaration, checks, options = {}) {
       violations.push({
         kind: VIOLATIONS.unproven,
         token: name,
-        message: `\`${name}\` is declared evidence-bearing but did not report on this pull request at all. A report of "no unresolved review threads" from this PR means NOBODY LOOKED, not that nothing was wrong — say which one you observed. (If the context was renamed, fix \`evidence_bearing_checks\`; names are compared byte for byte.)`,
+        message: `\`${name}\` is declared evidence-bearing but did not report on this pull request at all.${at} A report of "no unresolved review threads" from this PR means NOBODY LOOKED, not that nothing was wrong — say which one you observed. (If the context was renamed, fix \`evidence_bearing_checks\`; names are compared byte for byte.)`,
       });
       continue;
     }
@@ -1037,12 +1051,12 @@ export function evaluateVacuousChecks(declaration, checks, options = {}) {
             kind: VIOLATIONS.vacuous,
             token: name,
             contexts: [name],
-            message: `\`${name}\` reported ${state} with the description ${JSON.stringify(found.description ?? "")}, which says it DID NO WORK.${requiredNote} \`gh pr checks\` prints \`pass\` for this exactly as it does for a real review — the description is the only thing that tells them apart. Treat this PR as UNREVIEWED.`,
+            message: `\`${name}\` reported ${state} with the description ${JSON.stringify(found.description ?? "")}, which says it DID NO WORK.${at}${requiredNote} \`gh pr checks\` prints \`pass\` for this exactly as it does for a real review — the description is the only thing that tells them apart. Treat this PR as UNREVIEWED.`,
           }
         : {
             kind: VIOLATIONS.unproven,
             token: name,
-            message: `\`${name}\` reported ${state} with the description ${JSON.stringify(found.description ?? "")}, which proves neither that it reviewed anything nor that it did not.${requiredNote} Read the check itself before treating this PR as reviewed, or add the phrase to \`evidence_bearing_checks.${name}.proof\` once you have confirmed what it means.`,
+            message: `\`${name}\` reported ${state} with the description ${JSON.stringify(found.description ?? "")}, which proves neither that it reviewed anything nor that it did not.${at}${requiredNote} Read the check itself before treating this PR as reviewed, or add the phrase to \`evidence_bearing_checks.${name}.proof\` once you have confirmed what it means.`,
           }
     );
   }
@@ -1126,6 +1140,67 @@ export function resolveRepoSlug(repo, env = process.env) {
   }
 }
 
+/**
+ * The commit this pull request's checks were read at.
+ *
+ * Named in every finding, and that is not decoration. A status DESCRIPTION is
+ * not stable within a pull request: measured on CodySwannGT/lisa#3221, a head
+ * commit carried `Review rate limited` while a commit pushed to the same branch
+ * moments later carried `Review skipped: manual review required`, and whichever
+ * one is head at merge time is the one branch protection recorded. Two readers
+ * quoting the same pull request confidently disagreed; neither had misread
+ * anything, and both had omitted the only fact that reconciles them.
+ *
+ * The PR HEAD is the right commit to read, and that was measured rather than
+ * assumed: across the last 40 merged pull requests here, the MERGE COMMIT
+ * carried no `CodeRabbit` status at all, 40 times out of 40. A guard keyed on
+ * the merge commit would have read "absent" on every one of them.
+ *
+ * Returns undefined rather than throwing. Failing to name the commit must
+ * degrade the finding's provenance, never suppress the finding — an unnamed SHA
+ * is reported as unnamed, which is the honest shape.
+ *
+ * @param {string|number} pr - Pull request number
+ * @param {string} [repo] - `OWNER/NAME`, or undefined to resolve it
+ * @returns {string|undefined} The head commit SHA, or undefined when unresolvable
+ */
+export function resolveHeadSha(pr, repo) {
+  const slug = resolveRepoSlug(repo);
+  if (slug === undefined) return undefined;
+  try {
+    const raw = boundedExecFileSync(
+      "gh",
+      [
+        "pr",
+        "view",
+        String(pr),
+        "--repo",
+        slug,
+        "--json",
+        "headRefOid",
+        "--jq",
+        ".headRefOid",
+      ],
+      { encoding: "utf8" }
+    ).trim();
+    return raw === "" ? undefined : raw;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * How a finding cites the commit its evidence was read at.
+ *
+ * @param {string|undefined} headSha - The resolved head SHA, if any
+ * @returns {string} A sentence naming the commit, or naming its absence
+ */
+export function citeHeadSha(headSha) {
+  return headSha === undefined
+    ? " The head commit could NOT be resolved for this run, so this finding cannot be pinned to a SHA — treat it as unlocated evidence and re-read the pull request before quoting it."
+    : ` Read at head commit ${headSha}.`;
+}
+
 /**
  * Reads one `gh api` call that yields a JSON array, or throws with context.
  *
@@ -1544,9 +1619,9 @@ function readSecondsFlag(argv, name, fallback) {
  *
  * @param {ReadonlyArray<string>} argv - CLI arguments
  * @param {object} declaration - The per-repo declaration
- * @param {{trustRequiredContexts?: boolean, env?: NodeJS.ProcessEnv, fetch?: Function, probeBranch?: Function, now?: Function, sleep?: Function}} [options] -
+ * @param {{trustRequiredContexts?: boolean, env?: NodeJS.ProcessEnv, fetch?: Function, probeBranch?: Function, now?: Function, sleep?: Function, headSha?: Function}} [options] -
  *   Injection seams for the suite
- * @returns {{pr: string|undefined, prSource: string|null, checked: number, violations: object[], settled: boolean, refusal: {kind: string, reason: string}|null}|undefined} The inspection
+ * @returns {{pr: string|undefined, prSource: string|null, headSha: string|undefined, checked: number, violations: object[], settled: boolean, refusal: {kind: string, reason: string}|null}|undefined} The inspection
  */
 export function inspectVacuity(argv, declaration, options = {}) {
   const wired = argv.includes("--vacuity");
@@ -1594,12 +1669,18 @@ export function inspectVacuity(argv, declaration, options = {}) {
   if (refusal !== null) {
     return { ...empty, settled: read.settled, refusal };
   }
+  // Resolved here rather than inside a fetch path, so the commit is named
+  // whichever route read the checks — `gh pr checks` does not return it, and
+  // only the permission-light fallback was resolving it at all.
+  const headSha = (options.headSha ?? resolveHeadSha)(pr, repo);
   const evaluated = evaluateVacuousChecks(declaration, read.checks, {
     trustRequiredContexts: options.trustRequiredContexts,
+    headSha,
   });
   return {
     pr,
     prSource: source,
+    headSha,
     checked: evaluated.checked,
     violations: evaluated.violations,
     settled: read.settled,