@codyswann/lisa 4.29.10 → 4.29.12

package/scripts/check-third-party-action-pins.mjs

@@ -0,0 +1,455 @@
+#!/usr/bin/env node
+/**
+ * Deterministic gate that every THIRD-PARTY GitHub Action this repository uses
+ * — in its own workflows and in the workflow templates it seeds into consumers
+ * — is referenced by a full 40-character commit SHA (issue #3585).
+ *
+ * A mutable ref is not a version. `@master`, `@main`, `@v1` and `@v0.9.0` are
+ * pointers that resolve, at job start, to whatever the upstream owner has
+ * pushed at that instant, and for a third-party action that owner is a
+ * stranger. A tag is mutable by definition — `git push --force` moves it — and
+ * a floating major is *designed* to move.
+ *
+ * The exposure is not theoretical here. When this gate was written the tree
+ * held 19 distinct third-party references, none pinned, thirteen of them in
+ * jobs holding a credential:
+ *
+ *   - `snyk/actions/node@master` with the Snyk token in the same step's `env:`;
+ *   - `fossas/fossa-action@main` handed the FOSSA key as an input;
+ *   - `noliran/branch-based-secrets@v1`, whose ENTIRE FUNCTION is to resolve
+ *     repository secrets into the job environment, in the publish path;
+ *   - `GeoWerkstatt/create-jira-release@v1`, handed a webhook secret.
+ *
+ * `quality.yml` and `release.yml` are reusable workflows the fleet calls at
+ * `@main`, so a compromised upstream would execute inside every consumer's
+ * quality gate on their next run.
+ *
+ * Pinning today does not stop the next `uses:` line, which is why the durable
+ * artifact is this detector rather than the pins. A stale comment in a
+ * consumer repository nearly caused two load-bearing references to be
+ * unpinned; prose does not hold a boundary, an executable check does.
+ *
+ * ## What is deliberately NOT in scope
+ *
+ * **First-party references.** `uses: CodySwannGT/lisa/...@main` is a different
+ * threat model — a stability question about an upstream we own and can read,
+ * tracked separately in #3488 — and a check that reddens it is the wrong check
+ * and will be turned off. First-party references never appear in this gate's
+ * findings and are never named in its output. That exemption is a required
+ * property with its own test, not an afterthought.
+ *
+ * **`actions/*` and `github/*`.** These are published by GitHub itself, the
+ * same party that runs the job, so pinning them moves no trust boundary. The
+ * decision is recorded here so it stays consistent, and the exemption is
+ * narrow: it is an OWNER allowlist, not a substring match.
+ *
+ * **Fixture trees** (`tests/fixtures/`, `parity/fixtures/`). These are inputs
+ * to other checks, never executed by GitHub Actions, and pinning them would
+ * distort the shapes those checks exist to recognise. The count of skipped
+ * fixture files is REPORTED in every run rather than silently dropped, so the
+ * boundary is visible in the output instead of buried in this comment.
+ *
+ * **Consumer repositories already seeded from the `create-only` templates.**
+ * A `create-only` file is written once and never overwritten, so correcting a
+ * template changes what FUTURE repositories are seeded with and changes
+ * nothing about any repository that already exists. This gate runs in this
+ * repository only; it cannot and does not report seeded consumers as fixed.
+ *
+ * ## Why a bare SHA is also a finding
+ *
+ * `uses: <vendor>/<action>@<40-hex>` with no trailing comment is
+ * immutable but unreadable: nobody can tell what is installed, so nobody
+ * upgrades it, so it rots at whatever commit it was pinned at. The pin and the
+ * human-readable version are one artifact, and the gate treats them that way.
+ *
+ * Determinism guarantees (so the unit test is reproducible and CI is stable):
+ *   - zero third-party dependencies (Node built-ins only),
+ *   - no network access,
+ *   - no `Date` / `Math.random`,
+ *   - the file list comes from `git ls-files`, so the gate sees exactly what a
+ *     release would carry.
+ *
+ * CLI:
+ *   node scripts/check-third-party-action-pins.mjs [--root <dir>] [--json]
+ *
+ * Exit codes (mirroring the sibling gates in this directory):
+ *   0 — every third-party reference is a 40-hex SHA carrying a version comment.
+ *   1 — >=1 reference is mutable, or pinned without a version comment.
+ *   2 — operational/usage error: unknown flag, a flag missing its value,
+ *       `--root` absent or not a git repository, git unavailable, or zero
+ *       workflow files discovered. Finding nothing to check is a broken
+ *       invocation, not conformance.
+ *
+ * @module scripts/check-third-party-action-pins
+ */
+import fs from "node:fs";
+import path from "node:path";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+
+import { boundedExecFileSync } from "./lib/bounded-spawn.mjs";
+import { invokedAsScript } from "./lib/invoked-as-script.mjs";
+
+const REPO_ROOT = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  ".."
+);
+
+/**
+ * Any workflow file, at any depth: this repository's own `.github/workflows/`
+ * and every seeded template tree, so a mutable ref cannot be introduced by
+ * seeding rather than by running.
+ */
+const WORKFLOW_RE = /(^|\/)\.github\/workflows\/[^/]+\.ya?ml$/;
+
+/**
+ * Fixture trees: inputs to other checks, never executed by GitHub Actions.
+ * Prefix match on the repo-relative path, so a real workflow cannot hide
+ * behind a directory that merely contains the word "fixtures".
+ */
+const FIXTURE_PREFIXES = ["tests/fixtures/", "parity/fixtures/"];
+
+/**
+ * The owner of this repository's own reusable workflows. Compared
+ * case-insensitively because GitHub owners are case-insensitive and a `uses:`
+ * line may spell one either way.
+ */
+const FIRST_PARTY_OWNERS = ["codyswanngt"];
+
+/** GitHub's own action namespaces — the runner's publisher, see header. */
+const GITHUB_OWNED_OWNERS = ["actions", "github"];
+
+/** Classification of an action's owner against the two documented exemptions. */
+const FIRST_PARTY = "first-party";
+const GITHUB_OWNED = "github-owned";
+const THIRD_PARTY = "third-party";
+
+/** Verdicts `evaluateReference` returns; `MUTABLE` is the headline finding. */
+const OK = "ok";
+const MUTABLE = "mutable-ref";
+const NO_VERSION = "missing-version-comment";
+
+/**
+ * `uses: <owner>/<repo>[/<subpath>]@<ref>` with anything after it captured, so
+ * a trailing `# v1.2.3` can be read. Local (`./path`) and container
+ * (`docker://`) references have no owner and never match.
+ */
+const USES_RE =
+  /^\s*(?:-\s*)?uses:\s*(?<owner>[A-Za-z0-9][A-Za-z0-9-]*)\/(?<rest>[A-Za-z0-9_.][A-Za-z0-9_./-]*)@(?<ref>[^\s"'#]+)(?<trailer>.*)$/;
+
+/** A full commit SHA. Abbreviated SHAs are ambiguous and do not count. */
+const FULL_SHA_RE = /^[0-9a-f]{40}$/;
+
+/** A trailing `# <version>` comment with at least one non-space character. */
+const VERSION_COMMENT_RE = /#\s*(?<version>\S.*?)\s*$/;
+
+/** Max bytes of `git ls-files` output (7k+ tracked paths is ~0.3 MB today). */
+const MAX_GIT_OUTPUT_BYTES = 64 * 1024 * 1024;
+
+/**
+ * Usage error — thrown for an invalid invocation or an unusable state so
+ * `main` can distinguish it (exit 2) from a finding (exit 1).
+ */
+export class UsageError extends Error {}
+
+/**
+ * Classify an action owner against the two documented exemptions.
+ *
+ * @param {string} owner - the owner segment of a `uses:` reference.
+ * @returns {"first-party" | "github-owned" | "third-party"} the classification.
+ */
+export function classifyOwner(owner) {
+  const normalized = String(owner).toLowerCase();
+  if (FIRST_PARTY_OWNERS.includes(normalized)) return FIRST_PARTY;
+  if (GITHUB_OWNED_OWNERS.includes(normalized)) return GITHUB_OWNED;
+  return THIRD_PARTY;
+}
+
+/**
+ * Extract every action reference in `content`, already classified.
+ *
+ * Every reference is returned, first-party ones included, so the caller can
+ * count what it examined. Only third-party ones are ever eligible to become a
+ * finding — see `evaluateReference`.
+ *
+ * @param {string} content - full text of a workflow file.
+ * @returns {{ action: string, owner: string, ref: string, version: string | null, kind: string, line: number }[]}
+ *   one entry per reference, with 1-based line numbers, in file order.
+ */
+export function findActionRefs(content) {
+  const lines = String(content).split(/\r?\n/);
+  const refs = [];
+  for (let index = 0; index < lines.length; index++) {
+    const match = USES_RE.exec(lines[index]);
+    if (match === null) continue;
+    const { owner, ref, rest, trailer } = match.groups;
+    const comment = VERSION_COMMENT_RE.exec(trailer);
+    refs.push({
+      action: `${owner}/${rest}`,
+      kind: classifyOwner(owner),
+      line: index + 1,
+      owner,
+      ref,
+      version: comment === null ? null : comment.groups.version,
+    });
+  }
+  return refs;
+}
+
+/**
+ * Decide whether one reference is a finding.
+ *
+ * @param {{ kind: string, ref: string, version: string | null }} reference -
+ *   a reference from `findActionRefs`.
+ * @returns {"ok" | "mutable-ref" | "missing-version-comment"} the verdict.
+ */
+export function evaluateReference(reference) {
+  if (reference.kind !== THIRD_PARTY) return OK;
+  if (!FULL_SHA_RE.test(reference.ref)) return MUTABLE;
+  return reference.version === null ? NO_VERSION : OK;
+}
+
+/**
+ * Whether a repo-relative path is a workflow this gate governs.
+ *
+ * @param {string} file - repo-relative path.
+ * @returns {boolean} whether the file is an in-scope workflow.
+ */
+export function isGovernedWorkflow(file) {
+  return WORKFLOW_RE.test(file) && !isFixtureWorkflow(file);
+}
+
+/**
+ * Whether a repo-relative path is an excluded fixture workflow.
+ *
+ * @param {string} file - repo-relative path.
+ * @returns {boolean} whether the file sits in an excluded fixture tree.
+ */
+export function isFixtureWorkflow(file) {
+  return (
+    WORKFLOW_RE.test(file) &&
+    FIXTURE_PREFIXES.some(prefix => file.startsWith(prefix))
+  );
+}
+
+/**
+ * List every tracked file in `root`, relative to it. Throws `UsageError` when
+ * git is unavailable or `root` is not a repository.
+ *
+ * @param {string} root - the repository root.
+ * @returns {string[]} tracked paths, relative to `root`.
+ */
+function listTrackedFiles(root) {
+  let stdout;
+  try {
+    stdout = boundedExecFileSync("git", ["-C", root, "ls-files", "-z"], {
+      encoding: "utf8",
+      maxBuffer: MAX_GIT_OUTPUT_BYTES,
+      stdio: ["ignore", "pipe", "ignore"],
+    });
+  } catch (error) {
+    throw new UsageError(
+      `could not list tracked files in ${root}: ${error.message}`
+    );
+  }
+  return stdout.split("\0").filter(entry => entry !== "");
+}
+
+/**
+ * Assemble the machine-readable report.
+ *
+ * @param {ReadonlyArray<Record<string, unknown>>} findings - non-ok references.
+ * @param {{ root: string, workflows: number, fixturesSkipped: number, thirdParty: number, checked: number }} opts
+ *   resolved options plus scan size.
+ * @returns {Record<string, unknown>} the report object.
+ */
+export function buildReport(findings, opts) {
+  return {
+    findings,
+    root: opts.root,
+    schemaVersion: 1,
+    summary: {
+      checked: opts.checked,
+      fixturesSkipped: opts.fixturesSkipped,
+      missingVersionComment: findings.filter(
+        finding => finding.verdict === NO_VERSION
+      ).length,
+      mutable: findings.filter(finding => finding.verdict === MUTABLE).length,
+      thirdParty: opts.thirdParty,
+      workflows: opts.workflows,
+    },
+  };
+}
+
+/**
+ * The remediation text. A guard that reddens without teaching the fix gets
+ * bypassed, so the exact command that produces the SHA is in the output.
+ *
+ * @returns {string[]} lines of remediation guidance.
+ */
+function remediation() {
+  return [
+    "Pin every third-party action to a full 40-character commit SHA, with the",
+    "human-readable version in a trailing comment so the pin stays auditable:",
+    "",
+    "    - uses: <vendor>/<action>@<40-hex-sha>  # v1.2.3",
+    "",
+    "Resolve the SHA for the ref the workflow uses TODAY — pinning and",
+    "upgrading in one commit makes a regression unattributable:",
+    "",
+    "    gh api repos/<owner>/<repo>/commits/<ref> --jq .sha",
+    "",
+    "Then name the version that SHA carries:",
+    "",
+    "    gh api repos/<owner>/<repo>/tags --jq '.[] | select(.commit.sha==\"<sha>\") | .name'",
+    "",
+    "A branch head with no tag is pinned the same way; say so in the comment,",
+    "e.g. `# main (untagged, 2026-08-28)`.",
+    "",
+    "First-party workflow references and GitHub's own actions/* and github/*",
+    "are out of scope for this gate by design and are never reported here.",
+  ];
+}
+
+/**
+ * Render the human-readable report.
+ *
+ * @param {{ findings: ReadonlyArray<Record<string, unknown>>, summary: Record<string, number> }} report
+ *   the report object.
+ * @returns {string} the rendered report.
+ */
+function humanReport(report) {
+  const { summary } = report;
+  const scanned =
+    `${summary.thirdParty} third-party reference(s) across ` +
+    `${summary.workflows} workflow file(s) ` +
+    `(${summary.fixturesSkipped} fixture workflow(s) skipped)`;
+  if (report.findings.length === 0) {
+    return `✓ ${scanned} — all pinned to a full commit SHA with a version comment`;
+  }
+  const lines = report.findings.map(finding =>
+    finding.verdict === MUTABLE
+      ? `✗ ${finding.file}:${finding.line}\n    ${finding.action}@${finding.ref} — mutable ref, not a commit SHA`
+      : `✗ ${finding.file}:${finding.line}\n    ${finding.action}@${finding.ref} — pinned but no version comment`
+  );
+  return [
+    ...lines,
+    "",
+    `${report.findings.length} of ${scanned} are not safely pinned.`,
+    "",
+    ...remediation(),
+  ].join("\n");
+}
+
+/**
+ * Parse argv into resolved options. Throws `UsageError` on a bad invocation.
+ *
+ * @param {readonly string[]} argv - arguments (without node/script prefix).
+ * @returns {{ root: string, json: boolean }} options.
+ */
+export function parseArgs(argv) {
+  let root = null;
+  let json = false;
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--json") {
+      json = true;
+    } else if (arg === "--root") {
+      const next = argv[i + 1];
+      if (next === undefined || next.startsWith("--")) {
+        throw new UsageError("--root requires a value");
+      }
+      root = next;
+      i += 1;
+    } else {
+      throw new UsageError(`unknown argument: ${arg}`);
+    }
+  }
+  return { json, root: path.resolve(root ?? REPO_ROOT) };
+}
+
+/**
+ * Resolve the file list for a run, or throw `UsageError`.
+ *
+ * @param {string} root - the repository root.
+ * @returns {{ workflows: string[], fixturesSkipped: number }} the scan inputs.
+ */
+function collectWorkflows(root) {
+  if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) {
+    throw new UsageError(`--root is not a directory: ${root}`);
+  }
+  const tracked = listTrackedFiles(root);
+  const workflows = tracked.filter(file => isGovernedWorkflow(file));
+  if (workflows.length === 0) {
+    // Finding nothing to check is a broken invocation, not conformance.
+    throw new UsageError(
+      `no workflow files found under ${root} — expected paths like ` +
+        ".github/workflows/*.yml. Refusing to report a clean run for a scan " +
+        "that examined nothing."
+    );
+  }
+  return {
+    fixturesSkipped: tracked.filter(file => isFixtureWorkflow(file)).length,
+    workflows,
+  };
+}
+
+/**
+ * Run the gate. Returns the process exit code (does not call `exit`).
+ *
+ * @param {readonly string[]} argv - arguments (without node/script prefix).
+ * @param {{ stdout?: { write(s: string): void }, stderr?: { write(s: string): void } }} [io]
+ *   injectable streams (defaults to process streams).
+ * @returns {number} exit code (0 clean, 1 finding, 2 usage error).
+ */
+export function main(argv, io = {}) {
+  const out = io.stdout ?? process.stdout;
+  const err = io.stderr ?? process.stderr;
+  let opts;
+  let scan;
+  try {
+    opts = parseArgs(argv);
+    scan = collectWorkflows(opts.root);
+  } catch (error) {
+    err.write(`error: ${error.message}\n`);
+    return 2;
+  }
+
+  let checked = 0;
+  let thirdParty = 0;
+  const findings = [];
+  try {
+    for (const file of scan.workflows) {
+      const content = fs.readFileSync(path.join(opts.root, file), "utf8");
+      for (const reference of findActionRefs(content)) {
+        checked += 1;
+        if (reference.kind === THIRD_PARTY) thirdParty += 1;
+        const verdict = evaluateReference(reference);
+        if (verdict !== OK) {
+          findings.push({ ...reference, file, verdict });
+        }
+      }
+    }
+  } catch (error) {
+    err.write(`error: failed to scan workflows: ${error.message}\n`);
+    return 2;
+  }
+
+  const report = buildReport(findings, {
+    checked,
+    fixturesSkipped: scan.fixturesSkipped,
+    root: opts.root,
+    thirdParty,
+    workflows: scan.workflows.length,
+  });
+  out.write(
+    `${opts.json ? JSON.stringify(report, null, 2) : humanReport(report)}\n`
+  );
+  return report.findings.length === 0 ? 0 : 1;
+}
+
+if (invokedAsScript(import.meta.url)) {
+  // exitCode (not process.exit): when stdout is a pipe, writes are async and
+  // process.exit() truncates the report mid-flush.
+  process.exitCode = main(process.argv.slice(2));
+}

package/scripts/two-channel-couplings.json

@@ -7,7 +7,7 @@
     "workflows": 24,
     "steps": 680,
     "couplings": 33,
-    "inventory": 67
+    "inventory": 68
   },
   "counts": {
     "package-backed": 10,

package/typescript/copy-overwrite/.prettierignore

@@ -72,6 +72,7 @@ scripts/check-nightly-e2e-health.mjs
 scripts/check-release-package-identity.mjs
 scripts/check-skipped-required-checks.mjs
 scripts/check-state-classification.mjs
+scripts/check-third-party-review-evidence.mjs
 scripts/check-threshold-ratchet.mjs
 scripts/check-verification-coverage.mjs
 scripts/classify-maestro-failures.mjs

package/typescript/create-only/.github/workflows/third-party-review-evidence.yml

@@ -0,0 +1,114 @@
+# Seeded by Lisa on first setup — this file is YOURS.
+# Lisa will not overwrite it. (copy-overwrite assets ARE replaced each run.)
+# -----------------------------------------------------------------------------
+# Third-party review evidence — the RESPONSE arm (#3591)
+# -----------------------------------------------------------------------------
+# `review-evidence.yml` answers "can the gate SEE a review that never happened?"
+# This one answers "and then what?".
+#
+# A third-party review app posts a COMMIT STATUS on the pull request head. When
+# it is throttled or declines, it still reports `state: success` — it never
+# blocks. Only the `description` separates a review that ran from one that never
+# started. Measured over the last 30 merged pull requests in the repository this
+# was written against: 24 of 30 merges passed a required review gate that
+# reviewed nothing, every one of them `success`.
+#
+# WHY `on: status` AND NOT A POLLING LOOP. The verdict changes the moment the
+# reviewer posts, and only then. A schedule that polled for it would burn
+# billable minutes on every repository, every interval, to learn nothing on
+# almost every run. This wakes on the event that carries the answer, does one
+# API read, decides, and exits.
+#
+# WHY THE JOB DOES NOT CREATE A STATUS. Creating one would be a `status` event,
+# and this workflow triggers on `status`. Nothing here writes a status; the
+# verdict is the run's own exit code.
+#
+# WHAT MAKES IT RED. Neither real third-party review evidence at the head, nor a
+# posted adversarial-review substitute for that same head. A merge path that
+# substitutes a local review leaves a marked comment behind, and this reads it.
+#
+# THREE NO-OPS THAT SAY SO. No reviewer configured, a commit that is nobody's
+# pull request head, and a status from an app nobody declared are all no-ops —
+# and each PRINTS that it is one. A silent pass is the defect this exists to
+# end, so it is not reproduced here.
+#
+# AND ONE DEFERRAL. A declared reviewer still `pending` has not answered yet:
+# measured, a reviewer posts `Review queued` then `Review in progress` before
+# the verdict, up to seven statuses on one head. Judging the first would report
+# a finding on 100% of pull requests, and a guard that fires every time gets
+# deleted rather than read. The settled status fires its own event.
+#
+# THE TWO ARMS COVER DIFFERENT HALVES. This one wakes on a status and catches a
+# reviewer that ANSWERED HOLLOW. `review-evidence.yml` runs on the pull-request
+# event and catches a reviewer that NEVER ANSWERED, which produces no status and
+# therefore no event here. Neither replaces the other.
+#
+# NOTHING RE-REQUESTS A REVIEW. Re-requesting OVERWRITES the existing status
+# rather than adding to it, so under a throttle it destroys a real review,
+# one-way. Do not add one.
+
+name: 🕵️ Third-Party Review Evidence
+
+on:
+  status:
+  workflow_dispatch:
+    inputs:
+      sha:
+        description: 'Commit SHA to inspect'
+        required: true
+        type: string
+
+# `statuses: read` is the only route that carries a description at all;
+# `pull-requests: read` finds the head pull request and its comments.
+permissions:
+  contents: read
+  statuses: read
+  pull-requests: read
+
+concurrency:
+  group: third-party-review-evidence-${{ github.event.sha || inputs.sha }}
+  cancel-in-progress: true
+
+jobs:
+  evidence:
+    name: 🕵️ Was this head reviewed, or reviewed for?
+    # A pending status is the reviewer saying it has started, which is not yet
+    # an answer. Judging it would report a finding on every pull request, and a
+    # guard that fires every time gets deleted rather than read.
+    if: github.event_name == 'workflow_dispatch' || github.event.state != 'pending'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+      - name: 📥 Checkout repository
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.sha || inputs.sha }}
+          persist-credentials: false
+
+      - name: 🔧 Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          package-manager-cache: false
+          node-version: ${{ vars.NODE_VERSION || '22.21.1' }}
+
+      - name: 🕵️ Prove review evidence, or a posted substitute, at this head
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          EVIDENCE_SHA: ${{ github.event.sha || inputs.sha }}
+          # Empty on the manual path, where nothing woke the run and every
+          # declared reviewer is evaluated.
+          EVIDENCE_CONTEXT: ${{ github.event.context }}
+        run: |
+          set -eu
+          script=scripts/check-third-party-review-evidence.mjs
+          if [ ! -f "$script" ]; then
+            echo "::error title=Review-evidence prover missing::$script does not exist, so no review evidence was read for this commit. An absent prover is a FAILURE, never a skip — a job that passes because it found nothing to run with is the same false green this guard exists to refuse."
+            exit 1
+          fi
+          if [ -n "${EVIDENCE_CONTEXT:-}" ]; then
+            node "$script" --sha="$EVIDENCE_SHA" --context="$EVIDENCE_CONTEXT"
+          else
+            node "$script" --sha="$EVIDENCE_SHA"
+          fi