@codyswann/lisa 3.9.2 → 3.11.0

package/scripts/check-required-check-promotions.mjs

@@ -0,0 +1,683 @@
+#!/usr/bin/env node
+/**
+ * check-required-check-promotions — refuse a required status context whose
+ * safety nobody proved (CodySwannGT/lisa#2509).
+ *
+ * ## The rule
+ *
+ * **A check may only become a required status context if its budget has proven
+ * headroom.** "Proven" means measured against a run that actually REPRODUCED
+ * the failure the budget exists to prevent — not inferred from runs that
+ * passed. Sizing a budget from passing samples is circular: the sample already
+ * excludes the failure mode.
+ *
+ * ## Why this is a control and not a paragraph
+ *
+ * Promoting a check to required changes the cost of every marginal time budget
+ * on its path from "an agent re-runs" to "nothing merges." Nothing connected
+ * those two decisions: promotion happens in `<type>/github-rulesets/*.json` and
+ * in `.lisa.config.json`, while budgets live independently in the suites. A
+ * check can be promoted carrying a budget that loses occasionally at any load,
+ * and the first symptom is a permanently flaky merge gate, org-wide.
+ *
+ * It ships as a control rather than a rule document because that is what the
+ * measurement supports. In this repository, over one session, executable
+ * controls were obeyed 50 of 50 times and prose rules roughly 0 of 13 —
+ * including by the agents who had just written them. `.claude/rules` also
+ * explicitly excludes "prose restating a lint rule, hook, or CI gate."
+ *
+ * ## What it enforces, and which measured failure each clause came from
+ *
+ * Every declared required context must have an entry in
+ * `.github/required-check-promotions.json`. A context with no entry FAILS —
+ * that is the precondition itself. For each entry:
+ *
+ *  1. **The job must exist and be named exactly right.**
+ *     `rails/github-rulesets/quality-checks.json` names four contexts, three
+ *     non-emoji (`Quality Checks / Lint`) and one emoji
+ *     (`Quality Checks / 🔗 Work-Item Traceability`). Adding a context to that
+ *     file by symmetry with the TypeScript template produces a context no job
+ *     ever reports, which blocks every pull request in the repository forever.
+ *     So the ledger names the workflow and job id, and this guard reads the
+ *     YAML and compares the `name:` against the context.
+ *
+ *  2. **The reporting workflow may not be `paths:`-filtered.** Measured on PR
+ *     #2496: a filtered workflow does not run, so its context never reports and
+ *     GitHub shows "Expected — waiting for status to be reported" forever. Not
+ *     "runs and passes" — never reports at all.
+ *
+ *  3. **The headroom must be evidenced.** A `proven` entry must publish the
+ *     budget, the observed worst case, the machine conditions it was measured
+ *     under, and a description of the run that reproduced the failure. A
+ *     figure without its conditions is not a measurement:
+ *     `check-learnings-budget` was reported at 45.8s "in isolation" while ~56
+ *     sibling vitest processes were live, and that retracted number nearly
+ *     shipped as a permanent comment.
+ *
+ *  4. **The margin must be at least {@link MIN_HEADROOM_RATIO}x.**
+ *     `learnings-writer` failed at 10,259ms against a 10,000ms budget — 2.6%
+ *     over — then passed 5/5 on immediate re-run under UNCHANGED conditions.
+ *     If the box did not change and the result did, the budget is not measuring
+ *     the box; it is measuring nothing, because it has no headroom.
+ *     `plugin-sync-scripts` consumed 92% of its budget (1.09x) and failed 15 of
+ *     16 concurrent runs; at 60s (~2.5x) it failed 0 of 16. The floor therefore
+ *     sits inside (1.09, 2.54], and 2 is the conservative choice in that range.
+ *
+ *  5. **A budget may not be sized from a different subject.** #2490 raised five
+ *     budgets and sized `sonar-secrets`' 60s BY ANALOGY — its 16-way paired
+ *     probe ran against `plugin-sync-scripts`. Declaring `subject` and
+ *     `measured_on_subject` separately makes that mechanically visible instead
+ *     of a footnote in a report.
+ *
+ * ## The ratchet, and why incumbents are not simply exempted
+ *
+ * Contexts already required when this guard shipped may declare
+ * `"status": "grandfathered"`, which turns their problems into reported DEBT
+ * (exit 0) rather than violations. That is not an amnesty: the entry must state
+ * in `debt` exactly what is unproven, and the context must appear in the
+ * ledger's frozen `grandfathered_contexts` list. That list was fixed when the
+ * ledger was written, so a NEW promotion cannot buy its way in by claiming to
+ * be old. Reddening `main` to punish yesterday's promotions would only get the
+ * guard deleted; recording what each incumbent has not proven is the part that
+ * keeps working.
+ *
+ * ## Where it runs
+ *
+ * `tests/unit/scripts/required-check-promotions.repo.test.ts` calls
+ * {@link evaluate} against this repository, so it executes inside
+ * `🔍 Quality Checks / 🧪 Run Unit Tests` — itself a required context. An
+ * operator can also run `npm run check:required-check-promotions`.
+ *
+ * Usage:
+ *   node scripts/check-required-check-promotions.mjs [rootDir] [--json]
+ *
+ * @module scripts/check-required-check-promotions
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import yaml from "js-yaml";
+
+/** Integration id GitHub Actions reports status checks under. */
+export const ACTIONS_INTEGRATION_ID = 15_368;
+
+/** Ledger location, relative to the repository root. */
+export const LEDGER_RELATIVE_PATH = ".github/required-check-promotions.json";
+
+/**
+ * Smallest acceptable budget-to-observed-worst-case ratio.
+ *
+ * Derived, not chosen: 1.09x failed 15/16 and ~2.54x failed 0/16 on the only
+ * budget in this repository ever proven under load, so the floor lies in
+ * (1.09, 2.54]. See the module preamble.
+ */
+export const MIN_HEADROOM_RATIO = 2;
+
+/** Directory names never scanned for ruleset templates. */
+const SKIPPED_DIRECTORIES = new Set([
+  "node_modules",
+  "dist",
+  "coverage",
+  ".git",
+]);
+
+/** Separator GitHub puts between a caller job name and a called job name. */
+const CONTEXT_SEPARATOR = " / ";
+
+/** Raised for operator error (bad arguments, unreadable ledger). */
+export class UsageError extends Error {}
+
+/**
+ * Split a status context into the caller job name and the called job name.
+ *
+ * A reusable-workflow job reports as `<caller job name> / <called job name>`;
+ * a job in the calling workflow reports under its own name alone. Splitting on
+ * the FIRST separator only, because a called job name may itself contain one.
+ *
+ * @param {string} context - the status check context.
+ * @returns {{ callerName: string, calledName: string | null }} the two halves.
+ */
+export function splitContext(context) {
+  const index = context.indexOf(CONTEXT_SEPARATOR);
+  if (index === -1) return { callerName: context, calledName: null };
+  return {
+    callerName: context.slice(0, index),
+    calledName: context.slice(index + CONTEXT_SEPARATOR.length),
+  };
+}
+
+/**
+ * Read the required contexts declared by one ruleset template.
+ *
+ * @param {string} absolute - absolute path to the template JSON.
+ * @param {string} relative - path reported as the source.
+ * @returns {{ context: string, integrationId: number, source: string }[]} declarations.
+ */
+function contextsFromTemplate(absolute, relative) {
+  let parsed;
+  try {
+    parsed = JSON.parse(fs.readFileSync(absolute, "utf8"));
+  } catch {
+    return [];
+  }
+  const found = [];
+  for (const rule of parsed?.rules ?? []) {
+    if (rule?.type !== "required_status_checks") continue;
+    for (const check of rule?.parameters?.required_status_checks ?? []) {
+      if (typeof check?.context !== "string") continue;
+      found.push({
+        context: check.context,
+        integrationId:
+          typeof check.integration_id === "number"
+            ? check.integration_id
+            : ACTIONS_INTEGRATION_ID,
+        source: relative,
+      });
+    }
+  }
+  return found;
+}
+
+/**
+ * Read the per-repository `addRequiredChecks` opt-in.
+ *
+ * This surface is NOT templated and exists precisely because a Lisa-only
+ * context must never ship in a shared template — host projects would inherit a
+ * context they never report (the #2476 defect). A guard blind to it would clear
+ * every context added this way.
+ *
+ * @param {string} root - absolute repository root.
+ * @returns {{ context: string, integrationId: number, source: string }[]} declarations.
+ */
+function contextsFromConfig(root) {
+  const absolute = path.join(root, ".lisa.config.json");
+  if (!fs.existsSync(absolute)) return [];
+  let parsed;
+  try {
+    parsed = JSON.parse(fs.readFileSync(absolute, "utf8"));
+  } catch {
+    return [];
+  }
+  const added = parsed?.github?.rulesets?.addRequiredChecks ?? {};
+  const found = [];
+  for (const entries of Object.values(added)) {
+    for (const entry of Array.isArray(entries) ? entries : []) {
+      if (typeof entry?.context !== "string") continue;
+      found.push({
+        context: entry.context,
+        integrationId:
+          typeof entry.integration_id === "number"
+            ? entry.integration_id
+            : ACTIONS_INTEGRATION_ID,
+        source: ".lisa.config.json",
+      });
+    }
+  }
+  return found;
+}
+
+/**
+ * Collect every required context this repository declares, from both surfaces.
+ *
+ * @param {string} root - absolute repository root.
+ * @returns {{ context: string, integrationId: number, source: string }[]}
+ *   declarations, sorted by context then source.
+ */
+export function collectDeclaredContexts(root) {
+  const found = [];
+  for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
+    if (!entry.isDirectory() || SKIPPED_DIRECTORIES.has(entry.name)) continue;
+    const rulesetDir = path.join(root, entry.name, "github-rulesets");
+    if (!fs.existsSync(rulesetDir)) continue;
+    for (const file of fs.readdirSync(rulesetDir).sort()) {
+      if (!file.endsWith(".json")) continue;
+      found.push(
+        ...contextsFromTemplate(
+          path.join(rulesetDir, file),
+          `${entry.name}/github-rulesets/${file}`
+        )
+      );
+    }
+  }
+  found.push(...contextsFromConfig(root));
+  return found.sort(
+    (a, b) =>
+      a.context.localeCompare(b.context) || a.source.localeCompare(b.source)
+  );
+}
+
+/**
+ * Parse a workflow into the two facts a required context depends on: which
+ * jobs it declares, and whether it actually runs on every pull request.
+ *
+ * @param {string} absolute - absolute path to the workflow YAML.
+ * @returns {{ jobs: Map<string, string>, onPullRequest: boolean, pathFilterKeys: string[] } | null}
+ *   the parsed facts, or null when the file is absent or unparseable.
+ */
+export function readWorkflow(absolute) {
+  if (!fs.existsSync(absolute)) return null;
+  let doc;
+  try {
+    doc = yaml.load(fs.readFileSync(absolute, "utf8"));
+  } catch {
+    return null;
+  }
+  if (typeof doc !== "object" || doc === null) return null;
+  // `on:` is a YAML 1.1 boolean. js-yaml's default schema keeps it a string,
+  // but a schema change upstream would silently move the key to `true` and a
+  // guard reading only one spelling would then report "no pull_request
+  // trigger" for every workflow in the fleet.
+  const triggers = doc.on ?? doc[true] ?? {};
+  const pullRequest =
+    typeof triggers === "object" && triggers !== null
+      ? triggers.pull_request
+      : undefined;
+  const onPullRequest =
+    (typeof triggers === "object" &&
+      triggers !== null &&
+      "pull_request" in triggers) ||
+    triggers === "pull_request" ||
+    (Array.isArray(triggers) && triggers.includes("pull_request"));
+  const pathFilterKeys =
+    typeof pullRequest === "object" && pullRequest !== null
+      ? ["paths", "paths-ignore"].filter(key => key in pullRequest)
+      : [];
+  const jobs = new Map();
+  for (const [id, job] of Object.entries(doc.jobs ?? {})) {
+    jobs.set(id, typeof job?.name === "string" ? job.name : id);
+  }
+  return { jobs, onPullRequest, pathFilterKeys };
+}
+
+/**
+ * Validate one declared budget inside a headroom block.
+ *
+ * @param {object} budget - a `headroom.budgets[]` entry.
+ * @returns {{ rule: string, detail: string }[]} problems, empty when sound.
+ */
+function budgetProblems(budget) {
+  const subject = budget?.subject;
+  const measuredOn = budget?.measured_on_subject;
+  if (typeof subject !== "string" || typeof measuredOn !== "string") {
+    return [
+      {
+        rule: "headroom-evidence-missing",
+        detail:
+          "every headroom.budgets[] entry needs a subject and a measured_on_subject",
+      },
+    ];
+  }
+  if (subject !== measuredOn) {
+    return [
+      {
+        rule: "budget-sized-by-analogy",
+        detail: `budget for '${subject}' was measured on '${measuredOn}'; a budget must be measured on the subject that consumes it`,
+      },
+    ];
+  }
+  return ratioProblems(budget.budget_ms, budget.observed_worst_ms);
+}
+
+/**
+ * Compare an observed worst case against its budget.
+ *
+ * @param {unknown} budgetMs - the declared wall-clock budget.
+ * @param {unknown} observedMs - the worst case actually observed.
+ * @returns {{ rule: string, detail: string }[]} problems, empty when sound.
+ */
+function ratioProblems(budgetMs, observedMs) {
+  if (
+    typeof budgetMs !== "number" ||
+    typeof observedMs !== "number" ||
+    budgetMs <= 0 ||
+    observedMs <= 0
+  ) {
+    return [
+      {
+        rule: "headroom-evidence-missing",
+        detail:
+          "headroom needs a positive budget_ms and a positive observed_worst_ms",
+      },
+    ];
+  }
+  const ratio = budgetMs / observedMs;
+  if (ratio < MIN_HEADROOM_RATIO) {
+    return [
+      {
+        rule: "headroom-ratio-too-thin",
+        detail: `observed worst ${observedMs}ms against a ${budgetMs}ms budget is ${ratio.toFixed(2)}x; ${MIN_HEADROOM_RATIO}x is the floor`,
+      },
+    ];
+  }
+  return [];
+}
+
+/**
+ * Validate a headroom block.
+ *
+ * A `grandfathered` block is shape-checked by the caller instead, because its
+ * obligation is to NAME what is unproven rather than to prove it.
+ *
+ * @param {object} headroom - the `headroom` object from a ledger entry.
+ * @returns {{ rule: string, detail: string }[]} problems, empty when sound.
+ */
+export function headroomProblems(headroom) {
+  const status = headroom?.status;
+  if (status !== "proven" && status !== "grandfathered") {
+    return [
+      {
+        rule: "unknown-headroom-status",
+        detail: `headroom.status must be 'proven' or 'grandfathered', got '${String(status)}'`,
+      },
+    ];
+  }
+  if (status === "grandfathered") return [];
+  const problems = [];
+  const prose = [
+    [
+      "reproduced",
+      "headroom.reproduced must describe the run that reproduced the failure the budget prevents",
+    ],
+    [
+      "measured_on",
+      "headroom.measured_on must name what was measured, so the next person can re-measure it",
+    ],
+    [
+      "conditions",
+      "headroom.conditions must state the machine state the measurement was taken under",
+    ],
+  ];
+  for (const [field, detail] of prose) {
+    const value = headroom[field];
+    if (typeof value !== "string" || value.trim() === "") {
+      problems.push({ rule: "headroom-evidence-missing", detail });
+    }
+  }
+  if (problems.length > 0) return problems;
+  problems.push(
+    ...ratioProblems(headroom.budget_ms, headroom.observed_worst_ms)
+  );
+  if (problems.length > 0) return problems;
+  for (const budget of headroom.budgets ?? []) {
+    problems.push(...budgetProblems(budget));
+  }
+  return problems;
+}
+
+/**
+ * Verify that the workflow wiring a ledger entry declares actually reports the
+ * context it claims to.
+ *
+ * @param {object} entry - the ledger entry.
+ * @param {string} root - absolute repository root.
+ * @returns {{ rule: string, detail: string }[]} problems, empty when sound.
+ */
+export function wiringProblems(entry, root) {
+  const { callerName, calledName } = splitContext(entry.context);
+  if (typeof entry.caller_workflow !== "string") {
+    return [
+      {
+        rule: "caller-workflow-missing",
+        detail:
+          "an Actions-reported context must declare caller_workflow and caller_job",
+      },
+    ];
+  }
+  const caller = readWorkflow(path.join(root, entry.caller_workflow));
+  if (caller === null) {
+    return [
+      {
+        rule: "caller-workflow-missing",
+        detail: `caller_workflow '${entry.caller_workflow}' is absent or unparseable`,
+      },
+    ];
+  }
+  const problems = [];
+  if (!caller.onPullRequest) {
+    problems.push({
+      rule: "pull-request-trigger-missing",
+      detail: `'${entry.caller_workflow}' does not run on pull_request, so this context never reports on a pull request`,
+    });
+  }
+  if (caller.pathFilterKeys.length > 0) {
+    problems.push({
+      rule: "path-filtered-workflow",
+      detail: `'${entry.caller_workflow}' filters pull_request by ${caller.pathFilterKeys.join("/")}; a filtered workflow does not run, so a required context on it waits forever (#2496)`,
+    });
+  }
+  problems.push(
+    ...jobNameProblems(caller, entry.caller_job, callerName, "caller")
+  );
+  if (calledName !== null) {
+    problems.push(...calledJobProblems(entry, root, calledName));
+  }
+  return problems;
+}
+
+/**
+ * Compare a workflow job's display name against the half of the context it is
+ * supposed to report as.
+ *
+ * @param {{ jobs: Map<string, string> }} workflow - parsed workflow.
+ * @param {unknown} jobId - the job id the ledger declares.
+ * @param {string} expected - the context half the job must render as.
+ * @param {"caller" | "called"} side - which half, for the rule name.
+ * @returns {{ rule: string, detail: string }[]} problems, empty when sound.
+ */
+function jobNameProblems(workflow, jobId, expected, side) {
+  if (typeof jobId !== "string" || !workflow.jobs.has(jobId)) {
+    return [
+      {
+        rule: `${side}-job-missing`,
+        detail: `no job '${String(jobId)}' in the ${side} workflow; a context no job reports blocks every pull request forever`,
+      },
+    ];
+  }
+  const actual = workflow.jobs.get(jobId);
+  if (actual !== expected) {
+    return [
+      {
+        rule: `${side}-job-name-mismatch`,
+        detail: `job '${jobId}' is named '${actual}' but the context expects '${expected}'`,
+      },
+    ];
+  }
+  return [];
+}
+
+/**
+ * Verify the reusable workflow half of a `caller / called` context.
+ *
+ * @param {object} entry - the ledger entry.
+ * @param {string} root - absolute repository root.
+ * @param {string} calledName - the expected called-job display name.
+ * @returns {{ rule: string, detail: string }[]} problems, empty when sound.
+ */
+function calledJobProblems(entry, root, calledName) {
+  if (typeof entry.called_workflow !== "string") {
+    return [
+      {
+        rule: "called-workflow-missing",
+        detail: `context '${entry.context}' names a reusable-workflow job, so the entry must declare called_workflow and job_id`,
+      },
+    ];
+  }
+  const called = readWorkflow(path.join(root, entry.called_workflow));
+  if (called === null) {
+    return [
+      {
+        rule: "called-workflow-missing",
+        detail: `called_workflow '${entry.called_workflow}' is absent or unparseable`,
+      },
+    ];
+  }
+  return jobNameProblems(called, entry.job_id, calledName, "called");
+}
+
+/**
+ * Read the promotion ledger.
+ *
+ * @param {string} root - absolute repository root.
+ * @returns {object | null} the parsed ledger, or null when absent/unparseable.
+ */
+export function loadLedger(root) {
+  const absolute = path.join(root, LEDGER_RELATIVE_PATH);
+  if (!fs.existsSync(absolute)) return null;
+  try {
+    return JSON.parse(fs.readFileSync(absolute, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Check one matched ledger entry and file its problems as violations or debt.
+ *
+ * @param {object} args - entry, declaration, root, frozen set, and sinks.
+ * @returns {void}
+ */
+function evaluateEntry({ entry, integrationId, root, frozen, out }) {
+  const grandfathered = entry.headroom?.status === "grandfathered";
+  const file = (rule, detail) =>
+    (grandfathered ? out.debts : out.violations).push({
+      context: entry.context,
+      rule,
+      detail,
+    });
+  if (grandfathered) {
+    if (!frozen.has(entry.context)) {
+      out.violations.push({
+        context: entry.context,
+        rule: "grandfather-not-frozen",
+        detail:
+          "only contexts already required when the ledger was written may be grandfathered; a new promotion must prove its headroom",
+      });
+      return;
+    }
+    const debt = entry.headroom?.debt;
+    if (typeof debt !== "string" || debt.trim() === "") {
+      out.violations.push({
+        context: entry.context,
+        rule: "grandfather-missing-debt",
+        detail:
+          "a grandfathered entry must say in `debt` exactly what about its headroom is unproven",
+      });
+      return;
+    }
+    out.debts.push({
+      context: entry.context,
+      rule: "grandfathered-headroom",
+      detail: debt,
+    });
+  }
+  if (integrationId === ACTIONS_INTEGRATION_ID) {
+    for (const p of wiringProblems(entry, root)) file(p.rule, p.detail);
+  }
+  for (const p of headroomProblems(entry.headroom)) file(p.rule, p.detail);
+}
+
+/**
+ * Evaluate every required context this repository declares against the ledger.
+ *
+ * @param {string} root - absolute repository root.
+ * @returns {{ violations: {context: string|null, rule: string, detail: string}[],
+ *   debts: {context: string|null, rule: string, detail: string}[], covered: number }}
+ *   the report. A non-empty `violations` is a failed promotion precondition.
+ */
+export function evaluate(root) {
+  const out = { violations: [], debts: [], covered: 0 };
+  const ledger = loadLedger(root);
+  if (ledger === null) {
+    out.violations.push({
+      context: null,
+      rule: "ledger-missing",
+      detail: `${LEDGER_RELATIVE_PATH} is absent or unparseable; refusing to report that promotions are sound when nothing was read`,
+    });
+    return out;
+  }
+  const frozen = new Set(ledger.grandfathered_contexts ?? []);
+  const declared = new Map();
+  for (const d of collectDeclaredContexts(root)) {
+    if (!declared.has(d.context)) declared.set(d.context, d.integrationId);
+  }
+  const seen = new Set();
+  for (const entry of ledger.promotions ?? []) {
+    if (seen.has(entry.context)) {
+      out.violations.push({
+        context: entry.context,
+        rule: "duplicate-promotion-entry",
+        detail: "the ledger records this context more than once",
+      });
+      continue;
+    }
+    seen.add(entry.context);
+    if (!declared.has(entry.context)) {
+      out.violations.push({
+        context: entry.context,
+        rule: "orphan-promotion-entry",
+        detail:
+          "the ledger promotes a context no ruleset template or config declares; delete the entry or restore the declaration",
+      });
+      continue;
+    }
+    out.covered += 1;
+    evaluateEntry({
+      entry,
+      integrationId: declared.get(entry.context),
+      root,
+      frozen,
+      out,
+    });
+  }
+  for (const context of declared.keys()) {
+    if (seen.has(context)) continue;
+    out.violations.push({
+      context,
+      rule: "missing-promotion-entry",
+      detail: `'${context}' is required but has no entry in ${LEDGER_RELATIVE_PATH}; a promotion must record what proves it safe`,
+    });
+  }
+  return out;
+}
+
+/**
+ * Render a human-readable report.
+ *
+ * @param {ReturnType<typeof evaluate>} result - the evaluation.
+ * @returns {string} the report text.
+ */
+export function formatReport(result) {
+  const lines = [
+    `Required-check promotions: ${result.covered} recorded, ${result.violations.length} violation(s), ${result.debts.length} outstanding debt(s).`,
+  ];
+  for (const v of result.violations) {
+    lines.push(`  ✖ [${v.rule}] ${v.context ?? "(ledger)"}: ${v.detail}`);
+  }
+  for (const d of result.debts) {
+    lines.push(`  • [${d.rule}] ${d.context ?? "(ledger)"}: ${d.detail}`);
+  }
+  if (result.violations.length === 0) {
+    lines.push(
+      "No unproven promotion. Debt lines are recorded, not enforced — see #2509."
+    );
+  }
+  return lines.join("\n");
+}
+
+/**
+ * CLI entry point.
+ *
+ * @returns {void}
+ */
+function main() {
+  const args = process.argv.slice(2);
+  const json = args.includes("--json");
+  const root = path.resolve(args.find(a => !a.startsWith("--")) ?? ".");
+  const result = evaluate(root);
+  console.log(json ? JSON.stringify(result, null, 2) : formatReport(result));
+  if (result.violations.length > 0) process.exitCode = 1;
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  main();
+}

package/typescript/copy-overwrite/ast-grep/rule-tests/.gitkeep

@@ -1,3 +1,20 @@
-# This directory contains test cases for ast-grep rules.
-# Each test file should match its corresponding rule file name.
-# See: https://ast-grep.github.io/reference/yaml.html#rule-tests
+# Test cases for the ast-grep rules in ../rules/.
+#
+# These run. `ast-grep test` is wired to the `sg:test` package script and to
+# the AST Grep Scan job in Lisa's quality workflow, so a wrong assertion here
+# fails CI. The job counts the YAML files in this directory first: an empty
+# directory is reported as a skipped step with a warning, never as a pass,
+# because `ast-grep test` exits 0 when it finds nothing to run.
+#
+# A test file binds to a rule by its `id:` field, not by its filename — the
+# `<rule-id>-test.yml` convention is for humans reading the tree. Each case
+# listed under `valid:` must produce no diagnostic and each under `invalid:`
+# must produce one, so every rule is asserted in both directions.
+#
+# Every `invalid:` case also needs a snapshot under __snapshots__/ pinning
+# where the rule matched; a missing snapshot fails the run rather than being
+# skipped. Record them with:
+#
+#   ast-grep test --update-all      (or: <package manager> run sg:test -U)
+#
+# See: https://ast-grep.github.io/guide/test-rule.html