@codyswann/lisa 4.51.0 → 4.52.0

package/scripts/generate-two-channel-couplings.ts

@@ -228,9 +228,50 @@ export function buildDeliveryInventory(
   );
 }
 
+/**
+ * The only part of a report these two functions read.
+ *
+ * Narrower than the full report on purpose: they need a key and a verdict per
+ * entry and nothing else, and a parameter that asked for the whole measurement
+ * would force every caller — including a test — to build one it does not use.
+ */
+interface CouplingKeySource {
+  readonly entries: readonly {
+    readonly key: string;
+    readonly verdict: string;
+  }[];
+}
+
+/** One coupling's answer to "can this notice a STALE artifact?" (#3687). */
+export interface StalenessClassification {
+  /**
+   * What the coupling CAN detect today — an observation, not a decision.
+   *
+   * `version-handshake` asks the artifact for a contract version and compares
+   * it. `capability-probe` asks for one subcommand or flag and infers age from
+   * whether it answers, which sees staleness at exactly one point and is blind
+   * above that floor — the CodySwannGT/lisa#3477 defect. `existence-only`
+   * cannot see age at all.
+   */
+  readonly detects: "version-handshake" | "capability-probe" | "existence-only";
+  /**
+   * Why the current level is acceptable. Required unless a handshake exists.
+   *
+   * This is the DECISION half, kept separate from `detects` on purpose: a
+   * coupling that cannot see age and is fine that way (an advisory read, an
+   * artifact carrying no verdict-bearing logic) is a different fact from one
+   * that cannot see age and should. Collapsing them into a single
+   * "not-needed" token would lose exactly the distinction the sweep exists to
+   * record, and would let an unexamined default wear the same word as a
+   * considered exemption.
+   */
+  readonly reason?: string;
+}
+
 /** The ledger's hand-authored half plus its derived half. */
 interface Ledger {
   readonly ratified: Readonly<Record<string, string>>;
+  readonly staleness: Readonly<Record<string, StalenessClassification>>;
   readonly inspected: TwoChannelReport["inspected"];
   readonly counts: TwoChannelReport["counts"];
   readonly couplings: readonly {
@@ -259,6 +300,156 @@ function readRatified(ledgerPath: string): Readonly<Record<string, string>> {
   return parsed?.ratified ?? {};
 }
 
+/**
+ * Read the staleness classifications already recorded (#3687).
+ *
+ * Exactly the shape `readRatified` uses, and for the same reason: the ledger is
+ * a DERIVED artifact regenerated in-commit, so a hand-authored field survives
+ * only if the generator reads the previous version before rewriting it. This is
+ * the established local precedent for persisting a human decision inside a
+ * generated file, which is why the classification lives here rather than in a
+ * new sidecar nobody would remember to update.
+ * @param ledgerPath - Absolute path to the ledger
+ * @returns The classifications, keyed `<workflow>::<path>`
+ */
+function readStaleness(
+  ledgerPath: string
+): Readonly<Record<string, StalenessClassification>> {
+  if (!existsSync(ledgerPath)) return {};
+  const parsed = JSON.parse(readFileSync(ledgerPath, "utf8")) as
+    | Partial<Ledger>
+    | undefined;
+  return parsed?.staleness ?? {};
+}
+
+/**
+ * The verdict for a coupling the package channel already covers.
+ *
+ * Those are omitted from the ledger, so every question this file asks about a
+ * live coupling has to exclude them — named once so the three sites cannot
+ * disagree about which verdict that is.
+ */
+const PACKAGE_BACKED = "package-backed";
+
+/** The `detects` value meaning the coupling asks for a contract version. */
+const VERSION_HANDSHAKE = "version-handshake";
+
+/** The `detects` value meaning it infers age from one subcommand or flag. */
+const CAPABILITY_PROBE = "capability-probe";
+
+/** The `detects` value meaning it can see presence and nothing else. */
+const EXISTENCE_ONLY = "existence-only";
+
+/** The `detects` values a classification may carry. */
+const DETECTS = new Set([VERSION_HANDSHAKE, CAPABILITY_PROBE, EXISTENCE_ONLY]);
+
+/**
+ * The floor a stated reason must clear to be a reason rather than a label.
+ *
+ * A floor, not a proof — no check tells a real argument from filler. What it
+ * buys is that "n/a", "ok" and an empty string cannot stand in for the decision
+ * this field exists to record.
+ */
+const MIN_REASON_CHARACTERS = 40;
+
+/**
+ * Every problem with the staleness half, as operator-readable paragraphs.
+ *
+ * THREE refusals, and the first is the one that makes this a standing property
+ * rather than a one-time sweep: a coupling with no classification fails, so
+ * entry 25 cannot silently inherit "nobody looked". A sweep decays the moment
+ * somebody adds an entry; a required field does not.
+ * @param report - The measurement
+ * @param staleness - The recorded classifications
+ * @returns Problem paragraphs, empty when the staleness half is sound
+ */
+export function stalenessProblems(
+  report: CouplingKeySource,
+  staleness: Readonly<Record<string, StalenessClassification>>
+): readonly string[] {
+  const live = new Set(
+    report.entries
+      .filter(entry => entry.verdict !== PACKAGE_BACKED)
+      .map(entry => entry.key)
+  );
+  const missing = [...live]
+    .filter(key => staleness[key] === undefined)
+    .map(
+      key =>
+        `\nUNCLASSIFIED COUPLING ${key}\n  No staleness classification. Every ` +
+        `coupling must record whether it can detect a STALE artifact, not only ` +
+        `an absent one — absence skips and posts nothing, while present-and-old ` +
+        `posts a verdict from superseded logic that every reader believes. Add ` +
+        `a "staleness" entry with "detects" (version-handshake | ` +
+        `capability-probe | existence-only) and, unless it is a handshake, a ` +
+        `"reason" saying why that is acceptable here.`
+    );
+  const orphaned = Object.keys(staleness)
+    .filter(key => !live.has(key))
+    .map(
+      key =>
+        `\nSTALE CLASSIFICATION ${key}\n  Nothing reads this path any more, so ` +
+        `the classification is an unexamined judgement the next matching path ` +
+        `would inherit. Delete it from ${LEDGER_PATH}.`
+    );
+  const malformed = Object.entries(staleness)
+    .filter(([key]) => live.has(key))
+    .flatMap(([key, value]) => describeMalformed(key, value));
+  return [...missing, ...orphaned, ...malformed];
+}
+
+/**
+ * Why one recorded classification is not usable, if it is not.
+ * @param key - The coupling key
+ * @param value - The recorded classification
+ * @returns Problem paragraphs for this entry
+ */
+function describeMalformed(
+  key: string,
+  value: StalenessClassification
+): readonly string[] {
+  if (!DETECTS.has(value?.detects)) {
+    return [
+      `\nMALFORMED CLASSIFICATION ${key}\n  "detects" is ` +
+        `${JSON.stringify(value?.detects)}; expected one of ` +
+        `${[...DETECTS].join(", ")}.`,
+    ];
+  }
+  if (value.detects === VERSION_HANDSHAKE) return [];
+  const reason = typeof value.reason === "string" ? value.reason.trim() : "";
+  return reason.length >= MIN_REASON_CHARACTERS
+    ? []
+    : [
+        `\nUNREASONED CLASSIFICATION ${key}\n  "detects" is ` +
+          `"${value.detects}", which cannot see how old the artifact is, and no ` +
+          `"reason" of at least ${MIN_REASON_CHARACTERS} characters says why ` +
+          `that is acceptable here. A recorded decision and an unexamined ` +
+          `default must not look the same.`,
+      ];
+}
+
+/**
+ * The one-line fleet coverage sentence (#3687).
+ * @param report - The measurement
+ * @param staleness - The recorded classifications
+ * @returns A sentence naming how much of the fleet can see age
+ */
+export function stalenessCoverage(
+  report: CouplingKeySource,
+  staleness: Readonly<Record<string, StalenessClassification>>
+): string {
+  const live = report.entries.filter(entry => entry.verdict !== PACKAGE_BACKED);
+  const tally = (kind: string): number =>
+    live.filter(entry => staleness[entry.key]?.detects === kind).length;
+  const handshake = tally(VERSION_HANDSHAKE);
+  return (
+    `  staleness detection: ${handshake} of ${live.length} coupling(s) can see ` +
+    `how old the artifact is (${VERSION_HANDSHAKE}); ${tally(CAPABILITY_PROBE)} ` +
+    `detect it at one floor only (${CAPABILITY_PROBE}); ${tally(EXISTENCE_ONLY)} ` +
+    `detect absence only (${EXISTENCE_ONLY}).`
+  );
+}
+
 /**
  * Render the ledger a report implies.
  *
@@ -272,14 +463,16 @@ function readRatified(ledgerPath: string): Readonly<Record<string, string>> {
  */
 function toLedger(
   report: TwoChannelReport,
-  ratified: Readonly<Record<string, string>>
+  ratified: Readonly<Record<string, string>>,
+  staleness: Readonly<Record<string, StalenessClassification>>
 ): Ledger {
   return {
     ratified,
+    staleness,
     inspected: report.inspected,
     counts: report.counts,
     couplings: report.entries
-      .filter(entry => entry.verdict !== "package-backed")
+      .filter(entry => entry.verdict !== PACKAGE_BACKED)
       .map(entry => ({
         key: entry.key,
         workflow: entry.workflow,
@@ -302,6 +495,7 @@ function toLedger(
 export function measure(root: string): {
   readonly report: TwoChannelReport;
   readonly ratified: Readonly<Record<string, string>>;
+  readonly staleness: Readonly<Record<string, StalenessClassification>>;
 } {
   const workflowsDir = path.join(root, WORKFLOWS_DIR);
   const inventory = buildDeliveryInventory(root);
@@ -342,6 +536,7 @@ export function measure(root: string): {
       ratified: readRatified(path.join(root, LEDGER_PATH)),
     }),
     ratified: readRatified(path.join(root, LEDGER_PATH)),
+    staleness: readStaleness(path.join(root, LEDGER_PATH)),
   };
 }
 
@@ -350,7 +545,10 @@ export function measure(root: string): {
  * @param report - The measurement
  * @returns The report text
  */
-function humanReport(report: TwoChannelReport): string {
+function humanReport(
+  report: TwoChannelReport,
+  staleness: Readonly<Record<string, StalenessClassification>>
+): string {
   const counts = report.inspected;
   const header =
     `two-channel delivery: inspected ${counts.workflows} reusable workflow(s), ` +
@@ -370,7 +568,12 @@ function humanReport(report: TwoChannelReport): string {
         `inherit. Delete it from ${LEDGER_PATH}.`
     )
     .join("");
-  return `${header}\n${verdicts}${findings}${stale}`;
+  // The coverage line answers the question the registry could not answer at
+  // all before this: "how many of our couplings can detect staleness?" — in one
+  // line, without opening 24 workflows.
+  const coverage = stalenessCoverage(report, staleness);
+  const unclassified = stalenessProblems(report, staleness).join("");
+  return `${header}\n${verdicts}\n${coverage}${findings}${stale}${unclassified}`;
 }
 
 /**
@@ -384,9 +587,9 @@ export function main(argv: readonly string[]): number {
     process.stderr.write(`error: --root does not exist: ${options.root}\n`);
     return EXIT_OPERATIONAL;
   }
-  const { report, ratified } = measure(options.root);
+  const { report, ratified, staleness } = measure(options.root);
   process.stdout.write(
-    `${options.json ? JSON.stringify(report, null, 2) : humanReport(report)}\n`
+    `${options.json ? JSON.stringify(report, null, 2) : humanReport(report, staleness)}\n`
   );
   if (!report.measured) {
     process.stderr.write(
@@ -397,7 +600,7 @@ export function main(argv: readonly string[]): number {
     return EXIT_OPERATIONAL;
   }
   const ledgerPath = path.join(options.root, LEDGER_PATH);
-  const rendered = `${JSON.stringify(toLedger(report, ratified), null, 2)}\n`;
+  const rendered = `${JSON.stringify(toLedger(report, ratified, staleness), null, 2)}\n`;
   const current = existsSync(ledgerPath)
     ? readFileSync(ledgerPath, "utf8")
     : "";
@@ -413,9 +616,14 @@ export function main(argv: readonly string[]): number {
     mkdirSync(path.dirname(ledgerPath), { recursive: true });
     writeFileSync(ledgerPath, rendered);
   }
+  // The staleness half fails in BOTH modes, exactly as findings do. A gate you
+  // can regenerate your way past is not a gate: regenerating records a coupling,
+  // it does not classify one, and `toLedger` carries the map through untouched
+  // rather than inventing a default for a new entry (#3687).
   const failed =
     report.findings.length > 0 ||
     report.staleRatifications.length > 0 ||
+    stalenessProblems(report, staleness).length > 0 ||
     ledgerStale;
   return failed ? 1 : 0;
 }

package/scripts/two-channel-couplings.json

@@ -3,6 +3,100 @@
     "quality.yml::scripts/check-learnings-budget.ts": "Lisa-source-only branch, and the else arm is the delivery. The step tests for BOTH scripts/check-learnings-budget.ts and src/core/learnings-budget-check.ts, which co-exist only in this repository; every consumer takes the else arm and runs the prover as `bunx @codyswann/lisa@<their own declared version>`, which is the PACKAGE channel and not a caller-tree read at all. A trailing `grep -qE \"learnings budget (passed|saturated)|no learnings file\"` then refuses a run that printed no verdict, so the arm cannot pass vacuously. Ratified because the caller-tree read is unreachable outside this repository, not because the gap is tolerated.",
     "quality.yml::scripts/generate-nightly-e2e-guard-certificate.mjs": "Lisa-source-only branch. Both reusable jobs require `github.repository == 'CodySwannGT/lisa'` before the step can start, and the run body repeats that exact repository guard before checking this artifact path. Ordinary and private host callers therefore skip the step without reading or fetching from their own tree. The scoped branch fetches only the immutable refs needed to behavior-test Lisa's own retained certificate inputs. Ratified because the caller-tree read is unreachable outside this repository, not because an undelivered host dependency is tolerated."
   },
+  "staleness": {
+    "create-github-issue-on-failure.yml::scripts/nightly-e2e-provider-action.mjs": {
+      "detects": "existence-only",
+      "reason": "The read files a tracker issue after a nightly failure; it posts no required check and gates no merge. A stale copy files a slightly wrong issue body, which a human reads and corrects, rather than turning a merge green or red from superseded logic. The harm this classification exists to catch is a verdict nobody can see is old, and this coupling produces no verdict."
+    },
+    "create-jira-issue-on-failure.yml::scripts/nightly-e2e-provider-action.mjs": {
+      "detects": "existence-only",
+      "reason": "Same provider action as the GitHub arm, same reasoning: it files a tracker issue after a nightly failure, posts no required check, and gates no merge. A stale copy misformats a report a human reads; it cannot produce a merge verdict derived from superseded logic."
+    },
+    "create-linear-issue-on-failure.yml::scripts/nightly-e2e-provider-action.mjs": {
+      "detects": "existence-only",
+      "reason": "Same provider action as the GitHub and Jira arms, same reasoning: it files a tracker issue after a nightly failure, posts no required check, and gates no merge. A stale copy degrades a report rather than a verdict."
+    },
+    "create-sentry-issue-on-failure.yml::scripts/nightly-e2e-provider-action.mjs": {
+      "detects": "existence-only",
+      "reason": "Same provider action as the other three failure arms, same reasoning: it files an issue after a nightly failure, posts no required check, and gates no merge. A stale copy degrades a report rather than a verdict."
+    },
+    "maestro-native-e2e.yml::scripts/classify-maestro-failures.mjs": {
+      "detects": "existence-only",
+      "reason": "Classifies native E2E failures for the run summary so an operator can tell an infrastructure flake from a product regression. It labels a failure that has already been decided elsewhere; it does not decide one. A stale classifier mislabels a red the operator is already looking at, which is visible in the same output."
+    },
+    "nightly-e2e-health.yml::scripts/check-nightly-e2e-health.mjs": {
+      "detects": "version-handshake"
+    },
+    "nightly-e2e-health.yml::scripts/nightly-e2e-suites.schema.json": {
+      "detects": "existence-only",
+      "reason": "A JSON schema has no executable surface, so it cannot answer a contract-version call at all — no handshake is constructible here. It is covered transitively rather than left unguarded: the schema and check-nightly-e2e-health.mjs ship on the same typescript/copy-overwrite lane and are read by the same workflow, and that checker DOES handshake, so a tree old enough to carry a stale schema carries a stale checker and the existing handshake fails first."
+    },
+    "nightly-e2e-report.yml::scripts/check-nightly-e2e-health.mjs": {
+      "detects": "version-handshake"
+    },
+    "nightly-e2e-tracking.yml::scripts/reconcile-nightly-e2e-tracking.mjs": {
+      "detects": "existence-only",
+      "reason": "Reconciles the nightly tracking issue — opens, refreshes or closes it to match the latest health verdict. It consumes a verdict rather than producing one, so a stale copy mismanages a tracking issue a human reads and cannot make a merge gate green."
+    },
+    "publish-to-npm.yml::scripts/check-npm-publish-landed.mjs": {
+      "detects": "existence-only",
+      "reason": "A GENUINE GAP, recorded rather than waved through. It verifies a publish actually landed, so a stale copy can attest to a release that did not happen. It runs post-merge in the release workflow and posts no merge-blocking required context, which is why it is not the first handshake to build — but it is verdict-bearing and belongs at the front of the queue when this classification is next acted on."
+    },
+    "publish-to-npm.yml::scripts/check-release-package-identity.mjs": {
+      "detects": "existence-only",
+      "reason": "A GENUINE GAP, recorded rather than waved through, and the sibling of the publish-landed read: it attests that the published package is the one intended, so a stale copy can attest to an identity it no longer checks the same way. Post-merge and not merge-blocking, so it queues behind the merge gates rather than ahead of them."
+    },
+    "quality-rails.yml::scripts/check-threshold-ratchet.mjs": {
+      "detects": "existence-only",
+      "reason": "Same prover as the quality.yml read and the same GENUINE GAP, reached from the Rails caller tree. Recorded separately because the two callers can drift apart: a handshake added to one workflow does not reach the other, which is how a fleet ends up half-covered."
+    },
+    "quality-rails.yml::scripts/lisa-mutation.sh": {
+      "detects": "existence-only",
+      "reason": "A shell wrapper that shapes a mutation-testing invocation; the verdict comes from Stryker and its configured thresholds, not from this file. A stale wrapper passes older flags, which fails loudly as a runner usage error rather than producing a wrong mutation score."
+    },
+    "quality-rails.yml::scripts/lisa-scratch-run.sh": {
+      "detects": "existence-only",
+      "reason": "A shell wrapper that isolates a run's scratch namespace. It carries no verdict-bearing logic at all — it prepares a directory and execs the real command — so there is no superseded judgement for a stale copy to apply."
+    },
+    "quality-rails.yml::scripts/lisa-work-item.mjs": {
+      "detects": "version-handshake"
+    },
+    "quality.yml::scripts/check-e2e-coverage.mjs": {
+      "detects": "existence-only",
+      "reason": "Guarded by an existence test and scoped to Expo consumers. It reports route coverage against a declared threshold; a stale copy computes coverage by an older rule and says which rule it used in its own output, so the number it prints is interpretable rather than silently wrong."
+    },
+    "quality.yml::scripts/check-learnings-budget.ts": {
+      "detects": "existence-only",
+      "reason": "Already ratified as a Lisa-source-only branch: every consumer takes the else arm and runs the prover from the PACKAGE channel at their own declared version, which is a version pin rather than an unversioned copy. The two-channel skew this field is about does not arise on the path consumers actually take."
+    },
+    "quality.yml::scripts/check-skipped-required-checks.mjs": {
+      "detects": "existence-only",
+      "reason": "A GENUINE GAP and the sharpest one in this registry: it is the vacuity and review-evidence prover, so a stale copy posts a review verdict from superseded logic on a merge gate. Its own workflow resolves a second in-repo address when scripts/ has no copy, which bounds the absent case but says nothing about age. Highest-value handshake candidate here."
+    },
+    "quality.yml::scripts/check-state-classification.mjs": {
+      "detects": "existence-only",
+      "reason": "Guarded by an existence test and advisory in effect: it classifies demo/fixture state against a declared contract and reports what it found. A stale copy reports against an older contract version, which surfaces as a classification mismatch in its own output rather than as a green merge gate, so the failure is self-announcing."
+    },
+    "quality.yml::scripts/check-threshold-ratchet.mjs": {
+      "detects": "existence-only",
+      "reason": "A GENUINE GAP. The ratchet decides whether a threshold moved the wrong way, so a stale copy can pass a loosened threshold it does not yet know how to refuse. It is declared optional at push in this repository's own gate set, which lowers the blast radius but does not remove it."
+    },
+    "quality.yml::scripts/check-verification-coverage.mjs": {
+      "detects": "existence-only",
+      "reason": "Bespoke and conditional — it requires a verification-spec delta on feat/fix changes. A stale copy applies an older definition of what counts as a spec delta, which produces a visible disagreement with the spec in the pull request rather than a silent pass."
+    },
+    "quality.yml::scripts/generate-nightly-e2e-guard-certificate.mjs": {
+      "detects": "existence-only",
+      "reason": "Already ratified as undelivered: no lane ships this path, so no consumer tree carries a copy that could be stale. The staleness question is vacuous here for the same reason the delivery question was, and it is recorded rather than left blank so the entry is not mistaken for one nobody examined."
+    },
+    "quality.yml::scripts/lisa-floor-collisions.mjs": {
+      "detects": "existence-only",
+      "reason": "Reports collisions between declared security floors. A stale copy knows a smaller set of floors, so it under-reports rather than mis-decides, and the floors it does know it still checks correctly. Under-reporting is the benign direction for a collision detector."
+    },
+    "quality.yml::scripts/lisa-work-item.mjs": {
+      "detects": "version-handshake"
+    }
+  },
   "inspected": {
     "workflows": 23,
     "steps": 719,