@codyswann/lisa 4.6.3 → 4.6.5

package/scripts/check-learnings-budget.ts

@@ -16,9 +16,16 @@
  *
  * The two surfaces are genuinely different and both matter: the template is
  * what every adopting project starts from, and the ledger is what this
- * repository's agents actually read. A missing template is a failure — it is
- * committed here and always exists. A missing ledger is a pass, because a
- * project that has never recorded a learning has no file.
+ * repository's agents actually read. Both are committed in a source checkout,
+ * so a missing one there is a failure — see `checkDefaultSurfaces` for why the
+ * ledger's absence used to be a pass and no longer is. Running from the
+ * published package, where no tarball carries a ledger, its absence stays the
+ * ordinary quiet pass.
+ *
+ * A within-budget ledger reports one of TWO verdicts, not one: `learnings
+ * budget passed` while room remains, and `learnings budget saturated` once the
+ * next capture would not fit (#3089). Both exit 0. The reasoning for warning
+ * rather than failing lives on `describeLearningsSaturation` in the core.
  * @module scripts/check-learnings-budget
  */
 import { existsSync, readFileSync } from "node:fs";
@@ -28,7 +35,7 @@ import type * as BudgetCheckModule from "../src/core/learnings-budget-check.js";
 
 type BudgetChecker = Pick<
   typeof BudgetCheckModule,
-  "checkLearningsBudget" | "formatDiagnosticPath"
+  "checkLearningsBudget" | "formatBudgetVerdict" | "formatDiagnosticPath"
 >;
 
 const REPO_ROOT = path.resolve(import.meta.dir, "..");
@@ -69,6 +76,22 @@ function resolveLedger(): string {
   return path.resolve(REPO_ROOT, relative);
 }
 
+/**
+ * Whether this script is running from a source checkout rather than from the
+ * published package, decided by the presence of the TypeScript core the
+ * published tarball deliberately excludes.
+ *
+ * The same discriminator the quality workflow uses to choose between running
+ * this script and running the shipped CLI subcommand, so the two agree on what
+ * "Lisa's own repository" means by construction rather than by convention.
+ * @returns True when the in-tree TypeScript core is present
+ */
+function isSourceCheckout(): boolean {
+  return existsSync(
+    path.join(REPO_ROOT, "src", "core", "learnings-budget-check.ts")
+  );
+}
+
 /** Run the package-facing checker with zero or one explicit file path. */
 async function main(): Promise<void> {
   const arguments_ = process.argv.slice(2);
@@ -77,48 +100,87 @@ async function main(): Promise<void> {
   }
 
   const checker = await loadBudgetChecker();
-  if (arguments_.length === 1) {
-    await check(checker, path.resolve(process.cwd(), arguments_[0] as string), {
-      absenceIsFailure: true,
-    });
-    return;
+  const inspected =
+    arguments_.length === 1
+      ? await checkExplicitSurface(checker, arguments_[0] as string)
+      : await checkDefaultSurfaces(checker);
+
+  // VACUITY GUARD. Every path above either judges a document or exits, so this
+  // should be unreachable — which is the point. A run that inspected nothing
+  // and a run that inspected a healthy ledger produce identical output and an
+  // identical exit code, and that ambiguity is how this gate spent a release
+  // checking only a 0-entry template while looking like a gate on the ledger
+  // (#2932). Counting what was actually judged makes "nothing" say so instead
+  // of reporting all-clear.
+  if (inspected === 0) {
+    fail(
+      "inspected no learnings surface — an empty inspection is indistinguishable from a healthy ledger, so it fails rather than reporting all-clear"
+    );
   }
+}
 
-  await check(checker, TEMPLATE_LEARNINGS_FILE, { absenceIsFailure: true });
+/**
+ * Check the one document named on the command line.
+ * @param checker - The loaded budget checker
+ * @param argument - Caller-supplied learnings path
+ * @returns How many documents were judged
+ */
+async function checkExplicitSurface(
+  checker: BudgetChecker,
+  argument: string
+): Promise<number> {
+  await check(checker, path.resolve(process.cwd(), argument));
+  return 1;
+}
+
+/**
+ * Check the two default surfaces: the shipped template and this repository's
+ * own ledger.
+ * @param checker - The loaded budget checker
+ * @returns How many documents were judged
+ */
+async function checkDefaultSurfaces(checker: BudgetChecker): Promise<number> {
+  await check(checker, TEMPLATE_LEARNINGS_FILE);
   const ledger = resolveLedger();
-  if (!existsSync(ledger)) {
-    console.log(`${checker.formatDiagnosticPath(ledger)}: no learnings file`);
-    return;
+  if (existsSync(ledger)) {
+    await check(checker, ledger);
+    return 2;
   }
-  await check(checker, ledger, { absenceIsFailure: true });
+  // In a source checkout the ledger is a committed file, so its absence means
+  // the resolver drifted off it — and "no learnings file" is a verdict the CI
+  // marker grep accepts as green, which turns that drift into a gate on the
+  // template alone. Fail instead. From the published package the same absence
+  // is the ordinary case (no tarball carries a ledger) and stays a quiet pass,
+  // exactly as it does for host projects on the CLI subcommand.
+  if (isSourceCheckout()) {
+    fail(
+      `${checker.formatDiagnosticPath(ledger)}: resolved ledger does not exist, and this is a source checkout where the ledger is committed — the configured \`learnings.file\` no longer resolves to a document, so this run would have gated the shipped template alone`
+    );
+  }
+  console.log(`${checker.formatDiagnosticPath(ledger)}: no learnings file`);
+  return 1;
 }
 
 /**
  * Check one file and report, failing the process on any violation.
+ *
+ * Every document this script reaches must exist — an explicit argument names
+ * one, and both defaults are committed here — so a missing file is as much a
+ * failure as any other violation. Host projects use the
+ * `lisa check-learnings-budget` CLI subcommand instead, which treats a missing
+ * file as an expected pass.
  * @param checker - The loaded budget checker
  * @param file - Absolute path to the document to check
- * @param options - Whether a missing file is a violation
- * @param options.absenceIsFailure - True when the file must exist
  */
-async function check(
-  checker: BudgetChecker,
-  file: string,
-  options: { absenceIsFailure: boolean }
-): Promise<void> {
+async function check(checker: BudgetChecker, file: string): Promise<void> {
   const result = await checker.checkLearningsBudget(file);
   if (result.kind === "ok") {
-    console.log(
-      `${checker.formatDiagnosticPath(file)}: learnings budget passed (${result.entryCount}/${result.maxEntries} entries, ${result.measuredTokens}/${result.maxTokens} maxTokens)`
-    );
+    // Prints `learnings budget saturated` for a full-but-valid ledger and exits
+    // 0 all the same; the reasoning lives on describeLearningsSaturation.
+    console.log(checker.formatBudgetVerdict(file, result));
     return;
   }
-  // Both files this script checks are committed here, so a missing one is as
-  // much a failure as any other violation. Host projects use the
-  // `lisa check-learnings-budget` CLI subcommand instead, which treats a
-  // missing file as an expected pass.
-  if (options.absenceIsFailure) {
-    fail(`${checker.formatDiagnosticPath(file)}: ${result.detail}`);
-  }
+  fail(`${checker.formatDiagnosticPath(file)}: ${result.detail}`);
 }
 
 /**
@@ -146,6 +208,7 @@ async function loadBudgetChecker(): Promise<BudgetChecker> {
   );
   return {
     checkLearningsBudget: module_.checkLearningsBudget,
+    formatBudgetVerdict: module_.formatBudgetVerdict,
     formatDiagnosticPath: module_.formatDiagnosticPath,
   } as BudgetChecker;
 }

package/typescript/copy-overwrite/scripts/check-nightly-e2e-health.mjs

@@ -67,6 +67,18 @@
  * expiry timestamp on screen. Bootstrap forgives absence of evidence, never
  * evidence of failure — a red run is red inside the window too.
  *
+ * ## The bypass reads the pull request LIVE (contract 1.6.0, row 40)
+ *
+ * `github.event.pull_request` is a SNAPSHOT taken when the run was triggered,
+ * and a re-run replays it verbatim. Reading the bypass label and the
+ * `Nightly-E2E-Bypass:` trailer out of it made the gate's own printed remedy
+ * impossible to follow: applying the label fired no run, and re-running — the
+ * obvious next move — replayed a payload from before the label existed. The
+ * waiver was real, correctly recorded, and invisible to the gate that asked for
+ * it. `fetchPullRequestState` reads both halves from the API at gate time
+ * instead, which also means a label somebody REMOVED stops waiving. An
+ * unreadable pull request is a REJECTED bypass, never a granted one.
+ *
  * ## Inherited from three implementations, with one path closed
  *
  * `DECISIVE_CONCLUSIONS` comes from acmeorgb's `check-nightly-e2e.mjs` and is
@@ -89,7 +101,7 @@ import { invokedAsScript } from "./lib/invoked-as-script.mjs";
  * rather than running a contract neither half agrees on. See §8 of
  * `docs/nightly-e2e-gate.md` for what counts as major / minor / patch.
  */
-export const NIGHTLY_E2E_CONTRACT_VERSION = "1.5.0";
+export const NIGHTLY_E2E_CONTRACT_VERSION = "1.6.0";
 
 /**
  * The conclusions that constitute a verdict about the code.
@@ -1381,6 +1393,8 @@ const BYPASS_REJECTIONS = Object.freeze({
     "it was applied longer ago than `bypass_max_hours` allows. Bypasses auto-expire so a label nobody removes cannot become a permanent hole.",
   no_reason_or_ticket:
     "the PR body carries no `Nightly-E2E-Bypass: <TICKET> <reason>` line. A bypass without a reason and a ticket is not auditable.",
+  pr_state_unreadable:
+    "this pull request's live labels and body could not be read, so whether a waiver was requested is UNKNOWN. The gate stays closed: a bypass that fires when it could not read the request is worse than no bypass. On a private repository the caller job needs `pull-requests: read`; otherwise this is a transient API failure and a re-run resolves it.",
 });
 
 // ---------------------------------------------------------------------------
@@ -1591,8 +1605,17 @@ export function formatReport(verdict, context) {
     );
   } else if (verdict.blocked) {
     if (verdict.bypass && !verdict.bypass.valid) {
+      // `pr_state_unreadable` gets its own lead because the usual one asserts
+      // something this branch cannot know. When the live read failed, whether a
+      // label is present is precisely the unanswered question — saying "a label
+      // is present but was REJECTED" would send the reader to remove a label
+      // that may not exist.
+      const lead =
+        verdict.bypass.reason === "pr_state_unreadable"
+          ? "⛔ **The bypass could not be evaluated**"
+          : `⛔ **A \`${context.bypassLabel}\` label is present but was REJECTED**`;
       lines.push(
-        `⛔ **A \`${context.bypassLabel}\` label is present but was REJECTED** — ${BYPASS_REJECTIONS[verdict.bypass.reason] ?? verdict.bypass.reason}`,
+        `${lead} — ${BYPASS_REJECTIONS[verdict.bypass.reason] ?? verdict.bypass.reason}`,
         ""
       );
     }
@@ -2829,6 +2852,94 @@ export async function applyIssuePlan(api, plan, wait) {
   return Object.freeze(results);
 }
 
+/**
+ * Reads the pull request's CURRENT labels, body and author, LIVE from the API.
+ *
+ * THIS IS THE BYPASS'S ONLY SOURCE OF TRUTH, and the reason it exists is a
+ * measured defect. The gate used to read both halves of the bypass request from
+ * `github.event.pull_request` — `toJSON(...labels.*.name)` and `...body`. That
+ * object is the event payload CAPTURED WHEN THE RUN WAS TRIGGERED. It is a
+ * snapshot, not a live read, and a re-run REPLAYS the original payload. So the
+ * gate's own printed remedy — "add the trailer, then apply the label" — could
+ * not work: applying a label fires no run under the default activity types, and
+ * re-running (the obvious next move, and the one the failure message invites)
+ * replays a payload from before the label existed. Measured on two consumer
+ * repositories: the label sat on the pull request the whole time while the job
+ * logged `NIGHTLY_PR_LABELS: []`. The only thing that worked was an empty
+ * commit, to manufacture a `synchronize` whose payload happened to carry the
+ * label — a workaround nobody should have to discover.
+ *
+ * A live read is immune to both halves of that: it is correct on a re-run, and
+ * correct whether or not the caller subscribed to the `labeled` activity type.
+ *
+ * It also fixes the mirror-image hole, which matters more: a payload that still
+ * carries a label somebody has since REMOVED would waive a gate whose waiver was
+ * withdrawn. Reading live is the only way "the label is gone" can reach the
+ * gate at all.
+ *
+ * FAILS CLOSED, and never falls back to the payload. `null` means the pull
+ * request could not be read — 404, an unreadable API, a body with no `labels`
+ * array — and the caller turns that into a REJECTED bypass, never into a
+ * bypass. Falling back to the payload here would reintroduce exactly the
+ * stale-label waiver above, on the one path where nobody is watching. Note the
+ * distinction the caller depends on: `null` is "unreadable", while a readable
+ * pull request carrying no labels is `labels: []` — both stay gated, for
+ * reasons the report states differently.
+ *
+ * @param {object} api - API coordinates
+ * @param {number} prNumber - Pull request number
+ * @param {(ms: number) => Promise<void>} [wait] - Injectable sleep
+ * @returns {Promise<{labels: ReadonlyArray<string>, body: string, author: string|null}|null>} Live PR state, or `null` when unreadable
+ */
+export async function fetchPullRequestState(api, prNumber, wait) {
+  const result = await apiGet(
+    api,
+    `/repos/${api.repo}/pulls/${prNumber}`,
+    wait
+  ).catch(() => null);
+  const pr = result?.body;
+  // A missing `labels` array is UNREADABLE, not "no labels". The two must not
+  // collapse: one is a broken read and the other is a fact about the PR.
+  if (!pr || !Array.isArray(pr.labels)) return null;
+  const labels = pr.labels
+    .map(entry => (typeof entry === "string" ? entry : entry?.name))
+    .filter(name => typeof name === "string" && name.length > 0);
+  return Object.freeze({
+    labels: Object.freeze(labels),
+    body: typeof pr.body === "string" ? pr.body : "",
+    author: typeof pr.user?.login === "string" ? pr.user.login : null,
+  });
+}
+
+/**
+ * The emitted decision for "the pull request could not be read".
+ *
+ * Same keys as every `evaluateBypass` outcome, so `audit_json` has one shape
+ * whatever happened — §6's emitted record is asserted key for key.
+ *
+ * @param {{prNumber: number|null, label: string|null, prAuthor: string|null}} subject - What was being evaluated
+ * @returns {{valid: boolean, reason: string}} A rejected bypass
+ */
+export function unreadablePullRequestBypass({
+  prNumber = null,
+  label = null,
+  prAuthor = null,
+}) {
+  return Object.freeze({
+    label,
+    prAuthor,
+    prNumber,
+    actorPermission: null,
+    valid: false,
+    reason: "pr_state_unreadable",
+    actor: null,
+    appliedAt: null,
+    expiresAt: null,
+    ticket: null,
+    detail: null,
+  });
+}
+
 /**
  * Reads who most recently applied the bypass label, from the PR's issue events.
  *
@@ -2992,18 +3103,26 @@ export function resolveSettings(env) {
     // An ADDITIONAL project rule, never a replacement — see `evaluateBypass`.
     extraBypassReasonPattern: env.NIGHTLY_BYPASS_REASON_PATTERN || "",
     clamped,
+    // The ONLY thing taken from the event payload is the pull request NUMBER,
+    // and only because a PR's number is immutable — it is an address, not a
+    // fact about the PR. Everything the bypass decides on (labels, body, and
+    // the author carried onto the audit) is read LIVE by
+    // `fetchPullRequestState`, because `github.event` is a snapshot from
+    // trigger time that a re-run replays verbatim.
+    //
+    // `NIGHTLY_PR_LABELS` and `NIGHTLY_PR_BODY` are DELIBERATELY NOT READ from
+    // here as of contract 1.6.0. The reusable workflow still sets them, on
+    // purpose: a consumer that has taken the newer workflow ref but not yet run
+    // `lisa apply` is still running a pre-1.6.0 guard, and that guard needs
+    // them. Reading them here would defeat the fix — the payload is exactly
+    // what was wrong.
+    //
+    // `payloadAuthor` is a fallback for the audit record only. It never gates
+    // anything (see `evaluateBypass`: `prAuthor` is recorded, never used to
+    // reject), so a stale value cannot change a verdict.
     pr: {
       number: Number(env.NIGHTLY_PR_NUMBER) || null,
-      author: env.NIGHTLY_PR_AUTHOR || null,
-      body: env.NIGHTLY_PR_BODY || "",
-      labels: (() => {
-        try {
-          const parsed = JSON.parse(env.NIGHTLY_PR_LABELS || "[]");
-          return Array.isArray(parsed) ? parsed.map(String) : [];
-        } catch {
-          return [];
-        }
-      })(),
+      payloadAuthor: env.NIGHTLY_PR_AUTHOR || null,
     },
   };
 }
@@ -3044,28 +3163,49 @@ export async function runGate(env, wait) {
     return grace.firstSeen === null ? finding : { ...finding, grace };
   });
 
+  // The bypass reads the pull request LIVE and never consults `github.event`.
+  // Three outcomes, and the middle one is the vacuity guard: unreadable is a
+  // REJECTED bypass, never an absent one and never a granted one.
   let bypass = null;
-  if (settings.pr.number && settings.pr.labels.includes(settings.bypassLabel)) {
-    const labelEvent = await fetchLabelEvent(
+  if (settings.pr.number) {
+    const live = await fetchPullRequestState(
       settings.api,
       settings.pr.number,
-      settings.bypassLabel,
       wait
     );
-    const actorPermission = labelEvent
-      ? await fetchActorPermission(settings.api, labelEvent.actor, wait)
-      : null;
-    bypass = evaluateBypass({
-      labelEvent,
-      prAuthor: settings.pr.author,
-      prBody: settings.pr.body,
-      actorPermission,
+    const subject = {
       prNumber: settings.pr.number,
       label: settings.bypassLabel,
-      maxHours: settings.bypassMaxHours,
-      extraReasonPattern: settings.extraBypassReasonPattern,
-      now,
-    });
+      prAuthor: settings.pr.payloadAuthor,
+    };
+    if (live === null) {
+      bypass = unreadablePullRequestBypass(subject);
+    } else if (live.labels.includes(settings.bypassLabel)) {
+      const labelEvent = await fetchLabelEvent(
+        settings.api,
+        settings.pr.number,
+        settings.bypassLabel,
+        wait
+      );
+      const actorPermission = labelEvent
+        ? await fetchActorPermission(settings.api, labelEvent.actor, wait)
+        : null;
+      bypass = evaluateBypass({
+        labelEvent,
+        prAuthor: live.author ?? settings.pr.payloadAuthor,
+        prBody: live.body,
+        actorPermission,
+        prNumber: settings.pr.number,
+        label: settings.bypassLabel,
+        maxHours: settings.bypassMaxHours,
+        extraReasonPattern: settings.extraBypassReasonPattern,
+        now,
+      });
+    }
+    // The remaining case — read successfully, label not present — leaves
+    // `bypass` null. Nobody asked for a waiver, so there is nothing to report,
+    // and a label REMOVED since the run was triggered lands here rather than
+    // waiving anything.
   }
 
   return {