@biffo/cli 0.241.0 → 0.242.0

package/scripts/runner-drop-forensics.mjs

@@ -0,0 +1,344 @@
+#!/usr/bin/env node
+/**
+ * Was this red check a fleet fault, or did a gate actually reject the change?
+ * (#1021)
+ *
+ * ## The problem this closes
+ *
+ * A self-hosted job dies mid-run and GitHub reports a plain red check. The job
+ * log is a **404** by the time anyone looks, because the runner that was
+ * uploading it no longer exists. The only surviving trace is a check-run
+ * annotation saying the runner "lost communication with the server" — which is
+ * equally consistent with a reclaimed spot instance, a starved one, and a
+ * network fault. Every occurrence passes on a plain re-run, so the honest
+ * reading of a red branch and the lazy one are indistinguishable, and #982
+ * showed the estate had been counting these as broken code for months.
+ *
+ * `isRunnerKill` (in `practices-metrics.mjs`, imported below rather than
+ * re-implemented) already answers *"did a runner die?"* from the run's own step
+ * conclusions. It cannot answer *"why?"* — and "why" is what decides whether
+ * anyone should be looking at the code at all.
+ *
+ * ## The join nobody had written
+ *
+ * Two facts make the "why" recoverable without retaining a single job log:
+ *
+ * 1. **`runner_name` on a self-hosted job is the EC2 instance ID** — literally
+ *    `i-0b26948129cfd56f3`. The philips-labs module names runners after the
+ *    instance, so GitHub's own API hands you the fleet's primary key.
+ * 2. **CloudTrail already records every spot reclamation for 90 days**, as an
+ *    `AwsServiceEvent` named `BidEvictedEvent` whose
+ *    `serviceEventDetails.instanceIdSet` lists the instances AWS took back.
+ *    Nothing had to be enabled; management events are on by default.
+ *
+ * So the evidence was never actually missing. What was missing was the join.
+ * Measured across both fleets and all twelve repos under measurement,
+ * 2026-07-07 to 2026-08-03: **22 of 22 runner-killed jobs matched a spot
+ * eviction, and none were unexplained.** The cause is reclamation — not
+ * starvation, not network — which is why the fleet lever (spot allocation
+ * strategy) is the one that moves this number and log retention is not.
+ *
+ * ## Why the match is time-bounded and not just an ID lookup
+ *
+ * `biffo-runners` runs **pooled** runners (`enable_ephemeral_runners = false`,
+ * forced by a 4KB SSM limit on repo-level JIT configs), so one instance serves
+ * several jobs over its life. An eviction proves that instance died once; it
+ * does not tell you which of its jobs it was running at the time. Matching on
+ * ID alone would blame a reclamation for an unrelated earlier job on the same
+ * host that failed honestly.
+ *
+ * So an eviction counts only when it lands inside the job's own window. All 22
+ * observed matches did, comfortably — but the margins differ sharply by fleet,
+ * and the difference is worth knowing when reading output from this tool:
+ *
+ * - `biffo` (pooled, registration-token): the job ends **4–14 seconds** after
+ *   the eviction.
+ * - `tabsii` (ephemeral, JIT): the job hangs for **~9–10 minutes** before
+ *   GitHub gives up on the vanished runner.
+ *
+ * {@link MATCH_GRACE_MS} exists because of that 4-second floor.
+ */
+
+// @ts-check
+import { execFileSync } from 'node:child_process'
+import { isRunnerKill } from './practices-metrics.mjs'
+
+/**
+ * How far outside a job's own start/finish window an eviction may fall and
+ * still be counted as the thing that killed it.
+ *
+ * Zero tolerance is tempting and wrong. The tightest real margin observed was
+ * **4 seconds** (`i-0bfec05c84f2a7030`, `biffo-plugin-idea-scout`), so a few
+ * seconds of skew between GitHub's clock and CloudTrail's would push a genuine
+ * reclamation outside the window.
+ *
+ * The failure directions are not symmetric, which is what sets the value. Too
+ * tight and a reclaimed runner is reported `fleet-fault-unexplained`, sending
+ * someone to hunt instance logs that do not exist for a cause already known —
+ * the exact wild goose chase this module was written to end. Too loose and,
+ * on the pooled fleet only, a reclamation could be credited to a neighbouring
+ * job that failed honestly. A minute is far wider than any plausible clock
+ * skew and far narrower than the gap between consecutive jobs on one host.
+ */
+export const MATCH_GRACE_MS = 60_000
+
+/**
+ * Verdicts, in the order a reader should care about them.
+ *
+ * `FLEET_FAULT_UNEXPLAINED` is the interesting one and the reason this returns
+ * three values rather than a boolean. It means a runner demonstrably died and
+ * AWS did **not** reclaim it — starvation, a network fault, or the runner
+ * process being killed. That is the only case where retaining instance-level
+ * logs would buy anything, so it is also the trigger for reopening #1021's
+ * first checkbox. The corpus currently holds zero of them; if that changes,
+ * the conclusion above has an expiry date.
+ */
+export const VERDICT = {
+  /** A gate ran and rejected the change. Look at the code. */
+  REAL_FAILURE: 'real-failure',
+  /** AWS reclaimed the spot instance mid-job. Nothing to fix in the repo. */
+  SPOT_RECLAIMED: 'spot-reclaimed',
+  /** The runner died and no eviction explains it. This one needs a human. */
+  FLEET_FAULT_UNEXPLAINED: 'fleet-fault-unexplained',
+  /** Ran on a GitHub-hosted runner, so there is no fleet to correlate against. */
+  NOT_SELF_HOSTED: 'not-self-hosted',
+}
+
+/**
+ * EC2 instance IDs are `i-` plus 8 or 17 hex digits. A GitHub-hosted job
+ * reports something like `GitHub Actions 1000017440` instead, which must not be
+ * silently treated as an unmatched instance — "we could not see the input" is
+ * not "there was nothing there".
+ *
+ * @param {string | null | undefined} name a job's `runner_name`
+ * @returns {boolean}
+ */
+export function isInstanceId(name) {
+  return typeof name === 'string' && /^i-[0-9a-f]{8}([0-9a-f]{9})?$/.test(name)
+}
+
+/**
+ * Index CloudTrail's `lookup-events` output by the instances each event killed.
+ *
+ * The instance IDs are buried two layers deep: `CloudTrailEvent` is a JSON
+ * **string** holding the real record, and the IDs live in
+ * `serviceEventDetails.instanceIdSet`. The flat `Resources` array that the
+ * top-level event carries is empty for this event type, which is the obvious
+ * place to look and the wrong one.
+ *
+ * One event can name several instances — AWS reclaims a whole pool at once, and
+ * the largest single event observed took twelve runners in the same second.
+ *
+ * @param {Array<Record<string, any>>} events the `Events` array from
+ *   `aws cloudtrail lookup-events`
+ * @returns {Map<string, string>} instance ID to ISO-8601 eviction time
+ */
+export function parseEvictions(events) {
+  const byInstance = new Map()
+  for (const event of events ?? []) {
+    let record
+    try {
+      record = JSON.parse(event?.CloudTrailEvent ?? '{}')
+    } catch {
+      continue
+    }
+    const at = record?.eventTime
+    if (!at) continue
+    for (const id of record?.serviceEventDetails?.instanceIdSet ?? []) {
+      // Keep the earliest sighting: an instance is reclaimed once, and a
+      // duplicate page of results must not move the timestamp.
+      const seen = byInstance.get(id)
+      if (!seen || Date.parse(at) < Date.parse(seen)) byInstance.set(id, at)
+    }
+  }
+  return byInstance
+}
+
+/**
+ * Did `evictedAt` fall inside this job's run, allowing for {@link MATCH_GRACE_MS}?
+ *
+ * A job still running reports `completed_at: null`; treat that as "the window
+ * is still open" rather than as a zero-length window that matches nothing.
+ *
+ * @param {{ started_at?: string | null, completed_at?: string | null }} job
+ * @param {string} evictedAt
+ * @returns {boolean}
+ */
+export function evictionKilledJob(job, evictedAt) {
+  const evicted = Date.parse(evictedAt)
+  if (Number.isNaN(evicted)) return false
+  const started = Date.parse(job?.started_at ?? '')
+  if (Number.isNaN(started)) return false
+  const completedRaw = Date.parse(job?.completed_at ?? '')
+  const completed = Number.isNaN(completedRaw) ? Infinity : completedRaw + MATCH_GRACE_MS
+  return evicted >= started - MATCH_GRACE_MS && evicted <= completed
+}
+
+/**
+ * Adjudicate one failed run against the fleet's eviction record.
+ *
+ * Order matters: {@link isRunnerKill} is consulted **first**, so a run where a
+ * gate genuinely failed stays a real failure even if some unrelated instance
+ * happened to be reclaimed in the same window. A reclamation is only ever an
+ * explanation for a death that the run's own steps already evidence.
+ *
+ * @param {Array<Record<string, any>>} jobs the run's `jobs` array
+ * @param {Map<string, string>} evictions from {@link parseEvictions}
+ * @returns {{ verdict: string, jobs: Array<{ name: string, instance: string | null, verdict: string, evictedAt: string | null }> }}
+ */
+export function adjudicateRun(jobs, evictions) {
+  if (!isRunnerKill(jobs)) return { verdict: VERDICT.REAL_FAILURE, jobs: [] }
+
+  const lost = (jobs ?? []).filter((job) => job.conclusion === 'failure')
+  const detail = lost.map((job) => {
+    const instance = job.runner_name ?? null
+    if (!isInstanceId(instance)) {
+      return { name: job.name, instance, verdict: VERDICT.NOT_SELF_HOSTED, evictedAt: null }
+    }
+    const evictedAt = evictions.get(instance) ?? null
+    const reclaimed = evictedAt !== null && evictionKilledJob(job, evictedAt)
+    return {
+      name: job.name,
+      instance,
+      verdict: reclaimed ? VERDICT.SPOT_RECLAIMED : VERDICT.FLEET_FAULT_UNEXPLAINED,
+      evictedAt: reclaimed ? evictedAt : null,
+    }
+  })
+
+  // The run is only "explained" when every job that died has an eviction behind
+  // it. One unexplained death is the whole point of the tool, so it dominates —
+  // reporting the run as reclaimed because the *other* three jobs were would
+  // bury exactly the case that needs a person.
+  const explained = detail.every((d) => d.verdict === VERDICT.SPOT_RECLAIMED)
+  return {
+    verdict: explained ? VERDICT.SPOT_RECLAIMED : VERDICT.FLEET_FAULT_UNEXPLAINED,
+    jobs: detail,
+  }
+}
+
+/**
+ * Roll a set of adjudicated runs into the counts a human wants to read.
+ *
+ * @param {Array<{ verdict: string }>} runs
+ * @returns {Record<string, number>}
+ */
+export function summarise(runs) {
+  const counts = {
+    [VERDICT.REAL_FAILURE]: 0,
+    [VERDICT.SPOT_RECLAIMED]: 0,
+    [VERDICT.FLEET_FAULT_UNEXPLAINED]: 0,
+    [VERDICT.NOT_SELF_HOSTED]: 0,
+  }
+  for (const run of runs ?? []) counts[run.verdict] = (counts[run.verdict] ?? 0) + 1
+  return counts
+}
+
+// ---------------------------------------------------------------------------
+// I/O. Everything above is pure so the CLI's vitest suite can exercise it
+// without a network; everything below is the thin shell that feeds it.
+// ---------------------------------------------------------------------------
+
+/** @param {string[]} args */
+function gh(args) {
+  return JSON.parse(execFileSync('gh', args, { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }))
+}
+
+/**
+ * Fetch the fleet's eviction record.
+ *
+ * `lookup-events` pages at 50 regardless of `--max-results`, so this leans on
+ * the AWS CLI's own `--no-paginate`-free default, which follows `NextToken` to
+ * exhaustion. Asking for 1000 in one call silently returns 50 and looks like a
+ * quiet fleet — the first draft of this reported `tabsii` at 50 evictions when
+ * the true figure was 141.
+ *
+ * @param {{ profile: string, region: string, since: string }} opts
+ */
+function fetchEvictions({ profile, region, since }) {
+  const out = execFileSync(
+    'aws',
+    [
+      'cloudtrail',
+      'lookup-events',
+      '--profile',
+      profile,
+      '--region',
+      region,
+      '--lookup-attributes',
+      'AttributeKey=EventName,AttributeValue=BidEvictedEvent',
+      '--start-time',
+      since,
+      '--output',
+      'json',
+      '--no-cli-pager',
+    ],
+    { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 },
+  )
+  return parseEvictions(JSON.parse(out).Events ?? [])
+}
+
+/** @param {string[]} argv */
+function parseArgs(argv) {
+  const args = { repo: null, run: null, profile: 'default', region: 'us-east-2', since: null }
+  for (let i = 0; i < argv.length; i++) {
+    const next = () => argv[++i]
+    if (argv[i] === '--repo') args.repo = next()
+    else if (argv[i] === '--run') args.run = next()
+    else if (argv[i] === '--profile') args.profile = next()
+    else if (argv[i] === '--region') args.region = next()
+    else if (argv[i] === '--since') args.since = next()
+  }
+  return args
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2))
+  if (!args.repo || !args.run) {
+    console.error(
+      'usage: runner-drop-forensics.mjs --repo <owner/name> --run <run-id> [--profile <aws-profile>] [--region <region>] [--since <date>]\n\n' +
+        'Decides whether a red check was a fleet fault or a real failure, by joining the\n' +
+        "run's self-hosted jobs to CloudTrail's record of spot reclamations.",
+    )
+    process.exit(2)
+  }
+
+  const jobs = gh([
+    'api',
+    `repos/${args.repo}/actions/runs/${args.run}/jobs?per_page=100`,
+    '--paginate',
+    '--slurp',
+  ]).flatMap((/** @type {any} */ page) => page.jobs ?? [])
+
+  // Default the lookback to a day before the run started. CloudTrail keeps 90
+  // days of management events, so a wider window costs only latency, but there
+  // is no reason to page through three months to explain this morning.
+  const started = jobs.map((/** @type {any} */ j) => j.started_at).filter(Boolean).sort()[0]
+  const since =
+    args.since ?? new Date(Date.parse(started ?? new Date().toISOString()) - 86_400_000).toISOString()
+
+  const evictions = fetchEvictions({ profile: args.profile, region: args.region, since })
+  const result = adjudicateRun(jobs, evictions)
+
+  console.log(`run ${args.run} in ${args.repo}: ${result.verdict}`)
+  for (const job of result.jobs) {
+    const because = job.evictedAt ? `  (AWS reclaimed it at ${job.evictedAt})` : ''
+    console.log(`  ${job.verdict.padEnd(24)} ${job.instance ?? '—'}  ${job.name}${because}`)
+  }
+
+  // The estate's three-valued contract, and the mapping is not arbitrary:
+  // 0 the drop is explained and a plain re-run is justified; 1 a gate rejected
+  // the change, so go and read the code; 2 CANNOT TELL — a runner demonstrably
+  // died and nothing accounts for it.
+  //
+  // `fleet-fault-unexplained` earns 2 rather than a bespoke code precisely
+  // because **2 is never a pass** here as everywhere else. "The runner died for
+  // reasons unknown" must not read as "safe to re-run and move on"; that is the
+  // fail-open this whole tool exists to close.
+  if (result.verdict === VERDICT.REAL_FAILURE) process.exit(1)
+  if (result.verdict === VERDICT.SPOT_RECLAIMED) process.exit(0)
+  process.exit(2)
+}
+
+if (process.argv[1] && process.argv[1].endsWith('runner-drop-forensics.mjs')) {
+  main()
+}

package/scripts/verify.sh

@@ -720,7 +720,7 @@ else
 fi
 
 # Terraform, wherever this repo keeps it: modules/ in the template and
-# instances, infra/ and modules/ in siblings.
+# instances, infra/ and modules/ in siblings, terraform/ in the runner fleets.
 if [ -n "$LIST" ] || command -v terraform >/dev/null 2>&1; then
   # Scope must match this repo's CI, not exceed it. The template and instances
   # deliberately fmt-check modules/ ONLY: infra/environments/ is user-owned, and
@@ -731,11 +731,46 @@ if [ -n "$LIST" ] || command -v terraform >/dev/null 2>&1; then
   tf_dirs=""
   [ -d modules ] && tf_dirs="$tf_dirs modules/"
   [ -f biffo.sibling.json ] && [ -d infra ] && tf_dirs="$tf_dirs infra/"
+  # terraform/ is the whole of a runner fleet (#1239). Both fleets kept every
+  # .tf file there, which is neither of the two directories above, so `tf_dirs`
+  # came out empty and this gate printed `no terraform in this repo` -- in the
+  # two repos that are nothing BUT terraform. Their CI does check it
+  # (`terraform fmt -check -recursive terraform/`), so the gap was local only:
+  # the gate that exists to catch this before the push was the one thing not
+  # catching it. No repo in the estate holds both terraform/ and modules/, so
+  # adding it cannot widen scope anywhere that was already covered.
+  [ -d terraform ] && tf_dirs="$tf_dirs terraform/"
   if [ -n "$tf_dirs" ]; then
     # shellcheck disable=SC2086
     ci_has "terraform fmt" && run_check terraform-fmt terraform fmt -check -recursive $tf_dirs
   else
-    skip terraform-fmt "no terraform in this repo"
+    # Distinguish "this repo has no terraform" from "this repo has terraform
+    # somewhere I do not look". The old message asserted the first and was
+    # printed for the second, which is the difference between a considered skip
+    # and a blind spot wearing its clothes -- the same shape as a branch audit
+    # dropping the repos it could not read (#1145) and reporting the remainder
+    # as the whole.
+    # Pruned rather than filtered, and NOT capped: `| head -20 | wc -l` would
+    # silently report 20 for a repo with 200, and a count that stops counting is
+    # the denominator defect this estate keeps re-learning.
+    _tf_stray=$(find . \
+      \( -name .git -o -name .worktrees -o -name .terraform -o -name node_modules \) -prune \
+      -o -name '*.tf' -print 2>/dev/null | wc -l | tr -d ' ')
+    if [ "${_tf_stray:-0}" -gt 0 ] && [ -z "$LIST" ]; then
+      # A WARN, not a skip and not a failure -- exactly the posture pg-test
+      # takes above for "the repo HAS the thing and the gate is blind to it".
+      # Not a failure because the right scope depends on what this repo's CI
+      # covers, which this gate cannot decide for a layout nobody has declared.
+      NOT_RUN="$NOT_RUN terraform-fmt"
+      printf '  \033[33mWARN\033[0m %-16s NOT RUN - %s .tf file(s) present, none in a directory this gate checks\n' \
+        "terraform-fmt" "$_tf_stray"
+      printf '       \033[33m%s\033[0m\n' \
+        "it looks in modules/, infra/ (siblings) and terraform/ - this repo uses none of them"
+      printf '       \033[90m%s\033[0m\n' \
+        "add the directory to the tf_dirs block in scripts/verify.sh (biffo-template#1239)"
+    else
+      skip terraform-fmt "no .tf files in modules/, infra/ or terraform/"
+    fi
   fi
 else
   skip terraform-fmt "terraform not installed"