@1aboveio/skills 0.0.0-stage → 1.0.0

package/skills/cicd/github-actions/scripts/ci-run-audit-core.mjs

@@ -0,0 +1,612 @@
+// ci-run-audit core — audit a repo's GitHub Actions *run history* for waste. Report-only:
+// it never edits a workflow, and every finding is a candidate for a human to judge.
+//
+// Where ensure-coverage's ci-audit.mjs reads CI *config* (are the gates wired?), this
+// reads what CI actually *did* (what did it cost, and what did it catch?). Six checks:
+//
+//   1. re-proved trees   a job passing again on a byte-identical tree (same git tree
+//                        hash) it already passed: PR → queue → main re-runs
+//   2. catch yield       per job: code states it failed on (and never passed), how many of
+//                        those were its alone, and how many were flakes (failed, then
+//                        passed on the same tree)
+//   3. overlap           job A whose every real failure coincided with job B failing
+//   4. setup share       jobs whose time is mostly overhead: everything but their own
+//                        work steps (checkout, install, containers, runner gaps)
+//   5. critical path     PR wait time p50/p90 and which job finishes last
+//   6. stuck + queued    jobs that ran past 6h, and time waiting for a runner
+//
+// Usage:
+//   node ci-run-audit.mjs --repo <owner/name> [--runs 300] [--save runs.json] [--json]
+//   node ci-run-audit.mjs --from runs.json [--json]       # re-analyse a saved pull
+//   node ci-run-audit.mjs --repo <owner/name> --profile-run <runId> [--save runs.json] [--json]
+//   node ci-run-audit.mjs --from runs.json --profile-run <runId> [--json]
+// Options: --ignore-job <regex> (added to the default aggregator filter),
+//          --min-trees <n> (sample floor for a "no sole catch" candidate, default 20).
+// Needs an authenticated `gh`. Exit 0 with a report; 2 on usage or fetch errors.
+//
+// Import-only: `ci-run-audit.mjs` is the always-running CLI wrapper, so no main-module
+// detection lives here (skills/main-guard-hygiene.test.mjs).
+
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
+import fs from 'node:fs'
+
+const run = promisify(execFile)
+
+// Aggregator / umbrella jobs only restate other jobs' results; counting them would make
+// every real failure look shared. Extend per repo with --ignore-job.
+// "PR admission gate" is a roll-up; "smoke-gate" and "route-docs-gate" are real checks.
+export const DEFAULT_IGNORE = /^(all-gates|ci|.*\b(umbrella|verdict|timing report)\b.*|.* gate)$/i
+// Step names that are runner/setup overhead rather than the job's own check.
+export const SETUP_STEP = /^(set up job|initiali[sz]e containers|stop containers|complete job|post |run actions\/(checkout|setup-|cache)|checkout|check out|setup |set up |install |restore |.*\bcache\b|.*pnpm\/action-setup|.*corepack)/i
+const STUCK_MIN = 360 // GitHub's hosted-job ceiling; anything past it hung
+
+const min = (a, b) => (Date.parse(b) - Date.parse(a)) / 60000
+const median = (xs) => { if (!xs.length) return 0; const s = [...xs].sort((a, b) => a - b); const m = s.length >> 1; return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2 }
+const pct = (xs, q) => { if (!xs.length) return 0; const s = [...xs].sort((a, b) => a - b); return s[Math.floor(q * (s.length - 1))] }
+
+// Which lane a run belongs to. Merge-queue branches are recognised for Mergify and
+// GitHub's native queue; everything else falls back to the event name.
+export function lane(r) {
+  const b = r.branch || ''
+  if (b.startsWith('mergify/') || b.startsWith('gh-readonly-queue/')) return 'queue'
+  if (r.event === 'pull_request' || r.event === 'pull_request_target') return 'pr'
+  if (r.event === 'push' && (b === r.defaultBranch || b === 'main' || b === 'master')) return 'main'
+  return r.event
+}
+
+const timed = (j) => j.started && j.completed && j.conclusion !== 'skipped'
+const dur = (j) => min(j.started, j.completed)
+const workflowKey = (r) => JSON.stringify(r.workflowId != null ? ['id', r.workflowId]
+  : r.workflowPath ? ['path', r.workflowPath] : ['name', r.workflow])
+const jobKey = (r, j) => JSON.stringify([workflowKey(r), j.name])
+
+export function analyze(runs, { ignore = DEFAULT_IGNORE, minTrees = 20 } = {}) {
+  runs = [...runs].sort((a, b) => Date.parse(a.created) - Date.parse(b.created))
+  const jobsOf = (r) => r.jobs.filter((j) => timed(j) && dur(j) >= 0)
+  const gaps = dataGaps(runs)
+  const owners = new Map(), labels = new Map()
+  for (const r of runs) for (const j of r.jobs) {
+    if (!owners.has(j.name)) owners.set(j.name, new Set())
+    owners.get(j.name).add(workflowKey(r))
+  }
+  for (const r of runs) for (const j of r.jobs) labels.set(jobKey(r, j),
+    owners.get(j.name).size > 1 ? `${r.workflow} [${r.workflowId ?? r.workflowPath ?? r.workflow}] / ${j.name}` : j.name)
+
+  // 1. Re-proved trees. Keyed by workflow + job + tree, so a PR-only job and a main-only
+  // job never collide. Scheduled re-runs are deliberate (drift, dates, upstream), so they
+  // are reported apart and not counted as waste.
+  const firstPass = new Map()
+  let passMin = 0, reprovedMin = 0, scheduledRepeatMin = 0
+  const byLanePair = {}, byJob = {}
+  for (const r of runs) {
+    for (const j of jobsOf(r)) {
+      if (j.conclusion !== 'success' || dur(j) > STUCK_MIN) continue
+      const m = dur(j); passMin += m
+      const k = JSON.stringify([jobKey(r, j), r.tree])
+      if (!r.tree || !firstPass.has(k)) { if (r.tree) firstPass.set(k, lane(r)); continue }
+      if (lane(r) === 'schedule') { scheduledRepeatMin += m; continue }
+      reprovedMin += m
+      const pair = `${firstPass.get(k)}→${lane(r)}`
+      byLanePair[pair] = (byLanePair[pair] || 0) + m
+      const label = labels.get(jobKey(r, j))
+      byJob[label] = (byJob[label] || 0) + m
+    }
+  }
+
+  // 2 + 3. Yield and overlap, per code state (tree; sha if the tree is unknown). A job
+  // "really failed" on a tree only if it never passed there — fail-then-pass is a flake.
+  const states = new Map()
+  for (const r of runs) {
+    const key = r.tree ? `tree:${r.tree}` : r.sha ? `sha:${r.sha}` : r
+    if (!states.has(key)) states.set(key, new Map())
+    const s = states.get(key)
+    for (const j of r.jobs) {
+      if (!['success', 'failure'].includes(j.conclusion) || ignore.test(j.name)) continue
+      const identity = jobKey(r, j)
+      if (!s.has(identity)) s.set(identity, new Set())
+      s.get(identity).add(j.conclusion)
+    }
+  }
+  const y = new Map()
+  const Y = (n) => {
+    if (!y.has(n)) y.set(n, { job: labels.get(n), trees: 0, failed: 0, sole: 0, flaky: 0, coFail: {} })
+    return y.get(n)
+  }
+  for (const s of states.values()) {
+    const real = [...s].filter(([, c]) => c.has('failure') && !c.has('success')).map(([n]) => n)
+    for (const [n, c] of s) {
+      const e = Y(n); e.trees++
+      if (c.has('failure') && c.has('success')) e.flaky++
+    }
+    for (const n of real) {
+      const e = Y(n); e.failed++
+      if (real.length === 1) e.sole++
+      for (const o of real) if (o !== n) e.coFail[o] = (e.coFail[o] || 0) + 1
+    }
+  }
+  const yieldEntries = [...y.values()].sort((a, b) => b.trees - a.trees)
+  const yieldRows = yieldEntries.map((e) => ({ ...e,
+    coFail: Object.fromEntries(Object.entries(e.coFail).map(([n, count]) => [labels.get(n), count])),
+  }))
+  const noSoleCatch = yieldRows.filter((e) => e.trees >= minTrees && e.sole === 0)
+  const overlap = []
+  for (const e of yieldEntries) {
+    if (e.failed < 3) continue
+    for (const [o, n] of Object.entries(e.coFail)) {
+      if (n === e.failed) overlap.push({ job: e.job, alwaysWith: labels.get(o), failures: e.failed, otherFailures: y.get(o).failed })
+    }
+  }
+
+  // 4. Setup share, per workflow/job, over passing runs (a failed step skews the ratio).
+  // Measured as 1 − own-work / job time: in a short job much of the overhead sits
+  // *between* steps (runner start, post-steps), so summing named setup steps undercounts.
+  const setup = {}
+  for (const r of runs) for (const j of r.jobs) {
+    if (j.conclusion !== 'success' || dur(j) > STUCK_MIN || ignore.test(j.name)) continue
+    const timing = jobTiming(j)
+    if (!timing.completeTiming) {
+      gaps.timings.push({ runId: r.id, job: labels.get(jobKey(r, j)), reason: 'setup/work unavailable: missing, invalid, or inconsistent step/job timestamps' })
+      continue
+    }
+    const e = (setup[jobKey(r, j)] ||= { job: labels.get(jobKey(r, j)), n: 0, totalS: 0, setupS: 0, durs: [] })
+    e.n++; e.totalS += timing.elapsedS; e.setupS += timing.setupS; e.durs.push(timing.elapsedS / 60)
+  }
+  const setupRows = Object.values(setup)
+    .map((e) => ({ job: e.job, runs: e.n, medianMin: median(e.durs), setupShare: e.totalS ? e.setupS / e.totalS : 0, setupMin: e.setupS / 60 }))
+    .filter((e) => e.runs >= 3 && e.setupShare >= 0.7)
+    .sort((a, b) => b.setupMin - a.setupMin)
+
+  // 5. Critical path on the PR lane: per head sha, first run created → last job done.
+  const prBySha = new Map()
+  for (const r of runs) if (lane(r) === 'pr') {
+    const key = r.sha ?? r
+    if (!prBySha.has(key)) prBySha.set(key, [])
+    prBySha.get(key).push(r)
+  }
+  const waits = [], lastJob = {}
+  for (const rs of prBySha.values()) {
+    // Roll-up jobs finish last by construction; the job that set the wait is the last real one.
+    const js = rs.flatMap((r) => jobsOf(r).map((j) => ({ ...j, label: labels.get(jobKey(r, j)) })))
+      .filter((j) => dur(j) <= STUCK_MIN && !ignore.test(j.name))
+    if (!js.length) continue
+    const start = rs.map((r) => r.created).filter((t) => t && Number.isFinite(Date.parse(t)))
+      .sort((a, b) => Date.parse(a) - Date.parse(b))[0]
+    const last = js.reduce((a, j) => (Date.parse(j.completed) > Date.parse(a.completed) ? j : a))
+    const interval = seconds(start, last.completed)
+    if (interval === null || rs.some((r) => !r.created || !Number.isFinite(Date.parse(r.created)))) {
+      gaps.timings.push({ runId: rs[0].id, job: last.label, reason: 'PR creation-to-completion interval unavailable' })
+      continue
+    }
+    waits.push(interval / 60)
+    lastJob[last.label] = (lastJob[last.label] || 0) + 1
+  }
+
+  // 6. Stuck jobs and runner queue time.
+  const stuck = [], queued = []
+  for (const r of runs) for (const j of r.jobs) {
+    if (j.started && j.completed && dur(j) > STUCK_MIN) stuck.push({ workflow: r.workflow, job: j.name, hours: dur(j) / 60, conclusion: j.conclusion, started: j.started, branch: r.branch })
+    if (j.conclusion === 'skipped') continue
+    const interval = seconds(j.created, j.started)
+    if (interval !== null) queued.push(interval / 60)
+    else gaps.timings.push({ runId: r.id, job: labels.get(jobKey(r, j)), reason: 'created-to-start interval unavailable' })
+  }
+
+  return {
+    minTrees,
+    dataGaps: gaps,
+    window: { runs: runs.length, from: runs[0]?.created ?? null, to: runs.at(-1)?.created ?? null },
+    reproved: { identity: 'run-metadata-tree-candidates', testedCheckoutVerified: false,
+      passMin, reprovedMin, share: passMin ? reprovedMin / passMin : 0, scheduledRepeatMin, byLanePair, byJob },
+    yield: yieldRows, noSoleCatch, overlap,
+    setup: setupRows,
+    criticalPath: { prCommits: waits.length, p50: waits.length ? median(waits) : null, p90: waits.length ? pct(waits, 0.9) : null, lastJob },
+    stuck, queue: { samples: queued.length, p50: queued.length ? median(queued) : null, p90: queued.length ? pct(queued, 0.9) : null },
+  }
+}
+
+function dataGaps(runs) {
+  return {
+    unknownTrees: runs.filter((r) => !r.tree).map((r) => ({
+      runId: r.id, sha: r.sha ?? null, reason: r.treeUnknownReason ?? 'tree not recorded in saved data',
+    })),
+    testedCheckouts: runs.filter((r) => r.tree).map((r) => ({
+      runId: r.id, reason: 'Run tree metadata does not establish the tested checkout; PR synthetic merges and explicit checkout refs may differ. Matches are candidates, not reusable gate proof.',
+    })),
+    collection: runs.flatMap((r) => (r.collectionGaps || []).map((gap) => ({ runId: r.id, ...gap }))),
+    timings: [],
+  }
+}
+
+const seconds = (start, end) => {
+  if (!start || !end) return null
+  const s = (Date.parse(end) - Date.parse(start)) / 1000
+  return Number.isFinite(s) && s >= 0 ? s : null
+}
+
+function jobTiming(j) {
+  const steps = (j.steps || []).map((s) => ({
+    name: s.name, number: s.number ?? null, conclusion: s.conclusion ?? null,
+    started: s.started ?? null, completed: s.completed ?? null,
+    durationS: s.conclusion === 'skipped' ? 0 : seconds(s.started, s.completed),
+    kind: SETUP_STEP.test(s.name) ? 'setup-heuristic' : 'work',
+  })).sort((a, b) => (b.durationS ?? -1) - (a.durationS ?? -1))
+  const elapsedS = seconds(j.started, j.completed)
+  const intervals = steps.filter((s) => s.durationS > 0).sort((a, b) => Date.parse(a.started) - Date.parse(b.started))
+  const nonOverlapping = intervals.every((s, i) => i === 0 || Date.parse(s.started) >= Date.parse(intervals[i - 1].completed))
+  const measuredWork = steps.filter((s) => s.kind === 'work').reduce((n, s) => n + (s.durationS ?? 0), 0)
+  const completeTiming = steps.length > 0 && steps.every((s) => s.durationS !== null &&
+    (s.conclusion === 'skipped' || (seconds(j.started, s.started) !== null && seconds(s.completed, j.completed) !== null))) &&
+    elapsedS !== null && measuredWork <= elapsedS && nonOverlapping
+  return { steps, elapsedS, completeTiming, workS: completeTiming ? measuredWork : null,
+    setupS: completeTiming ? Math.max(0, elapsedS - measuredWork) : null }
+}
+
+// Setup is a name-based heuristic: elapsed minus timed non-setup work includes gaps.
+// REST timestamps do not expose DAG eligibility or the instant a runner was requested.
+export function profileRun(runs, runId) {
+  const matches = runs.filter((r) => String(r.id) === String(runId))
+  if (matches.length !== 1) throw new Error(matches.length ? `multiple records for run ${runId}` : `run ${runId} not found in saved data`)
+  const r = matches[0]
+  const jobs = r.jobs.map((j) => {
+    const { steps, elapsedS, completeTiming, workS, setupS } = jobTiming(j)
+    return {
+      id: j.id ?? null, name: j.name, attempt: j.attempt ?? r.attempt ?? null,
+      url: j.url ?? null, runUrl: j.runUrl ?? r.url ?? null, conclusion: j.conclusion ?? null,
+      status: j.status ?? null, created: j.created ?? null, started: j.started ?? null, completed: j.completed ?? null,
+      runnerId: j.runnerId ?? null, runnerName: j.runnerName ?? null,
+      runnerGroupId: j.runnerGroupId ?? null, runnerGroupName: j.runnerGroupName ?? null,
+      runnerLabels: j.runnerLabels ?? null,
+      dispatchDelayS: seconds(r.created, j.created), queueS: seconds(j.created, j.started),
+      elapsedS, workS, setupS,
+      timingGaps: [
+        ...(!completeTiming ? ['setup/work unavailable: missing, invalid, or inconsistent step/job timestamps'] : []),
+        ...(seconds(j.created, j.started) === null ? ['queue interval unavailable'] : []),
+        ...(seconds(r.created, j.created) === null ? ['dispatch interval unavailable'] : []),
+      ],
+      steps,
+    }
+  })
+  return {
+    mode: 'profile', runId: r.id, attempt: r.attempt ?? null, runUrl: r.url ?? null,
+    workflow: r.workflow, workflowId: r.workflowId ?? null, workflowPath: r.workflowPath ?? null,
+    event: r.event, branch: r.branch, sha: r.sha ?? null, tree: r.tree ?? null,
+    status: r.status ?? null, conclusion: r.conclusion ?? null,
+    created: r.created ?? null, started: r.started ?? null, dataGaps: dataGaps([r]),
+    timingNotes: {
+      units: 'seconds; null means unknown',
+      setup: 'Heuristic: job elapsed minus timed non-setup steps; includes setup, teardown and inter-step gaps. Step names classify setup, not measured resource use.',
+      queue: 'Job created to started is an observed delay, not proven runner queue time; REST does not separate runner availability, dispatch or DAG dependencies.',
+      dispatch: 'Run created to job created may include dispatch/DAG delay. On reruns it can include time before the selected attempt; it is not runner queue time.',
+      scope: 'Latest recorded attempt only; no logs, billing or workflow DAG collected. Recommendations only; preserve required gates.',
+    },
+    jobs,
+  }
+}
+
+export function renderProfile(p) {
+  const show = (s) => s === null ? 'unknown' : `${s.toFixed(1)}s`
+  const L = [`# CI run profile: ${p.workflow} / ${p.runId}`, '',
+    `Attempt: ${p.attempt ?? 'unknown'}; conclusion: ${p.conclusion ?? 'unknown'}; run: ${p.runUrl ?? 'unknown'}.`, '',
+    ...Object.values(p.timingNotes), '',
+    '| job | dispatch/DAG delay | created-to-start delay (queue uncertain) | setup heuristic | work | job elapsed | conclusion | runner |',
+    '|---|---|---|---|---|---|---|---|']
+  for (const j of p.jobs) L.push(`| ${j.name} | ${show(j.dispatchDelayS)} | ${show(j.queueS)} | ${show(j.setupS)} | ${show(j.workS)} | ${show(j.elapsedS)} | ${j.conclusion ?? 'unknown'} | ${j.runnerName ?? 'unknown'} (${j.runnerLabels?.join(', ') ?? 'labels unknown'}) |`)
+  for (const j of p.jobs) {
+    L.push('', `## ${j.name} (attempt ${j.attempt ?? 'unknown'})`, '', `Job: ${j.url ?? 'unknown'}; run: ${j.runUrl ?? 'unknown'}.`)
+    for (const gap of j.timingGaps) L.push(`- ${gap}`)
+    L.push('', '| step (longest first) | duration | classification | conclusion |', '|---|---|---|---|')
+    for (const s of j.steps) L.push(`| ${s.name} | ${show(s.durationS)} | ${s.kind} | ${s.conclusion ?? 'unknown'} |`)
+  }
+  appendGaps(L, p.dataGaps)
+  return L.join('\n')
+}
+
+function appendGaps(L, gaps) {
+  if (!gaps.unknownTrees.length && !gaps.testedCheckouts.length && !gaps.collection.length && !gaps.timings.length) return
+  L.push('', '## Data gaps', '')
+  L.push(`${gaps.unknownTrees.length} run(s) have no verified tree; excluded from identical-tree repeat evidence. Catch yield falls back to commit SHA, not verified tree equality.`)
+  for (const g of gaps.unknownTrees) L.push(`- Run ${g.runId}, SHA ${g.sha ?? 'unknown'}: ${g.reason}`)
+  if (gaps.testedCheckouts.length) L.push(`- ${gaps.testedCheckouts.length} run(s): ${gaps.testedCheckouts[0].reason}`)
+  for (const g of gaps.collection) L.push(`- Run ${g.runId}: ${g.reason}`)
+  if (gaps.timings.length) L.push(`- ${gaps.timings.length} unavailable timing measurement(s); excluded rather than counted as zero.`)
+  for (const g of gaps.timings.slice(0, 6)) L.push(`- Run ${g.runId}, ${g.job}: ${g.reason}`)
+}
+
+const f1 = (x) => x.toFixed(1)
+const minutes = (m) => m === null ? 'unknown' : `${f1(m)}m`
+const top = (o, n = 6) => Object.entries(o).sort((a, b) => b[1] - a[1]).slice(0, n)
+
+export function render(a, repo = '') {
+  const L = []
+  L.push(`# CI run audit${repo ? ` — ${repo}` : ''}`, '')
+  L.push(`${a.window.runs} completed runs, ${a.window.from?.slice(0, 10)} → ${a.window.to?.slice(0, 10)}. Report-only: each row is a candidate with its evidence, not a verdict.`, '')
+
+  L.push('## 1. Re-proved trees', '')
+  L.push(`**${f1(a.reproved.reprovedMin)} of ${f1(a.reproved.passMin)} passing job-minutes (${Math.round(a.reproved.share * 100)}%)** matched prior workflow/job run-tree metadata; these are repeat candidates, not verified repeated tested checkouts.` +
+    (a.reproved.scheduledRepeatMin ? ` Scheduled repeats (${f1(a.reproved.scheduledRepeatMin)} min) are left out as deliberate.` : ''), '')
+  if (top(a.reproved.byLanePair).length) {
+    L.push('| first passed → again on | minutes |', '|---|---|')
+    for (const [k, v] of top(a.reproved.byLanePair)) L.push(`| ${k} | ${f1(v)} |`)
+    L.push('', '| job | re-proved minutes |', '|---|---|')
+    for (const [k, v] of top(a.reproved.byJob)) L.push(`| ${k} | ${f1(v)} |`)
+    L.push('', 'Inspect actual checkout refs and execution inputs before proposing reuse. A PR head tree can differ from its tested synthetic merge. Changed combined queue trees still need proof; main reuse requires complete trusted queue evidence. Date- or network-dependent jobs can legitimately repeat.')
+  }
+  L.push('')
+
+  L.push('## 2. Catch yield', '')
+  L.push('Per metadata code state (resolved run tree, otherwise commit SHA only; tested checkout unverified). *failed* = failed and never passed on that state; *sole* = the only job that failed there; *flaky* = failed and passed on the same state, an instability lead rather than a diagnosed test flake.', '')
+  L.push('| job | trees | failed | sole | flaky | fails most often with |', '|---|---|---|---|---|---|')
+  for (const e of a.yield) {
+    const w = top(e.coFail, 1)[0]
+    L.push(`| ${e.job} | ${e.trees} | ${e.failed} | ${e.sole} | ${e.flaky} | ${w ? `${w[0]} ×${w[1]}` : ''} |`)
+  }
+  L.push('')
+  if (a.noSoleCatch.length) {
+    L.push(`**No sole catch** in this window: ${a.noSoleCatch.map((e) => `\`${e.job}\` (${e.trees} trees${e.failed ? `, ${e.failed} shared failures` : ', never failed'})`).join(', ')}.`)
+    L.push('Candidates to merge into another job, move off the PR lane, or retire — unless the job guards something rare and costly (a security or money path), where zero catches is the point. Widen the window before deciding.', '')
+  } else if (!a.yield.some((e) => e.trees >= a.minTrees)) {
+    L.push(`No job ran on ${a.minTrees}+ code states, too few to call any job zero-yield. Widen the window (\`--runs\`) or lower \`--min-trees\`.`, '')
+  }
+
+  L.push('## 3. Overlap', '')
+  if (!a.overlap.length) L.push('No job with ≥3 failures whose every failure coincided with another job failing.', '')
+  else {
+    L.push('| job | every failure also failed | its failures | other job\'s failures |', '|---|---|---|---|')
+    for (const o of a.overlap) L.push(`| ${o.job} | ${o.alwaysWith} | ${o.failures} | ${o.otherFailures} |`)
+    L.push('', 'Either the two test the same thing, or both depend on one broken fixture (shared boot, DB, build). Read one failure log of each before merging them.', '')
+  }
+
+  L.push('## 4. Setup share', '')
+  if (!a.setup.length) L.push('No measured job meets the ≥70% setup candidate threshold across at least three runs.', '')
+  else {
+    L.push('| job | runs | median | setup share | setup minutes |', '|---|---|---|---|---|')
+    for (const e of a.setup) L.push(`| ${e.job} | ${e.runs} | ${f1(e.medianMin)}m | ${Math.round(e.setupShare * 100)}% | ${f1(e.setupMin)} |`)
+    L.push('', 'Jobs that are mostly setup are cheap to merge: several such checks can share one job, one checkout, one install.', '')
+  }
+
+  L.push('## 5. Critical path (PR lane)', '')
+  const c = a.criticalPath
+  L.push(`${c.prCommits} PR commits: wait p50 **${minutes(c.p50)}**, p90 **${minutes(c.p90)}**. Last job to finish:`, '')
+  for (const [k, v] of top(c.lastJob, 5)) L.push(`- ${k}: ${v}`)
+  L.push('', 'The last-finishing job is a lead, not the whole critical path. Inspect its prerequisites: reducing an upstream build or dispatch delay can shorten PR latency too. This timestamp report does not resolve the workflow DAG.', '')
+
+  L.push('## 6. Stuck jobs and runner queue', '')
+  L.push(`Queue wait (job created → started, not proven runner-only queue): ${a.queue.samples} measured job(s), p50 ${minutes(a.queue.p50)}, p90 ${minutes(a.queue.p90)}. REST does not separate runner queue from dispatch/DAG delay.`)
+  if (a.stuck.length) {
+    L.push('', `**${a.stuck.length} job(s) ran past 6h**, so they hung rather than ran slowly (check the runner, and set \`timeout-minutes\`):`, '')
+    for (const s of a.stuck.slice(0, 10)) L.push(`- ${s.workflow} / ${s.job}: ${f1(s.hours)}h, ${s.conclusion}, ${s.started.slice(0, 16)}, ${s.branch}`)
+  }
+  appendGaps(L, a.dataGaps)
+  L.push('')
+  return L.join('\n')
+}
+
+// ---- fetch (thin; everything above is pure) ----
+
+const MAX_RUNS = 2000
+const MAX_JOB_PAGES = 100 // 10,000 jobs; refuse instead of silently truncating.
+const GH_TIMEOUT_MS = 60000
+
+async function gh(path) {
+  try {
+    const { stdout } = await run('gh', ['api', path], {
+      timeout: GH_TIMEOUT_MS, killSignal: 'SIGKILL', maxBuffer: 16 * 1024 * 1024,
+      env: { ...process.env, GH_PROMPT_DISABLED: '1' },
+    })
+    return JSON.parse(stdout)
+  } catch (e) {
+    const status = /HTTP\s+(\d{3})/.exec(e.stderr || '')?.[1]
+    if (status) e.status = Number(status)
+    throw e
+  }
+}
+
+async function pool(items, n, fn) {
+  const out = new Array(items.length); let i = 0, failure
+  // Stop assigning work on error, but drain in-flight gh calls before returning.
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (!failure && i < items.length) {
+      const k = i++
+      try { out[k] = await fn(items[k]) } catch (e) { failure ||= e }
+    }
+  }))
+  if (failure) throw failure
+  return out
+}
+
+function validRepo(repo) {
+  if (typeof repo !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(repo)) throw new Error('--repo must be owner/name')
+}
+
+function positiveInt(value, label, max = Number.MAX_SAFE_INTEGER) {
+  if (!/^[1-9]\d*$/.test(String(value)) || !Number.isSafeInteger(Number(value)) || Number(value) > max) throw new Error(`${label} must be an integer from 1 to ${max}`)
+  return Number(value)
+}
+
+function collector(repo, api) {
+  validRepo(repo)
+  const trees = new Map()
+  async function treeOf(r) {
+    if (r.head_commit?.tree_id) return { tree: r.head_commit.tree_id, treeSource: 'actions.head_commit.tree_id', treeUnknownReason: null }
+    if (!r.head_sha) return { tree: null, treeSource: null, treeUnknownReason: 'head SHA unavailable' }
+    // Cache the promise, including unavailable results, before concurrent runs await it.
+    if (!trees.has(r.head_sha)) trees.set(r.head_sha, (async () => {
+      try {
+        const commit = await api(`repos/${repo}/commits/${encodeURIComponent(r.head_sha)}`)
+        const tree = commit.commit?.tree?.sha
+        return tree ? { tree, treeSource: 'commits.commit.tree.sha', treeUnknownReason: null }
+          : { tree: null, treeSource: null, treeUnknownReason: 'commit response has no tree SHA' }
+      } catch (e) {
+        if (![404, 410, 422].includes(e.status)) throw e
+        return { tree: null, treeSource: null, treeUnknownReason: `commit unavailable (HTTP ${e.status}); deleted queue commits may no longer resolve` }
+      }
+    })())
+    return trees.get(r.head_sha)
+  }
+  async function collect(r, defaultBranch) {
+    positiveInt(r.id, 'run ID')
+    const attempt = r.run_attempt == null ? null : positiveInt(r.run_attempt, 'run attempt')
+    const endpoint = attempt === null ? `repos/${repo}/actions/runs/${r.id}/jobs?filter=latest&`
+      : `repos/${repo}/actions/runs/${r.id}/attempts/${attempt}/jobs?`
+    const jobs = []
+    let complete = false, total
+    for (let page = 1; page <= MAX_JOB_PAGES; page++) {
+      const result = await api(`${endpoint}per_page=100&page=${page}`)
+      if (!Array.isArray(result.jobs) || result.jobs.length > 100) throw new Error(`invalid jobs response for run ${r.id}`)
+      if (result.total_count != null) {
+        if (!Number.isSafeInteger(result.total_count) || result.total_count < 0 || result.total_count > MAX_JOB_PAGES * 100) throw new Error(`job pagination limit or invalid total for run ${r.id}`)
+        if (total !== undefined && total !== result.total_count) throw new Error(`job total changed during collection for run ${r.id}`)
+        total = result.total_count
+      }
+      jobs.push(...result.jobs)
+      if (total !== undefined && jobs.length >= total) {
+        if (jobs.length !== total) throw new Error(`inconsistent job total for run ${r.id}`)
+        complete = true; break
+      }
+      if (result.jobs.length < 100) {
+        if (total !== undefined && jobs.length < total) throw new Error(`incomplete jobs response for run ${r.id}`)
+        complete = true; break
+      }
+    }
+    if (!complete) throw new Error(`job pagination limit reached for run ${r.id}; refusing partial data`)
+    const seen = new Set()
+    for (const j of jobs) {
+      positiveInt(j.id, 'job ID')
+      if (seen.has(j.id)) throw new Error(`duplicate job ${j.id} while collecting run ${r.id}`)
+      seen.add(j.id)
+      if (attempt !== null && j.run_attempt != null && j.run_attempt !== attempt) throw new Error(`job attempt mismatch for run ${r.id}`)
+    }
+    return {
+      id: r.id, attempt, url: r.html_url ?? null,
+      workflow: r.name, workflowId: r.workflow_id ?? null, workflowPath: r.path ?? null,
+      event: r.event, branch: r.head_branch, defaultBranch,
+      status: r.status ?? null, conclusion: r.conclusion ?? null,
+      sha: r.head_sha ?? null, ...await treeOf(r), created: r.created_at, started: r.run_started_at ?? null,
+      collectionScope: 'latest-attempt',
+      collectionGaps: attempt === null ? [{ reason: 'run attempt unavailable; collected latest jobs only' }]
+        : attempt > 1 ? [{ reason: `only attempt ${attempt} collected; earlier attempts are not included` }] : [],
+      jobs: jobs.map((j) => ({
+        id: j.id, name: j.name, attempt: j.run_attempt ?? attempt, url: j.html_url ?? null, runUrl: r.html_url ?? null,
+        status: j.status ?? null, conclusion: j.conclusion ?? null,
+        created: j.created_at ?? null, started: j.started_at ?? null, completed: j.completed_at ?? null,
+        runnerId: j.runner_id ?? null, runnerName: j.runner_name ?? null,
+        runnerGroupId: j.runner_group_id ?? null, runnerGroupName: j.runner_group_name ?? null,
+        runnerLabels: j.labels ?? null,
+        steps: (j.steps || []).map((s) => ({
+          number: s.number ?? null, name: s.name, conclusion: s.conclusion ?? null,
+          started: s.started_at ?? null, completed: s.completed_at ?? null,
+        })),
+      })),
+    }
+  }
+  return collect
+}
+
+// The optional API function is the gh/provider boundary for fixture tests, never a CLI flag.
+export async function fetchRuns(repo, want = 300, api = gh) {
+  want = positiveInt(want, '--runs', MAX_RUNS)
+  const collect = collector(repo, api)
+  const meta = await api(`repos/${repo}`)
+  const raw = []
+  for (let page = 1; raw.length < want && page <= 20; page++) {
+    const result = await api(`repos/${repo}/actions/runs?status=completed&per_page=100&page=${page}`)
+    if (!Array.isArray(result.workflow_runs) || result.workflow_runs.length > 100) throw new Error('invalid workflow runs response')
+    raw.push(...result.workflow_runs)
+    if (result.workflow_runs.length < 100) break
+  }
+  const selected = raw.slice(0, want)
+  if (new Set(selected.map((r) => String(r.id))).size !== selected.length) throw new Error('duplicate run ID during history pagination; refusing partial evidence')
+  const runs = await pool(selected, 8, (r) => collect(r, meta.default_branch))
+  validateRuns(runs)
+  return runs
+}
+
+export async function fetchRun(repo, runId, api = gh) {
+  runId = positiveInt(runId, '--profile-run')
+  const collect = collector(repo, api)
+  const meta = await api(`repos/${repo}`)
+  const raw = await api(`repos/${repo}/actions/runs/${runId}`)
+  if (String(raw.id) !== String(runId)) throw new Error('requested run ID does not match response')
+  const runs = [await collect(raw, meta.default_branch)]
+  validateRuns(runs)
+  return runs
+}
+
+function validateRuns(runs) {
+  if (!Array.isArray(runs)) throw new Error('saved data must be an array of runs')
+  const timestamp = (value, label) => {
+    if (value != null && (typeof value !== 'string' || !Number.isFinite(Date.parse(value)))) throw new Error(`invalid ${label} timestamp`)
+  }
+  for (const r of runs) {
+    if (!r || typeof r !== 'object' || !Array.isArray(r.jobs) || typeof r.workflow !== 'string') throw new Error('each run must have a workflow name and jobs array')
+    positiveInt(r.id, 'run ID')
+    if (r.attempt != null) positiveInt(r.attempt, 'run attempt')
+    if (r.tree != null && (typeof r.tree !== 'string' || !r.tree)) throw new Error('tree must be a nonempty hash string or null')
+    if (r.sha != null && typeof r.sha !== 'string') throw new Error('sha must be a string or null')
+    if (r.collectionGaps != null && (!Array.isArray(r.collectionGaps) || r.collectionGaps.some((g) => !g || typeof g.reason !== 'string'))) throw new Error('invalid collectionGaps')
+    timestamp(r.created, 'run created'); timestamp(r.started, 'run started')
+    for (const j of r.jobs) {
+      if (!j || typeof j.name !== 'string' || (j.steps != null && !Array.isArray(j.steps))) throw new Error('each job must have a name and optional steps array')
+      if (j.attempt != null) positiveInt(j.attempt, 'job attempt')
+      if (j.runnerLabels != null && (!Array.isArray(j.runnerLabels) || j.runnerLabels.some((s) => typeof s !== 'string'))) throw new Error('runnerLabels must be an array of strings or null')
+      timestamp(j.created, 'job created'); timestamp(j.started, 'job started'); timestamp(j.completed, 'job completed')
+      for (const s of j.steps || []) {
+        if (!s || typeof s.name !== 'string') throw new Error('each step must have a name')
+        timestamp(s.started, 'step started'); timestamp(s.completed, 'step completed')
+      }
+    }
+  }
+}
+
+const USAGE = 'usage: ci-run-audit.mjs (--repo <owner/name> [--runs 300] | --from f.json) [--profile-run <runId>] [--save f.json] [--json] [--ignore-job <regex>] [--min-trees 20]'
+
+function args(argv) {
+  const o = { runs: 300, json: false, minTrees: 20 }, seen = new Set()
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i]
+    if (seen.has(a)) throw new Error(`duplicate argument: ${a}`)
+    seen.add(a)
+    const v = () => {
+      const value = argv[++i]
+      if (!value || value.startsWith('-')) throw new Error(`missing value for ${a}`)
+      return value
+    }
+    if (a === '--repo') o.repo = v()
+    else if (a === '--runs') o.runs = positiveInt(v(), '--runs', MAX_RUNS)
+    else if (a === '--profile-run') o.profileRun = positiveInt(v(), '--profile-run')
+    else if (a === '--save') o.save = v()
+    else if (a === '--from') o.from = v()
+    else if (a === '--json') o.json = true
+    else if (a === '--ignore-job') o.ignore = v()
+    else if (a === '--min-trees') o.minTrees = positiveInt(v(), '--min-trees')
+    else if (a === '--help' || a === '-h') o.help = true
+    else throw new Error(`unknown argument: ${a}`)
+  }
+  if (o.help) return o
+  if (Boolean(o.repo) === Boolean(o.from)) throw new Error('provide exactly one of --repo or --from')
+  if (o.repo) validRepo(o.repo)
+  if (seen.has('--runs') && (o.from || o.profileRun)) throw new Error('--runs is only supported for live history collection')
+  if (o.profileRun && (seen.has('--ignore-job') || seen.has('--min-trees'))) throw new Error('--ignore-job and --min-trees are audit-only; profiles include all jobs')
+  o.ignoreRegex = o.ignore ? new RegExp(`${DEFAULT_IGNORE.source}|${o.ignore}`, 'i') : DEFAULT_IGNORE
+  return o
+}
+
+export async function runCli(argv) {
+  try {
+    const o = args(argv)
+    if (o.help) { console.error(USAGE); return 0 }
+    const runs = o.from ? JSON.parse(fs.readFileSync(o.from, 'utf8'))
+      : o.profileRun ? await fetchRun(o.repo, o.profileRun) : await fetchRuns(o.repo, o.runs)
+    validateRuns(runs)
+    const report = o.profileRun ? profileRun(runs, o.profileRun) : analyze(runs, { ignore: o.ignoreRegex, minTrees: o.minTrees })
+    const output = o.json ? JSON.stringify(report, null, 2) : o.profileRun ? renderProfile(report) : render(report, o.repo || o.from)
+    // Save the collected array, never the derived report; saved profiles remain re-auditable.
+    if (o.save) fs.writeFileSync(o.save, JSON.stringify(runs))
+    process.stdout.write(output + '\n')
+    return 0
+  } catch (e) {
+    console.error(`ci-run-audit: ${e.message}`)
+    return 2
+  }
+}

package/skills/cicd/github-actions/scripts/ci-run-audit.mjs

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { runCli } from './ci-run-audit-core.mjs'
+
+process.exitCode = await runCli(process.argv.slice(2))