@futurelastic/colab-handbook 1.16.0-rc.18 → 1.16.0-rc.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -33,5 +33,5 @@
33
33
  "engines": {
34
34
  "node": ">=18"
35
35
  },
36
- "version": "1.16.0-rc.18"
36
+ "version": "1.16.0-rc.19"
37
37
  }
package/tools/colab CHANGED
@@ -4797,7 +4797,8 @@ CI grant merges into a repo whose own test suite is known-failing:
4797
4797
  "Red" here (#155) is the SAME verdict \`ship\`'s trunk-CI-green check uses — an ordinary
4798
4798
  still-running run is NOT red (nothing for a grant to exempt; \`ship\` would clear it on its own),
4799
4799
  but a WEDGED run (queued with zero jobs, or stuck past a generous age backstop — the shape a
4800
- billing lockout or dead runner pool leaves) reclassifies to red, because \`ship\` cannot clear
4800
+ billing lockout or dead runner pool leaves; or every job completed and the forge never concluded
4801
+ the run, which cancel and rerun cannot move, #650) reclassifies to red, because \`ship\` cannot clear
4801
4802
  that on its own either and this is the sanctioned door for exactly that case. Before #155 the
4802
4803
  two commands computed this independently and disagreed: a grant could be minted for a state
4803
4804
  \`ship\` would never consult it for, reading live in \`--list\` while doing nothing.
@@ -11836,12 +11837,15 @@ function classifyCiRunRaw(repoAbs, run, at) {
11836
11837
  }
11837
11838
  // Non-completed, non-none: running/queued/etc. Ordinarily SELF_CLEARING — but #155's wedged
11838
11839
  // shape (queued, zero jobs, unmoving for hours) can never clear on its own, so check for it.
11839
- const jobCount = run.databaseId != null ? git.ghRunJobCount(repoAbs, run.databaseId) : null;
11840
+ // #650: one `gh run view --json jobs` — the same single call the job count used to cost — so the
11841
+ // jobs' statuses can tell a run the forge never concluded (every job done) from #155's shapes.
11842
+ const jobs = run.databaseId != null ? git.ghRunJobs(repoAbs, run.databaseId) : null;
11843
+ const jobCount = Array.isArray(jobs) ? jobs.length : null;
11840
11844
  // #559: wedge age and zero-jobs floor follow the repo's measured CI (cached profile; bootstrap = 6h / 2m).
11841
11845
  const th = ciProfileFor(repoAbs).th;
11842
- const wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount }, { wedgeAgeSec: th.wedgeAgeSec, zeroJobsFloorSec: th.zeroJobsFloorSec });
11846
+ const wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount, jobs }, { wedgeAgeSec: th.wedgeAgeSec, zeroJobsFloorSec: th.zeroJobsFloorSec });
11843
11847
  if (wedge.wedged) {
11844
- return { ok: false, class: HUMAN_GATED, detail: `run for ${at}: status=${run.status} — WEDGED (${wedge.reason}); a human must look, not retry` };
11848
+ return { ok: false, class: HUMAN_GATED, wedge: wedge.shape, detail: `run for ${at}: status=${run.status} — WEDGED (${wedge.reason}); ${ciVerdict.wedgeExit(wedge)}` };
11845
11849
  }
11846
11850
  return { ok: false, class: SELF_CLEARING, detail: `run for ${at}: status=${run.status} — still running, retry later` };
11847
11851
  }
@@ -15932,7 +15936,8 @@ The first output word is one verdict:
15932
15936
  GREEN every judged workflow's newest run at the sha succeeded — ship's row would pass
15933
15937
  RED a judged run completed and did not succeed — trunk is red
15934
15938
  PENDING a judged run is still in flight (or the sha is minutes old with no run yet)
15935
- WEDGED a run that will never move on its own (zero jobs / hours unmoving, #155)
15939
+ WEDGED a run that will never move on its own (zero jobs / hours unmoving, #155; every job
15940
+ completed but the forge never concluded the run, #650 — the detail names the exits)
15936
15941
  NONE no repo-owned run exists at the sha — CI not wired or never started
15937
15942
  UNREADABLE gh unusable or the read failed — never "green"
15938
15943
 
@@ -15951,7 +15956,8 @@ checkout (COLAB_TRUNK_CI_TTL=<sec> overrides, 0 disables; --fresh skips the cach
15951
15956
  To WAIT for CI, use \`colab ci-wait\` — never a loop around this command.
15952
15957
 
15953
15958
  --fresh ignore a cached verdict (still refreshes the cache)
15954
- --json { verdict, sha, ok, class, detail, trunk[, cited: [<run url>…] on a #493 skip][, cachedAgeSec] }
15959
+ --json { verdict, sha, ok, class, detail, trunk[, wedge: zero-jobs|unconcluded|age on a WEDGED (#650)]
15960
+ [, cited: [<run url>…] on a #493 skip][, cachedAgeSec] }
15955
15961
  Exit 0 whenever a verdict was read (whatever it is); 2 on UNREADABLE.`;
15956
15962
 
15957
15963
  /** Map readCiVerdict's result to the closed trunk-ci vocabulary. Pure — exported for tests. */
@@ -15992,6 +15998,7 @@ function cmdTrunkCi(argv) {
15992
15998
  let detail = r.verdict.detail;
15993
15999
  if (verdict === 'UNREADABLE' && gh && !git.ghState().ok) detail += ` (${ghUnusableWhy(repoAbs)})`;
15994
16000
  out = { verdict, trunk, sha, ok: r.verdict.ok, class: r.verdict.class || null, detail,
16001
+ ...(r.verdict.wedge ? { wedge: r.verdict.wedge } : {}),
15995
16002
  ...(r.verdict.treeGreen ? { cited: r.verdict.treeGreen } : {}),
15996
16003
  ...(r.run && Array.isArray(r.run.setAside) && r.run.setAside.length ? { setAside: r.run.setAside } : {}),
15997
16004
  ...(r.run && Array.isArray(r.run.dispatchCounted) && r.run.dispatchCounted.length ? { dispatchCounted: r.run.dispatchCounted } : {}) };
@@ -37,6 +37,20 @@
37
37
  * materialization window was not re-measured empirically here. Costs nothing against the signal's
38
38
  * actual target: a billing lockout or dead runner pool leaves a run at zero jobs for far longer than
39
39
  * two minutes, so the floor never masks a real wedge, only the moment right after creation.
40
+ *
41
+ * Amendment (#650): a THIRD shape, told apart from both of #155's — every job reached `completed`,
42
+ * yet the forge never concluded the run (run and check suite stay `in_progress`, `updated_at`
43
+ * frozen). Measured during a forge incident that delayed Actions result processing: the skipped
44
+ * jobs' check runs were written ~6 minutes after the job they `needs:`, and from then on the REST
45
+ * API refused every exit at once — cancel/force-cancel 409 "not in progress", every rerun and the
46
+ * check-suite rerequest 403 "already running". Nothing CI can still do will move it; only the forge
47
+ * (or a human) can. Before this the shape had no signal of its own: the jobs are non-empty, so it
48
+ * read "still running, retry later" until the 6h age backstop, then a WEDGED that named no exit.
49
+ * Signal: the job list is non-empty, every job is `completed`, and the newest job timestamp is past
50
+ * UNCONCLUDED_FLOOR_MINUTES. Every job needs a readable timestamp — a missing one never manufactures
51
+ * a positive (same posture as the two signals above). Its verdict is still HUMAN_GATED, never green,
52
+ * even when every job succeeded: the forge has not said the run passed (epic #652, fail closed).
53
+ * `wedgeExit` below names the exits that work for each shape, so the verdict is actionable.
40
54
  */
41
55
 
42
56
  // Generous on purpose (see doc comment above) — an ordinary run finishes in minutes to tens of
@@ -50,6 +64,11 @@ const WEDGE_AGE_HOURS = 6;
50
64
  // hours, not minutes).
51
65
  const ZERO_JOBS_AGE_FLOOR_MINUTES = 2;
52
66
 
67
+ // #650: how long after its newest job finished a run may still be waiting for the forge to conclude
68
+ // it. Normally that takes seconds; the measured incident wrote check runs ~6 minutes late, so 15
69
+ // minutes is more than twice the worst lag seen before the read says "the forge, not CI".
70
+ const UNCONCLUDED_FLOOR_MINUTES = 15;
71
+
53
72
  function hoursSince(iso, nowMs) {
54
73
  if (!iso) return null;
55
74
  const t = Date.parse(iso);
@@ -59,7 +78,41 @@ function hoursSince(iso, nowMs) {
59
78
  }
60
79
 
61
80
  /**
62
- * `run` is the `{status, conclusion, sha, createdAt, jobCount}` shape — `createdAt`/`jobCount` may be
81
+ * #650: did every job of a non-completed run finish, long enough ago that the forge, not CI, is what
82
+ * is holding it? `jobs` is tools/lib/git.js `ghRunJobs`'s shape (`[{name, status, conclusion,
83
+ * startedAt, completedAt}]`), or null when unread. Returns `{ unconcluded, minutes, failed, counts }`:
84
+ * `minutes` since the newest job timestamp (null when not measurable), `failed` the names of jobs
85
+ * that concluded anything but success/skipped/neutral, `counts` conclusion → number of jobs.
86
+ * Never a positive from a missing signal: null/empty jobs, any job not `completed`, or any job with
87
+ * no readable timestamp all read `unconcluded: false`.
88
+ */
89
+ function unconcludedJobs(jobs, opts = {}) {
90
+ const no = { unconcluded: false, minutes: null, failed: [], counts: {} };
91
+ if (!Array.isArray(jobs) || jobs.length === 0) return no;
92
+ if (!jobs.every((j) => j && j.status === 'completed')) return no;
93
+ let newest = -Infinity;
94
+ for (const j of jobs) {
95
+ // The measured incident wrote a skipped job's started_at AFTER its completed_at — so take the
96
+ // later of the two, per job, and the latest over all jobs.
97
+ const ts = [j.startedAt, j.completedAt].map((x) => (x ? Date.parse(x) : NaN)).filter((t) => !Number.isNaN(t));
98
+ if (!ts.length) return no;
99
+ newest = Math.max(newest, ...ts);
100
+ }
101
+ const now = opts.nowMs != null ? opts.nowMs : Date.now();
102
+ const minutes = (now - newest) / 60000;
103
+ const floorMin = Number.isFinite(opts.floorSec) ? opts.floorSec / 60 : UNCONCLUDED_FLOOR_MINUTES;
104
+ const counts = {};
105
+ const failed = [];
106
+ for (const j of jobs) {
107
+ const c = j.conclusion || 'none';
108
+ counts[c] = (counts[c] || 0) + 1;
109
+ if (!['success', 'skipped', 'neutral'].includes(c)) failed.push(`${j.name || 'unnamed'} (${c})`);
110
+ }
111
+ return { unconcluded: minutes >= floorMin, minutes, failed, counts, floorMin };
112
+ }
113
+
114
+ /**
115
+ * `run` is the `{status, conclusion, sha, createdAt, jobCount, jobs}` shape — `createdAt`/`jobCount` may be
63
116
  * null when not measured (job count is fetched lazily by the caller, only for a run already known to
64
117
  * be non-completed; see tools/lib/git.js `ghRunJobCount`'s doc comment for why).
65
118
  *
@@ -67,10 +120,15 @@ function hoursSince(iso, nowMs) {
67
120
  * never wedged by definition (it finished, however it finished), and `'none'` means no run exists at
68
121
  * all, a different and already-handled case (HUMAN_GATED — "CI is not wired for this ref").
69
122
  *
70
- * Returns `{ wedged, reason }`. `reason` is null when `wedged` is false.
123
+ * `jobs` (#650) is optional — `ghRunJobs`'s list, read by the same lazy call; when present it is what
124
+ * tells the forge-stuck shape (every job completed, run never concluded) from #155's two.
125
+ *
126
+ * Returns `{ wedged, reason, shape }`. `reason` and `shape` are null when `wedged` is false; `shape` is
127
+ * `'zero-jobs'` | `'unconcluded'` | `'age'` (which signal fired). An `'unconcluded'` verdict also
128
+ * carries `failed` — the jobs that concluded red, if any.
71
129
  */
72
130
  function wedgedVerdict(run, opts = {}) {
73
- if (!run || run.status === 'completed' || run.status === 'none') return { wedged: false, reason: null };
131
+ if (!run || run.status === 'completed' || run.status === 'none') return { wedged: false, reason: null, shape: null };
74
132
  const age = hoursSince(run.createdAt, opts.nowMs);
75
133
  // #559: both bounds may come from the repo's measured CI (tools/lib/ci-profile.js); absent → today's.
76
134
  const floorMin = Number.isFinite(opts.zeroJobsFloorSec) ? opts.zeroJobsFloorSec / 60 : ZERO_JOBS_AGE_FLOOR_MINUTES;
@@ -82,15 +140,48 @@ function wedgedVerdict(run, opts = {}) {
82
140
  if (age !== null && age >= floorHours) {
83
141
  return {
84
142
  wedged: true,
143
+ shape: 'zero-jobs',
85
144
  reason: `zero jobs — no runner has ever picked it up (unmoving for ${age.toFixed(2)}h, past the ${fmtMin(floorMin)} floor)`,
86
145
  };
87
146
  }
88
- return { wedged: false, reason: null };
147
+ return { wedged: false, reason: null, shape: null };
148
+ }
149
+ // #650: checked BEFORE the age backstop — a run in this shape is old by the time anyone looks, and
150
+ // the backstop's reason ("status=in_progress for 7h") would hide the one fact that decides the exit.
151
+ const u = unconcludedJobs(run.jobs, { nowMs: opts.nowMs, floorSec: opts.unconcludedFloorSec });
152
+ if (u.unconcluded) {
153
+ const tally = Object.keys(u.counts).sort().map((c) => `${u.counts[c]} ${c}`).join(', ');
154
+ return {
155
+ wedged: true,
156
+ shape: 'unconcluded',
157
+ failed: u.failed,
158
+ reason: `every job completed (${tally}) ${u.minutes.toFixed(0)}m ago, but the forge never concluded the run — past the ${fmtMin(u.floorMin)} floor`,
159
+ };
89
160
  }
90
161
  if (age !== null && age >= wedgeHours) {
91
- return { wedged: true, reason: `status=${run.status} for ${age.toFixed(1)}h (>= ${fmtHours(wedgeHours)} backstop)` };
162
+ return { wedged: true, shape: 'age', reason: `status=${run.status} for ${age.toFixed(1)}h (>= ${fmtHours(wedgeHours)} backstop)` };
163
+ }
164
+ return { wedged: false, reason: null, shape: null };
165
+ }
166
+
167
+ /**
168
+ * The exit a WEDGED verdict names (#650) — the tail of the detail line after the reason. #155's two
169
+ * shapes keep their words: a human must look (billing, a runner pool, a dead runner). The forge-stuck
170
+ * shape names what works and what was measured not to, so nobody retries an exit the forge refuses.
171
+ * The ci-grant authority line does not move: no exit here lets an agent's own run stand in for
172
+ * trunk's (epic #652).
173
+ */
174
+ function wedgeExit(wedge) {
175
+ if (!wedge || wedge.shape !== 'unconcluded') return 'a human must look, not retry';
176
+ if (wedge.failed && wedge.failed.length) {
177
+ return `red in substance — ${wedge.failed.join(', ')} did not succeed, whatever the run's status says; ` +
178
+ "fix trunk (the fix's own push gets a new run), and only the branch that IS that fix may take a human ci-grant";
92
179
  }
93
- return { wedged: false, reason: null };
180
+ return 'nothing CI can do moves it, and nothing here makes it green. Works: wait for the forge to conclude the run ' +
181
+ '(re-read with `colab trunk-ci --fresh`), or a human `colab ci-grant <issue> --branch <branch>` per candidate. ' +
182
+ 'Does not work: cancel / force-cancel (409 "not in progress"), rerun / rerun-failed-jobs / check-suite rerequest ' +
183
+ '(403 "already running"), a workflow_dispatch at this sha (the lost-push rescue skips it while this push run exists, #567). ' +
184
+ "Deleting the run cannot be undone — the owner's call";
94
185
  }
95
186
 
96
187
  const fmtMin = (m) => `${Number.isInteger(m) ? m : m.toFixed(1)}m`;
@@ -118,4 +209,4 @@ function emptyReadVerdict(committedAtMs, opts = {}) {
118
209
  return { fresh: minutes >= 0 && minutes < graceMinutes, minutes, graceMinutes };
119
210
  }
120
211
 
121
- module.exports = { WEDGE_AGE_HOURS, ZERO_JOBS_AGE_FLOOR_MINUTES, EMPTY_READ_GRACE_MINUTES, wedgedVerdict, emptyReadVerdict };
212
+ module.exports = { WEDGE_AGE_HOURS, ZERO_JOBS_AGE_FLOOR_MINUTES, UNCONCLUDED_FLOOR_MINUTES, EMPTY_READ_GRACE_MINUTES, wedgedVerdict, unconcludedJobs, wedgeExit, emptyReadVerdict };
package/tools/lib/git.js CHANGED
@@ -996,8 +996,8 @@ function ghRunsForCommit(repo, branch, sha, limit = 10) {
996
996
 
997
997
  // createdAt + databaseId are additive (#155): a NON-completed pick needs both to let a caller
998
998
  // judge whether it is merely slow or structurally WEDGED (tools/lib/ci-verdict.js) — createdAt for
999
- // the age backstop, databaseId to look up its job count (ghRunJobCount, below — a second call, made
1000
- // lazily by the caller, never here: an ordinary green check must not pay for a read it never needs).
999
+ // the age backstop, databaseId to look up its jobs (ghRunJobs, below — a second call, made lazily
1000
+ // by the caller, never here: an ordinary green check must not pay for a read it never needs).
1001
1001
  const r = gh(['run', 'list', '--branch', branch, '-L', String(limit),
1002
1002
  // workflowName is additive (#338): `colab release cut`'s full-suite condition groups the rows by
1003
1003
  // workflow — a workflow whose only run at the sha was cancelled never ran its suite.
@@ -1341,8 +1341,9 @@ function summarizeRepoOwnedRuns(forSha, sha) {
1341
1341
  /**
1342
1342
  * Job count for one workflow run (#155) — how many jobs GitHub has attached to it. A queued run
1343
1343
  * with ZERO jobs has not been picked up by any runner: the measured shape of a billing lockout or a
1344
- * dead runner pool, and the PRIMARY signal tools/lib/ci-verdict.js uses to call a non-completed run
1345
- * WEDGED rather than merely slow. Deliberately a separate call from `ghRunForSha` (never folded into
1344
+ * dead runner pool. The wedge read in tools/colab `classifyCiRunRaw` now takes the count from
1345
+ * `ghRunJobs` below (#650 needs the jobs' statuses too, from the same one call); this stays for the
1346
+ * callers that need only the count (release-finalize's displaced-run check). Deliberately a separate call from `ghRunForSha` (never folded into
1346
1347
  * its `--json` field list): job counts require `gh run view`, one call per run, and the ordinary
1347
1348
  * green-trunk ship makes zero of these — callers fetch it lazily, only for a run that is already
1348
1349
  * non-completed and therefore a wedge candidate.
@@ -1364,14 +1365,15 @@ function ghRunJobCount(repo, runDatabaseId) {
1364
1365
  * failure, run gone, unparseable body). `null` must NOT be read as "no jobs"; every consumer of
1365
1366
  * this fails closed on it.
1366
1367
  *
1367
- * A SIBLING of `ghRunJobCount` above, deliberately not a refactor of it: that function sits on the
1368
- * wedge-detection hot path, and rewriting it as `ghRunJobs(...).length` would be churn with real
1369
- * regression surface for no gain here.
1368
+ * A SIBLING of `ghRunJobCount` above, deliberately not a refactor of it — the release-finalize
1369
+ * caller needs only the count. Since #650 this is also the wedge read: `classifyCiRunRaw` calls it
1370
+ * (once, for a non-completed run only) instead of `ghRunJobCount`, because telling "every job
1371
+ * completed, the forge never concluded the run" from #155's shapes needs the jobs' statuses and
1372
+ * timestamps, and `gh run view --json jobs` costs the same one call either way.
1370
1373
  *
1371
- * Costs one `gh run view` per run, so callers fetch it LAZILY — tools/lib/ci-cure.js's cure rule
1372
- * (condition 2b, #297, and the #321 carve-out) is the only caller today, and `colab ship` reaches
1373
- * it only for a branch that is already blocked by a red trunk and has passed containment and a
1374
- * green run of its own. Every other ship path makes zero of these calls.
1374
+ * Costs one `gh run view` per run, so callers fetch it LAZILY — the wedge read above, the cure rule
1375
+ * (tools/lib/ci-cure.js, condition 2b, #297, and the #321 carve-out), and the release paths. An
1376
+ * ordinary green or plain-failed trunk read makes zero of these calls.
1375
1377
  *
1376
1378
  * Step-level detail is the whole point: GitHub reports a job whose steps were all SKIPPED as
1377
1379
  * `conclusion: success`, so run- and job-level conclusions are blind to exactly the fast-exit the