@futurelastic/colab-handbook 1.16.0-rc.18 → 1.16.0-rc.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/tools/colab +13 -6
- package/tools/lib/ci-verdict.js +98 -7
- package/tools/lib/git.js +13 -11
package/package.json
CHANGED
package/tools/colab
CHANGED
|
@@ -4797,7 +4797,8 @@ CI grant merges into a repo whose own test suite is known-failing:
|
|
|
4797
4797
|
"Red" here (#155) is the SAME verdict \`ship\`'s trunk-CI-green check uses — an ordinary
|
|
4798
4798
|
still-running run is NOT red (nothing for a grant to exempt; \`ship\` would clear it on its own),
|
|
4799
4799
|
but a WEDGED run (queued with zero jobs, or stuck past a generous age backstop — the shape a
|
|
4800
|
-
billing lockout or dead runner pool leaves
|
|
4800
|
+
billing lockout or dead runner pool leaves; or every job completed and the forge never concluded
|
|
4801
|
+
the run, which cancel and rerun cannot move, #650) reclassifies to red, because \`ship\` cannot clear
|
|
4801
4802
|
that on its own either and this is the sanctioned door for exactly that case. Before #155 the
|
|
4802
4803
|
two commands computed this independently and disagreed: a grant could be minted for a state
|
|
4803
4804
|
\`ship\` would never consult it for, reading live in \`--list\` while doing nothing.
|
|
@@ -11836,12 +11837,15 @@ function classifyCiRunRaw(repoAbs, run, at) {
|
|
|
11836
11837
|
}
|
|
11837
11838
|
// Non-completed, non-none: running/queued/etc. Ordinarily SELF_CLEARING — but #155's wedged
|
|
11838
11839
|
// shape (queued, zero jobs, unmoving for hours) can never clear on its own, so check for it.
|
|
11839
|
-
|
|
11840
|
+
// #650: one `gh run view --json jobs` — the same single call the job count used to cost — so the
|
|
11841
|
+
// jobs' statuses can tell a run the forge never concluded (every job done) from #155's shapes.
|
|
11842
|
+
const jobs = run.databaseId != null ? git.ghRunJobs(repoAbs, run.databaseId) : null;
|
|
11843
|
+
const jobCount = Array.isArray(jobs) ? jobs.length : null;
|
|
11840
11844
|
// #559: wedge age and zero-jobs floor follow the repo's measured CI (cached profile; bootstrap = 6h / 2m).
|
|
11841
11845
|
const th = ciProfileFor(repoAbs).th;
|
|
11842
|
-
const wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount }, { wedgeAgeSec: th.wedgeAgeSec, zeroJobsFloorSec: th.zeroJobsFloorSec });
|
|
11846
|
+
const wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount, jobs }, { wedgeAgeSec: th.wedgeAgeSec, zeroJobsFloorSec: th.zeroJobsFloorSec });
|
|
11843
11847
|
if (wedge.wedged) {
|
|
11844
|
-
return { ok: false, class: HUMAN_GATED, detail: `run for ${at}: status=${run.status} — WEDGED (${wedge.reason});
|
|
11848
|
+
return { ok: false, class: HUMAN_GATED, wedge: wedge.shape, detail: `run for ${at}: status=${run.status} — WEDGED (${wedge.reason}); ${ciVerdict.wedgeExit(wedge)}` };
|
|
11845
11849
|
}
|
|
11846
11850
|
return { ok: false, class: SELF_CLEARING, detail: `run for ${at}: status=${run.status} — still running, retry later` };
|
|
11847
11851
|
}
|
|
@@ -15932,7 +15936,8 @@ The first output word is one verdict:
|
|
|
15932
15936
|
GREEN every judged workflow's newest run at the sha succeeded — ship's row would pass
|
|
15933
15937
|
RED a judged run completed and did not succeed — trunk is red
|
|
15934
15938
|
PENDING a judged run is still in flight (or the sha is minutes old with no run yet)
|
|
15935
|
-
WEDGED a run that will never move on its own (zero jobs / hours unmoving, #155
|
|
15939
|
+
WEDGED a run that will never move on its own (zero jobs / hours unmoving, #155; every job
|
|
15940
|
+
completed but the forge never concluded the run, #650 — the detail names the exits)
|
|
15936
15941
|
NONE no repo-owned run exists at the sha — CI not wired or never started
|
|
15937
15942
|
UNREADABLE gh unusable or the read failed — never "green"
|
|
15938
15943
|
|
|
@@ -15951,7 +15956,8 @@ checkout (COLAB_TRUNK_CI_TTL=<sec> overrides, 0 disables; --fresh skips the cach
|
|
|
15951
15956
|
To WAIT for CI, use \`colab ci-wait\` — never a loop around this command.
|
|
15952
15957
|
|
|
15953
15958
|
--fresh ignore a cached verdict (still refreshes the cache)
|
|
15954
|
-
--json { verdict, sha, ok, class, detail, trunk[,
|
|
15959
|
+
--json { verdict, sha, ok, class, detail, trunk[, wedge: zero-jobs|unconcluded|age on a WEDGED (#650)]
|
|
15960
|
+
[, cited: [<run url>…] on a #493 skip][, cachedAgeSec] }
|
|
15955
15961
|
Exit 0 whenever a verdict was read (whatever it is); 2 on UNREADABLE.`;
|
|
15956
15962
|
|
|
15957
15963
|
/** Map readCiVerdict's result to the closed trunk-ci vocabulary. Pure — exported for tests. */
|
|
@@ -15992,6 +15998,7 @@ function cmdTrunkCi(argv) {
|
|
|
15992
15998
|
let detail = r.verdict.detail;
|
|
15993
15999
|
if (verdict === 'UNREADABLE' && gh && !git.ghState().ok) detail += ` (${ghUnusableWhy(repoAbs)})`;
|
|
15994
16000
|
out = { verdict, trunk, sha, ok: r.verdict.ok, class: r.verdict.class || null, detail,
|
|
16001
|
+
...(r.verdict.wedge ? { wedge: r.verdict.wedge } : {}),
|
|
15995
16002
|
...(r.verdict.treeGreen ? { cited: r.verdict.treeGreen } : {}),
|
|
15996
16003
|
...(r.run && Array.isArray(r.run.setAside) && r.run.setAside.length ? { setAside: r.run.setAside } : {}),
|
|
15997
16004
|
...(r.run && Array.isArray(r.run.dispatchCounted) && r.run.dispatchCounted.length ? { dispatchCounted: r.run.dispatchCounted } : {}) };
|
package/tools/lib/ci-verdict.js
CHANGED
|
@@ -37,6 +37,20 @@
|
|
|
37
37
|
* materialization window was not re-measured empirically here. Costs nothing against the signal's
|
|
38
38
|
* actual target: a billing lockout or dead runner pool leaves a run at zero jobs for far longer than
|
|
39
39
|
* two minutes, so the floor never masks a real wedge, only the moment right after creation.
|
|
40
|
+
*
|
|
41
|
+
* Amendment (#650): a THIRD shape, told apart from both of #155's — every job reached `completed`,
|
|
42
|
+
* yet the forge never concluded the run (run and check suite stay `in_progress`, `updated_at`
|
|
43
|
+
* frozen). Measured during a forge incident that delayed Actions result processing: the skipped
|
|
44
|
+
* jobs' check runs were written ~6 minutes after the job they `needs:`, and from then on the REST
|
|
45
|
+
* API refused every exit at once — cancel/force-cancel 409 "not in progress", every rerun and the
|
|
46
|
+
* check-suite rerequest 403 "already running". Nothing CI can still do will move it; only the forge
|
|
47
|
+
* (or a human) can. Before this the shape had no signal of its own: the jobs are non-empty, so it
|
|
48
|
+
* read "still running, retry later" until the 6h age backstop, then a WEDGED that named no exit.
|
|
49
|
+
* Signal: the job list is non-empty, every job is `completed`, and the newest job timestamp is past
|
|
50
|
+
* UNCONCLUDED_FLOOR_MINUTES. Every job needs a readable timestamp — a missing one never manufactures
|
|
51
|
+
* a positive (same posture as the two signals above). Its verdict is still HUMAN_GATED, never green,
|
|
52
|
+
* even when every job succeeded: the forge has not said the run passed (epic #652, fail closed).
|
|
53
|
+
* `wedgeExit` below names the exits that work for each shape, so the verdict is actionable.
|
|
40
54
|
*/
|
|
41
55
|
|
|
42
56
|
// Generous on purpose (see doc comment above) — an ordinary run finishes in minutes to tens of
|
|
@@ -50,6 +64,11 @@ const WEDGE_AGE_HOURS = 6;
|
|
|
50
64
|
// hours, not minutes).
|
|
51
65
|
const ZERO_JOBS_AGE_FLOOR_MINUTES = 2;
|
|
52
66
|
|
|
67
|
+
// #650: how long after its newest job finished a run may still be waiting for the forge to conclude
|
|
68
|
+
// it. Normally that takes seconds; the measured incident wrote check runs ~6 minutes late, so 15
|
|
69
|
+
// minutes is more than twice the worst lag seen before the read says "the forge, not CI".
|
|
70
|
+
const UNCONCLUDED_FLOOR_MINUTES = 15;
|
|
71
|
+
|
|
53
72
|
function hoursSince(iso, nowMs) {
|
|
54
73
|
if (!iso) return null;
|
|
55
74
|
const t = Date.parse(iso);
|
|
@@ -59,7 +78,41 @@ function hoursSince(iso, nowMs) {
|
|
|
59
78
|
}
|
|
60
79
|
|
|
61
80
|
/**
|
|
62
|
-
*
|
|
81
|
+
* #650: did every job of a non-completed run finish, long enough ago that the forge, not CI, is what
|
|
82
|
+
* is holding it? `jobs` is tools/lib/git.js `ghRunJobs`'s shape (`[{name, status, conclusion,
|
|
83
|
+
* startedAt, completedAt}]`), or null when unread. Returns `{ unconcluded, minutes, failed, counts }`:
|
|
84
|
+
* `minutes` since the newest job timestamp (null when not measurable), `failed` the names of jobs
|
|
85
|
+
* that concluded anything but success/skipped/neutral, `counts` conclusion → number of jobs.
|
|
86
|
+
* Never a positive from a missing signal: null/empty jobs, any job not `completed`, or any job with
|
|
87
|
+
* no readable timestamp all read `unconcluded: false`.
|
|
88
|
+
*/
|
|
89
|
+
function unconcludedJobs(jobs, opts = {}) {
|
|
90
|
+
const no = { unconcluded: false, minutes: null, failed: [], counts: {} };
|
|
91
|
+
if (!Array.isArray(jobs) || jobs.length === 0) return no;
|
|
92
|
+
if (!jobs.every((j) => j && j.status === 'completed')) return no;
|
|
93
|
+
let newest = -Infinity;
|
|
94
|
+
for (const j of jobs) {
|
|
95
|
+
// The measured incident wrote a skipped job's started_at AFTER its completed_at — so take the
|
|
96
|
+
// later of the two, per job, and the latest over all jobs.
|
|
97
|
+
const ts = [j.startedAt, j.completedAt].map((x) => (x ? Date.parse(x) : NaN)).filter((t) => !Number.isNaN(t));
|
|
98
|
+
if (!ts.length) return no;
|
|
99
|
+
newest = Math.max(newest, ...ts);
|
|
100
|
+
}
|
|
101
|
+
const now = opts.nowMs != null ? opts.nowMs : Date.now();
|
|
102
|
+
const minutes = (now - newest) / 60000;
|
|
103
|
+
const floorMin = Number.isFinite(opts.floorSec) ? opts.floorSec / 60 : UNCONCLUDED_FLOOR_MINUTES;
|
|
104
|
+
const counts = {};
|
|
105
|
+
const failed = [];
|
|
106
|
+
for (const j of jobs) {
|
|
107
|
+
const c = j.conclusion || 'none';
|
|
108
|
+
counts[c] = (counts[c] || 0) + 1;
|
|
109
|
+
if (!['success', 'skipped', 'neutral'].includes(c)) failed.push(`${j.name || 'unnamed'} (${c})`);
|
|
110
|
+
}
|
|
111
|
+
return { unconcluded: minutes >= floorMin, minutes, failed, counts, floorMin };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* `run` is the `{status, conclusion, sha, createdAt, jobCount, jobs}` shape — `createdAt`/`jobCount` may be
|
|
63
116
|
* null when not measured (job count is fetched lazily by the caller, only for a run already known to
|
|
64
117
|
* be non-completed; see tools/lib/git.js `ghRunJobCount`'s doc comment for why).
|
|
65
118
|
*
|
|
@@ -67,10 +120,15 @@ function hoursSince(iso, nowMs) {
|
|
|
67
120
|
* never wedged by definition (it finished, however it finished), and `'none'` means no run exists at
|
|
68
121
|
* all, a different and already-handled case (HUMAN_GATED — "CI is not wired for this ref").
|
|
69
122
|
*
|
|
70
|
-
*
|
|
123
|
+
* `jobs` (#650) is optional — `ghRunJobs`'s list, read by the same lazy call; when present it is what
|
|
124
|
+
* tells the forge-stuck shape (every job completed, run never concluded) from #155's two.
|
|
125
|
+
*
|
|
126
|
+
* Returns `{ wedged, reason, shape }`. `reason` and `shape` are null when `wedged` is false; `shape` is
|
|
127
|
+
* `'zero-jobs'` | `'unconcluded'` | `'age'` (which signal fired). An `'unconcluded'` verdict also
|
|
128
|
+
* carries `failed` — the jobs that concluded red, if any.
|
|
71
129
|
*/
|
|
72
130
|
function wedgedVerdict(run, opts = {}) {
|
|
73
|
-
if (!run || run.status === 'completed' || run.status === 'none') return { wedged: false, reason: null };
|
|
131
|
+
if (!run || run.status === 'completed' || run.status === 'none') return { wedged: false, reason: null, shape: null };
|
|
74
132
|
const age = hoursSince(run.createdAt, opts.nowMs);
|
|
75
133
|
// #559: both bounds may come from the repo's measured CI (tools/lib/ci-profile.js); absent → today's.
|
|
76
134
|
const floorMin = Number.isFinite(opts.zeroJobsFloorSec) ? opts.zeroJobsFloorSec / 60 : ZERO_JOBS_AGE_FLOOR_MINUTES;
|
|
@@ -82,15 +140,48 @@ function wedgedVerdict(run, opts = {}) {
|
|
|
82
140
|
if (age !== null && age >= floorHours) {
|
|
83
141
|
return {
|
|
84
142
|
wedged: true,
|
|
143
|
+
shape: 'zero-jobs',
|
|
85
144
|
reason: `zero jobs — no runner has ever picked it up (unmoving for ${age.toFixed(2)}h, past the ${fmtMin(floorMin)} floor)`,
|
|
86
145
|
};
|
|
87
146
|
}
|
|
88
|
-
return { wedged: false, reason: null };
|
|
147
|
+
return { wedged: false, reason: null, shape: null };
|
|
148
|
+
}
|
|
149
|
+
// #650: checked BEFORE the age backstop — a run in this shape is old by the time anyone looks, and
|
|
150
|
+
// the backstop's reason ("status=in_progress for 7h") would hide the one fact that decides the exit.
|
|
151
|
+
const u = unconcludedJobs(run.jobs, { nowMs: opts.nowMs, floorSec: opts.unconcludedFloorSec });
|
|
152
|
+
if (u.unconcluded) {
|
|
153
|
+
const tally = Object.keys(u.counts).sort().map((c) => `${u.counts[c]} ${c}`).join(', ');
|
|
154
|
+
return {
|
|
155
|
+
wedged: true,
|
|
156
|
+
shape: 'unconcluded',
|
|
157
|
+
failed: u.failed,
|
|
158
|
+
reason: `every job completed (${tally}) ${u.minutes.toFixed(0)}m ago, but the forge never concluded the run — past the ${fmtMin(u.floorMin)} floor`,
|
|
159
|
+
};
|
|
89
160
|
}
|
|
90
161
|
if (age !== null && age >= wedgeHours) {
|
|
91
|
-
return { wedged: true, reason: `status=${run.status} for ${age.toFixed(1)}h (>= ${fmtHours(wedgeHours)} backstop)` };
|
|
162
|
+
return { wedged: true, shape: 'age', reason: `status=${run.status} for ${age.toFixed(1)}h (>= ${fmtHours(wedgeHours)} backstop)` };
|
|
163
|
+
}
|
|
164
|
+
return { wedged: false, reason: null, shape: null };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* The exit a WEDGED verdict names (#650) — the tail of the detail line after the reason. #155's two
|
|
169
|
+
* shapes keep their words: a human must look (billing, a runner pool, a dead runner). The forge-stuck
|
|
170
|
+
* shape names what works and what was measured not to, so nobody retries an exit the forge refuses.
|
|
171
|
+
* The ci-grant authority line does not move: no exit here lets an agent's own run stand in for
|
|
172
|
+
* trunk's (epic #652).
|
|
173
|
+
*/
|
|
174
|
+
function wedgeExit(wedge) {
|
|
175
|
+
if (!wedge || wedge.shape !== 'unconcluded') return 'a human must look, not retry';
|
|
176
|
+
if (wedge.failed && wedge.failed.length) {
|
|
177
|
+
return `red in substance — ${wedge.failed.join(', ')} did not succeed, whatever the run's status says; ` +
|
|
178
|
+
"fix trunk (the fix's own push gets a new run), and only the branch that IS that fix may take a human ci-grant";
|
|
92
179
|
}
|
|
93
|
-
return
|
|
180
|
+
return 'nothing CI can do moves it, and nothing here makes it green. Works: wait for the forge to conclude the run ' +
|
|
181
|
+
'(re-read with `colab trunk-ci --fresh`), or a human `colab ci-grant <issue> --branch <branch>` per candidate. ' +
|
|
182
|
+
'Does not work: cancel / force-cancel (409 "not in progress"), rerun / rerun-failed-jobs / check-suite rerequest ' +
|
|
183
|
+
'(403 "already running"), a workflow_dispatch at this sha (the lost-push rescue skips it while this push run exists, #567). ' +
|
|
184
|
+
"Deleting the run cannot be undone — the owner's call";
|
|
94
185
|
}
|
|
95
186
|
|
|
96
187
|
const fmtMin = (m) => `${Number.isInteger(m) ? m : m.toFixed(1)}m`;
|
|
@@ -118,4 +209,4 @@ function emptyReadVerdict(committedAtMs, opts = {}) {
|
|
|
118
209
|
return { fresh: minutes >= 0 && minutes < graceMinutes, minutes, graceMinutes };
|
|
119
210
|
}
|
|
120
211
|
|
|
121
|
-
module.exports = { WEDGE_AGE_HOURS, ZERO_JOBS_AGE_FLOOR_MINUTES, EMPTY_READ_GRACE_MINUTES, wedgedVerdict, emptyReadVerdict };
|
|
212
|
+
module.exports = { WEDGE_AGE_HOURS, ZERO_JOBS_AGE_FLOOR_MINUTES, UNCONCLUDED_FLOOR_MINUTES, EMPTY_READ_GRACE_MINUTES, wedgedVerdict, unconcludedJobs, wedgeExit, emptyReadVerdict };
|
package/tools/lib/git.js
CHANGED
|
@@ -996,8 +996,8 @@ function ghRunsForCommit(repo, branch, sha, limit = 10) {
|
|
|
996
996
|
|
|
997
997
|
// createdAt + databaseId are additive (#155): a NON-completed pick needs both to let a caller
|
|
998
998
|
// judge whether it is merely slow or structurally WEDGED (tools/lib/ci-verdict.js) — createdAt for
|
|
999
|
-
// the age backstop, databaseId to look up its
|
|
1000
|
-
//
|
|
999
|
+
// the age backstop, databaseId to look up its jobs (ghRunJobs, below — a second call, made lazily
|
|
1000
|
+
// by the caller, never here: an ordinary green check must not pay for a read it never needs).
|
|
1001
1001
|
const r = gh(['run', 'list', '--branch', branch, '-L', String(limit),
|
|
1002
1002
|
// workflowName is additive (#338): `colab release cut`'s full-suite condition groups the rows by
|
|
1003
1003
|
// workflow — a workflow whose only run at the sha was cancelled never ran its suite.
|
|
@@ -1341,8 +1341,9 @@ function summarizeRepoOwnedRuns(forSha, sha) {
|
|
|
1341
1341
|
/**
|
|
1342
1342
|
* Job count for one workflow run (#155) — how many jobs GitHub has attached to it. A queued run
|
|
1343
1343
|
* with ZERO jobs has not been picked up by any runner: the measured shape of a billing lockout or a
|
|
1344
|
-
* dead runner pool
|
|
1345
|
-
*
|
|
1344
|
+
* dead runner pool. The wedge read in tools/colab `classifyCiRunRaw` now takes the count from
|
|
1345
|
+
* `ghRunJobs` below (#650 needs the jobs' statuses too, from the same one call); this stays for the
|
|
1346
|
+
* callers that need only the count (release-finalize's displaced-run check). Deliberately a separate call from `ghRunForSha` (never folded into
|
|
1346
1347
|
* its `--json` field list): job counts require `gh run view`, one call per run, and the ordinary
|
|
1347
1348
|
* green-trunk ship makes zero of these — callers fetch it lazily, only for a run that is already
|
|
1348
1349
|
* non-completed and therefore a wedge candidate.
|
|
@@ -1364,14 +1365,15 @@ function ghRunJobCount(repo, runDatabaseId) {
|
|
|
1364
1365
|
* failure, run gone, unparseable body). `null` must NOT be read as "no jobs"; every consumer of
|
|
1365
1366
|
* this fails closed on it.
|
|
1366
1367
|
*
|
|
1367
|
-
* A SIBLING of `ghRunJobCount` above, deliberately not a refactor of it
|
|
1368
|
-
*
|
|
1369
|
-
*
|
|
1368
|
+
* A SIBLING of `ghRunJobCount` above, deliberately not a refactor of it — the release-finalize
|
|
1369
|
+
* caller needs only the count. Since #650 this is also the wedge read: `classifyCiRunRaw` calls it
|
|
1370
|
+
* (once, for a non-completed run only) instead of `ghRunJobCount`, because telling "every job
|
|
1371
|
+
* completed, the forge never concluded the run" from #155's shapes needs the jobs' statuses and
|
|
1372
|
+
* timestamps, and `gh run view --json jobs` costs the same one call either way.
|
|
1370
1373
|
*
|
|
1371
|
-
* Costs one `gh run view` per run, so callers fetch it LAZILY —
|
|
1372
|
-
* (condition 2b, #297, and the #321 carve-out)
|
|
1373
|
-
*
|
|
1374
|
-
* green run of its own. Every other ship path makes zero of these calls.
|
|
1374
|
+
* Costs one `gh run view` per run, so callers fetch it LAZILY — the wedge read above, the cure rule
|
|
1375
|
+
* (tools/lib/ci-cure.js, condition 2b, #297, and the #321 carve-out), and the release paths. An
|
|
1376
|
+
* ordinary green or plain-failed trunk read makes zero of these calls.
|
|
1375
1377
|
*
|
|
1376
1378
|
* Step-level detail is the whole point: GitHub reports a job whose steps were all SKIPPED as
|
|
1377
1379
|
* `conclusion: success`, so run- and job-level conclusions are blind to exactly the fast-exit the
|