@futurelastic/colab-handbook 1.20.3-rc.2 → 1.20.3-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/tools/colab +17 -2
- package/tools/lib/ci-verdict.js +57 -2
- package/tools/lib/ci-wait.js +3 -0
- package/tools/lib/git.js +44 -3
package/package.json
CHANGED
package/tools/colab
CHANGED
|
@@ -12289,10 +12289,23 @@ function classifyCiRunRaw(repoAbs, run, at) {
|
|
|
12289
12289
|
const jobCount = Array.isArray(jobs) ? jobs.length : null;
|
|
12290
12290
|
// #559: wedge age and zero-jobs floor follow the repo's measured CI (cached profile; bootstrap = 6h / 2m).
|
|
12291
12291
|
const th = ciProfileFor(repoAbs).th;
|
|
12292
|
-
const
|
|
12292
|
+
const wedgeOpts = { wedgeAgeSec: th.wedgeAgeSec, zeroJobsFloorSec: th.zeroJobsFloorSec };
|
|
12293
|
+
let wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount, jobs }, wedgeOpts);
|
|
12294
|
+
// #705: a `pending` run with zero jobs may only be waiting for its concurrency group — an older run
|
|
12295
|
+
// of the same workflow on the same ref still in progress. One more read, made only here, for a run
|
|
12296
|
+
// that would otherwise read WEDGED on zero jobs; an unreadable answer keeps the verdict above.
|
|
12297
|
+
if (wedge.shape === 'zero-jobs' && run.status === 'pending' && run.workflowId != null && run.headBranch) {
|
|
12298
|
+
const active = git.ghActiveRunsOfWorkflow(repoAbs, run.workflowId, run.headBranch);
|
|
12299
|
+
const queuedBehind = ciVerdict.concurrencyBlocker(run, active);
|
|
12300
|
+
if (queuedBehind) wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount, jobs, queuedBehind }, wedgeOpts);
|
|
12301
|
+
}
|
|
12293
12302
|
if (wedge.wedged) {
|
|
12294
12303
|
return { ok: false, class: HUMAN_GATED, wedge: wedge.shape, detail: `run for ${at}: status=${run.status} — WEDGED (${wedge.reason}); ${ciVerdict.wedgeExit(wedge)}` };
|
|
12295
12304
|
}
|
|
12305
|
+
if (wedge.queuedBehind) {
|
|
12306
|
+
const b = wedge.queuedBehind;
|
|
12307
|
+
return { ok: false, class: SELF_CLEARING, queuedBehind: b, detail: `run for ${at}: status=pending — queued behind run ${b.databaseId}${b.headSha ? ` (${String(b.headSha).slice(0, 7)})` : ''} of the same workflow on ${run.headBranch} (concurrency), retry later` };
|
|
12308
|
+
}
|
|
12296
12309
|
return { ok: false, class: SELF_CLEARING, detail: `run for ${at}: status=${run.status} — still running, retry later` };
|
|
12297
12310
|
}
|
|
12298
12311
|
|
|
@@ -16526,7 +16539,9 @@ the detail (--json: setAside). Override in .github/project.yml, by workflow name
|
|
|
16526
16539
|
The first output word is one verdict:
|
|
16527
16540
|
GREEN every judged workflow's newest run at the sha succeeded — ship's row would pass
|
|
16528
16541
|
RED a judged run completed and did not succeed — trunk is red
|
|
16529
|
-
PENDING a judged run is still in flight (or the sha is minutes old with no run yet
|
|
16542
|
+
PENDING a judged run is still in flight (or the sha is minutes old with no run yet, or
|
|
16543
|
+
its run is queued behind an older in-progress run of the same workflow on
|
|
16544
|
+
trunk — its concurrency group, #705 — and the detail names that run)
|
|
16530
16545
|
WEDGED a run that will never move on its own (zero jobs / hours unmoving, #155; every job
|
|
16531
16546
|
completed but the forge never concluded the run, #650 — the detail names the exits)
|
|
16532
16547
|
NONE no repo-owned run exists at the sha — CI not wired or never started
|
package/tools/lib/ci-verdict.js
CHANGED
|
@@ -51,6 +51,18 @@
|
|
|
51
51
|
* a positive (same posture as the two signals above). Its verdict is still HUMAN_GATED, never green,
|
|
52
52
|
* even when every job succeeded: the forge has not said the run passed (epic #652, fail closed).
|
|
53
53
|
* `wedgeExit` below names the exits that work for each shape, so the verdict is actionable.
|
|
54
|
+
*
|
|
55
|
+
* Amendment (#705): signal B's zero jobs has one ordinary cause it must not wedge on — a run WAITING
|
|
56
|
+
* FOR ITS CONCURRENCY GROUP. A workflow with `concurrency: { group: ci-${{ github.ref }} }` that does
|
|
57
|
+
* not cancel in progress on trunk leaves the newer trunk run at `status: pending` with zero jobs until
|
|
58
|
+
* the older run holding the group finishes. Measured: a trunk read WEDGED / "a human must look" while
|
|
59
|
+
* the previous trunk run of the same workflow was in progress with jobs running, and branch runs
|
|
60
|
+
* started after it were already in progress (so runners were there). Signal: the run is `pending`
|
|
61
|
+
* (the status GitHub gives a run held by its group — never `queued`, which is the runner wait #155's
|
|
62
|
+
* shape lives in), and an OLDER run of the same workflow on the same ref is `in_progress`
|
|
63
|
+
* (`concurrencyBlocker` below; the caller reads the in-progress runs lazily, only for a run about to
|
|
64
|
+
* wedge on zero jobs). That reads as waiting — not wedged — and the age backstop still bounds it, so
|
|
65
|
+
* a group held forever by a dead run still reaches a human.
|
|
54
66
|
*/
|
|
55
67
|
|
|
56
68
|
// Generous on purpose (see doc comment above) — an ordinary run finishes in minutes to tens of
|
|
@@ -123,9 +135,14 @@ function unconcludedJobs(jobs, opts = {}) {
|
|
|
123
135
|
* `jobs` (#650) is optional — `ghRunJobs`'s list, read by the same lazy call; when present it is what
|
|
124
136
|
* tells the forge-stuck shape (every job completed, run never concluded) from #155's two.
|
|
125
137
|
*
|
|
138
|
+
* `queuedBehind` (#705) is optional — `concurrencyBlocker`'s answer: the older in-progress run of the
|
|
139
|
+
* same workflow on the same ref that a `pending` run is waiting on. Present, it turns zero jobs from a
|
|
140
|
+
* wedge into waiting (age backstop still applies).
|
|
141
|
+
*
|
|
126
142
|
* Returns `{ wedged, reason, shape }`. `reason` and `shape` are null when `wedged` is false; `shape` is
|
|
127
143
|
* `'zero-jobs'` | `'unconcluded'` | `'age'` (which signal fired). An `'unconcluded'` verdict also
|
|
128
|
-
* carries `failed` — the jobs that concluded red, if any.
|
|
144
|
+
* carries `failed` — the jobs that concluded red, if any. A not-wedged verdict for a run waiting on its
|
|
145
|
+
* concurrency group carries `queuedBehind` back, so the caller can name the run it waits on.
|
|
129
146
|
*/
|
|
130
147
|
function wedgedVerdict(run, opts = {}) {
|
|
131
148
|
if (!run || run.status === 'completed' || run.status === 'none') return { wedged: false, reason: null, shape: null };
|
|
@@ -134,6 +151,15 @@ function wedgedVerdict(run, opts = {}) {
|
|
|
134
151
|
const floorMin = Number.isFinite(opts.zeroJobsFloorSec) ? opts.zeroJobsFloorSec / 60 : ZERO_JOBS_AGE_FLOOR_MINUTES;
|
|
135
152
|
const wedgeHours = Number.isFinite(opts.wedgeAgeSec) ? opts.wedgeAgeSec / 3600 : WEDGE_AGE_HOURS;
|
|
136
153
|
if (run.jobCount === 0) {
|
|
154
|
+
// #705: held by its own concurrency group behind an older in-progress run — ordinary queueing,
|
|
155
|
+
// bounded only by the age backstop (a group held for hours means the holder is what is stuck).
|
|
156
|
+
if (run.status === 'pending' && run.queuedBehind) {
|
|
157
|
+
const b = run.queuedBehind;
|
|
158
|
+
if (age !== null && age >= wedgeHours) {
|
|
159
|
+
return { wedged: true, shape: 'age', reason: `status=pending for ${age.toFixed(1)}h queued behind run ${b.databaseId} of the same workflow (concurrency), (>= ${fmtHours(wedgeHours)} backstop)` };
|
|
160
|
+
}
|
|
161
|
+
return { wedged: false, reason: null, shape: null, queuedBehind: b };
|
|
162
|
+
}
|
|
137
163
|
// #171: no createdAt (age unknown) never manufactures a positive here, same posture as the age
|
|
138
164
|
// backstop below when createdAt is missing/unparseable — a missing signal must not wedge on its own.
|
|
139
165
|
const floorHours = floorMin / 60;
|
|
@@ -164,6 +190,35 @@ function wedgedVerdict(run, opts = {}) {
|
|
|
164
190
|
return { wedged: false, reason: null, shape: null };
|
|
165
191
|
}
|
|
166
192
|
|
|
193
|
+
/**
|
|
194
|
+
* #705: the older in-progress run of the same workflow on the same ref that `run` waits behind, or
|
|
195
|
+
* null. `run` is the picked run (`{status, databaseId, createdAt}`); `active` is the in-progress runs
|
|
196
|
+
* of its workflow on its ref (tools/lib/git.js `ghActiveRunsOfWorkflow`), or null when unread.
|
|
197
|
+
* Only a `pending` run is ever queued behind its group (see the #705 amendment above). "Older" is the
|
|
198
|
+
* earlier createdAt, else the lower run id; a row that cannot be ordered against `run` is not a
|
|
199
|
+
* blocker — a missing signal never manufactures one. Returns `{ databaseId, createdAt, headSha }` of
|
|
200
|
+
* the newest such run.
|
|
201
|
+
*/
|
|
202
|
+
function concurrencyBlocker(run, active) {
|
|
203
|
+
if (!run || run.status !== 'pending' || !Array.isArray(active)) return null;
|
|
204
|
+
const tOf = (x) => (x && x.createdAt ? Date.parse(x.createdAt) : NaN);
|
|
205
|
+
const idOf = (x) => (x && x.databaseId != null ? Number(x.databaseId) : NaN);
|
|
206
|
+
const rt = tOf(run);
|
|
207
|
+
const rid = idOf(run);
|
|
208
|
+
const older = active.filter((x) => {
|
|
209
|
+
if (!x || x.status !== 'in_progress') return false;
|
|
210
|
+
if (!Number.isNaN(rid) && idOf(x) === rid) return false;
|
|
211
|
+
const t = tOf(x);
|
|
212
|
+
if (!Number.isNaN(rt) && !Number.isNaN(t) && t !== rt) return t < rt;
|
|
213
|
+
const id = idOf(x);
|
|
214
|
+
return !Number.isNaN(rid) && !Number.isNaN(id) && id < rid;
|
|
215
|
+
});
|
|
216
|
+
if (!older.length) return null;
|
|
217
|
+
older.sort((a, b) => (tOf(b) || 0) - (tOf(a) || 0) || (idOf(b) || 0) - (idOf(a) || 0));
|
|
218
|
+
const b = older[0];
|
|
219
|
+
return { databaseId: b.databaseId != null ? b.databaseId : null, createdAt: b.createdAt || null, headSha: b.headSha || null };
|
|
220
|
+
}
|
|
221
|
+
|
|
167
222
|
/**
|
|
168
223
|
* The exit a WEDGED verdict names (#650) — the tail of the detail line after the reason. #155's two
|
|
169
224
|
* shapes keep their words: a human must look (billing, a runner pool, a dead runner). The forge-stuck
|
|
@@ -209,4 +264,4 @@ function emptyReadVerdict(committedAtMs, opts = {}) {
|
|
|
209
264
|
return { fresh: minutes >= 0 && minutes < graceMinutes, minutes, graceMinutes };
|
|
210
265
|
}
|
|
211
266
|
|
|
212
|
-
module.exports = { WEDGE_AGE_HOURS, ZERO_JOBS_AGE_FLOOR_MINUTES, UNCONCLUDED_FLOOR_MINUTES, EMPTY_READ_GRACE_MINUTES, wedgedVerdict, unconcludedJobs, wedgeExit, emptyReadVerdict };
|
|
267
|
+
module.exports = { WEDGE_AGE_HOURS, ZERO_JOBS_AGE_FLOOR_MINUTES, UNCONCLUDED_FLOOR_MINUTES, EMPTY_READ_GRACE_MINUTES, wedgedVerdict, unconcludedJobs, concurrencyBlocker, wedgeExit, emptyReadVerdict };
|
package/tools/lib/ci-wait.js
CHANGED
|
@@ -102,6 +102,9 @@ function restRow(x) {
|
|
|
102
102
|
event: x.event || null, url: x.html_url || null,
|
|
103
103
|
// #510: the workflow FILE's id — the cure rule's same-workflow test, which a display name cannot answer.
|
|
104
104
|
workflowId: x.workflow_id ?? null,
|
|
105
|
+
// #705: the ref the run is on — a zero-jobs run is told apart from one queued behind its own
|
|
106
|
+
// concurrency group by an older in-progress run of the same workflow on the SAME ref.
|
|
107
|
+
headBranch: x.head_branch || null,
|
|
105
108
|
};
|
|
106
109
|
}
|
|
107
110
|
|
package/tools/lib/git.js
CHANGED
|
@@ -1014,7 +1014,8 @@ function ghRunsForCommit(repo, branch, sha, limit = 10) {
|
|
|
1014
1014
|
// workflowDatabaseId is additive (#510): the cure rule's same-workflow test reads the workflow's
|
|
1015
1015
|
// id, never its display name; mapped to `workflowId`, the name the REST row (ci-wait restRow) uses.
|
|
1016
1016
|
// updatedAt is additive (#555): a finished run's completion time, the lone-member wait's anchor.
|
|
1017
|
-
|
|
1017
|
+
// headBranch is additive (#705): the ref the concurrency-queue read (ghActiveRunsOfWorkflow) keys on.
|
|
1018
|
+
'--json', 'headSha,status,conclusion,createdAt,updatedAt,databaseId,workflowName,workflowDatabaseId,event,headBranch'], { cwd: repo });
|
|
1018
1019
|
if (!r.ok) return null;
|
|
1019
1020
|
let runs;
|
|
1020
1021
|
try { runs = JSON.parse(r.stdout); } catch (_) { return null; }
|
|
@@ -1458,7 +1459,14 @@ function summarizeRepoOwnedRuns(forSha, sha) {
|
|
|
1458
1459
|
// forSha[0] is the newest remaining row either way).
|
|
1459
1460
|
const pending = forSha.find((x) => x.status !== 'completed');
|
|
1460
1461
|
const pick = pending || forSha[0];
|
|
1461
|
-
|
|
1462
|
+
// #705: a still-running pick carries its workflow id and ref, when the row had them, so the wedge
|
|
1463
|
+
// read can ask whether it is only queued behind an older run of the same workflow on the same ref
|
|
1464
|
+
// (its concurrency group) — additive, and absent on rows that never had them.
|
|
1465
|
+
const queueKeys = pending ? {
|
|
1466
|
+
...(pick.workflowId != null ? { workflowId: pick.workflowId } : {}),
|
|
1467
|
+
...(pick.headBranch ? { headBranch: pick.headBranch } : {}),
|
|
1468
|
+
} : {};
|
|
1469
|
+
return { status: pick.status, conclusion: pick.conclusion || null, sha, createdAt: pick.createdAt || null, databaseId: pick.databaseId || null, runCount, ...queueKeys };
|
|
1462
1470
|
}
|
|
1463
1471
|
|
|
1464
1472
|
/**
|
|
@@ -1482,6 +1490,39 @@ function ghRunJobCount(repo, runDatabaseId) {
|
|
|
1482
1490
|
return Number.isFinite(n) ? n : null;
|
|
1483
1491
|
}
|
|
1484
1492
|
|
|
1493
|
+
/**
|
|
1494
|
+
* The IN-PROGRESS runs of one workflow on one ref (#705) — `[{databaseId, status, createdAt, headSha,
|
|
1495
|
+
* headBranch, workflowId}]` (ci-wait restRow shape), or null when it could not be read. null must NOT
|
|
1496
|
+
* be read as "none in progress": the caller then keeps the verdict it had without this read.
|
|
1497
|
+
*
|
|
1498
|
+
* Why it exists: a run waiting for its concurrency group (`concurrency: { group: …${github.ref} }`
|
|
1499
|
+
* with no cancel-in-progress on trunk) sits at `status: pending` with ZERO jobs until the older run
|
|
1500
|
+
* holding the group finishes — exactly #155's wedge shape, while it is ordinary queueing. The older
|
|
1501
|
+
* run in progress on the same workflow and ref is what tells the two apart.
|
|
1502
|
+
*
|
|
1503
|
+
* One REST call (`actions/workflows/<id>/runs?branch=&status=in_progress`), made lazily: only for a
|
|
1504
|
+
* pending run that would otherwise read WEDGED on zero jobs. Falls back to `gh run list` on any
|
|
1505
|
+
* failure but a rate limit, the same posture as ghRunsAtShaRest.
|
|
1506
|
+
*/
|
|
1507
|
+
function ghActiveRunsOfWorkflow(repo, workflowId, branch) {
|
|
1508
|
+
if (workflowId == null || !branch) return null;
|
|
1509
|
+
const ciWait = require('./ci-wait');
|
|
1510
|
+
const q = `branch=${encodeURIComponent(branch)}&status=in_progress&per_page=20`;
|
|
1511
|
+
const r = ghApiConditional(repo, `repos/{owner}/{repo}/actions/workflows/${encodeURIComponent(String(workflowId))}/runs?${q}`);
|
|
1512
|
+
if (r.kind === 'rate-limited') return null;
|
|
1513
|
+
if (r.kind === 'ok' && r.json && Array.isArray(r.json.workflow_runs)) {
|
|
1514
|
+
return r.json.workflow_runs.map(ciWait.restRow).filter((x) => x.status === 'in_progress' && (!x.headBranch || x.headBranch === branch));
|
|
1515
|
+
}
|
|
1516
|
+
const l = gh(['run', 'list', '--workflow', String(workflowId), '--branch', branch, '--status', 'in_progress', '-L', '20',
|
|
1517
|
+
'--json', 'databaseId,status,createdAt,headSha,headBranch,workflowDatabaseId'], { cwd: repo });
|
|
1518
|
+
if (!l.ok) return null;
|
|
1519
|
+
let rows;
|
|
1520
|
+
try { rows = JSON.parse(l.stdout); } catch (_) { return null; }
|
|
1521
|
+
if (!Array.isArray(rows)) return null;
|
|
1522
|
+
return rows.filter((x) => x && x.status === 'in_progress')
|
|
1523
|
+
.map(({ workflowDatabaseId, ...x }) => ({ ...x, workflowId: workflowDatabaseId ?? workflowId }));
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1485
1526
|
/**
|
|
1486
1527
|
* The JOBS of one workflow run, with their steps (#321) — `[{name, status, conclusion, startedAt,
|
|
1487
1528
|
* completedAt, steps: [{name, status, conclusion}]}]`, or `null` if it could not be read (gh
|
|
@@ -1584,7 +1625,7 @@ module.exports = {
|
|
|
1584
1625
|
dirtyTracked, dirtyUntracked, dirtyAny,
|
|
1585
1626
|
ghAvailable, ghState, ghInstalled, ghApiConditional, ghRunsAtShaRest, ghRunsAtShaUnfiltered, fullSha, ghIssueEdit, ghListLabels, ghOpenIssueNumbersByLabel, ghAssignedIssues,
|
|
1586
1627
|
ghCurrentLogin, ghIssueView, ghIssueComment, ghRunForSha, ghRunForCommit, ghRunsForCommit, ghRunsForRef, ghRunsAtCommit, ghRunForCommitAnyRef, commitTimeMs, ghRunsSince, summarizeRunsForCommit, isRepoOwnedRun,
|
|
1587
|
-
ghRunJobCount, ghRunJobs, ghWorkflowDispatch,
|
|
1628
|
+
ghRunJobCount, ghRunJobs, ghActiveRunsOfWorkflow, ghWorkflowDispatch,
|
|
1588
1629
|
ghIssueListByLabel, ghLabelDelete, ghLabelCreate, ghListLabelsDetailed, ghLabelEditDescription,
|
|
1589
1630
|
ghApi, isGraphqlRateLimit, ghIssueRelease, ghClaimHolder, ghIssueLabelEvents, ghIssueLabelActors,
|
|
1590
1631
|
ghPrForBranch, ghOpenPrsForBranch, ghOpenPrs, ghPrCreate, ghPrClose, ghRepoVisibility, ghRunCancel,
|