@futurelastic/colab-handbook 1.20.3-rc.3 → 1.20.3-rc.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -33,5 +33,5 @@
33
33
  "engines": {
34
34
  "node": ">=18"
35
35
  },
36
- "version": "1.20.3-rc.3"
36
+ "version": "1.20.3-rc.5"
37
37
  }
package/tools/colab CHANGED
@@ -12289,10 +12289,23 @@ function classifyCiRunRaw(repoAbs, run, at) {
12289
12289
  const jobCount = Array.isArray(jobs) ? jobs.length : null;
12290
12290
  // #559: wedge age and zero-jobs floor follow the repo's measured CI (cached profile; bootstrap = 6h / 2m).
12291
12291
  const th = ciProfileFor(repoAbs).th;
12292
- const wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount, jobs }, { wedgeAgeSec: th.wedgeAgeSec, zeroJobsFloorSec: th.zeroJobsFloorSec });
12292
+ const wedgeOpts = { wedgeAgeSec: th.wedgeAgeSec, zeroJobsFloorSec: th.zeroJobsFloorSec };
12293
+ let wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount, jobs }, wedgeOpts);
12294
+ // #705: a `pending` run with zero jobs may only be waiting for its concurrency group — an older run
12295
+ // of the same workflow on the same ref still in progress. One more read, made only here, for a run
12296
+ // that would otherwise read WEDGED on zero jobs; an unreadable answer keeps the verdict above.
12297
+ if (wedge.shape === 'zero-jobs' && run.status === 'pending' && run.workflowId != null && run.headBranch) {
12298
+ const active = git.ghActiveRunsOfWorkflow(repoAbs, run.workflowId, run.headBranch);
12299
+ const queuedBehind = ciVerdict.concurrencyBlocker(run, active);
12300
+ if (queuedBehind) wedge = ciVerdict.wedgedVerdict({ status: run.status, createdAt: run.createdAt, jobCount, jobs, queuedBehind }, wedgeOpts);
12301
+ }
12293
12302
  if (wedge.wedged) {
12294
12303
  return { ok: false, class: HUMAN_GATED, wedge: wedge.shape, detail: `run for ${at}: status=${run.status} — WEDGED (${wedge.reason}); ${ciVerdict.wedgeExit(wedge)}` };
12295
12304
  }
12305
+ if (wedge.queuedBehind) {
12306
+ const b = wedge.queuedBehind;
12307
+ return { ok: false, class: SELF_CLEARING, queuedBehind: b, detail: `run for ${at}: status=pending — queued behind run ${b.databaseId}${b.headSha ? ` (${String(b.headSha).slice(0, 7)})` : ''} of the same workflow on ${run.headBranch} (concurrency), retry later` };
12308
+ }
12296
12309
  return { ok: false, class: SELF_CLEARING, detail: `run for ${at}: status=${run.status} — still running, retry later` };
12297
12310
  }
12298
12311
 
@@ -16526,7 +16539,9 @@ the detail (--json: setAside). Override in .github/project.yml, by workflow name
16526
16539
  The first output word is one verdict:
16527
16540
  GREEN every judged workflow's newest run at the sha succeeded — ship's row would pass
16528
16541
  RED a judged run completed and did not succeed — trunk is red
16529
- PENDING a judged run is still in flight (or the sha is minutes old with no run yet)
16542
+ PENDING a judged run is still in flight (or the sha is minutes old with no run yet, or
16543
+ its run is queued behind an older in-progress run of the same workflow on
16544
+ trunk — its concurrency group, #705 — and the detail names that run)
16530
16545
  WEDGED a run that will never move on its own (zero jobs / hours unmoving, #155; every job
16531
16546
  completed but the forge never concluded the run, #650 — the detail names the exits)
16532
16547
  NONE no repo-owned run exists at the sha — CI not wired or never started
@@ -16654,7 +16669,7 @@ function trunkCiCache(repoAbs) {
16654
16669
  // thresholds are NOT cached — they are recomputed from the cached profile and the CURRENT
16655
16670
  // `ci-wait-factor`, so editing the factor applies at once with no refetch.
16656
16671
  const CI_PROFILE_TTL_DEFAULT_SEC = 24 * 3600;
16657
- const CI_PROFILE_CACHE_V = 1;
16672
+ const CI_PROFILE_CACHE_V = 2; // 2 (#706): profiles read before guard skips left the trunk sample are refetched
16658
16673
 
16659
16674
  function ciProfileTtlSec() {
16660
16675
  const env = process.env.COLAB_CI_PROFILE_TTL;
@@ -16668,11 +16683,18 @@ function ciProfileCacheFile(repoAbs) {
16668
16683
 
16669
16684
  /**
16670
16685
  * The repo's completed CI runs in the profile window — up to 3 pages of the whole list plus one page
16671
- * of trunk pushes, so a branch-heavy repo still yields trunk samples. Returns `{ rows, runsRead }`,
16672
- * or `{ error }` when the list cannot be read (the caller falls back, never fails a wait on it).
16686
+ * of trunk pushes, so a branch-heavy repo still yields trunk samples. Returns `{ rows, runsRead,
16687
+ * jobsRead, jobsFailed }`, or `{ error }` when the list cannot be read (the caller falls back, never
16688
+ * fails a wait on it).
16689
+ *
16690
+ * #706: then the check suites of the newest trunk shas (at most `ciProfile.MAX_SHA_READS`, batched
16691
+ * GraphQL, `SHA_BATCH` per call) — the run list carries no job-level signal, and a guard-skipped trunk
16692
+ * run is only visible as "one job succeeded, the rest skipped". Each trunk candidate at a read sha gets
16693
+ * `jobs` from the suite matching its `check_suite_id`; a failed or partial read leaves `jobs: null` (the
16694
+ * row is kept, as before #706); a row at a sha past the cap gets no `jobs` key at all.
16673
16695
  */
16674
16696
  function fetchCiProfileRows(repoAbs, trunk, sinceIso) {
16675
- const jq = '.workflow_runs[] | [.id, .head_branch, .head_sha, .event, .name, .status, .conclusion, .run_attempt, .created_at, .updated_at, .head_commit.timestamp] | @json';
16697
+ const jq = '.workflow_runs[] | [.id, .head_branch, .head_sha, .event, .name, .status, .conclusion, .run_attempt, .created_at, .updated_at, .head_commit.timestamp, .check_suite_id] | @json';
16676
16698
  const day = sinceIso.slice(0, 10);
16677
16699
  const paths = [1, 2, 3].map((n) => `repos/{owner}/{repo}/actions/runs?per_page=100&status=completed&created=%3E%3D${day}&page=${n}`);
16678
16700
  paths.push(`repos/{owner}/{repo}/actions/runs?per_page=100&status=completed&created=%3E%3D${day}&branch=${encodeURIComponent(trunk)}&event=push`);
@@ -16688,11 +16710,37 @@ function fetchCiProfileRows(repoAbs, trunk, sinceIso) {
16688
16710
  seen.set(a[0], {
16689
16711
  id: a[0], branch: a[1], sha: a[2], event: a[3], workflowName: a[4], status: a[5], conclusion: a[6],
16690
16712
  attempt: a[7], createdMs: Date.parse(a[8]), updatedMs: Date.parse(a[9]), headCommitMs: a[10] ? Date.parse(a[10]) : null,
16713
+ checkSuiteId: a[11] === undefined ? null : a[11],
16691
16714
  });
16692
16715
  }
16693
16716
  if (i < 3 && n < 100) i = 2; // a short page ends the paged list; the trunk page still runs
16694
16717
  }
16695
- return { rows: [...seen.values()], runsRead: seen.size };
16718
+ const rows = [...seen.values()];
16719
+ const shas = ciProfile.jobReadShas(rows, { trunk });
16720
+ const suitesAt = new Map();
16721
+ for (let i = 0; i < shas.length; i += ciProfile.SHA_BATCH) {
16722
+ const batch = shas.slice(i, i + ciProfile.SHA_BATCH).filter((x) => /^[0-9a-f]{7,40}$/i.test(x));
16723
+ if (!batch.length) continue;
16724
+ const query = `query($owner:String!,$name:String!){repository(owner:$owner,name:$name){${batch.map((x, k) => `c${k}:object(oid:"${x}"){...C}`).join(' ')}}} `
16725
+ + 'fragment C on Commit{checkSuites(first:30){nodes{databaseId checkRuns(first:100,filterBy:{checkType:LATEST}){totalCount nodes{conclusion}}}}}';
16726
+ const r = git.ghApi(repoAbs, ['graphql', '-F', 'owner={owner}', '-F', 'name={repo}', '-f', `query=${query}`,
16727
+ '--jq', '.data.repository | to_entries[] | [.key, [(.value.checkSuites.nodes // [])[] | [.databaseId, .checkRuns.totalCount, [.checkRuns.nodes[].conclusion]]]] | @json']);
16728
+ if (!r.ok) continue; // every row at these shas stays unread (jobs: null) — kept, as before #706
16729
+ for (const line of String(r.stdout || '').split('\n')) {
16730
+ let a; try { a = JSON.parse(line); } catch (_) { continue; }
16731
+ const k = Array.isArray(a) ? Number(String(a[0]).slice(1)) : NaN;
16732
+ if (!Number.isInteger(k) || !batch[k] || !Array.isArray(a[1])) continue;
16733
+ suitesAt.set(batch[k], a[1].map(([id, total, conclusions]) => ({ id, total, conclusions })));
16734
+ }
16735
+ }
16736
+ let jobsRead = 0, jobsFailed = 0;
16737
+ const read = new Set(shas);
16738
+ for (const row of rows) {
16739
+ if (!read.has(row.sha) || !ciProfile.isTrunkCandidate(row, trunk)) continue;
16740
+ row.jobs = ciProfile.jobsFromSuites(suitesAt.get(row.sha), row.checkSuiteId);
16741
+ if (row.jobs) jobsRead++; else jobsFailed++;
16742
+ }
16743
+ return { rows, runsRead: seen.size, jobsRead, jobsFailed };
16696
16744
  }
16697
16745
 
16698
16746
  const ciProfileMemo = new Map();
@@ -16730,8 +16778,9 @@ function ciProfileFor(repoAbs, { fetch = 'never' } = {}) {
16730
16778
  if (got.error) notes.push(`could not read CI history (${got.error}) — ${entry ? 'using the stale cached profile' : 'using the bootstrap bounds'}`);
16731
16779
  else {
16732
16780
  const policy = verifyRuns.parsePolicy(doc);
16733
- const prof = ciProfile.profile(ciProfile.shaDurations(got.rows, { trunk, policy }), { sinceMs });
16734
- entry = { v: CI_PROFILE_CACHE_V, trunk, fetchedAt: nowMs, since: new Date(sinceMs).toISOString(), runsRead: got.runsRead, profile: prof };
16781
+ const prof = ciProfile.profile(ciProfile.shaDurations(got.rows, { trunk, policy, probed: true }), { sinceMs });
16782
+ entry = { v: CI_PROFILE_CACHE_V, trunk, fetchedAt: nowMs, since: new Date(sinceMs).toISOString(), runsRead: got.runsRead,
16783
+ jobsRead: got.jobsRead, jobsFailed: got.jobsFailed, profile: prof };
16735
16784
  state = 'fetched';
16736
16785
  try {
16737
16786
  const tmp = `${file}.${process.pid}.tmp`;
@@ -16745,7 +16794,8 @@ function ciProfileFor(repoAbs, { fetch = 'never' } = {}) {
16745
16794
  const out = {
16746
16795
  trunk, factor, profile, th: ciProfile.thresholds(profile, factor.value),
16747
16796
  cache: { state, ageSec: state === 'fetched' ? 0 : ageSec, fetchedAt: entry ? new Date(entry.fetchedAt).toISOString() : null,
16748
- since: entry ? entry.since : null, runsRead: entry ? entry.runsRead : null },
16797
+ since: entry ? entry.since : null, runsRead: entry ? entry.runsRead : null,
16798
+ jobsRead: entry ? entry.jobsRead : null, jobsFailed: entry ? entry.jobsFailed : null },
16749
16799
  notes,
16750
16800
  };
16751
16801
  ciProfileMemo.set(memoKey, out);
@@ -16856,7 +16906,7 @@ function cmdCiWait(argv) {
16856
16906
  deadlineSec = cp.th.ciWaitDefaultSec[kind];
16857
16907
  schedule = cp.th.scheduleSec[kind];
16858
16908
  const b = kind === 'any' ? (cp.th.basis.branchWait === 'measured' || cp.th.basis.trunkWait === 'measured' ? 'measured' : 'bootstrap') : cp.th.basis[`${kind}Wait`];
16859
- deadlineBasis = `${kind === 'any' ? 'larger' : kind} bound, ${b}`;
16909
+ deadlineBasis = `${kind === 'any' ? 'larger' : kind} bound, ${b === 'branch' ? 'measured branch bound (#706: never shorter than it)' : b}`;
16860
16910
  if (!opts.json) for (const n of cp.notes) process.stderr.write(`ci-wait: ${n}\n`);
16861
16911
  }
16862
16912
 
@@ -17135,9 +17185,15 @@ finish, over clean first-attempt green shas in the last ${ciProfile.WINDOW_DAYS}
17135
17185
  runs — plus the push → run lag on trunk. A kind with fewer than ${ciProfile.MIN_SAMPLES} samples stays on its
17136
17186
  bootstrap value, which is today's fleet value exactly (15m waits, 6h wedge, 10m grace, 2m floor).
17137
17187
 
17188
+ Guard skips leave the trunk sample (#706): a trunk sha whose every green run had one job succeed and
17189
+ the rest skipped — the CI templates' "already tested at this sha or tree?" guard — measured the
17190
+ guard, not the suite. The newest ${ciProfile.MAX_SHA_READS} trunk shas get their check runs read to tell; an older one is
17191
+ not counted, and one whose read failed is kept. The dropped shas are listed; their lag still counts.
17192
+
17138
17193
  Declared: only \`ci-wait-factor: <number ≥ 1>\` in .github/project.yml (default ${ciProfile.DEFAULT_FACTOR}) — how many times
17139
17194
  the measured p95 a repo will wait. Every bound:
17140
- branch / trunk wait 2m + factor × p95 of that kind, rounded up to the minute
17195
+ branch / trunk wait 2m + factor × p95 of that kind, rounded up to the minute; the trunk wait is
17196
+ never below a measured branch wait (a full-suite trunk run runs that suite)
17141
17197
  wedged-run age factor × the longer wait, never above 6h
17142
17198
  empty-read grace factor × push→run p95, never below 2m
17143
17199
  zero-jobs floor 2m (not measurable from the run list)
@@ -17178,10 +17234,18 @@ function cmdCiProfile(argv) {
17178
17234
  const k = (name, x) => print(` ${name.padEnd(11)} ${x.source} · ${x.samples} sha(s)${x.samples ? ` · p50 ${dur(x.p50Sec)} · p95 ${dur(x.p95Sec)} · max ${dur(x.maxSec)}` : ''}`);
17179
17235
  k('branch', p.branch);
17180
17236
  k('trunk', p.trunk);
17237
+ const dr = p.trunk.dropped;
17238
+ if (dr && dr.guardSkip) {
17239
+ const shas = dr.guardSkipShas.map((s) => String(s).slice(0, 7)).join(' ');
17240
+ print(` ${''.padEnd(11)} dropped ${dr.guardSkip} guard-skip sha(s), every job but the guard skipped (max ${dur(dr.guardSkipMaxSec)}): ${shas}${dr.guardSkip > dr.guardSkipShas.length ? ' …' : ''}`);
17241
+ }
17242
+ if (dr && dr.unprobed) print(` ${''.padEnd(11)} not counted: ${dr.unprobed} older trunk sha(s) past the ${ciProfile.MAX_SHA_READS}-sha jobs read`);
17243
+ if (dr && dr.jobsUnread) print(` ${''.padEnd(11)} kept unverified: ${dr.jobsUnread} trunk sha(s) whose jobs could not be read — a guard skip among them would shorten the trunk p95`);
17181
17244
  print(` ${'push→run'.padEnd(11)} ${p.lag.source} · ${p.lag.samples} sample(s)${p.lag.samples ? ` · p95 ${dur(p.lag.p95Sec)}` : ''}`);
17182
17245
  const row = (name, sec, basis, how, boot) => print(` ${name.padEnd(11)} ${dur(sec).padEnd(7)} ${basis === 'measured' ? how : basis === 'default' ? how : 'bootstrap'}${basis === 'measured' && boot ? ` (bootstrap ${boot})` : ''}`);
17183
17246
  row('branch wait', th.branchWaitSec, th.basis.branchWait, `2m + ${th.factor} × p95`, '15m');
17184
- row('trunk wait', th.trunkWaitSec, th.basis.trunkWait, `2m + ${th.factor} × p95`, '15m');
17247
+ if (th.basis.trunkWait === 'branch') print(` ${'trunk wait'.padEnd(11)} ${dur(th.trunkWaitSec).padEnd(7)} the branch wait — trunk's own bound (${dur(th.trunkWaitOwnSec)}) is shorter, and a full-suite trunk run runs the branch suite`);
17248
+ else row('trunk wait', th.trunkWaitSec, th.basis.trunkWait, `2m + ${th.factor} × p95`, '15m');
17185
17249
  print(` ${'ci-wait'.padEnd(11)} ${dur(th.ciWaitDefaultSec.branch)} branch · ${dur(th.ciWaitDefaultSec.trunk)} trunk · ${dur(th.ciWaitDefaultSec.any)} run id (its default with no --timeout)`);
17186
17250
  row('wedged age', th.wedgeAgeSec, th.basis.wedgeAge, `${th.factor} × the longer wait, ≤ 6h`, '6h');
17187
17251
  row('empty read', th.emptyReadGraceSec, th.basis.emptyReadGrace, `${th.factor} × push→run p95, ≥ 2m`, '10m');
@@ -21,6 +21,15 @@
21
21
  * `ci-wait --sha` waits on). That is exactly the wait a session sits through: "every run at the sha
22
22
  * has finished". Only clean first-attempt green shas count — a re-run measures an incident, a red
23
23
  * run measures how fast it failed.
24
+ *
25
+ * Guard skips are not samples of the trunk suite (#706). The CI templates' "already tested at this sha
26
+ * or tree?" guard (#418, #493) skips every job on a trunk push whose tree a branch run already passed —
27
+ * on a repo that ships by squash merge that is nearly every trunk run, so a trunk sample taken from
28
+ * the run list measured the guard (seconds), not the suite (minutes), and a full-suite trunk run then
29
+ * outlived its own wait. The run list carries no job-level signal, so `tools/colab` reads the check
30
+ * runs of the newest trunk shas (capped, `MAX_SHA_READS`) onto each row as `jobs`; a sha whose every
31
+ * green run has the guard-skip shape leaves the trunk sample (its push→run lag still counts), and the
32
+ * trunk bound is never shorter than the measured branch bound.
24
33
  */
25
34
 
26
35
  const ciWait = require('./ci-wait');
@@ -33,6 +42,14 @@ const DEFAULT_FACTOR = 2;
33
42
  /** A kind with fewer clean green shas than this in the window stays on its bootstrap value. */
34
43
  const MIN_SAMPLES = 10;
35
44
  const WINDOW_DAYS = 30;
45
+ /**
46
+ * At most this many trunk shas get their jobs read per profile refresh (#706) — batched GraphQL, `SHA_BATCH`
47
+ * shas per call, so two calls, once a day per checkout through the cache. The newest win; an older trunk
48
+ * sha past the cap is not counted, so the trunk sample stays the newest shas whose shape is known.
49
+ * Measured on this repo: one REST call per run spent 78 s on 60 runs that covered only 25 shas.
50
+ */
51
+ const MAX_SHA_READS = 50;
52
+ const SHA_BATCH = 25;
36
53
 
37
54
  /**
38
55
  * The one platform constant: how long GitHub may take to start a run and materialize its job list
@@ -73,14 +90,60 @@ function parseFactor(doc) {
73
90
  return { value: n, declared: true, valid: true, reason: `${FACTOR_KEY}: ${n}` };
74
91
  }
75
92
 
93
+ /** A run the guard skipped: exactly one job succeeded (the guard) and every other job (≥ 1) was skipped.
94
+ * Name-independent, like tree-green's `treeSkipCheckRuns` — adopters rename the guard (#511) — but
95
+ * without its annotation test: the #418 sha skip leaves no notice. `row.jobs` is
96
+ * `{ total, success, skipped }`; absent or null (not read, or the read failed) is not a skip. */
97
+ function isGuardSkip(row) {
98
+ const j = row && row.jobs;
99
+ if (!j || typeof j !== 'object') return false;
100
+ const { total, success, skipped } = j;
101
+ return [total, success, skipped].every(Number.isInteger) && success === 1 && skipped >= 1 && total === success + skipped;
102
+ }
103
+
104
+ /** Could this row decide a trunk sample? The runs worth a jobs read: trunk, finished green, first attempt. */
105
+ function isTrunkCandidate(r, trunk) {
106
+ return !!r && r.branch === trunk && r.event !== 'dynamic' && r.status === 'completed' && r.conclusion === 'success' && !(Number(r.attempt) > 1);
107
+ }
108
+
109
+ /** The trunk shas to read jobs for: those of the newest candidates, distinct, at most `max` (#706). */
110
+ function jobReadShas(rows, { trunk, max = MAX_SHA_READS } = {}) {
111
+ const out = [];
112
+ for (const r of (rows || []).filter((x) => isTrunkCandidate(x, trunk))
113
+ .sort((a, b) => (b.createdMs - a.createdMs) || (Number(b.id) - Number(a.id)))) {
114
+ if (out.length >= max) break;
115
+ if (!out.includes(r.sha)) out.push(r.sha);
116
+ }
117
+ return out;
118
+ }
119
+
120
+ /**
121
+ * One run's jobs from its commit's check suites (#706): `suites` is `[{ id, total, conclusions }]` as
122
+ * GraphQL returns them (conclusions upper-case), `suiteId` the run's `check_suite_id`. Null — unread —
123
+ * when the suite is missing or its check runs did not fit the page: never a guess.
124
+ */
125
+ function jobsFromSuites(suites, suiteId) {
126
+ if (!Array.isArray(suites) || suiteId === null || suiteId === undefined) return null;
127
+ const s = suites.find((x) => x && String(x.id) === String(suiteId));
128
+ if (!s || !Array.isArray(s.conclusions) || s.total !== s.conclusions.length) return null;
129
+ const c = s.conclusions.map((x) => String(x || '').toLowerCase());
130
+ return { total: s.total, success: c.filter((x) => x === 'success').length, skipped: c.filter((x) => x === 'skipped').length };
131
+ }
132
+
76
133
  /**
77
134
  * Rows → one wall time per clean green sha. A row: `{ sha, branch, event, workflowName, status,
78
135
  * conclusion, attempt, createdMs, updatedMs, headCommitMs }`.
79
136
  *
80
137
  * A branch group at a sha that is also a trunk sha is dropped: a branch cut moments ago re-runs
81
138
  * trunk's content, and those near-instant runs would drag the branch duration down.
139
+ *
140
+ * `probed` (#706): the trunk rows carry a jobs read (`jobs`: an object, or null when the read failed).
141
+ * A trunk sha is then marked `drop: 'guard-skip'` when every green run at it has the guard-skip shape,
142
+ * or `drop: 'unprobed'` when its sha was past the read cap (no `jobs` key at all). A failed
143
+ * read keeps the sha — the pre-#706 reading, never a guess. `profile()` leaves dropped entries out of
144
+ * the trunk sample and counts them; their push→run lag still counts.
82
145
  */
83
- function shaDurations(rows, { trunk, policy = null } = {}) {
146
+ function shaDurations(rows, { trunk, policy = null, probed = false } = {}) {
84
147
  const groups = new Map();
85
148
  for (const r of rows || []) {
86
149
  if (!r || !r.sha || r.event === 'dynamic') continue;
@@ -108,7 +171,14 @@ function shaDurations(rows, { trunk, policy = null } = {}) {
108
171
  const c = Math.min(...counted.map((r) => r.headCommitMs).filter(Number.isFinite));
109
172
  if (Number.isFinite(c) && created >= c) lagSec = Math.round((created - c) / 1000);
110
173
  }
111
- out.push({ sha: g[0].sha, kind, sec: Math.round((ended - created) / 1000), lagSec, endMs: ended });
174
+ const e = { sha: g[0].sha, kind, sec: Math.round((ended - created) / 1000), lagSec, endMs: ended };
175
+ if (kind === 'trunk' && probed) {
176
+ const green = counted.filter((r) => r.conclusion === 'success');
177
+ if (green.some((r) => !Object.prototype.hasOwnProperty.call(r, 'jobs'))) e.drop = 'unprobed';
178
+ else if (green.every(isGuardSkip)) e.drop = 'guard-skip';
179
+ else if (green.some((r) => r.jobs === null)) e.jobsUnread = true;
180
+ }
181
+ out.push(e);
112
182
  }
113
183
  return out;
114
184
  }
@@ -124,11 +194,29 @@ function stats(values) {
124
194
  };
125
195
  }
126
196
 
127
- /** Durations → `{ branch, trunk, lag, source }`; only samples ending at or after `sinceMs` count. */
197
+ /** How many dropped shas the profile names; the count is always whole. */
198
+ const DROPPED_SHAS_LISTED = 20;
199
+
200
+ /**
201
+ * Durations → `{ branch, trunk, lag, source }`; only samples ending at or after `sinceMs` count.
202
+ * `trunk.dropped` (#706) says what left the trunk sample: guard skips (newest shas named), runs past
203
+ * the jobs-read cap, and how many kept shas had a failed jobs read.
204
+ */
128
205
  function profile(durations, { sinceMs = null } = {}) {
129
206
  const d = (durations || []).filter((x) => sinceMs === null || x.endMs >= sinceMs);
130
207
  const branch = stats(d.filter((x) => x.kind === 'branch').map((x) => x.sec));
131
- const trunk = stats(d.filter((x) => x.kind === 'trunk').map((x) => x.sec));
208
+ const trunkAll = d.filter((x) => x.kind === 'trunk');
209
+ const skips = trunkAll.filter((x) => x.drop === 'guard-skip').sort((a, b) => b.endMs - a.endMs);
210
+ const trunk = {
211
+ ...stats(trunkAll.filter((x) => !x.drop).map((x) => x.sec)),
212
+ dropped: {
213
+ guardSkip: skips.length,
214
+ guardSkipShas: skips.slice(0, DROPPED_SHAS_LISTED).map((x) => x.sha),
215
+ guardSkipMaxSec: skips.length ? Math.max(...skips.map((x) => x.sec)) : null,
216
+ unprobed: trunkAll.filter((x) => x.drop === 'unprobed').length,
217
+ jobsUnread: trunkAll.filter((x) => !x.drop && x.jobsUnread).length,
218
+ },
219
+ };
132
220
  const lagV = d.filter((x) => x.kind === 'trunk' && x.lagSec !== null).map((x) => x.lagSec);
133
221
  const lag = { source: lagV.length >= MIN_SAMPLES ? 'measured' : 'bootstrap', samples: lagV.length, p95Sec: quantile(lagV, 0.95) };
134
222
  const n = [branch, trunk].filter((k) => k.source === 'measured').length;
@@ -157,7 +245,11 @@ function thresholds(prof, factor = DEFAULT_FACTOR) {
157
245
  const f = Number.isFinite(factor) && factor >= 1 ? factor : DEFAULT_FACTOR;
158
246
  const p = prof || {};
159
247
  const branch = boundFor(p.branch, f, BOOTSTRAP.branchWaitSec);
160
- const trunk = boundFor(p.trunk, f, BOOTSTRAP.trunkWaitSec);
248
+ const trunkOwn = boundFor(p.trunk, f, BOOTSTRAP.trunkWaitSec);
249
+ // #706: a trunk run that does run the suite runs the branch's suite, so its wait is never shorter
250
+ // than the measured branch wait — whatever a thin or skip-heavy trunk sample says. A longer deadline
251
+ // costs nothing when the run finishes; it only reports a genuinely stuck trunk run later.
252
+ const trunk = branch.basis === 'measured' && branch.sec > trunkOwn.sec ? { sec: branch.sec, basis: 'branch' } : trunkOwn;
161
253
  const anyMeasured = branch.basis === 'measured' || trunk.basis === 'measured';
162
254
  const wedgeAgeSec = anyMeasured ? Math.min(BOOTSTRAP.wedgeAgeSec, ceilMinute(f * Math.max(branch.sec, trunk.sec))) : BOOTSTRAP.wedgeAgeSec;
163
255
  const lagMeasured = p.lag && p.lag.source === 'measured' && Number.isFinite(p.lag.p95Sec);
@@ -166,6 +258,7 @@ function thresholds(prof, factor = DEFAULT_FACTOR) {
166
258
  factor: f,
167
259
  branchWaitSec: branch.sec,
168
260
  trunkWaitSec: trunk.sec,
261
+ trunkWaitOwnSec: trunkOwn.sec,
169
262
  ciWaitDefaultSec: { branch: branch.sec, trunk: trunk.sec, any: Math.max(branch.sec, trunk.sec) },
170
263
  wedgeAgeSec,
171
264
  emptyReadGraceSec,
@@ -190,6 +283,6 @@ function kindFor({ branch = null, trunk = null, onTrunk = null } = {}) {
190
283
  }
191
284
 
192
285
  module.exports = {
193
- FACTOR_KEY, DEFAULT_FACTOR, MIN_SAMPLES, WINDOW_DAYS, PLATFORM_FLOOR_SEC, BOOTSTRAP,
194
- parseFactor, shaDurations, profile, thresholds, scheduleFor, kindFor,
286
+ FACTOR_KEY, DEFAULT_FACTOR, MIN_SAMPLES, WINDOW_DAYS, MAX_SHA_READS, SHA_BATCH, PLATFORM_FLOOR_SEC, BOOTSTRAP,
287
+ parseFactor, isGuardSkip, isTrunkCandidate, jobReadShas, jobsFromSuites, shaDurations, profile, thresholds, scheduleFor, kindFor,
195
288
  };
@@ -51,6 +51,18 @@
51
51
  * a positive (same posture as the two signals above). Its verdict is still HUMAN_GATED, never green,
52
52
  * even when every job succeeded: the forge has not said the run passed (epic #652, fail closed).
53
53
  * `wedgeExit` below names the exits that work for each shape, so the verdict is actionable.
54
+ *
55
+ * Amendment (#705): signal B's zero jobs has one ordinary cause it must not wedge on — a run WAITING
56
+ * FOR ITS CONCURRENCY GROUP. A workflow with `concurrency: { group: ci-${{ github.ref }} }` that does
57
+ * not cancel in progress on trunk leaves the newer trunk run at `status: pending` with zero jobs until
58
+ * the older run holding the group finishes. Measured: a trunk read WEDGED / "a human must look" while
59
+ * the previous trunk run of the same workflow was in progress with jobs running, and branch runs
60
+ * started after it were already in progress (so runners were there). Signal: the run is `pending`
61
+ * (the status GitHub gives a run held by its group — never `queued`, which is the runner wait #155's
62
+ * shape lives in), and an OLDER run of the same workflow on the same ref is `in_progress`
63
+ * (`concurrencyBlocker` below; the caller reads the in-progress runs lazily, only for a run about to
64
+ * wedge on zero jobs). That reads as waiting — not wedged — and the age backstop still bounds it, so
65
+ * a group held forever by a dead run still reaches a human.
54
66
  */
55
67
 
56
68
  // Generous on purpose (see doc comment above) — an ordinary run finishes in minutes to tens of
@@ -123,9 +135,14 @@ function unconcludedJobs(jobs, opts = {}) {
123
135
  * `jobs` (#650) is optional — `ghRunJobs`'s list, read by the same lazy call; when present it is what
124
136
  * tells the forge-stuck shape (every job completed, run never concluded) from #155's two.
125
137
  *
138
+ * `queuedBehind` (#705) is optional — `concurrencyBlocker`'s answer: the older in-progress run of the
139
+ * same workflow on the same ref that a `pending` run is waiting on. Present, it turns zero jobs from a
140
+ * wedge into waiting (age backstop still applies).
141
+ *
126
142
  * Returns `{ wedged, reason, shape }`. `reason` and `shape` are null when `wedged` is false; `shape` is
127
143
  * `'zero-jobs'` | `'unconcluded'` | `'age'` (which signal fired). An `'unconcluded'` verdict also
128
- * carries `failed` — the jobs that concluded red, if any.
144
+ * carries `failed` — the jobs that concluded red, if any. A not-wedged verdict for a run waiting on its
145
+ * concurrency group carries `queuedBehind` back, so the caller can name the run it waits on.
129
146
  */
130
147
  function wedgedVerdict(run, opts = {}) {
131
148
  if (!run || run.status === 'completed' || run.status === 'none') return { wedged: false, reason: null, shape: null };
@@ -134,6 +151,15 @@ function wedgedVerdict(run, opts = {}) {
134
151
  const floorMin = Number.isFinite(opts.zeroJobsFloorSec) ? opts.zeroJobsFloorSec / 60 : ZERO_JOBS_AGE_FLOOR_MINUTES;
135
152
  const wedgeHours = Number.isFinite(opts.wedgeAgeSec) ? opts.wedgeAgeSec / 3600 : WEDGE_AGE_HOURS;
136
153
  if (run.jobCount === 0) {
154
+ // #705: held by its own concurrency group behind an older in-progress run — ordinary queueing,
155
+ // bounded only by the age backstop (a group held for hours means the holder is what is stuck).
156
+ if (run.status === 'pending' && run.queuedBehind) {
157
+ const b = run.queuedBehind;
158
+ if (age !== null && age >= wedgeHours) {
159
+ return { wedged: true, shape: 'age', reason: `status=pending for ${age.toFixed(1)}h queued behind run ${b.databaseId} of the same workflow (concurrency), (>= ${fmtHours(wedgeHours)} backstop)` };
160
+ }
161
+ return { wedged: false, reason: null, shape: null, queuedBehind: b };
162
+ }
137
163
  // #171: no createdAt (age unknown) never manufactures a positive here, same posture as the age
138
164
  // backstop below when createdAt is missing/unparseable — a missing signal must not wedge on its own.
139
165
  const floorHours = floorMin / 60;
@@ -164,6 +190,35 @@ function wedgedVerdict(run, opts = {}) {
164
190
  return { wedged: false, reason: null, shape: null };
165
191
  }
166
192
 
193
+ /**
194
+ * #705: the older in-progress run of the same workflow on the same ref that `run` waits behind, or
195
+ * null. `run` is the picked run (`{status, databaseId, createdAt}`); `active` is the in-progress runs
196
+ * of its workflow on its ref (tools/lib/git.js `ghActiveRunsOfWorkflow`), or null when unread.
197
+ * Only a `pending` run is ever queued behind its group (see the #705 amendment above). "Older" is the
198
+ * earlier createdAt, else the lower run id; a row that cannot be ordered against `run` is not a
199
+ * blocker — a missing signal never manufactures one. Returns `{ databaseId, createdAt, headSha }` of
200
+ * the newest such run.
201
+ */
202
+ function concurrencyBlocker(run, active) {
203
+ if (!run || run.status !== 'pending' || !Array.isArray(active)) return null;
204
+ const tOf = (x) => (x && x.createdAt ? Date.parse(x.createdAt) : NaN);
205
+ const idOf = (x) => (x && x.databaseId != null ? Number(x.databaseId) : NaN);
206
+ const rt = tOf(run);
207
+ const rid = idOf(run);
208
+ const older = active.filter((x) => {
209
+ if (!x || x.status !== 'in_progress') return false;
210
+ if (!Number.isNaN(rid) && idOf(x) === rid) return false;
211
+ const t = tOf(x);
212
+ if (!Number.isNaN(rt) && !Number.isNaN(t) && t !== rt) return t < rt;
213
+ const id = idOf(x);
214
+ return !Number.isNaN(rid) && !Number.isNaN(id) && id < rid;
215
+ });
216
+ if (!older.length) return null;
217
+ older.sort((a, b) => (tOf(b) || 0) - (tOf(a) || 0) || (idOf(b) || 0) - (idOf(a) || 0));
218
+ const b = older[0];
219
+ return { databaseId: b.databaseId != null ? b.databaseId : null, createdAt: b.createdAt || null, headSha: b.headSha || null };
220
+ }
221
+
167
222
  /**
168
223
  * The exit a WEDGED verdict names (#650) — the tail of the detail line after the reason. #155's two
169
224
  * shapes keep their words: a human must look (billing, a runner pool, a dead runner). The forge-stuck
@@ -209,4 +264,4 @@ function emptyReadVerdict(committedAtMs, opts = {}) {
209
264
  return { fresh: minutes >= 0 && minutes < graceMinutes, minutes, graceMinutes };
210
265
  }
211
266
 
212
- module.exports = { WEDGE_AGE_HOURS, ZERO_JOBS_AGE_FLOOR_MINUTES, UNCONCLUDED_FLOOR_MINUTES, EMPTY_READ_GRACE_MINUTES, wedgedVerdict, unconcludedJobs, wedgeExit, emptyReadVerdict };
267
+ module.exports = { WEDGE_AGE_HOURS, ZERO_JOBS_AGE_FLOOR_MINUTES, UNCONCLUDED_FLOOR_MINUTES, EMPTY_READ_GRACE_MINUTES, wedgedVerdict, unconcludedJobs, concurrencyBlocker, wedgeExit, emptyReadVerdict };
@@ -102,6 +102,9 @@ function restRow(x) {
102
102
  event: x.event || null, url: x.html_url || null,
103
103
  // #510: the workflow FILE's id — the cure rule's same-workflow test, which a display name cannot answer.
104
104
  workflowId: x.workflow_id ?? null,
105
+ // #705: the ref the run is on — a zero-jobs run is told apart from one queued behind its own
106
+ // concurrency group by an older in-progress run of the same workflow on the SAME ref.
107
+ headBranch: x.head_branch || null,
105
108
  };
106
109
  }
107
110
 
package/tools/lib/git.js CHANGED
@@ -1014,7 +1014,8 @@ function ghRunsForCommit(repo, branch, sha, limit = 10) {
1014
1014
  // workflowDatabaseId is additive (#510): the cure rule's same-workflow test reads the workflow's
1015
1015
  // id, never its display name; mapped to `workflowId`, the name the REST row (ci-wait restRow) uses.
1016
1016
  // updatedAt is additive (#555): a finished run's completion time, the lone-member wait's anchor.
1017
- '--json', 'headSha,status,conclusion,createdAt,updatedAt,databaseId,workflowName,workflowDatabaseId,event'], { cwd: repo });
1017
+ // headBranch is additive (#705): the ref the concurrency-queue read (ghActiveRunsOfWorkflow) keys on.
1018
+ '--json', 'headSha,status,conclusion,createdAt,updatedAt,databaseId,workflowName,workflowDatabaseId,event,headBranch'], { cwd: repo });
1018
1019
  if (!r.ok) return null;
1019
1020
  let runs;
1020
1021
  try { runs = JSON.parse(r.stdout); } catch (_) { return null; }
@@ -1458,7 +1459,14 @@ function summarizeRepoOwnedRuns(forSha, sha) {
1458
1459
  // forSha[0] is the newest remaining row either way).
1459
1460
  const pending = forSha.find((x) => x.status !== 'completed');
1460
1461
  const pick = pending || forSha[0];
1461
- return { status: pick.status, conclusion: pick.conclusion || null, sha, createdAt: pick.createdAt || null, databaseId: pick.databaseId || null, runCount };
1462
+ // #705: a still-running pick carries its workflow id and ref, when the row had them, so the wedge
1463
+ // read can ask whether it is only queued behind an older run of the same workflow on the same ref
1464
+ // (its concurrency group) — additive, and absent on rows that never had them.
1465
+ const queueKeys = pending ? {
1466
+ ...(pick.workflowId != null ? { workflowId: pick.workflowId } : {}),
1467
+ ...(pick.headBranch ? { headBranch: pick.headBranch } : {}),
1468
+ } : {};
1469
+ return { status: pick.status, conclusion: pick.conclusion || null, sha, createdAt: pick.createdAt || null, databaseId: pick.databaseId || null, runCount, ...queueKeys };
1462
1470
  }
1463
1471
 
1464
1472
  /**
@@ -1482,6 +1490,39 @@ function ghRunJobCount(repo, runDatabaseId) {
1482
1490
  return Number.isFinite(n) ? n : null;
1483
1491
  }
1484
1492
 
1493
+ /**
1494
+ * The IN-PROGRESS runs of one workflow on one ref (#705) — `[{databaseId, status, createdAt, headSha,
1495
+ * headBranch, workflowId}]` (ci-wait restRow shape), or null when it could not be read. null must NOT
1496
+ * be read as "none in progress": the caller then keeps the verdict it had without this read.
1497
+ *
1498
+ * Why it exists: a run waiting for its concurrency group (`concurrency: { group: …${github.ref} }`
1499
+ * with no cancel-in-progress on trunk) sits at `status: pending` with ZERO jobs until the older run
1500
+ * holding the group finishes — exactly #155's wedge shape, while it is ordinary queueing. The older
1501
+ * run in progress on the same workflow and ref is what tells the two apart.
1502
+ *
1503
+ * One REST call (`actions/workflows/<id>/runs?branch=&status=in_progress`), made lazily: only for a
1504
+ * pending run that would otherwise read WEDGED on zero jobs. Falls back to `gh run list` on any
1505
+ * failure but a rate limit, the same posture as ghRunsAtShaRest.
1506
+ */
1507
+ function ghActiveRunsOfWorkflow(repo, workflowId, branch) {
1508
+ if (workflowId == null || !branch) return null;
1509
+ const ciWait = require('./ci-wait');
1510
+ const q = `branch=${encodeURIComponent(branch)}&status=in_progress&per_page=20`;
1511
+ const r = ghApiConditional(repo, `repos/{owner}/{repo}/actions/workflows/${encodeURIComponent(String(workflowId))}/runs?${q}`);
1512
+ if (r.kind === 'rate-limited') return null;
1513
+ if (r.kind === 'ok' && r.json && Array.isArray(r.json.workflow_runs)) {
1514
+ return r.json.workflow_runs.map(ciWait.restRow).filter((x) => x.status === 'in_progress' && (!x.headBranch || x.headBranch === branch));
1515
+ }
1516
+ const l = gh(['run', 'list', '--workflow', String(workflowId), '--branch', branch, '--status', 'in_progress', '-L', '20',
1517
+ '--json', 'databaseId,status,createdAt,headSha,headBranch,workflowDatabaseId'], { cwd: repo });
1518
+ if (!l.ok) return null;
1519
+ let rows;
1520
+ try { rows = JSON.parse(l.stdout); } catch (_) { return null; }
1521
+ if (!Array.isArray(rows)) return null;
1522
+ return rows.filter((x) => x && x.status === 'in_progress')
1523
+ .map(({ workflowDatabaseId, ...x }) => ({ ...x, workflowId: workflowDatabaseId ?? workflowId }));
1524
+ }
1525
+
1485
1526
  /**
1486
1527
  * The JOBS of one workflow run, with their steps (#321) — `[{name, status, conclusion, startedAt,
1487
1528
  * completedAt, steps: [{name, status, conclusion}]}]`, or `null` if it could not be read (gh
@@ -1584,7 +1625,7 @@ module.exports = {
1584
1625
  dirtyTracked, dirtyUntracked, dirtyAny,
1585
1626
  ghAvailable, ghState, ghInstalled, ghApiConditional, ghRunsAtShaRest, ghRunsAtShaUnfiltered, fullSha, ghIssueEdit, ghListLabels, ghOpenIssueNumbersByLabel, ghAssignedIssues,
1586
1627
  ghCurrentLogin, ghIssueView, ghIssueComment, ghRunForSha, ghRunForCommit, ghRunsForCommit, ghRunsForRef, ghRunsAtCommit, ghRunForCommitAnyRef, commitTimeMs, ghRunsSince, summarizeRunsForCommit, isRepoOwnedRun,
1587
- ghRunJobCount, ghRunJobs, ghWorkflowDispatch,
1628
+ ghRunJobCount, ghRunJobs, ghActiveRunsOfWorkflow, ghWorkflowDispatch,
1588
1629
  ghIssueListByLabel, ghLabelDelete, ghLabelCreate, ghListLabelsDetailed, ghLabelEditDescription,
1589
1630
  ghApi, isGraphqlRateLimit, ghIssueRelease, ghClaimHolder, ghIssueLabelEvents, ghIssueLabelActors,
1590
1631
  ghPrForBranch, ghOpenPrsForBranch, ghOpenPrs, ghPrCreate, ghPrClose, ghRepoVisibility, ghRunCancel,