@futurelastic/colab-handbook 1.13.0-rc.23 → 1.13.0-rc.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/audit/audit.mjs CHANGED
@@ -1304,6 +1304,17 @@ function auditRepo(target, ctx) {
1304
1304
  }
1305
1305
  }
1306
1306
 
1307
+ // ---- ship-batch-wait (#555) -------------------------------------------------
1308
+ // How long a lone ready candidate waits for a partner (tools/lib/ship-batch.js parseShipBatchWait,
1309
+ // the reading `colab ship --batch` uses). Malformed fails here and in the CI templates' descriptor
1310
+ // check, the #416 pattern — ship fails closed to no wait, silently. A window with no batch to fill
1311
+ // (ship-batch absent or 1) is inert, so it warns. No ceiling: the repo chooses the value.
1312
+ if ("ship-batch-wait" in (cfg || {}) && cfg["ship-batch-wait"] !== null) {
1313
+ const wCfg = shipBatch.parseShipBatchWait(cfg);
1314
+ if (!wCfg.valid) fail(`${wCfg.reason} — see project.schema.md, ship-batch-wait`);
1315
+ else if (wCfg.sec > 0 && shipBatch.parseShipBatch(cfg).n <= 1) warn(`${wCfg.reason} is inert without ship-batch > 1 — there is no batch for a partner to join`);
1316
+ }
1317
+
1307
1318
  // ---- migrations (#383) ----------------------------------------------------
1308
1319
  // Where the repo's migrations live, beyond the two defaults (database/migrations/,
1309
1320
  // prisma/migrations/) — tools/lib/migration-paths.js is the one reading, shared with
package/package.json CHANGED
@@ -33,5 +33,5 @@
33
33
  "engines": {
34
34
  "node": ">=18"
35
35
  },
36
- "version": "1.13.0-rc.23"
36
+ "version": "1.13.0-rc.25"
37
37
  }
@@ -263,23 +263,39 @@ jobs:
263
263
  # green; only a ship dry run noticed. So the value fails HERE, where the person
264
264
  # who changed project.yml is looking. Same rule as `colab ship` and the audit
265
265
  # (tools/lib/ship-batch.js parseShipBatch): absent/empty/null, or 1–3. Why the
266
- # cap is 3: project.schema.md, `ship-batch`.
266
+ # cap is 3: project.schema.md, `ship-batch`. `ship-batch-wait` (#555) the same way
267
+ # (parseShipBatchWait): absent/empty/null, or a whole number with a unit (s, m, h).
267
268
  # ---------------------------------------------------------------------------
268
269
  - name: Descriptor check (.github/project.yml)
269
270
  working-directory: ${{ github.workspace }}
270
271
  run: |
271
272
  [ -f .github/project.yml ] || exit 0
272
- line="$(grep -E '^ship-batch[[:space:]]*:' .github/project.yml | head -1 || true)"
273
- [ -n "$line" ] || exit 0
274
- v="$(printf '%s\n' "$line" | sed -E -e 's/^ship-batch[[:space:]]*:[[:space:]]*//' \
273
+ # The scalar after `<key>:` — comment, trailing blanks and one level of quotes stripped.
274
+ field() {
275
+ grep -E "^$1[[:space:]]*:" .github/project.yml | head -1 \
276
+ | sed -E -e "s/^$1[[:space:]]*:[[:space:]]*//" \
275
277
  -e 's/[[:space:]]+#.*$//' -e 's/[[:space:]]+$//' \
276
- -e 's/^"(.*)"$/\1/' -e "s/^'(.*)'\$/\\1/")"
278
+ -e 's/^"(.*)"$/\1/' -e "s/^'(.*)'\$/\\1/" || true
279
+ }
280
+ v="$(field ship-batch)"
277
281
  case "$v" in
278
282
  ''|'~'|null|1|2|3) echo "ship-batch: ${v:-absent} — ok" ;;
279
283
  *)
280
284
  echo "::error file=.github/project.yml::ship-batch is '$v', expected an integer 1-3 (omit for serial). colab ship ignores any other value and lands serially. Why the cap is 3: project.schema.md, ship-batch."
281
285
  exit 1 ;;
282
286
  esac
287
+ # ship-batch-wait (#555): a whole number with a unit — 90s, 6m, 1h. No ceiling: the repo chooses.
288
+ w="$(field ship-batch-wait)"
289
+ case "$w" in
290
+ ''|'~'|null) echo "ship-batch-wait: absent — ok" ;;
291
+ *)
292
+ if printf '%s\n' "$w" | grep -Eq '^[0-9]+[smh]$'; then
293
+ echo "ship-batch-wait: $w — ok"
294
+ else
295
+ echo "::error file=.github/project.yml::ship-batch-wait is '$w', expected a whole number with a unit: 90s, 6m, 1h (omit for no wait). colab ship ignores any other value and does not wait."
296
+ exit 1
297
+ fi ;;
298
+ esac
283
299
 
284
300
  # ---------------------------------------------------------------------------
285
301
  # Toolchain resolution. Precedence, highest first:
@@ -13,7 +13,8 @@
13
13
  # `php artisan wayfinder:generate` and needs PHP on the runner. Use ci-laravel.yml.
14
14
  #
15
15
  # What it does:
16
- # 0. Fail on a .github/project.yml value colab would silently ignore (ship-batch, #416).
16
+ # 0. Fail on a .github/project.yml value colab would silently ignore (ship-batch, #416;
17
+ # ship-batch-wait, #555).
17
18
  # 1. Resolve the Node version from declared sources — never a hardcoded default.
18
19
  # 2. Secret scan with gitleaks (pinned release binary, see note at the step).
19
20
  # 3. Install dependencies with the package manager the lockfile implies.
@@ -257,23 +258,39 @@ jobs:
257
258
  # green; only a ship dry run noticed. So the value fails HERE, where the person
258
259
  # who changed project.yml is looking. Same rule as `colab ship` and the audit
259
260
  # (tools/lib/ship-batch.js parseShipBatch): absent/empty/null, or 1–3. Why the
260
- # cap is 3: project.schema.md, `ship-batch`.
261
+ # cap is 3: project.schema.md, `ship-batch`. `ship-batch-wait` (#555) the same way
262
+ # (parseShipBatchWait): absent/empty/null, or a whole number with a unit (s, m, h).
261
263
  # ---------------------------------------------------------------------------
262
264
  - name: Descriptor check (.github/project.yml)
263
265
  working-directory: ${{ github.workspace }}
264
266
  run: |
265
267
  [ -f .github/project.yml ] || exit 0
266
- line="$(grep -E '^ship-batch[[:space:]]*:' .github/project.yml | head -1 || true)"
267
- [ -n "$line" ] || exit 0
268
- v="$(printf '%s\n' "$line" | sed -E -e 's/^ship-batch[[:space:]]*:[[:space:]]*//' \
268
+ # The scalar after `<key>:` — comment, trailing blanks and one level of quotes stripped.
269
+ field() {
270
+ grep -E "^$1[[:space:]]*:" .github/project.yml | head -1 \
271
+ | sed -E -e "s/^$1[[:space:]]*:[[:space:]]*//" \
269
272
  -e 's/[[:space:]]+#.*$//' -e 's/[[:space:]]+$//' \
270
- -e 's/^"(.*)"$/\1/' -e "s/^'(.*)'\$/\\1/")"
273
+ -e 's/^"(.*)"$/\1/' -e "s/^'(.*)'\$/\\1/" || true
274
+ }
275
+ v="$(field ship-batch)"
271
276
  case "$v" in
272
277
  ''|'~'|null|1|2|3) echo "ship-batch: ${v:-absent} — ok" ;;
273
278
  *)
274
279
  echo "::error file=.github/project.yml::ship-batch is '$v', expected an integer 1-3 (omit for serial). colab ship ignores any other value and lands serially. Why the cap is 3: project.schema.md, ship-batch."
275
280
  exit 1 ;;
276
281
  esac
282
+ # ship-batch-wait (#555): a whole number with a unit — 90s, 6m, 1h. No ceiling: the repo chooses.
283
+ w="$(field ship-batch-wait)"
284
+ case "$w" in
285
+ ''|'~'|null) echo "ship-batch-wait: absent — ok" ;;
286
+ *)
287
+ if printf '%s\n' "$w" | grep -Eq '^[0-9]+[smh]$'; then
288
+ echo "ship-batch-wait: $w — ok"
289
+ else
290
+ echo "::error file=.github/project.yml::ship-batch-wait is '$w', expected a whole number with a unit: 90s, 6m, 1h (omit for no wait). colab ship ignores any other value and does not wait."
291
+ exit 1
292
+ fi ;;
293
+ esac
277
294
 
278
295
  # ---------------------------------------------------------------------------
279
296
  # Toolchain resolution. Precedence, highest first:
@@ -27,7 +27,8 @@
27
27
  # NOT for: Laravel + Inertia fullstack apps — use ci-laravel.yml.
28
28
  #
29
29
  # What it does:
30
- # 0. Fail on a .github/project.yml value colab would silently ignore (ship-batch, #416).
30
+ # 0. Fail on a .github/project.yml value colab would silently ignore (ship-batch, #416;
31
+ # ship-batch-wait, #555).
31
32
  # 1. Resolve the Python version from declared sources — never a hardcoded default.
32
33
  # 2. Secret scan with gitleaks (pinned release binary, see note at the step).
33
34
  # 3. Install dependencies from whichever manifest the repo actually has.
@@ -248,23 +249,39 @@ jobs:
248
249
  # green; only a ship dry run noticed. So the value fails HERE, where the person
249
250
  # who changed project.yml is looking. Same rule as `colab ship` and the audit
250
251
  # (tools/lib/ship-batch.js parseShipBatch): absent/empty/null, or 1–3. Why the
251
- # cap is 3: project.schema.md, `ship-batch`.
252
+ # cap is 3: project.schema.md, `ship-batch`. `ship-batch-wait` (#555) the same way
253
+ # (parseShipBatchWait): absent/empty/null, or a whole number with a unit (s, m, h).
252
254
  # ---------------------------------------------------------------------------
253
255
  - name: Descriptor check (.github/project.yml)
254
256
  working-directory: ${{ github.workspace }}
255
257
  run: |
256
258
  [ -f .github/project.yml ] || exit 0
257
- line="$(grep -E '^ship-batch[[:space:]]*:' .github/project.yml | head -1 || true)"
258
- [ -n "$line" ] || exit 0
259
- v="$(printf '%s\n' "$line" | sed -E -e 's/^ship-batch[[:space:]]*:[[:space:]]*//' \
259
+ # The scalar after `<key>:` — comment, trailing blanks and one level of quotes stripped.
260
+ field() {
261
+ grep -E "^$1[[:space:]]*:" .github/project.yml | head -1 \
262
+ | sed -E -e "s/^$1[[:space:]]*:[[:space:]]*//" \
260
263
  -e 's/[[:space:]]+#.*$//' -e 's/[[:space:]]+$//' \
261
- -e 's/^"(.*)"$/\1/' -e "s/^'(.*)'\$/\\1/")"
264
+ -e 's/^"(.*)"$/\1/' -e "s/^'(.*)'\$/\\1/" || true
265
+ }
266
+ v="$(field ship-batch)"
262
267
  case "$v" in
263
268
  ''|'~'|null|1|2|3) echo "ship-batch: ${v:-absent} — ok" ;;
264
269
  *)
265
270
  echo "::error file=.github/project.yml::ship-batch is '$v', expected an integer 1-3 (omit for serial). colab ship ignores any other value and lands serially. Why the cap is 3: project.schema.md, ship-batch."
266
271
  exit 1 ;;
267
272
  esac
273
+ # ship-batch-wait (#555): a whole number with a unit — 90s, 6m, 1h. No ceiling: the repo chooses.
274
+ w="$(field ship-batch-wait)"
275
+ case "$w" in
276
+ ''|'~'|null) echo "ship-batch-wait: absent — ok" ;;
277
+ *)
278
+ if printf '%s\n' "$w" | grep -Eq '^[0-9]+[smh]$'; then
279
+ echo "ship-batch-wait: $w — ok"
280
+ else
281
+ echo "::error file=.github/project.yml::ship-batch-wait is '$w', expected a whole number with a unit: 90s, 6m, 1h (omit for no wait). colab ship ignores any other value and does not wait."
282
+ exit 1
283
+ fi ;;
284
+ esac
268
285
 
269
286
  # ---------------------------------------------------------------------------
270
287
  # Toolchain resolution. Precedence, highest first:
package/tools/colab CHANGED
@@ -63,6 +63,7 @@ const codec = require('./lib/codec');
63
63
  const machine = require('./lib/machine');
64
64
  const targetSync = require('./lib/ship-target-sync');
65
65
  const shipBatch = require('./lib/ship-batch');
66
+ const batchStats = require('./lib/batch-stats');
66
67
  const conflictMarkers = require('./lib/conflict-markers');
67
68
  const docsOnly = require('./lib/docs-only');
68
69
  const cureDiff = require('./lib/cure-diff');
@@ -10351,6 +10352,12 @@ at build, added since) is NOT landed with it — the command names both lists an
10351
10352
  --dry pushes, deletes, lands, comments and releases nothing on EVERY path (#415): a fresh batch is built
10352
10353
  locally and printed; a staged green batch prints what would land (members, combined run, target sha).
10353
10354
  Consumer CI must opt in: add 'ship-batch/**' to a CI workflow's push: branches:.
10355
+ PARTNER WAIT (#555) — opt-in per repo: \`ship-batch-wait: <duration>\` in project.yml (90s, 6m, 1h; the
10356
+ unit is required). Absent, zero or malformed: no wait, exactly as above. With it, a call where exactly
10357
+ ONE member can join pauses (exit 3, "⏸ PARTNER-WAIT", the seconds left) until that window has passed
10358
+ since the member became ready (its head CI's last update, else its head commit's date) — then the same
10359
+ call declines it to serial (exit 4). The caller waits, gathers every ready candidate again and calls
10360
+ --batch with all of them. A single named branch is accepted only when the field is set.
10354
10361
 
10355
10362
  B3 KEEPS the branch by default. Deleting refs from a shared remote is not something this command
10356
10363
  does unasked, however certain it is that the content landed: reporting is safe and reversible,
@@ -13909,7 +13916,10 @@ function cmdShipBatch(opts) {
13909
13916
  if (String(doc.autonomy || '') !== 'auto-trunk') {
13910
13917
  return serial(`project.yml does not grant autonomy: auto-trunk — batch landing needs it (every member lands in one unattended push)`);
13911
13918
  }
13912
- if (members.length < 2) return serial(`--batch names ${members.length} branch(es) — a batch needs at least two`);
13919
+ // #555: a declared ship-batch-wait lets ONE named member through — it may wait for a partner below.
13920
+ const waitCfg = shipBatch.parseShipBatchWait(doc);
13921
+ if (!waitCfg.valid) warn(`${waitCfg.reason} — ignored: a lone member does not wait (fails closed, #555)`);
13922
+ if (members.length < 2 && !(members.length === 1 && waitCfg.sec > 0)) return serial(`--batch names ${members.length} branch(es) — a batch needs at least two`);
13913
13923
  if (members.length > cfg.n) warn(`--batch names ${members.length} branches; ship-batch: ${cfg.n} caps the batch — the rest wait for the next one`);
13914
13924
 
13915
13925
  const remote = git.remoteFor(repoAbs);
@@ -14048,6 +14058,22 @@ function cmdShipBatch(opts) {
14048
14058
  e.ok && !selected.includes(e) ? `eligible — waits for the next batch (cap ${cfg.n})` : e.why]), ['', 'MEMBER', 'DETAIL']));
14049
14059
  const notJoining = evaluated.filter((e) => !selected.includes(e)).map((e) => e.branch);
14050
14060
  step = shipBatch.nextStep({ enabled: true, wired: true, trunkCi, eligibleCount: selected.length, existing: null, trunkNow });
14061
+ if (step.step === 'serial' && selected.length === 1 && waitCfg.sec > 0) {
14062
+ // #555: the lane is idle here (trunk's run finished, no batch in flight at this base), and one
14063
+ // member is ready. Wait for a partner until the repo's window, counted from when it became ready.
14064
+ const lone = selected[0];
14065
+ const commitIso = git.git(['log', '-1', '--format=%cI', lone.sha], repoAbs);
14066
+ const since = shipBatch.readySince(git.ghRunsForCommit(repoAbs, lone.branch, lone.sha), commitIso.ok ? commitIso.stdout.trim() : null);
14067
+ const pw = shipBatch.partnerWait({ n: cfg.n, waitSec: waitCfg.sec, eligibleCount: 1, readySinceMs: since, nowMs: Date.now() });
14068
+ if (pw.wait) {
14069
+ print(`⏸ PARTNER-WAIT ${lone.branch} — the only ready member; ${waitCfg.reason}, ready since ${new Date(since).toISOString()}, ${ciWait.fmtDuration(pw.leftSec * 1000)} left. Nothing landed.`);
14070
+ print(` Wait up to ${pw.leftSec}s (a wait on another candidate's run counts toward it), gather EVERY ready candidate again,`);
14071
+ print(' then run colab ship --batch with all of them. Past the window, this same call lands it alone (serial).');
14072
+ return 3;
14073
+ }
14074
+ const why = { 'window-elapsed': `${waitCfg.reason} has elapsed since it became ready`, 'ready-time-unread': 'its ready time could not be read, so it does not wait' }[pw.reason] || pw.reason;
14075
+ return serial(`1 member can join and no partner arrived — ${why}`);
14076
+ }
14051
14077
  if (step.step === 'serial') return serial(`${selected.length} member(s) can join — a batch needs at least two`);
14052
14078
 
14053
14079
  // ---- (e) build: trunk head + one squash commit per member, in an ephemeral detached worktree ----
@@ -14069,9 +14095,9 @@ function cmdShipBatch(opts) {
14069
14095
  // regenerated by .colab/hooks/pre-ship, never hand-merged and never a reason to leave the batch.
14070
14096
  // Only an overlap outside `generated:` (or no hook to regenerate with) drops the member.
14071
14097
  const nonGen = unmerged.filter((f) => !isGeneratedPath(f, generated));
14072
- let why = null;
14073
- if (nonGen.length) why = `conflicts with the members already in (${nonGen.join(', ')}) — next batch`;
14074
- else if (!hasPreShip) why = `conflicts with the members already in only in generated file(s) (${unmerged.join(', ')}), but there is no .colab/hooks/pre-ship to regenerate them — next batch`;
14098
+ let why = null, cls = null;
14099
+ if (nonGen.length) { why = `conflicts with the members already in (${nonGen.join(', ')}) — next batch`; cls = 'conflict'; }
14100
+ else if (!hasPreShip) { cls = 'generated-no-hook'; why = `conflicts with the members already in only in generated file(s) (${unmerged.join(', ')}), but there is no .colab/hooks/pre-ship to regenerate them — next batch`; }
14075
14101
  else {
14076
14102
  const hk = runHook(repoAbs, 'pre-ship', {
14077
14103
  wtPath: dir, name: m.branch, branch: m.branch, ports: [], trunk, base: trunk,
@@ -14082,24 +14108,26 @@ function cmdShipBatch(opts) {
14082
14108
  // state, so `left` reads empty — read the staged content itself before trusting it.
14083
14109
  const markers = hk.ran && hk.ok && !left.length ? stagedConflictMarkers(dir, unmerged) : [];
14084
14110
  if (!hk.ran || !hk.ok || left.length) {
14111
+ cls = 'hook-failed';
14085
14112
  why = `.colab/hooks/pre-ship could not regenerate ${unmerged.join(', ')} on the combined head${left.length && hk.ok ? ` (still unmerged: ${left.join(', ')})` : ''} — next batch`;
14086
14113
  } else if (markers.length) {
14114
+ cls = 'hook-markers';
14087
14115
  why = `.colab/hooks/pre-ship exited 0 but left conflict markers staged in ${conflictMarkers.describe(markers)} on the combined head — treated as a failing hook, next batch`;
14088
14116
  } else regenerated = unmerged;
14089
14117
  }
14090
14118
  if (why) {
14091
14119
  git.git(['reset', '--hard', 'HEAD'], dir);
14092
- dropped.push({ branch: m.branch, why });
14120
+ dropped.push({ branch: m.branch, sha: m.sha, cls, why });
14093
14121
  continue;
14094
14122
  }
14095
14123
  } else if (!sq.ok) {
14096
14124
  git.git(['reset', '--hard', 'HEAD'], dir);
14097
- dropped.push({ branch: m.branch, why: `squash failed: ${git.gitFailureLine(sq.stderr)}` });
14125
+ dropped.push({ branch: m.branch, sha: m.sha, cls: 'squash-failed', why: `squash failed: ${git.gitFailureLine(sq.stderr)}` });
14098
14126
  continue;
14099
14127
  }
14100
14128
  if (!git.git(['diff', '--cached', '--name-only'], dir).stdout.trim()) {
14101
14129
  git.git(['reset', '--hard', 'HEAD'], dir);
14102
- dropped.push({ branch: m.branch, why: 'stages nothing on top of the members already in' });
14130
+ dropped.push({ branch: m.branch, sha: m.sha, cls: 'empty', why: 'stages nothing on top of the members already in' });
14103
14131
  continue;
14104
14132
  }
14105
14133
  const closeIssues = m.report.closeIssues || [];
@@ -14111,14 +14139,25 @@ function cmdShipBatch(opts) {
14111
14139
  const inherited = msg && refsIssues.length ? squash.inheritedClosingKeywordConflicts(msg, refsIssues) : [];
14112
14140
  if (!msg || !coverage.ok || inherited.length) {
14113
14141
  git.git(['reset', '--hard', 'HEAD'], dir);
14114
- dropped.push({ branch: m.branch, why: !msg ? 'no commit of its own to take a squash subject from' : 'its composed message does not close exactly what it claims — ship it alone to see why' });
14142
+ dropped.push({ branch: m.branch, sha: m.sha, cls: 'message', why: !msg ? 'no commit of its own to take a squash subject from' : 'its composed message does not close exactly what it claims — ship it alone to see why' });
14115
14143
  continue;
14116
14144
  }
14117
14145
  const c = git.run('git', ['commit', '-q', '-m', msg], { cwd: dir });
14118
- if (!c.ok) { git.git(['reset', '--hard', 'HEAD'], dir); dropped.push({ branch: m.branch, why: `commit failed: ${git.gitFailureLine(c.stderr)}` }); continue; }
14146
+ if (!c.ok) { git.git(['reset', '--hard', 'HEAD'], dir); dropped.push({ branch: m.branch, sha: m.sha, cls: 'commit-failed', why: `commit failed: ${git.gitFailureLine(c.stderr)}` }); continue; }
14119
14147
  built.push({ ...m, regenerated, commit: git.git(['rev-parse', 'HEAD'], dir).stdout });
14120
14148
  }
14121
14149
  for (const d of dropped) print(` ✗ ${d.branch}: ${d.why}`);
14150
+ // #554: a drop leaves a durable trace — one Ship-Batch-Dropped trailer per dropped member on the
14151
+ // batch head, so it reaches trunk with a landed batch and rides a red combined run's head message.
14152
+ // Only when the batch is built (≥2 members): a build that collapses to serial pushes nothing.
14153
+ if (dropped.length && built.length >= 2) {
14154
+ const last = built[built.length - 1];
14155
+ const cur = git.git(['log', '-1', '--format=%B'], dir);
14156
+ const lines = dropped.map((d) => shipBatch.droppedTrailer({ ref, branch: d.branch, sha: d.sha, cls: d.cls }));
14157
+ const am = cur.ok ? git.run('git', ['commit', '-q', '--amend', '-m', shipBatch.appendTrailers(cur.stdout, lines)], { cwd: dir }) : cur;
14158
+ if (am.ok) last.commit = git.git(['rev-parse', 'HEAD'], dir).stdout;
14159
+ else warn(`could not record the dropped member(s) on the batch head: ${git.gitFailureLine(am.stderr)} — the drop is on this terminal only`);
14160
+ }
14122
14161
  for (const b of built.filter((x) => x.regenerated)) {
14123
14162
  print(` ↻ ${b.branch}: combined despite a generated-file overlap — ${b.regenerated.join(', ')} regenerated by .colab/hooks/pre-ship on the combined head (the combined run is the gate)`);
14124
14163
  }
@@ -14619,6 +14658,141 @@ function landedBaseTrunk(repoAbs) {
14619
14658
  'gets measured against the wrong one.)');
14620
14659
  }
14621
14660
 
14661
+ const HELP_BATCH_STATS = `colab batch-stats [--repo <path>] [--since <30d | 12h | YYYY-MM-DD>] [--json]
14662
+
14663
+ How batch landing (#373) actually performs in this repo — read-only, derived from GitHub so it
14664
+ answers the same on any machine (#554). It is how a repo tunes its batch knobs: ship-batch: <N>
14665
+ and ship-batch-wait: <duration> have no handbook default (CONVENTIONS §4, Batch landing).
14666
+
14667
+ Sources: trunk's first-parent log since --since (default 30d) — Ship-Batch: trailers mark batch
14668
+ members, Ship-Batch-Dropped: trailers mark members dropped at build, anything else with a
14669
+ Closes #N is a serial landing — and the repo's CI runs in the same window (REST, paginated):
14670
+ ship-batch/** runs are the combined runs, feature-branch runs say when a branch went green at
14671
+ its head, trunk runs say when a landing was pushed and when its trunk-CI cycle ended.
14672
+
14673
+ Reports:
14674
+ batches landed, and their fill · combined-run builds: first-attempt green, green after the
14675
+ one re-run, red, pending
14676
+ red red builds, and how many of their members then landed serially (the fallback)
14677
+ dropped members dropped at build, by class, and the eviction rate
14678
+ serial serial landings, and MISSED PARTNERS — another change, landed later, that was
14679
+ already green when this one landed alone (could have joined) or went green inside
14680
+ this landing's trunk-CI cycle (a near miss; its wait is what ship-batch-wait buys)
14681
+ queue wait green-at-head → landing, per landed change
14682
+
14683
+ No batch history in the window is an answer, not an error: it says so, and still reports the
14684
+ serial side — missed partners are the evidence for turning ship-batch on at all. Not the optional
14685
+ machine-local journal: that is off by default and per-machine, so it cannot be the source.
14686
+ --json is the input for an audit; its shape is batch-stats.js derive()'s return value.
14687
+ Exit 0 on a report (empty or not); 1 when trunk's log cannot be read.
14688
+ `;
14689
+
14690
+ /** `--since`: `<N>d` / `<N>h` / `<N>m` back from now, or an ISO date. → epoch ms, or null. */
14691
+ function batchStatsSince(v, nowMs) {
14692
+ const s = String(v || '30d').trim();
14693
+ const rel = /^(\d+)([dhm])$/.exec(s);
14694
+ if (rel) return nowMs - Number(rel[1]) * ({ d: 86400, h: 3600, m: 60 })[rel[2]] * 1000;
14695
+ const t = Date.parse(s);
14696
+ return Number.isFinite(t) ? t : null;
14697
+ }
14698
+
14699
+ function cmdBatchStats(argv) {
14700
+ const opts = parseArgs(argv, {
14701
+ values: { '--repo': 'repo', '--since': 'since' },
14702
+ flags: { '--json': 'json', '-h': 'help', '--help': 'help' },
14703
+ });
14704
+ if (opts.help) return print(HELP_BATCH_STATS);
14705
+ const repoAbs = resolveRepo(opts);
14706
+ if (!git.repoRoot(repoAbs)) throw new UserError(`${repoAbs} is not a git repo — pass --repo <path> to a clone`);
14707
+ const doc = projectDoc(repoAbs);
14708
+ const trunk = resolveTrunk(repoAbs, doc);
14709
+ if (!trunk) throw new UserError('Could not determine trunk branch — add `trunk: <branch>` to .github/project.yml, then run `colab batch-stats` again');
14710
+ const nowMs = Date.now();
14711
+ const sinceMs = batchStatsSince(opts.since, nowMs);
14712
+ if (sinceMs === null) throw new UserError(`--since ${JSON.stringify(opts.since)}: expected 30d, 12h, 90m or an ISO date`);
14713
+ const sinceIso = new Date(sinceMs).toISOString();
14714
+ const notes = [];
14715
+
14716
+ const remote = git.remoteFor(repoAbs);
14717
+ const f = git.git(['fetch', remote, trunk], repoAbs);
14718
+ if (!f.ok) notes.push(`could not fetch ${remote}/${trunk} (${git.gitFailureLine(f.stderr)}) — read from the local copy, which may be behind`);
14719
+ const log = git.git(['log', `${remote}/${trunk}`, '--first-parent', `--since=${sinceIso}`, '--format=%H%x1f%cI%x1f%B%x1e'], repoAbs);
14720
+ if (!log.ok) throw new UserError(`could not read ${remote}/${trunk}: ${git.gitFailureLine(log.stderr)} — run \`git fetch ${remote} ${trunk}\`, then run this again`);
14721
+ const commits = log.stdout.split('\x1e').map((r) => r.replace(/^\n/, '')).filter((r) => r.trim()).map((r) => {
14722
+ const [sha, iso, body] = r.split('\x1f');
14723
+ return { sha: sha.trim(), dateMs: Date.parse(iso), body: body || '' };
14724
+ });
14725
+
14726
+ let runs = [];
14727
+ const builds = {};
14728
+ if (!isGhUsable(repoAbs)) {
14729
+ notes.push(`gh not usable (${ghUnusableWhy(repoAbs)}) — trunk commits only: no combined runs, no green times, no partner matching`);
14730
+ } else {
14731
+ const jq = '.workflow_runs[] | [.id, .head_branch, .head_sha, .event, .status, .conclusion, .run_attempt, .created_at, .updated_at, (if ((.head_branch // "") | startswith("ship-batch/")) then .head_commit.message else null end)] | @json';
14732
+ // One query per UTC day: GitHub answers at most 1000 rows for a filtered run list, and a busy
14733
+ // repo passes that inside a fortnight. A day that still hits the cap is named, never hidden.
14734
+ const days = [];
14735
+ for (let d = Date.parse(sinceIso.slice(0, 10)); d <= nowMs; d += 86_400_000) days.push(new Date(d).toISOString().slice(0, 10));
14736
+ let failed = null;
14737
+ const capped = [];
14738
+ for (const day of days) {
14739
+ const r = git.ghApi(repoAbs, ['--paginate', `repos/{owner}/{repo}/actions/runs?per_page=100&created=${day}`, '--jq', jq]);
14740
+ if (!r.ok) { failed = git.gitFailureLine(r.stderr); break; }
14741
+ let n = 0;
14742
+ for (const line of String(r.stdout || '').split('\n')) {
14743
+ if (!line.trim()) continue;
14744
+ let a; try { a = JSON.parse(line); } catch (_) { continue; }
14745
+ n++;
14746
+ runs.push({ id: a[0], branch: a[1], sha: a[2], event: a[3], status: a[4], conclusion: a[5], attempt: a[6], createdMs: Date.parse(a[7]), updatedMs: Date.parse(a[8]), message: a[9] });
14747
+ }
14748
+ if (n >= 1000) capped.push(day);
14749
+ }
14750
+ if (failed) { notes.push(`could not list CI runs (${failed}) — trunk commits only`); runs = []; }
14751
+ else {
14752
+ if (capped.length) notes.push(`GitHub caps a filtered run list at 1000 and ${capped.join(', ')} hit it — those days are incomplete`);
14753
+ // An unlanded combined run's ref is usually gone: read every commit of the build, so a red
14754
+ // batch's members and its drops are known, not just its head commit's.
14755
+ const landedShas = new Set(commits.map((c) => c.sha));
14756
+ const unlanded = [...new Set(runs.filter((x) => String(x.branch || '').startsWith(shipBatch.REF_PREFIX) && !landedShas.has(x.sha)).map((x) => `${x.branch} ${x.sha}`))];
14757
+ for (const k of unlanded) {
14758
+ const [ref, sha] = k.split(' ');
14759
+ const base7 = (shipBatch.parseBatchRef(ref) || {}).base7;
14760
+ if (!base7) continue;
14761
+ const c = git.ghApi(repoAbs, [`repos/{owner}/{repo}/compare/${base7}...${sha}`, '--jq', '[.commits[].commit.message]']);
14762
+ if (c.ok) { try { builds[sha] = JSON.parse(c.stdout); } catch (_) { /* head message only */ } }
14763
+ }
14764
+ }
14765
+ }
14766
+
14767
+ const cfg = shipBatch.parseShipBatch(doc);
14768
+ const waitCfg = shipBatch.parseShipBatchWait(doc);
14769
+ const rep = batchStats.derive({ commits, runs, builds, trunk, config: { n: cfg.n, waitSec: waitCfg.sec } });
14770
+ rep.repo = shortRepo(repoAbs);
14771
+ rep.since = sinceIso;
14772
+ rep.until = new Date(nowMs).toISOString();
14773
+ rep.notes = [...notes, ...rep.notes];
14774
+ if (opts.json) { print(JSON.stringify(rep, null, 2)); return 0; }
14775
+
14776
+ const pct = (x) => (x === null ? '—' : `${Math.round(x * 100)}%`);
14777
+ const dur = (s) => (s === null ? '—' : ciWait.fmtDuration(s * 1000));
14778
+ const q = (w) => (w.samples ? `p50 ${dur(w.p50Sec)} · p90 ${dur(w.p90Sec)} · max ${dur(w.maxSec)} (${w.samples})` : 'no samples');
14779
+ print(`batch-stats ${rep.repo} → ${trunk} ${rep.since.slice(0, 10)} … ${rep.until.slice(0, 10)} (ship-batch: ${cfg.n}${waitCfg.sec ? ` · ship-batch-wait: ${dur(waitCfg.sec)}` : ''})`);
14780
+ print(` landings ${rep.landings.total} (${rep.landings.batch} batch · ${rep.landings.serial} serial)`);
14781
+ if (!rep.history) print(' batches no batch history in this window — nothing has landed or been built through colab ship --batch');
14782
+ else {
14783
+ const fill = Object.keys(rep.batches.fill).sort().map((k) => `${rep.batches.fill[k]}×${k}`).join(' · ') || '—';
14784
+ print(` batches ${rep.batches.landed} landed (fill ${fill}) · ${rep.batches.builds} combined-run build(s): ${rep.batches.firstAttemptGreen} first-attempt green (${pct(rep.batches.firstAttemptGreenRate)}), ${rep.batches.greenAfterRerun} after a re-run, ${rep.batches.red} red, ${rep.batches.pending} pending`);
14785
+ print(` red ${rep.red.builds} build(s) · ${rep.red.members} member(s), ${rep.red.relandedSerially} then landed serially`);
14786
+ const cls = Object.entries(rep.dropped.byClass).map(([k, v]) => `${v} ${k}`).join(', ');
14787
+ print(` dropped ${rep.dropped.count} of ${rep.dropped.attempted} attempted (${pct(rep.dropped.rate)})${cls ? ` — ${cls}` : ''}`);
14788
+ }
14789
+ print(` serial ${rep.serial.landings} landed alone · ${rep.serial.withMissedPartner} with a missed partner (already green) · ${rep.serial.withNearMiss} more with a near miss (green within its trunk-CI cycle)`);
14790
+ if (rep.serial.nearMissWait.samples) print(` near misses per pair, the wait from the lone change's own green that would have caught its partner: ${q(rep.serial.nearMissWait)}`);
14791
+ print(` queue wait green-at-head → landing: ${q(rep.queueWait)}`);
14792
+ for (const n of rep.notes) print(` ! ${n}`);
14793
+ return 0;
14794
+ }
14795
+
14622
14796
  function cmdLanded(argv) {
14623
14797
  const opts = parseArgs(argv, {
14624
14798
  values: { '--repo': 'repo', '--worktree': 'worktree', '--branch': 'branch', '--base': 'base' },
@@ -15403,6 +15577,9 @@ Commands:
15403
15577
  landed [--worktree N | --branch B | --all] [--base B] [--json] has this work already landed?
15404
15578
  (the shared rule behind code-sweep's buckets — correct
15405
15579
  across squash merges, where "commits ahead" is not)
15580
+ batch-stats [--since 30d] [--json] how batch landing performs here, from git + CI history —
15581
+ fill, first-attempt green, drops, missed partners, queue
15582
+ wait; how a repo tunes ship-batch / ship-batch-wait (#554)
15406
15583
  holders <path> [--base B] [--json] who else is holding <path> — refs that touched it and
15407
15584
  have not landed on the base (filters the raw git-log
15408
15585
  one-liner through \`landed\`'s content classification)
@@ -15487,6 +15664,7 @@ function main(argv) {
15487
15664
  case 'worktrees': return cmdWorktrees(rest);
15488
15665
  case 'ship': return cmdShip(rest);
15489
15666
  case 'landed': return cmdLanded(rest);
15667
+ case 'batch-stats': return cmdBatchStats(rest);
15490
15668
  case 'trunk-ci': return cmdTrunkCi(rest);
15491
15669
  case 'ci-wait': return cmdCiWait(rest);
15492
15670
  case 'holders': return cmdHolders(rest);
@@ -0,0 +1,273 @@
1
+ 'use strict';
2
+ /**
3
+ * tools/lib/batch-stats.js — how batch landing (#373) actually performs in ONE repo (#554).
4
+ *
5
+ * The batch knobs (`ship-batch: <N>`, `ship-batch-wait: <duration>`) have no handbook default: a repo
6
+ * picks them from its own history. This module is that history, derived from two sources every
7
+ * machine can read — never from the optional machine-local journal, which is off by default and
8
+ * per-machine:
9
+ *
10
+ * commits trunk's first-parent log. A batch member's squash carries `Ship-Batch: <ref>
11
+ * <branch>@<sha>` (ship-batch.js memberTrailer); a member dropped at build is recorded on
12
+ * the batch head as `Ship-Batch-Dropped: <ref> <branch>@<sha> <class>`. A trunk commit
13
+ * with neither is a serial landing, known by its `Closes #N`.
14
+ * runs the repo's CI runs (REST rows). Runs on `ship-batch/**` are the combined runs; runs on
15
+ * a feature branch say when that branch went green at its head; runs on trunk say when a
16
+ * landing was pushed (created) and when its trunk-CI cycle ended (updated).
17
+ *
18
+ * Pure: facts in, a report out — no git, no gh, no clock. `tools/colab`'s `cmdBatchStats` reads.
19
+ *
20
+ * What it measures, per window:
21
+ * batches landed and their fill; every combined-run build, first-attempt green, green after
22
+ * the one re-run, red, still pending
23
+ * red red builds and how many of their members then landed serially (the fallback)
24
+ * dropped members dropped at build, by class, and the eviction rate among attempted members
25
+ * serial serial landings, and MISSED PARTNERS: another candidate that landed later but was
26
+ * already green when this one landed alone (could have joined), or turned green
27
+ * within this landing's trunk-CI cycle (a partner wait of `waitSec` would have caught
28
+ * it — the number `ship-batch-wait` is tuned from)
29
+ * queueWait per landed change, green-at-head → landing
30
+ */
31
+
32
+ const branchName = require('./branch-name');
33
+ const shipBatch = require('./ship-batch');
34
+
35
+ /** `Closes #N` / `Fixes #N` / `Resolves #N` numbers in a commit body (GitHub's own keywords). */
36
+ function closesIssues(body) {
37
+ const out = new Set();
38
+ const re = /\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s+#(\d+)\b/gi;
39
+ let m;
40
+ while ((m = re.exec(String(body || '')))) out.add(Number(m[1]));
41
+ return [...out];
42
+ }
43
+
44
+ /** Value at the q-th quantile (nearest-rank) of numbers, or null for none. */
45
+ function quantile(values, q) {
46
+ const v = (values || []).filter(Number.isFinite).sort((a, b) => a - b);
47
+ if (!v.length) return null;
48
+ return v[Math.min(v.length - 1, Math.max(0, Math.ceil(q * v.length) - 1))];
49
+ }
50
+
51
+ function summary(values) {
52
+ const v = (values || []).filter(Number.isFinite);
53
+ return { samples: v.length, p50Sec: quantile(v, 0.5), p90Sec: quantile(v, 0.9), maxSec: v.length ? Math.max(...v) : null };
54
+ }
55
+
56
+ const rate = (num, den) => (den > 0 ? Math.round((num / den) * 1000) / 1000 : null);
57
+ const isBatchRef = (b) => String(b || '').startsWith(shipBatch.REF_PREFIX);
58
+
59
+ /**
60
+ * When did `branch` go green at the head it had at time `atMs`? Its runs created before `atMs` are
61
+ * read; the newest sha among them is the head at that moment. Green only when every run at that sha
62
+ * finished success (cancelled aside, as combinedVerdict reads it) by `atMs` → the latest finish.
63
+ * Null when there is no run, or the head was not green yet — unknown, never a guess.
64
+ */
65
+ function greenAt(runsByBranch, branch, atMs, sha = null) {
66
+ const rows = (runsByBranch.get(branch) || []).filter((r) => r.createdMs <= atMs);
67
+ if (!rows.length) return null;
68
+ const headSha = sha
69
+ ? (rows.some((r) => r.sha.startsWith(sha) || sha.startsWith(r.sha)) ? rows.find((r) => r.sha.startsWith(sha) || sha.startsWith(r.sha)).sha : null)
70
+ : rows.reduce((a, b) => (b.createdMs > a.createdMs ? b : a)).sha;
71
+ if (!headSha) return null;
72
+ const at = rows.filter((r) => r.sha === headSha);
73
+ const v = shipBatch.combinedVerdict(at.map((r) => ({ status: r.status, conclusion: r.conclusion, attempt: r.attempt })));
74
+ if (v.state !== 'green') return null;
75
+ const done = Math.max(...at.filter((r) => r.conclusion === 'success').map((r) => r.updatedMs));
76
+ return done <= atMs ? done : null;
77
+ }
78
+
79
+ /** Which feature branch carried a serial landing that closed `issues`, as of `atMs`? Newest run wins. */
80
+ function branchForIssues(runsByBranch, issues, atMs) {
81
+ if (!issues.length) return null;
82
+ let best = null;
83
+ for (const [b, rows] of runsByBranch) {
84
+ const nums = branchName.branchIssueNumbers(b);
85
+ if (!nums.some((n) => issues.includes(n))) continue;
86
+ const before = rows.filter((r) => r.createdMs <= atMs);
87
+ if (!before.length) continue;
88
+ const t = Math.max(...before.map((r) => r.createdMs));
89
+ if (!best || t > best.t) best = { branch: b, t };
90
+ }
91
+ return best ? best.branch : null;
92
+ }
93
+
94
+ /**
95
+ * The report. Inputs:
96
+ * commits [{ sha, dateMs, body }] — trunk first-parent, inside the window, any order
97
+ * runs [{ id, branch, sha, event, status, conclusion, attempt, createdMs, updatedMs, message? }]
98
+ * `message` is the head commit's message, needed only for ship-batch/** rows
99
+ * builds optional { [sha]: [message, …] } — every commit message of an UNLANDED build, when the
100
+ * caller could read them (compare API); a landed build is read from trunk instead
101
+ * trunk trunk branch name
102
+ * config { n, waitSec } — the repo's declared knobs, echoed so the report says what it tunes
103
+ */
104
+ function derive({ commits = [], runs = [], builds = {}, trunk = 'main', config = {} } = {}) {
105
+ const notes = [];
106
+ const runsByBranch = new Map();
107
+ const trunkRuns = new Map(); // sha → rows
108
+ const batchRuns = new Map(); // `${ref} ${sha}` → rows
109
+ for (const r of runs || []) {
110
+ if (!r || !r.branch || !r.sha) continue;
111
+ if (r.branch === trunk) { (trunkRuns.get(r.sha) || trunkRuns.set(r.sha, []).get(r.sha)).push(r); continue; }
112
+ if (isBatchRef(r.branch)) { const k = `${r.branch} ${r.sha}`; (batchRuns.get(k) || batchRuns.set(k, []).get(k)).push(r); continue; }
113
+ (runsByBranch.get(r.branch) || runsByBranch.set(r.branch, []).get(r.branch)).push(r);
114
+ }
115
+ // When a landing reached trunk: its trunk push run was created then. Fallback: the commit date
116
+ // (a fast-forward keeps the build-time date, so it reads early by the combined run's length).
117
+ const landedAt = (sha, fallback) => {
118
+ const rows = trunkRuns.get(sha) || [];
119
+ return rows.length ? Math.min(...rows.map((r) => r.createdMs)) : fallback;
120
+ };
121
+ const cycleEnd = (sha) => {
122
+ const rows = (trunkRuns.get(sha) || []).filter((r) => r.status === 'completed');
123
+ return rows.length ? Math.max(...rows.map((r) => r.updatedMs)) : null;
124
+ };
125
+
126
+ // ---- landings, oldest first ----
127
+ const sorted = [...(commits || [])].sort((a, b) => a.dateMs - b.dateMs);
128
+ const batches = new Map(); // ref → { ref, commits: [], members: [], dropped: [] }
129
+ const landings = []; // { kind, sha, atMs, members: [{ branch, sha }], issues }
130
+ for (const c of sorted) {
131
+ const members = shipBatch.parseMemberTrailers([c.body]);
132
+ if (members.length) {
133
+ const ref = members[0].ref;
134
+ const b = batches.get(ref) || batches.set(ref, { ref, commits: [], members: [], dropped: [] }).get(ref);
135
+ b.commits.push(c);
136
+ b.members.push(...members);
137
+ b.dropped.push(...shipBatch.parseDroppedTrailers([c.body]));
138
+ } else {
139
+ landings.push({ kind: 'serial', sha: c.sha, atMs: landedAt(c.sha, c.dateMs), commitMs: c.dateMs, issues: closesIssues(c.body) });
140
+ }
141
+ }
142
+ for (const b of batches.values()) {
143
+ const head = b.commits[b.commits.length - 1];
144
+ b.headSha = head.sha;
145
+ landings.push({ kind: 'batch', ref: b.ref, sha: head.sha, atMs: landedAt(head.sha, head.dateMs), commitMs: head.dateMs, members: b.members });
146
+ }
147
+ landings.sort((a, b) => a.atMs - b.atMs);
148
+
149
+ // ---- every landed change, with its green-at-head time ----
150
+ const changes = []; // { landing, branch, sha, issues, greenMs, atMs }
151
+ for (const l of landings) {
152
+ if (l.kind === 'batch') {
153
+ for (const m of l.members) {
154
+ changes.push({ landing: l, branch: m.branch, issues: branchName.branchIssueNumbers(m.branch), greenMs: greenAt(runsByBranch, m.branch, l.atMs, m.sha), atMs: l.atMs });
155
+ }
156
+ } else {
157
+ const br = branchForIssues(runsByBranch, l.issues, l.atMs);
158
+ l.branch = br;
159
+ changes.push({ landing: l, branch: br, issues: l.issues, greenMs: br ? greenAt(runsByBranch, br, l.atMs) : null, atMs: l.atMs });
160
+ }
161
+ }
162
+
163
+ // ---- combined-run builds ----
164
+ const landedHeads = new Set([...batches.values()].map((b) => b.headSha));
165
+ const buildRows = [];
166
+ for (const [k, rows] of batchRuns) {
167
+ const [ref, sha] = k.split(' ');
168
+ const v = shipBatch.combinedVerdict(rows.map((r) => ({ status: r.status, conclusion: r.conclusion, attempt: r.attempt, databaseId: r.id })));
169
+ const landed = landedHeads.has(sha);
170
+ const messages = landed ? null : (builds[sha] || rows.map((r) => r.message).filter(Boolean).slice(0, 1));
171
+ buildRows.push({ ref, sha, state: v.state, attempt: v.attempt, landed, createdMs: Math.min(...rows.map((r) => r.createdMs)), messages });
172
+ }
173
+ const finished = buildRows.filter((b) => b.state === 'green' || b.state === 'red');
174
+ const firstGreen = buildRows.filter((b) => b.state === 'green' && b.attempt === 1).length;
175
+ const rerunGreen = buildRows.filter((b) => b.state === 'green' && b.attempt > 1).length;
176
+ const redBuilds = buildRows.filter((b) => b.state === 'red');
177
+
178
+ // ---- red builds → did their members then land serially? ----
179
+ const serialChanges = changes.filter((c) => c.landing.kind === 'serial');
180
+ let redMembers = 0, relanded = 0, redMembersUnread = 0;
181
+ for (const b of redBuilds) {
182
+ const members = b.messages && b.messages.length ? shipBatch.parseMemberTrailers(b.messages) : [];
183
+ if (!members.length) { redMembersUnread++; continue; }
184
+ for (const m of members) {
185
+ redMembers++;
186
+ const nums = branchName.branchIssueNumbers(m.branch);
187
+ if (serialChanges.some((c) => c.atMs >= b.createdMs && (c.branch === m.branch || c.issues.some((n) => nums.includes(n))))) relanded++;
188
+ }
189
+ }
190
+ if (redMembersUnread) notes.push(`${redMembersUnread} red build(s) whose members could not be read (the ref is gone and its commits were not readable)`);
191
+
192
+ // ---- dropped members: landed batches from trunk, unlanded builds from their messages ----
193
+ const dropped = [];
194
+ const seenDrop = new Set();
195
+ const addDrop = (d, landed) => {
196
+ const k = `${d.ref} ${d.branch}@${d.sha}`;
197
+ if (seenDrop.has(k)) return;
198
+ seenDrop.add(k);
199
+ dropped.push({ ...d, landed });
200
+ };
201
+ for (const b of batches.values()) for (const d of b.dropped) addDrop(d, true);
202
+ for (const b of buildRows) if (!b.landed && b.messages) for (const d of shipBatch.parseDroppedTrailers(b.messages)) addDrop(d, false);
203
+ const builtMembers = [...batches.values()].reduce((n, b) => n + b.members.length, 0)
204
+ + buildRows.filter((b) => !b.landed && b.messages).reduce((n, b) => n + shipBatch.parseMemberTrailers(b.messages).length, 0);
205
+ const byClass = {};
206
+ for (const d of dropped) byClass[d.cls] = (byClass[d.cls] || 0) + 1;
207
+
208
+ // ---- missed partners: serial landings that landed alone while a partner was (nearly) ready ----
209
+ const pairs = [];
210
+ let withMissed = 0, withNear = 0, cyclesUnread = 0;
211
+ // A partner is judged by ITS state at this landing's moment, never by the head it finally landed
212
+ // with: a serial ship syncs the other branches, so their final heads all post-date it. The cycle
213
+ // ends when this landing's trunk run finishes — or at the next landing, whichever is first (a
214
+ // re-run moves a run's updated_at hours later, and the next landing moves trunk anyway).
215
+ const landingTimes = landings.map((l) => l.atMs).sort((a, b) => a - b);
216
+ for (const c of serialChanges) {
217
+ const next = landingTimes.find((t) => t > c.atMs);
218
+ let end = cycleEnd(c.landing.sha);
219
+ if (end === null) cyclesUnread++;
220
+ else if (next !== undefined) end = Math.min(end, next);
221
+ let missed = false, near = false;
222
+ const label = c.branch || c.landing.sha.slice(0, 7);
223
+ for (const o of changes) {
224
+ if (o === c || o.landing === c.landing || o.atMs <= c.atMs || !o.branch || o.branch === c.branch) continue;
225
+ const gNow = greenAt(runsByBranch, o.branch, c.atMs);
226
+ if (gNow !== null) {
227
+ missed = true;
228
+ pairs.push({ kind: 'missed', landed: label, partner: o.branch });
229
+ continue;
230
+ }
231
+ if (end === null) continue;
232
+ const gEnd = greenAt(runsByBranch, o.branch, end);
233
+ if (gEnd !== null && gEnd > c.atMs) {
234
+ near = true;
235
+ const from = c.greenMs !== null ? c.greenMs : c.atMs;
236
+ pairs.push({ kind: 'near', landed: label, partner: o.branch, waitSec: Math.round((gEnd - from) / 1000) });
237
+ }
238
+ }
239
+ if (missed) withMissed++;
240
+ else if (near) withNear++;
241
+ }
242
+ if (cyclesUnread) notes.push(`${cyclesUnread} serial landing(s) with no finished trunk run in the window — near misses not judged for them`);
243
+ const unknownGreen = changes.filter((c) => c.greenMs === null).length;
244
+ if (unknownGreen) notes.push(`${unknownGreen} landed change(s) with no readable green-at-head run — left out of queue wait and partner matching`);
245
+
246
+ const fill = {};
247
+ for (const b of batches.values()) fill[b.members.length] = (fill[b.members.length] || 0) + 1;
248
+ const queueWait = summary(changes.filter((c) => c.greenMs !== null).map((c) => Math.round((c.atMs - c.greenMs) / 1000)));
249
+
250
+ return {
251
+ trunk,
252
+ config: { n: config.n || 1, waitSec: config.waitSec || 0 },
253
+ history: batches.size > 0 || buildRows.length > 0,
254
+ landings: { total: landings.length, batch: batches.size, serial: landings.length - batches.size },
255
+ batches: {
256
+ landed: batches.size, fill,
257
+ builds: buildRows.length, firstAttemptGreen: firstGreen, greenAfterRerun: rerunGreen, red: redBuilds.length,
258
+ pending: buildRows.length - finished.length,
259
+ firstAttemptGreenRate: rate(firstGreen, finished.length),
260
+ },
261
+ red: { builds: redBuilds.length, members: redMembers, relandedSerially: relanded },
262
+ dropped: { count: dropped.length, attempted: builtMembers + dropped.length, rate: rate(dropped.length, builtMembers + dropped.length), byClass, members: dropped },
263
+ serial: {
264
+ landings: serialChanges.length, withMissedPartner: withMissed, withNearMiss: withNear,
265
+ nearMissWait: summary(pairs.filter((p) => p.kind === 'near').map((p) => p.waitSec)),
266
+ pairs,
267
+ },
268
+ queueWait,
269
+ notes,
270
+ };
271
+ }
272
+
273
+ module.exports = { derive, closesIssues, quantile, greenAt, branchForIssues };
@@ -94,6 +94,8 @@ function restRow(x) {
94
94
  return {
95
95
  headSha: x.head_sha, status: x.status, conclusion: x.conclusion || null,
96
96
  createdAt: x.created_at || null, databaseId: x.id, workflowName: x.name || null,
97
+ // #555: a finished run's last update is its completion — when a lone batch member became ready.
98
+ updatedAt: x.updated_at || null,
97
99
  event: x.event || null, url: x.html_url || null,
98
100
  // #510: the workflow FILE's id — the cure rule's same-workflow test, which a display name cannot answer.
99
101
  workflowId: x.workflow_id ?? null,
package/tools/lib/git.js CHANGED
@@ -960,7 +960,8 @@ function ghRunsForCommit(repo, branch, sha, limit = 10) {
960
960
  // does not own) before it computes a verdict, and can only do that if the row says so.
961
961
  // workflowDatabaseId is additive (#510): the cure rule's same-workflow test reads the workflow's
962
962
  // id, never its display name; mapped to `workflowId`, the name the REST row (ci-wait restRow) uses.
963
- '--json', 'headSha,status,conclusion,createdAt,databaseId,workflowName,workflowDatabaseId,event'], { cwd: repo });
963
+ // updatedAt is additive (#555): a finished run's completion time, the lone-member wait's anchor.
964
+ '--json', 'headSha,status,conclusion,createdAt,updatedAt,databaseId,workflowName,workflowDatabaseId,event'], { cwd: repo });
964
965
  if (!r.ok) return null;
965
966
  let runs;
966
967
  try { runs = JSON.parse(r.stdout); } catch (_) { return null; }
@@ -46,6 +46,70 @@ function parseShipBatch(doc) {
46
46
  return { n: 1, declared: true, valid: false, reason: `ship-batch is ${JSON.stringify(v)}, expected an integer 1–${MAX_BATCH} (omit for serial)` };
47
47
  }
48
48
 
49
+ const WAIT_KEY = 'ship-batch-wait';
50
+ const WAIT_UNITS = { s: 1, m: 60, h: 3600 };
51
+
52
+ /**
53
+ * #555: `ship-batch-wait:` from project.yml — how long a LONE ready candidate waits for a partner
54
+ * before it lands alone. A whole number plus a unit: `90s`, `6m`, `1h`. The unit is required, so a
55
+ * bare `6` (seconds? minutes?) is refused, never guessed. `0s`/`0m` is valid and means no wait.
56
+ *
57
+ * Absent → `sec: 0`, today's behaviour exactly. The handbook gives NO default value and NO ceiling
58
+ * (owner, 2026-10-07: "Should not set any hard code number") — the repo chooses, from its own
59
+ * history. Anything malformed FAILS CLOSED to no wait, with the reason: a bad value must never delay
60
+ * a landing. The audit and the CI templates' descriptor check fail the same values.
61
+ */
62
+ function parseShipBatchWait(doc) {
63
+ const has = !!doc && Object.prototype.hasOwnProperty.call(doc, WAIT_KEY);
64
+ if (!has || doc[WAIT_KEY] === null || doc[WAIT_KEY] === undefined) {
65
+ return { sec: 0, declared: false, valid: true, reason: `${WAIT_KEY} absent` };
66
+ }
67
+ const v = doc[WAIT_KEY];
68
+ const m = typeof v === 'string' ? /^(\d+)([smh])$/.exec(v.trim()) : null;
69
+ if (m) {
70
+ const sec = Number(m[1]) * WAIT_UNITS[m[2]];
71
+ if (Number.isSafeInteger(sec)) return { sec, declared: true, valid: true, reason: `${WAIT_KEY}: ${v.trim()}` };
72
+ }
73
+ return { sec: 0, declared: true, valid: false, reason: `${WAIT_KEY} is ${JSON.stringify(v)}, expected a whole number with a unit — 90s, 6m, 1h (omit for no wait)` };
74
+ }
75
+
76
+ /**
77
+ * #555: does a batch call with ONE eligible member wait for a partner, or let it go serial now?
78
+ * Facts in (all already read by the caller, after it has ruled out a pending trunk run and a batch
79
+ * already in flight — the lane is otherwise idle by then):
80
+ * n parseShipBatch(doc).n
81
+ * waitSec parseShipBatchWait(doc).sec (0 = absent, zero, or malformed)
82
+ * eligibleCount members that passed their own gates
83
+ * readySinceMs when the lone member became ready — its head CI's last update, else its head
84
+ * commit's date; null = unreadable
85
+ * nowMs the clock
86
+ * Out: `{ wait: true, leftSec }` or `{ wait: false, reason }`. The window is counted from the member's
87
+ * own ready time, not from this call, so the command stays stateless and re-entrant: every call
88
+ * re-derives how much is left, on any machine, and the window can never restart.
89
+ */
90
+ function partnerWait({ n, waitSec, eligibleCount, readySinceMs, nowMs }) {
91
+ if (!(n > 1)) return { wait: false, reason: 'not-enabled' };
92
+ if (!(waitSec > 0)) return { wait: false, reason: 'no-window' };
93
+ if (eligibleCount !== 1) return { wait: false, reason: eligibleCount > 1 ? 'has-partner' : 'none-ready' };
94
+ if (!Number.isFinite(readySinceMs) || !Number.isFinite(nowMs)) return { wait: false, reason: 'ready-time-unread' };
95
+ const leftSec = Math.ceil(waitSec - Math.max(0, nowMs - readySinceMs) / 1000);
96
+ if (leftSec <= 0) return { wait: false, reason: 'window-elapsed' };
97
+ return { wait: true, leftSec };
98
+ }
99
+
100
+ /**
101
+ * When did a member become ready? The newest `updatedAt` among its finished head runs (a run's last
102
+ * update is its completion), else — no run, because none can arrive — its head commit's date.
103
+ * Null when neither reads; the caller then does not wait.
104
+ */
105
+ function readySince(rows, commitIso) {
106
+ const done = (Array.isArray(rows) ? rows : []).filter((r) => r && r.status === 'completed')
107
+ .map((r) => Date.parse(r.updatedAt || '')).filter(Number.isFinite);
108
+ if (done.length) return Math.max(...done);
109
+ const c = Date.parse(commitIso || '');
110
+ return Number.isFinite(c) ? c : null;
111
+ }
112
+
49
113
  function batchRefName(baseSha) {
50
114
  return `${REF_PREFIX}${String(baseSha || '').slice(0, 7)}`;
51
115
  }
@@ -77,6 +141,48 @@ function parseMemberTrailers(messages) {
77
141
  return out;
78
142
  }
79
143
 
144
+ /**
145
+ * #554: a member dropped at build used to leave no trace in git — only a `✗ <member>: …` line on
146
+ * one terminal — so eviction rate could not be measured. The batch head now records each drop:
147
+ *
148
+ * Ship-Batch-Dropped: <ref> <branch>@<sha> <class>
149
+ *
150
+ * appended to the LAST built member's message (the head commit), so it reaches trunk with a landed
151
+ * batch and rides the head commit's message of a combined run that went red. `class` is one token
152
+ * from DROP_CLASSES; the human reason stays on the terminal. A branch (already pushed, so already
153
+ * public) and a sha, never a host — the same rule as memberTrailer.
154
+ */
155
+ const DROP_TRAILER_KEY = 'Ship-Batch-Dropped';
156
+ const DROP_CLASSES = ['conflict', 'generated-no-hook', 'hook-failed', 'hook-markers', 'squash-failed', 'empty', 'message', 'commit-failed'];
157
+
158
+ function droppedTrailer({ ref, branch, sha, cls }) {
159
+ const c = DROP_CLASSES.includes(cls) ? cls : 'other';
160
+ return `${DROP_TRAILER_KEY}: ${ref} ${branch}@${String(sha || '').slice(0, 12)} ${c}`;
161
+ }
162
+
163
+ /** Every `Ship-Batch-Dropped:` trailer across commit messages → [{ ref, branch, sha, cls }]. */
164
+ function parseDroppedTrailers(messages) {
165
+ const out = [];
166
+ for (const msg of messages || []) {
167
+ for (const line of String(msg || '').split('\n')) {
168
+ const m = new RegExp(`^${DROP_TRAILER_KEY}:\\s+(\\S+)\\s+(\\S+)@([0-9a-f]{7,40})\\s+([a-z-]+)\\s*$`).exec(line.trim());
169
+ if (m) out.push({ ref: m[1], branch: m[2], sha: m[3], cls: m[4] });
170
+ }
171
+ }
172
+ return out;
173
+ }
174
+
175
+ /**
176
+ * `msg` with `lines` appended to its final trailer block — never as a new paragraph, which would
177
+ * split the block and hide the earlier trailers from `git interpret-trailers`. Lines already
178
+ * present are not repeated (a rebuild re-composes the same message).
179
+ */
180
+ function appendTrailers(msg, lines) {
181
+ const body = String(msg || '').replace(/\s+$/, '');
182
+ const add = (lines || []).filter((l) => l && !body.split('\n').includes(l));
183
+ return add.length ? `${body}\n${add.join('\n')}\n` : `${body}\n`;
184
+ }
185
+
80
186
  /**
81
187
  * A member's OWN head CI, read as a joining class (§4 Branch CI):
82
188
  * green every run at its head finished, all success
@@ -315,8 +421,9 @@ function landPushFailure({ stderr, remoteNow, base, max = 8 } = {}) {
315
421
  }
316
422
 
317
423
  module.exports = {
318
- MAX_BATCH, REF_PREFIX, PROBE_REF, TRAILER_KEY,
319
- parseShipBatch, batchRefName, parseBatchRef, memberTrailer, parseMemberTrailers,
424
+ MAX_BATCH, REF_PREFIX, PROBE_REF, TRAILER_KEY, WAIT_KEY, DROP_TRAILER_KEY, DROP_CLASSES,
425
+ parseShipBatch, parseShipBatchWait, partnerWait, readySince, batchRefName, parseBatchRef, memberTrailer, parseMemberTrailers,
426
+ droppedTrailer, parseDroppedTrailers, appendTrailers,
320
427
  branchCiClass, memberEligibility, selectMembers, wiring, combinedVerdict, nextStep, foreignBatchStep,
321
428
  batchGreenCoversTrunk, evidenceSuffix, serialLine, notStaged, landPushFailure,
322
429
  };