@futurelastic/colab-handbook 1.13.0-rc.24 → 1.13.0-rc.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -33,5 +33,5 @@
33
33
  "engines": {
34
34
  "node": ">=18"
35
35
  },
36
- "version": "1.13.0-rc.24"
36
+ "version": "1.13.0-rc.25"
37
37
  }
package/tools/colab CHANGED
@@ -63,6 +63,7 @@ const codec = require('./lib/codec');
63
63
  const machine = require('./lib/machine');
64
64
  const targetSync = require('./lib/ship-target-sync');
65
65
  const shipBatch = require('./lib/ship-batch');
66
+ const batchStats = require('./lib/batch-stats');
66
67
  const conflictMarkers = require('./lib/conflict-markers');
67
68
  const docsOnly = require('./lib/docs-only');
68
69
  const cureDiff = require('./lib/cure-diff');
@@ -14094,9 +14095,9 @@ function cmdShipBatch(opts) {
14094
14095
  // regenerated by .colab/hooks/pre-ship, never hand-merged and never a reason to leave the batch.
14095
14096
  // Only an overlap outside `generated:` (or no hook to regenerate with) drops the member.
14096
14097
  const nonGen = unmerged.filter((f) => !isGeneratedPath(f, generated));
14097
- let why = null;
14098
- if (nonGen.length) why = `conflicts with the members already in (${nonGen.join(', ')}) — next batch`;
14099
- else if (!hasPreShip) why = `conflicts with the members already in only in generated file(s) (${unmerged.join(', ')}), but there is no .colab/hooks/pre-ship to regenerate them — next batch`;
14098
+ let why = null, cls = null;
14099
+ if (nonGen.length) { why = `conflicts with the members already in (${nonGen.join(', ')}) — next batch`; cls = 'conflict'; }
14100
+ else if (!hasPreShip) { cls = 'generated-no-hook'; why = `conflicts with the members already in only in generated file(s) (${unmerged.join(', ')}), but there is no .colab/hooks/pre-ship to regenerate them — next batch`; }
14100
14101
  else {
14101
14102
  const hk = runHook(repoAbs, 'pre-ship', {
14102
14103
  wtPath: dir, name: m.branch, branch: m.branch, ports: [], trunk, base: trunk,
@@ -14107,24 +14108,26 @@ function cmdShipBatch(opts) {
14107
14108
  // state, so `left` reads empty — read the staged content itself before trusting it.
14108
14109
  const markers = hk.ran && hk.ok && !left.length ? stagedConflictMarkers(dir, unmerged) : [];
14109
14110
  if (!hk.ran || !hk.ok || left.length) {
14111
+ cls = 'hook-failed';
14110
14112
  why = `.colab/hooks/pre-ship could not regenerate ${unmerged.join(', ')} on the combined head${left.length && hk.ok ? ` (still unmerged: ${left.join(', ')})` : ''} — next batch`;
14111
14113
  } else if (markers.length) {
14114
+ cls = 'hook-markers';
14112
14115
  why = `.colab/hooks/pre-ship exited 0 but left conflict markers staged in ${conflictMarkers.describe(markers)} on the combined head — treated as a failing hook, next batch`;
14113
14116
  } else regenerated = unmerged;
14114
14117
  }
14115
14118
  if (why) {
14116
14119
  git.git(['reset', '--hard', 'HEAD'], dir);
14117
- dropped.push({ branch: m.branch, why });
14120
+ dropped.push({ branch: m.branch, sha: m.sha, cls, why });
14118
14121
  continue;
14119
14122
  }
14120
14123
  } else if (!sq.ok) {
14121
14124
  git.git(['reset', '--hard', 'HEAD'], dir);
14122
- dropped.push({ branch: m.branch, why: `squash failed: ${git.gitFailureLine(sq.stderr)}` });
14125
+ dropped.push({ branch: m.branch, sha: m.sha, cls: 'squash-failed', why: `squash failed: ${git.gitFailureLine(sq.stderr)}` });
14123
14126
  continue;
14124
14127
  }
14125
14128
  if (!git.git(['diff', '--cached', '--name-only'], dir).stdout.trim()) {
14126
14129
  git.git(['reset', '--hard', 'HEAD'], dir);
14127
- dropped.push({ branch: m.branch, why: 'stages nothing on top of the members already in' });
14130
+ dropped.push({ branch: m.branch, sha: m.sha, cls: 'empty', why: 'stages nothing on top of the members already in' });
14128
14131
  continue;
14129
14132
  }
14130
14133
  const closeIssues = m.report.closeIssues || [];
@@ -14136,14 +14139,25 @@ function cmdShipBatch(opts) {
14136
14139
  const inherited = msg && refsIssues.length ? squash.inheritedClosingKeywordConflicts(msg, refsIssues) : [];
14137
14140
  if (!msg || !coverage.ok || inherited.length) {
14138
14141
  git.git(['reset', '--hard', 'HEAD'], dir);
14139
- dropped.push({ branch: m.branch, why: !msg ? 'no commit of its own to take a squash subject from' : 'its composed message does not close exactly what it claims — ship it alone to see why' });
14142
+ dropped.push({ branch: m.branch, sha: m.sha, cls: 'message', why: !msg ? 'no commit of its own to take a squash subject from' : 'its composed message does not close exactly what it claims — ship it alone to see why' });
14140
14143
  continue;
14141
14144
  }
14142
14145
  const c = git.run('git', ['commit', '-q', '-m', msg], { cwd: dir });
14143
- if (!c.ok) { git.git(['reset', '--hard', 'HEAD'], dir); dropped.push({ branch: m.branch, why: `commit failed: ${git.gitFailureLine(c.stderr)}` }); continue; }
14146
+ if (!c.ok) { git.git(['reset', '--hard', 'HEAD'], dir); dropped.push({ branch: m.branch, sha: m.sha, cls: 'commit-failed', why: `commit failed: ${git.gitFailureLine(c.stderr)}` }); continue; }
14144
14147
  built.push({ ...m, regenerated, commit: git.git(['rev-parse', 'HEAD'], dir).stdout });
14145
14148
  }
14146
14149
  for (const d of dropped) print(` ✗ ${d.branch}: ${d.why}`);
14150
+ // #554: a drop leaves a durable trace — one Ship-Batch-Dropped trailer per dropped member on the
14151
+ // batch head, so it reaches trunk with a landed batch and rides a red combined run's head message.
14152
+ // Only when the batch is built (≥2 members): a build that collapses to serial pushes nothing.
14153
+ if (dropped.length && built.length >= 2) {
14154
+ const last = built[built.length - 1];
14155
+ const cur = git.git(['log', '-1', '--format=%B'], dir);
14156
+ const lines = dropped.map((d) => shipBatch.droppedTrailer({ ref, branch: d.branch, sha: d.sha, cls: d.cls }));
14157
+ const am = cur.ok ? git.run('git', ['commit', '-q', '--amend', '-m', shipBatch.appendTrailers(cur.stdout, lines)], { cwd: dir }) : cur;
14158
+ if (am.ok) last.commit = git.git(['rev-parse', 'HEAD'], dir).stdout;
14159
+ else warn(`could not record the dropped member(s) on the batch head: ${git.gitFailureLine(am.stderr)} — the drop is on this terminal only`);
14160
+ }
14147
14161
  for (const b of built.filter((x) => x.regenerated)) {
14148
14162
  print(` ↻ ${b.branch}: combined despite a generated-file overlap — ${b.regenerated.join(', ')} regenerated by .colab/hooks/pre-ship on the combined head (the combined run is the gate)`);
14149
14163
  }
@@ -14644,6 +14658,141 @@ function landedBaseTrunk(repoAbs) {
14644
14658
  'gets measured against the wrong one.)');
14645
14659
  }
14646
14660
 
14661
+ const HELP_BATCH_STATS = `colab batch-stats [--repo <path>] [--since <30d | 12h | YYYY-MM-DD>] [--json]
14662
+
14663
+ How batch landing (#373) actually performs in this repo — read-only, derived from GitHub so it
14664
+ answers the same on any machine (#554). It is how a repo tunes its batch knobs: ship-batch: <N>
14665
+ and ship-batch-wait: <duration> have no handbook default (CONVENTIONS §4, Batch landing).
14666
+
14667
+ Sources: trunk's first-parent log since --since (default 30d) — Ship-Batch: trailers mark batch
14668
+ members, Ship-Batch-Dropped: trailers mark members dropped at build, anything else with a
14669
+ Closes #N is a serial landing — and the repo's CI runs in the same window (REST, paginated):
14670
+ ship-batch/** runs are the combined runs, feature-branch runs say when a branch went green at
14671
+ its head, trunk runs say when a landing was pushed and when its trunk-CI cycle ended.
14672
+
14673
+ Reports:
14674
+ batches landed, and their fill · combined-run builds: first-attempt green, green after the
14675
+ one re-run, red, pending
14676
+ red red builds, and how many of their members then landed serially (the fallback)
14677
+ dropped members dropped at build, by class, and the eviction rate
14678
+ serial serial landings, and MISSED PARTNERS — another change, landed later, that was
14679
+ already green when this one landed alone (could have joined) or went green inside
14680
+ this landing's trunk-CI cycle (a near miss; its wait is what ship-batch-wait buys)
14681
+ queue wait green-at-head → landing, per landed change
14682
+
14683
+ No batch history in the window is an answer, not an error: it says so, and still reports the
14684
+ serial side — missed partners are the evidence for turning ship-batch on at all. Not the optional
14685
+ machine-local journal: that is off by default and per-machine, so it cannot be the source.
14686
+ --json is the input for an audit; its shape is batch-stats.js derive()'s return value.
14687
+ Exit 0 on a report (empty or not); 1 when trunk's log cannot be read.
14688
+ `;
14689
+
14690
+ /** `--since`: `<N>d` / `<N>h` / `<N>m` back from now, or an ISO date. → epoch ms, or null. */
14691
+ function batchStatsSince(v, nowMs) {
14692
+ const s = String(v || '30d').trim();
14693
+ const rel = /^(\d+)([dhm])$/.exec(s);
14694
+ if (rel) return nowMs - Number(rel[1]) * ({ d: 86400, h: 3600, m: 60 })[rel[2]] * 1000;
14695
+ const t = Date.parse(s);
14696
+ return Number.isFinite(t) ? t : null;
14697
+ }
14698
+
14699
+ function cmdBatchStats(argv) {
14700
+ const opts = parseArgs(argv, {
14701
+ values: { '--repo': 'repo', '--since': 'since' },
14702
+ flags: { '--json': 'json', '-h': 'help', '--help': 'help' },
14703
+ });
14704
+ if (opts.help) return print(HELP_BATCH_STATS);
14705
+ const repoAbs = resolveRepo(opts);
14706
+ if (!git.repoRoot(repoAbs)) throw new UserError(`${repoAbs} is not a git repo — pass --repo <path> to a clone`);
14707
+ const doc = projectDoc(repoAbs);
14708
+ const trunk = resolveTrunk(repoAbs, doc);
14709
+ if (!trunk) throw new UserError('Could not determine trunk branch — add `trunk: <branch>` to .github/project.yml, then run `colab batch-stats` again');
14710
+ const nowMs = Date.now();
14711
+ const sinceMs = batchStatsSince(opts.since, nowMs);
14712
+ if (sinceMs === null) throw new UserError(`--since ${JSON.stringify(opts.since)}: expected 30d, 12h, 90m or an ISO date`);
14713
+ const sinceIso = new Date(sinceMs).toISOString();
14714
+ const notes = [];
14715
+
14716
+ const remote = git.remoteFor(repoAbs);
14717
+ const f = git.git(['fetch', remote, trunk], repoAbs);
14718
+ if (!f.ok) notes.push(`could not fetch ${remote}/${trunk} (${git.gitFailureLine(f.stderr)}) — read from the local copy, which may be behind`);
14719
+ const log = git.git(['log', `${remote}/${trunk}`, '--first-parent', `--since=${sinceIso}`, '--format=%H%x1f%cI%x1f%B%x1e'], repoAbs);
14720
+ if (!log.ok) throw new UserError(`could not read ${remote}/${trunk}: ${git.gitFailureLine(log.stderr)} — run \`git fetch ${remote} ${trunk}\`, then run this again`);
14721
+ const commits = log.stdout.split('\x1e').map((r) => r.replace(/^\n/, '')).filter((r) => r.trim()).map((r) => {
14722
+ const [sha, iso, body] = r.split('\x1f');
14723
+ return { sha: sha.trim(), dateMs: Date.parse(iso), body: body || '' };
14724
+ });
14725
+
14726
+ let runs = [];
14727
+ const builds = {};
14728
+ if (!isGhUsable(repoAbs)) {
14729
+ notes.push(`gh not usable (${ghUnusableWhy(repoAbs)}) — trunk commits only: no combined runs, no green times, no partner matching`);
14730
+ } else {
14731
+ const jq = '.workflow_runs[] | [.id, .head_branch, .head_sha, .event, .status, .conclusion, .run_attempt, .created_at, .updated_at, (if ((.head_branch // "") | startswith("ship-batch/")) then .head_commit.message else null end)] | @json';
14732
+ // One query per UTC day: GitHub answers at most 1000 rows for a filtered run list, and a busy
14733
+ // repo passes that inside a fortnight. A day that still hits the cap is named, never hidden.
14734
+ const days = [];
14735
+ for (let d = Date.parse(sinceIso.slice(0, 10)); d <= nowMs; d += 86_400_000) days.push(new Date(d).toISOString().slice(0, 10));
14736
+ let failed = null;
14737
+ const capped = [];
14738
+ for (const day of days) {
14739
+ const r = git.ghApi(repoAbs, ['--paginate', `repos/{owner}/{repo}/actions/runs?per_page=100&created=${day}`, '--jq', jq]);
14740
+ if (!r.ok) { failed = git.gitFailureLine(r.stderr); break; }
14741
+ let n = 0;
14742
+ for (const line of String(r.stdout || '').split('\n')) {
14743
+ if (!line.trim()) continue;
14744
+ let a; try { a = JSON.parse(line); } catch (_) { continue; }
14745
+ n++;
14746
+ runs.push({ id: a[0], branch: a[1], sha: a[2], event: a[3], status: a[4], conclusion: a[5], attempt: a[6], createdMs: Date.parse(a[7]), updatedMs: Date.parse(a[8]), message: a[9] });
14747
+ }
14748
+ if (n >= 1000) capped.push(day);
14749
+ }
14750
+ if (failed) { notes.push(`could not list CI runs (${failed}) — trunk commits only`); runs = []; }
14751
+ else {
14752
+ if (capped.length) notes.push(`GitHub caps a filtered run list at 1000 and ${capped.join(', ')} hit it — those days are incomplete`);
14753
+ // An unlanded combined run's ref is usually gone: read every commit of the build, so a red
14754
+ // batch's members and its drops are known, not just its head commit's.
14755
+ const landedShas = new Set(commits.map((c) => c.sha));
14756
+ const unlanded = [...new Set(runs.filter((x) => String(x.branch || '').startsWith(shipBatch.REF_PREFIX) && !landedShas.has(x.sha)).map((x) => `${x.branch} ${x.sha}`))];
14757
+ for (const k of unlanded) {
14758
+ const [ref, sha] = k.split(' ');
14759
+ const base7 = (shipBatch.parseBatchRef(ref) || {}).base7;
14760
+ if (!base7) continue;
14761
+ const c = git.ghApi(repoAbs, [`repos/{owner}/{repo}/compare/${base7}...${sha}`, '--jq', '[.commits[].commit.message]']);
14762
+ if (c.ok) { try { builds[sha] = JSON.parse(c.stdout); } catch (_) { /* head message only */ } }
14763
+ }
14764
+ }
14765
+ }
14766
+
14767
+ const cfg = shipBatch.parseShipBatch(doc);
14768
+ const waitCfg = shipBatch.parseShipBatchWait(doc);
14769
+ const rep = batchStats.derive({ commits, runs, builds, trunk, config: { n: cfg.n, waitSec: waitCfg.sec } });
14770
+ rep.repo = shortRepo(repoAbs);
14771
+ rep.since = sinceIso;
14772
+ rep.until = new Date(nowMs).toISOString();
14773
+ rep.notes = [...notes, ...rep.notes];
14774
+ if (opts.json) { print(JSON.stringify(rep, null, 2)); return 0; }
14775
+
14776
+ const pct = (x) => (x === null ? '—' : `${Math.round(x * 100)}%`);
14777
+ const dur = (s) => (s === null ? '—' : ciWait.fmtDuration(s * 1000));
14778
+ const q = (w) => (w.samples ? `p50 ${dur(w.p50Sec)} · p90 ${dur(w.p90Sec)} · max ${dur(w.maxSec)} (${w.samples})` : 'no samples');
14779
+ print(`batch-stats ${rep.repo} → ${trunk} ${rep.since.slice(0, 10)} … ${rep.until.slice(0, 10)} (ship-batch: ${cfg.n}${waitCfg.sec ? ` · ship-batch-wait: ${dur(waitCfg.sec)}` : ''})`);
14780
+ print(` landings ${rep.landings.total} (${rep.landings.batch} batch · ${rep.landings.serial} serial)`);
14781
+ if (!rep.history) print(' batches no batch history in this window — nothing has landed or been built through colab ship --batch');
14782
+ else {
14783
+ const fill = Object.keys(rep.batches.fill).sort().map((k) => `${rep.batches.fill[k]}×${k}`).join(' · ') || '—';
14784
+ print(` batches ${rep.batches.landed} landed (fill ${fill}) · ${rep.batches.builds} combined-run build(s): ${rep.batches.firstAttemptGreen} first-attempt green (${pct(rep.batches.firstAttemptGreenRate)}), ${rep.batches.greenAfterRerun} after a re-run, ${rep.batches.red} red, ${rep.batches.pending} pending`);
14785
+ print(` red ${rep.red.builds} build(s) · ${rep.red.members} member(s), ${rep.red.relandedSerially} then landed serially`);
14786
+ const cls = Object.entries(rep.dropped.byClass).map(([k, v]) => `${v} ${k}`).join(', ');
14787
+ print(` dropped ${rep.dropped.count} of ${rep.dropped.attempted} attempted (${pct(rep.dropped.rate)})${cls ? ` — ${cls}` : ''}`);
14788
+ }
14789
+ print(` serial ${rep.serial.landings} landed alone · ${rep.serial.withMissedPartner} with a missed partner (already green) · ${rep.serial.withNearMiss} more with a near miss (green within its trunk-CI cycle)`);
14790
+ if (rep.serial.nearMissWait.samples) print(` near misses per pair, the wait from the lone change's own green that would have caught its partner: ${q(rep.serial.nearMissWait)}`);
14791
+ print(` queue wait green-at-head → landing: ${q(rep.queueWait)}`);
14792
+ for (const n of rep.notes) print(` ! ${n}`);
14793
+ return 0;
14794
+ }
14795
+
14647
14796
  function cmdLanded(argv) {
14648
14797
  const opts = parseArgs(argv, {
14649
14798
  values: { '--repo': 'repo', '--worktree': 'worktree', '--branch': 'branch', '--base': 'base' },
@@ -15428,6 +15577,9 @@ Commands:
15428
15577
  landed [--worktree N | --branch B | --all] [--base B] [--json] has this work already landed?
15429
15578
  (the shared rule behind code-sweep's buckets — correct
15430
15579
  across squash merges, where "commits ahead" is not)
15580
+ batch-stats [--since 30d] [--json] how batch landing performs here, from git + CI history —
15581
+ fill, first-attempt green, drops, missed partners, queue
15582
+ wait; how a repo tunes ship-batch / ship-batch-wait (#554)
15431
15583
  holders <path> [--base B] [--json] who else is holding <path> — refs that touched it and
15432
15584
  have not landed on the base (filters the raw git-log
15433
15585
  one-liner through \`landed\`'s content classification)
@@ -15512,6 +15664,7 @@ function main(argv) {
15512
15664
  case 'worktrees': return cmdWorktrees(rest);
15513
15665
  case 'ship': return cmdShip(rest);
15514
15666
  case 'landed': return cmdLanded(rest);
15667
+ case 'batch-stats': return cmdBatchStats(rest);
15515
15668
  case 'trunk-ci': return cmdTrunkCi(rest);
15516
15669
  case 'ci-wait': return cmdCiWait(rest);
15517
15670
  case 'holders': return cmdHolders(rest);
@@ -0,0 +1,273 @@
1
+ 'use strict';
2
+ /**
3
+ * tools/lib/batch-stats.js — how batch landing (#373) actually performs in ONE repo (#554).
4
+ *
5
+ * The batch knobs (`ship-batch: <N>`, `ship-batch-wait: <duration>`) have no handbook default: a repo
6
+ * picks them from its own history. This module is that history, derived from two sources every
7
+ * machine can read — never from the optional machine-local journal, which is off by default and
8
+ * per-machine:
9
+ *
10
+ * commits trunk's first-parent log. A batch member's squash carries `Ship-Batch: <ref>
11
+ * <branch>@<sha>` (ship-batch.js memberTrailer); a member dropped at build is recorded on
12
+ * the batch head as `Ship-Batch-Dropped: <ref> <branch>@<sha> <class>`. A trunk commit
13
+ * with neither is a serial landing, known by its `Closes #N`.
14
+ * runs the repo's CI runs (REST rows). Runs on `ship-batch/**` are the combined runs; runs on
15
+ * a feature branch say when that branch went green at its head; runs on trunk say when a
16
+ * landing was pushed (created) and when its trunk-CI cycle ended (updated).
17
+ *
18
+ * Pure: facts in, a report out — no git, no gh, no clock. `tools/colab`'s `cmdBatchStats` reads.
19
+ *
20
+ * What it measures, per window:
21
+ * batches landed and their fill; every combined-run build, first-attempt green, green after
22
+ * the one re-run, red, still pending
23
+ * red red builds and how many of their members then landed serially (the fallback)
24
+ * dropped members dropped at build, by class, and the eviction rate among attempted members
25
+ * serial serial landings, and MISSED PARTNERS: another candidate that landed later but was
26
+ * already green when this one landed alone (could have joined), or turned green
27
+ * within this landing's trunk-CI cycle (a partner wait of `waitSec` would have caught
28
+ * it — the number `ship-batch-wait` is tuned from)
29
+ * queueWait per landed change, green-at-head → landing
30
+ */
31
+
32
+ const branchName = require('./branch-name');
33
+ const shipBatch = require('./ship-batch');
34
+
35
+ /** `Closes #N` / `Fixes #N` / `Resolves #N` numbers in a commit body (GitHub's own keywords). */
36
+ function closesIssues(body) {
37
+ const out = new Set();
38
+ const re = /\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s+#(\d+)\b/gi;
39
+ let m;
40
+ while ((m = re.exec(String(body || '')))) out.add(Number(m[1]));
41
+ return [...out];
42
+ }
43
+
44
+ /** Value at the q-th quantile (nearest-rank) of numbers, or null for none. */
45
+ function quantile(values, q) {
46
+ const v = (values || []).filter(Number.isFinite).sort((a, b) => a - b);
47
+ if (!v.length) return null;
48
+ return v[Math.min(v.length - 1, Math.max(0, Math.ceil(q * v.length) - 1))];
49
+ }
50
+
51
+ function summary(values) {
52
+ const v = (values || []).filter(Number.isFinite);
53
+ return { samples: v.length, p50Sec: quantile(v, 0.5), p90Sec: quantile(v, 0.9), maxSec: v.length ? Math.max(...v) : null };
54
+ }
55
+
56
+ const rate = (num, den) => (den > 0 ? Math.round((num / den) * 1000) / 1000 : null);
57
+ const isBatchRef = (b) => String(b || '').startsWith(shipBatch.REF_PREFIX);
58
+
59
+ /**
60
+ * When did `branch` go green at the head it had at time `atMs`? Its runs created before `atMs` are
61
+ * read; the newest sha among them is the head at that moment. Green only when every run at that sha
62
+ * finished success (cancelled aside, as combinedVerdict reads it) by `atMs` → the latest finish.
63
+ * Null when there is no run, or the head was not green yet — unknown, never a guess.
64
+ */
65
+ function greenAt(runsByBranch, branch, atMs, sha = null) {
66
+ const rows = (runsByBranch.get(branch) || []).filter((r) => r.createdMs <= atMs);
67
+ if (!rows.length) return null;
68
+ const headSha = sha
69
+ ? (rows.some((r) => r.sha.startsWith(sha) || sha.startsWith(r.sha)) ? rows.find((r) => r.sha.startsWith(sha) || sha.startsWith(r.sha)).sha : null)
70
+ : rows.reduce((a, b) => (b.createdMs > a.createdMs ? b : a)).sha;
71
+ if (!headSha) return null;
72
+ const at = rows.filter((r) => r.sha === headSha);
73
+ const v = shipBatch.combinedVerdict(at.map((r) => ({ status: r.status, conclusion: r.conclusion, attempt: r.attempt })));
74
+ if (v.state !== 'green') return null;
75
+ const done = Math.max(...at.filter((r) => r.conclusion === 'success').map((r) => r.updatedMs));
76
+ return done <= atMs ? done : null;
77
+ }
78
+
79
+ /** Which feature branch carried a serial landing that closed `issues`, as of `atMs`? Newest run wins. */
80
+ function branchForIssues(runsByBranch, issues, atMs) {
81
+ if (!issues.length) return null;
82
+ let best = null;
83
+ for (const [b, rows] of runsByBranch) {
84
+ const nums = branchName.branchIssueNumbers(b);
85
+ if (!nums.some((n) => issues.includes(n))) continue;
86
+ const before = rows.filter((r) => r.createdMs <= atMs);
87
+ if (!before.length) continue;
88
+ const t = Math.max(...before.map((r) => r.createdMs));
89
+ if (!best || t > best.t) best = { branch: b, t };
90
+ }
91
+ return best ? best.branch : null;
92
+ }
93
+
94
+ /**
95
+ * The report. Inputs:
96
+ * commits [{ sha, dateMs, body }] — trunk first-parent, inside the window, any order
97
+ * runs [{ id, branch, sha, event, status, conclusion, attempt, createdMs, updatedMs, message? }]
98
+ * `message` is the head commit's message, needed only for ship-batch/** rows
99
+ * builds optional { [sha]: [message, …] } — every commit message of an UNLANDED build, when the
100
+ * caller could read them (compare API); a landed build is read from trunk instead
101
+ * trunk trunk branch name
102
+ * config { n, waitSec } — the repo's declared knobs, echoed so the report says what it tunes
103
+ */
104
+ function derive({ commits = [], runs = [], builds = {}, trunk = 'main', config = {} } = {}) {
105
+ const notes = [];
106
+ const runsByBranch = new Map();
107
+ const trunkRuns = new Map(); // sha → rows
108
+ const batchRuns = new Map(); // `${ref} ${sha}` → rows
109
+ for (const r of runs || []) {
110
+ if (!r || !r.branch || !r.sha) continue;
111
+ if (r.branch === trunk) { (trunkRuns.get(r.sha) || trunkRuns.set(r.sha, []).get(r.sha)).push(r); continue; }
112
+ if (isBatchRef(r.branch)) { const k = `${r.branch} ${r.sha}`; (batchRuns.get(k) || batchRuns.set(k, []).get(k)).push(r); continue; }
113
+ (runsByBranch.get(r.branch) || runsByBranch.set(r.branch, []).get(r.branch)).push(r);
114
+ }
115
+ // When a landing reached trunk: its trunk push run was created then. Fallback: the commit date
116
+ // (a fast-forward keeps the build-time date, so it reads early by the combined run's length).
117
+ const landedAt = (sha, fallback) => {
118
+ const rows = trunkRuns.get(sha) || [];
119
+ return rows.length ? Math.min(...rows.map((r) => r.createdMs)) : fallback;
120
+ };
121
+ const cycleEnd = (sha) => {
122
+ const rows = (trunkRuns.get(sha) || []).filter((r) => r.status === 'completed');
123
+ return rows.length ? Math.max(...rows.map((r) => r.updatedMs)) : null;
124
+ };
125
+
126
+ // ---- landings, oldest first ----
127
+ const sorted = [...(commits || [])].sort((a, b) => a.dateMs - b.dateMs);
128
+ const batches = new Map(); // ref → { ref, commits: [], members: [], dropped: [] }
129
+ const landings = []; // { kind, sha, atMs, members: [{ branch, sha }], issues }
130
+ for (const c of sorted) {
131
+ const members = shipBatch.parseMemberTrailers([c.body]);
132
+ if (members.length) {
133
+ const ref = members[0].ref;
134
+ const b = batches.get(ref) || batches.set(ref, { ref, commits: [], members: [], dropped: [] }).get(ref);
135
+ b.commits.push(c);
136
+ b.members.push(...members);
137
+ b.dropped.push(...shipBatch.parseDroppedTrailers([c.body]));
138
+ } else {
139
+ landings.push({ kind: 'serial', sha: c.sha, atMs: landedAt(c.sha, c.dateMs), commitMs: c.dateMs, issues: closesIssues(c.body) });
140
+ }
141
+ }
142
+ for (const b of batches.values()) {
143
+ const head = b.commits[b.commits.length - 1];
144
+ b.headSha = head.sha;
145
+ landings.push({ kind: 'batch', ref: b.ref, sha: head.sha, atMs: landedAt(head.sha, head.dateMs), commitMs: head.dateMs, members: b.members });
146
+ }
147
+ landings.sort((a, b) => a.atMs - b.atMs);
148
+
149
+ // ---- every landed change, with its green-at-head time ----
150
+ const changes = []; // { landing, branch, sha, issues, greenMs, atMs }
151
+ for (const l of landings) {
152
+ if (l.kind === 'batch') {
153
+ for (const m of l.members) {
154
+ changes.push({ landing: l, branch: m.branch, issues: branchName.branchIssueNumbers(m.branch), greenMs: greenAt(runsByBranch, m.branch, l.atMs, m.sha), atMs: l.atMs });
155
+ }
156
+ } else {
157
+ const br = branchForIssues(runsByBranch, l.issues, l.atMs);
158
+ l.branch = br;
159
+ changes.push({ landing: l, branch: br, issues: l.issues, greenMs: br ? greenAt(runsByBranch, br, l.atMs) : null, atMs: l.atMs });
160
+ }
161
+ }
162
+
163
+ // ---- combined-run builds ----
164
+ const landedHeads = new Set([...batches.values()].map((b) => b.headSha));
165
+ const buildRows = [];
166
+ for (const [k, rows] of batchRuns) {
167
+ const [ref, sha] = k.split(' ');
168
+ const v = shipBatch.combinedVerdict(rows.map((r) => ({ status: r.status, conclusion: r.conclusion, attempt: r.attempt, databaseId: r.id })));
169
+ const landed = landedHeads.has(sha);
170
+ const messages = landed ? null : (builds[sha] || rows.map((r) => r.message).filter(Boolean).slice(0, 1));
171
+ buildRows.push({ ref, sha, state: v.state, attempt: v.attempt, landed, createdMs: Math.min(...rows.map((r) => r.createdMs)), messages });
172
+ }
173
+ const finished = buildRows.filter((b) => b.state === 'green' || b.state === 'red');
174
+ const firstGreen = buildRows.filter((b) => b.state === 'green' && b.attempt === 1).length;
175
+ const rerunGreen = buildRows.filter((b) => b.state === 'green' && b.attempt > 1).length;
176
+ const redBuilds = buildRows.filter((b) => b.state === 'red');
177
+
178
+ // ---- red builds → did their members then land serially? ----
179
+ const serialChanges = changes.filter((c) => c.landing.kind === 'serial');
180
+ let redMembers = 0, relanded = 0, redMembersUnread = 0;
181
+ for (const b of redBuilds) {
182
+ const members = b.messages && b.messages.length ? shipBatch.parseMemberTrailers(b.messages) : [];
183
+ if (!members.length) { redMembersUnread++; continue; }
184
+ for (const m of members) {
185
+ redMembers++;
186
+ const nums = branchName.branchIssueNumbers(m.branch);
187
+ if (serialChanges.some((c) => c.atMs >= b.createdMs && (c.branch === m.branch || c.issues.some((n) => nums.includes(n))))) relanded++;
188
+ }
189
+ }
190
+ if (redMembersUnread) notes.push(`${redMembersUnread} red build(s) whose members could not be read (the ref is gone and its commits were not readable)`);
191
+
192
+ // ---- dropped members: landed batches from trunk, unlanded builds from their messages ----
193
+ const dropped = [];
194
+ const seenDrop = new Set();
195
+ const addDrop = (d, landed) => {
196
+ const k = `${d.ref} ${d.branch}@${d.sha}`;
197
+ if (seenDrop.has(k)) return;
198
+ seenDrop.add(k);
199
+ dropped.push({ ...d, landed });
200
+ };
201
+ for (const b of batches.values()) for (const d of b.dropped) addDrop(d, true);
202
+ for (const b of buildRows) if (!b.landed && b.messages) for (const d of shipBatch.parseDroppedTrailers(b.messages)) addDrop(d, false);
203
+ const builtMembers = [...batches.values()].reduce((n, b) => n + b.members.length, 0)
204
+ + buildRows.filter((b) => !b.landed && b.messages).reduce((n, b) => n + shipBatch.parseMemberTrailers(b.messages).length, 0);
205
+ const byClass = {};
206
+ for (const d of dropped) byClass[d.cls] = (byClass[d.cls] || 0) + 1;
207
+
208
+ // ---- missed partners: serial landings that landed alone while a partner was (nearly) ready ----
209
+ const pairs = [];
210
+ let withMissed = 0, withNear = 0, cyclesUnread = 0;
211
+ // A partner is judged by ITS state at this landing's moment, never by the head it finally landed
212
+ // with: a serial ship syncs the other branches, so their final heads all post-date it. The cycle
213
+ // ends when this landing's trunk run finishes — or at the next landing, whichever is first (a
214
+ // re-run moves a run's updated_at hours later, and the next landing moves trunk anyway).
215
+ const landingTimes = landings.map((l) => l.atMs).sort((a, b) => a - b);
216
+ for (const c of serialChanges) {
217
+ const next = landingTimes.find((t) => t > c.atMs);
218
+ let end = cycleEnd(c.landing.sha);
219
+ if (end === null) cyclesUnread++;
220
+ else if (next !== undefined) end = Math.min(end, next);
221
+ let missed = false, near = false;
222
+ const label = c.branch || c.landing.sha.slice(0, 7);
223
+ for (const o of changes) {
224
+ if (o === c || o.landing === c.landing || o.atMs <= c.atMs || !o.branch || o.branch === c.branch) continue;
225
+ const gNow = greenAt(runsByBranch, o.branch, c.atMs);
226
+ if (gNow !== null) {
227
+ missed = true;
228
+ pairs.push({ kind: 'missed', landed: label, partner: o.branch });
229
+ continue;
230
+ }
231
+ if (end === null) continue;
232
+ const gEnd = greenAt(runsByBranch, o.branch, end);
233
+ if (gEnd !== null && gEnd > c.atMs) {
234
+ near = true;
235
+ const from = c.greenMs !== null ? c.greenMs : c.atMs;
236
+ pairs.push({ kind: 'near', landed: label, partner: o.branch, waitSec: Math.round((gEnd - from) / 1000) });
237
+ }
238
+ }
239
+ if (missed) withMissed++;
240
+ else if (near) withNear++;
241
+ }
242
+ if (cyclesUnread) notes.push(`${cyclesUnread} serial landing(s) with no finished trunk run in the window — near misses not judged for them`);
243
+ const unknownGreen = changes.filter((c) => c.greenMs === null).length;
244
+ if (unknownGreen) notes.push(`${unknownGreen} landed change(s) with no readable green-at-head run — left out of queue wait and partner matching`);
245
+
246
+ const fill = {};
247
+ for (const b of batches.values()) fill[b.members.length] = (fill[b.members.length] || 0) + 1;
248
+ const queueWait = summary(changes.filter((c) => c.greenMs !== null).map((c) => Math.round((c.atMs - c.greenMs) / 1000)));
249
+
250
+ return {
251
+ trunk,
252
+ config: { n: config.n || 1, waitSec: config.waitSec || 0 },
253
+ history: batches.size > 0 || buildRows.length > 0,
254
+ landings: { total: landings.length, batch: batches.size, serial: landings.length - batches.size },
255
+ batches: {
256
+ landed: batches.size, fill,
257
+ builds: buildRows.length, firstAttemptGreen: firstGreen, greenAfterRerun: rerunGreen, red: redBuilds.length,
258
+ pending: buildRows.length - finished.length,
259
+ firstAttemptGreenRate: rate(firstGreen, finished.length),
260
+ },
261
+ red: { builds: redBuilds.length, members: redMembers, relandedSerially: relanded },
262
+ dropped: { count: dropped.length, attempted: builtMembers + dropped.length, rate: rate(dropped.length, builtMembers + dropped.length), byClass, members: dropped },
263
+ serial: {
264
+ landings: serialChanges.length, withMissedPartner: withMissed, withNearMiss: withNear,
265
+ nearMissWait: summary(pairs.filter((p) => p.kind === 'near').map((p) => p.waitSec)),
266
+ pairs,
267
+ },
268
+ queueWait,
269
+ notes,
270
+ };
271
+ }
272
+
273
+ module.exports = { derive, closesIssues, quantile, greenAt, branchForIssues };
@@ -141,6 +141,48 @@ function parseMemberTrailers(messages) {
141
141
  return out;
142
142
  }
143
143
 
144
+ /**
145
+ * #554: a member dropped at build used to leave no trace in git — only a `✗ <member>: …` line on
146
+ * one terminal — so eviction rate could not be measured. The batch head now records each drop:
147
+ *
148
+ * Ship-Batch-Dropped: <ref> <branch>@<sha> <class>
149
+ *
150
+ * appended to the LAST built member's message (the head commit), so it reaches trunk with a landed
151
+ * batch and rides the head commit's message of a combined run that went red. `class` is one token
152
+ * from DROP_CLASSES; the human reason stays on the terminal. A branch (already pushed, so already
153
+ * public) and a sha, never a host — the same rule as memberTrailer.
154
+ */
155
+ const DROP_TRAILER_KEY = 'Ship-Batch-Dropped';
156
+ const DROP_CLASSES = ['conflict', 'generated-no-hook', 'hook-failed', 'hook-markers', 'squash-failed', 'empty', 'message', 'commit-failed'];
157
+
158
+ function droppedTrailer({ ref, branch, sha, cls }) {
159
+ const c = DROP_CLASSES.includes(cls) ? cls : 'other';
160
+ return `${DROP_TRAILER_KEY}: ${ref} ${branch}@${String(sha || '').slice(0, 12)} ${c}`;
161
+ }
162
+
163
+ /** Every `Ship-Batch-Dropped:` trailer across commit messages → [{ ref, branch, sha, cls }]. */
164
+ function parseDroppedTrailers(messages) {
165
+ const out = [];
166
+ for (const msg of messages || []) {
167
+ for (const line of String(msg || '').split('\n')) {
168
+ const m = new RegExp(`^${DROP_TRAILER_KEY}:\\s+(\\S+)\\s+(\\S+)@([0-9a-f]{7,40})\\s+([a-z-]+)\\s*$`).exec(line.trim());
169
+ if (m) out.push({ ref: m[1], branch: m[2], sha: m[3], cls: m[4] });
170
+ }
171
+ }
172
+ return out;
173
+ }
174
+
175
+ /**
176
+ * `msg` with `lines` appended to its final trailer block — never as a new paragraph, which would
177
+ * split the block and hide the earlier trailers from `git interpret-trailers`. Lines already
178
+ * present are not repeated (a rebuild re-composes the same message).
179
+ */
180
+ function appendTrailers(msg, lines) {
181
+ const body = String(msg || '').replace(/\s+$/, '');
182
+ const add = (lines || []).filter((l) => l && !body.split('\n').includes(l));
183
+ return add.length ? `${body}\n${add.join('\n')}\n` : `${body}\n`;
184
+ }
185
+
144
186
  /**
145
187
  * A member's OWN head CI, read as a joining class (§4 Branch CI):
146
188
  * green every run at its head finished, all success
@@ -379,8 +421,9 @@ function landPushFailure({ stderr, remoteNow, base, max = 8 } = {}) {
379
421
  }
380
422
 
381
423
  module.exports = {
382
- MAX_BATCH, REF_PREFIX, PROBE_REF, TRAILER_KEY, WAIT_KEY,
424
+ MAX_BATCH, REF_PREFIX, PROBE_REF, TRAILER_KEY, WAIT_KEY, DROP_TRAILER_KEY, DROP_CLASSES,
383
425
  parseShipBatch, parseShipBatchWait, partnerWait, readySince, batchRefName, parseBatchRef, memberTrailer, parseMemberTrailers,
426
+ droppedTrailer, parseDroppedTrailers, appendTrailers,
384
427
  branchCiClass, memberEligibility, selectMembers, wiring, combinedVerdict, nextStep, foreignBatchStep,
385
428
  batchGreenCoversTrunk, evidenceSuffix, serialLine, notStaged, landPushFailure,
386
429
  };