@biffo/cli 0.241.1 → 0.242.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/_skeletons/plugin-template/AGENTS.md +14 -0
- package/_skeletons/sibling-template/AGENTS.md +14 -0
- package/dist/index.js +16 -2
- package/package.json +5 -2
- package/scripts/practices-corpus.mjs +172 -0
- package/scripts/practices-metrics.mjs +2682 -0
- package/scripts/runner-drop-forensics.mjs +344 -0
|
@@ -0,0 +1,2682 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Collect development-practices metrics across every Biffo and tabsii repo.
|
|
3
|
+
*
|
|
4
|
+
* ## Why this exists
|
|
5
|
+
*
|
|
6
|
+
* `docs/guides/development-practices.md` is a 38-row corpus of real failures,
|
|
7
|
+
* and it cannot rank any of them. Exactly one row carries a cost figure and none
|
|
8
|
+
* carries a date, so "highest impact first" is currently an opinion. Worse, the
|
|
9
|
+
* page's own headline conclusion ("fail-open is the dominant shape — three of
|
|
10
|
+
* the five filed issues") was written against a 5-row sample and never revised;
|
|
11
|
+
* across all 38 rows `fail-open` is the *least* common class. Hand-narrated
|
|
12
|
+
* conclusions drift from their own evidence.
|
|
13
|
+
*
|
|
14
|
+
* This script is the other half of the fix: numbers nobody has to remember to
|
|
15
|
+
* write down. It reads what GitHub and git already know, so a snapshot cannot be
|
|
16
|
+
* biased by the agent being measured.
|
|
17
|
+
*
|
|
18
|
+
* ## Why plain .mjs rather than TypeScript
|
|
19
|
+
*
|
|
20
|
+
* Same reasoning as `destructive-plan.mjs`: it runs on bare node with no
|
|
21
|
+
* dependency install, so it can be invoked from a scheduled workflow that sets
|
|
22
|
+
* up nothing. The pure logic is exported and tested from
|
|
23
|
+
* `cli/src/lib/practices-metrics.test.ts`, so it has one home rather than a
|
|
24
|
+
* TypeScript copy that can drift.
|
|
25
|
+
*
|
|
26
|
+
* ## The one rule this file obeys about its own results
|
|
27
|
+
*
|
|
28
|
+
* **"Could not measure" is never reported as zero.** The corpus's most valuable
|
|
29
|
+
* lesson is that a gate which passes when it cannot run makes "green" and
|
|
30
|
+
* "checked" different things. A metrics collector has the identical failure
|
|
31
|
+
* mode: a repo whose runs did not come back would otherwise score a perfect
|
|
32
|
+
* 0% CI failure rate and quietly drag the average down. Every metric here is
|
|
33
|
+
* either a number or `null`, and `null` propagates into the snapshot as
|
|
34
|
+
* `unmeasured` rather than being averaged in.
|
|
35
|
+
*
|
|
36
|
+
* Usage:
|
|
37
|
+
* node scripts/practices-metrics.mjs --out docs/practices/data
|
|
38
|
+
* node scripts/practices-metrics.mjs --window 30 --repo keiranholloway/biffo-template
|
|
39
|
+
* node scripts/practices-metrics.mjs --gate-lookback 7 # cheaper jobs fetch
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
import { execFileSync } from 'node:child_process'
|
|
43
|
+
import { mkdirSync, writeFileSync } from 'node:fs'
|
|
44
|
+
import { join } from 'node:path'
|
|
45
|
+
// The corpus is legacy `.jsonl` + one-file-per-entry directory, merged and
|
|
46
|
+
// sorted (#1132) — `readCorpusStrict` throws rather than reading a corpus
|
|
47
|
+
// that half-parses as if it were a smaller valid one, matching this file's
|
|
48
|
+
// own "unmeasured, never zero" rule below.
|
|
49
|
+
import { readCorpusStrict } from './practices-corpus.mjs'
|
|
50
|
+
|
|
51
|
+
/** Snapshot schema version. Bump when a field's meaning changes, never when one is added. */
|
|
52
|
+
export const SCHEMA_VERSION = 2
|
|
53
|
+
|
|
54
|
+
/** Default observation window. 90 days is long enough to survive a quiet fortnight. */
|
|
55
|
+
export const DEFAULT_WINDOW_DAYS = 90
|
|
56
|
+
|
|
57
|
+
/** Windows the daily dashboard shows side by side: yesterday, this week, the baseline. */
|
|
58
|
+
export const DEFAULT_WINDOWS = [1, 7, 90]
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The repos under measurement, and where their working clone lives.
|
|
62
|
+
*
|
|
63
|
+
* `path` is only needed for the rework metric, which reads file-level history
|
|
64
|
+
* from git rather than paying one API call per pull request. A repo with no
|
|
65
|
+
* local clone still gets every other metric; its rework rate reports `null`.
|
|
66
|
+
*
|
|
67
|
+
* This list is explicit rather than derived from what happens to be on disk:
|
|
68
|
+
* a second checkout of a repo already here would double every one of its rows.
|
|
69
|
+
*/
|
|
70
|
+
export const REPOS = [
|
|
71
|
+
{ slug: 'keiranholloway/biffo-template', path: 'biffo-template', role: 'template', side: 'platform' },
|
|
72
|
+
{ slug: 'keiranholloway/biffo-platform', path: 'biffo-platform', role: 'instance', side: 'platform' },
|
|
73
|
+
{ slug: 'keiranholloway/biffo-platform-app', path: 'biffo-platform-app', role: 'sibling', side: 'platform' },
|
|
74
|
+
{ slug: 'keiranholloway/biffo-plugin-ideation', path: 'biffo-plugin-ideation', role: 'plugin', side: 'platform' },
|
|
75
|
+
{ slug: 'keiranholloway/biffo-plugin-idea-scout', path: 'biffo-plugin-idea-scout', role: 'plugin', side: 'platform' },
|
|
76
|
+
{ slug: 'keiranholloway/biffo-runners', path: 'biffo-runners', role: 'infra', side: 'platform' },
|
|
77
|
+
{ slug: 'tabsii-com/tabsii-platform', path: 'tabsii-platform', role: 'instance', side: 'product' },
|
|
78
|
+
{ slug: 'tabsii-com/tabsii-crm', path: 'tabsii-crm', role: 'sibling', side: 'product' },
|
|
79
|
+
{ slug: 'tabsii-com/tabsii-intake', path: 'tabsii-intake', role: 'sibling', side: 'product' },
|
|
80
|
+
{ slug: 'tabsii-com/tabsii-map', path: 'tabsii-map', role: 'package', side: 'product' },
|
|
81
|
+
{ slug: 'tabsii-com/tabsii-geo', path: 'tabsii-geo', role: 'sibling', side: 'product' },
|
|
82
|
+
{ slug: 'tabsii-com/tabsii-marketplace', path: 'tabsii-marketplace', role: 'sibling', side: 'product' },
|
|
83
|
+
{ slug: 'tabsii-com/tabsii-app', path: 'tabsii-app', role: 'sibling', side: 'product' },
|
|
84
|
+
{ slug: 'tabsii-com/tabsii-runners', path: 'tabsii-runners', role: 'infra', side: 'product' },
|
|
85
|
+
{
|
|
86
|
+
slug: 'tabsii-com/tabsii-data-model-design',
|
|
87
|
+
path: 'tabsii-data-model-design',
|
|
88
|
+
role: 'design',
|
|
89
|
+
side: 'product',
|
|
90
|
+
},
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Conclusions that mean a gate ran and rejected the change.
|
|
95
|
+
*
|
|
96
|
+
* `cancelled` is excluded on purpose and counted separately: most cancellations
|
|
97
|
+
* here are a newer push superseding an in-flight run, which is ordinary
|
|
98
|
+
* iteration rather than a gate finding a defect. Folding the two together would
|
|
99
|
+
* inflate the failure rate every time someone pushes twice in quick succession.
|
|
100
|
+
*/
|
|
101
|
+
export const FAILING_CONCLUSIONS = new Set(['failure', 'timed_out', 'startup_failure'])
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Step conclusions that mean the step **stopped without a verdict**.
|
|
105
|
+
*
|
|
106
|
+
* A dying runner produces two different signatures and #982 caught only the
|
|
107
|
+
* first, so `biffo-platform` kept two failures it had not earned:
|
|
108
|
+
*
|
|
109
|
+
* - `null` — the step was still executing when the lights went out. A deploy
|
|
110
|
+
* frozen on "Package and deploy Lambda", six steps left `pending`.
|
|
111
|
+
* - `cancelled` — the step was stopped, and every later step reads `skipped`.
|
|
112
|
+
* Two `biffo-platform` CI runs died 64 seconds in this way, on "Type check"
|
|
113
|
+
* and "Lint".
|
|
114
|
+
*
|
|
115
|
+
* ## Why `cancelled` here is not an ordinary cancellation
|
|
116
|
+
*
|
|
117
|
+
* The obvious objection is that this launders someone hitting cancel, or a
|
|
118
|
+
* `cancel-in-progress` supersession. It does not, and the reason is structural:
|
|
119
|
+
* **those conclude the run `cancelled`**, which `FAILING_CONCLUSIONS` already
|
|
120
|
+
* excludes. This function is only ever reached for a run that concluded
|
|
121
|
+
* `failure`. A run that concluded `failure` while no step ever returned a
|
|
122
|
+
* verdict was therefore stopped by something that is not a cancellation.
|
|
123
|
+
*
|
|
124
|
+
* The evidence agreed rather than merely permitting it: neither run was
|
|
125
|
+
* superseded — the next CI run came 32 minutes later, long after both had died.
|
|
126
|
+
*
|
|
127
|
+
* A step that hits `timeout-minutes` (20 since #980) is expected to be marked
|
|
128
|
+
* `failure` and so stays a real failure. No timed-out run exists in the corpus
|
|
129
|
+
* yet to confirm that from data rather than from documentation.
|
|
130
|
+
*/
|
|
131
|
+
const STOPPED_SHORT = new Set([null, undefined, 'cancelled'])
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Did this run fail because a **runner died**, rather than because a gate
|
|
135
|
+
* rejected the change? (#982)
|
|
136
|
+
*
|
|
137
|
+
* ## The hole this closes
|
|
138
|
+
*
|
|
139
|
+
* `FAILING_CONCLUSIONS` above excludes `cancelled` because a killed or
|
|
140
|
+
* superseded run is not a defect. That reasoning is right and its coverage is
|
|
141
|
+
* only partial: **a runner killed mid-job reports `cancelled` only sometimes.**
|
|
142
|
+
* The rest of the time GitHub concludes the run `failure` with *no failing
|
|
143
|
+
* step* — the same physical event, a different label, and the second label was
|
|
144
|
+
* counted as if code had broken.
|
|
145
|
+
*
|
|
146
|
+
* Measured on `tabsii-com/tabsii-platform`, 2026-07-31: **all six** `dev`
|
|
147
|
+
* failures inspected had zero failing steps and 3–21 steps left incomplete. One
|
|
148
|
+
* deploy succeeded through thirteen steps and froze on "Package and deploy
|
|
149
|
+
* Lambda". Not one gate rejected a change, while the estate's board reported 8
|
|
150
|
+
* integration failures and 111.7 red minutes on that branch.
|
|
151
|
+
*
|
|
152
|
+
* ## Why this is not cosmetic
|
|
153
|
+
*
|
|
154
|
+
* H3's counter-metric refutes the experiment on `integration.failures > 2` or
|
|
155
|
+
* `redMinutes > 60`. `tabsii-platform` joined its treatment arm on 2026-07-31
|
|
156
|
+
* already past both — entirely on runner kills, which have nothing to do with
|
|
157
|
+
* `strict`. Left alone, the experiment gets refuted for something it never
|
|
158
|
+
* touched, four days later.
|
|
159
|
+
*
|
|
160
|
+
* ## The rule, and why it errs the way it does
|
|
161
|
+
*
|
|
162
|
+
* A failed run is a runner kill when **no job reports a failing step** and **at
|
|
163
|
+
* least one failed job has a step that stopped without a verdict** — see
|
|
164
|
+
* {@link STOPPED_SHORT} for the two signatures that means, and why `cancelled`
|
|
165
|
+
* among them is not an ordinary cancellation. Both halves matter: the first
|
|
166
|
+
* says nothing rejected the change, the second says work was still outstanding
|
|
167
|
+
* when the lights went out.
|
|
168
|
+
*
|
|
169
|
+
* A failed run with no steps recorded at all is deliberately **not** classified
|
|
170
|
+
* as a kill. It stays a failure. That is the conservative direction for a
|
|
171
|
+
* counter-metric — it can still refute an experiment the author would prefer to
|
|
172
|
+
* confirm — and this file's whole purpose is to make that the default.
|
|
173
|
+
*
|
|
174
|
+
* A job that hits its `timeout-minutes` (20 since #980) marks the offending step
|
|
175
|
+
* `failure`, so a genuine hang stays a genuine failure and is not laundered
|
|
176
|
+
* through here.
|
|
177
|
+
*
|
|
178
|
+
* @param {Array<Record<string, any>>} jobs the `jobs` array of one run
|
|
179
|
+
* @returns {boolean}
|
|
180
|
+
*/
|
|
181
|
+
export function isRunnerKill(jobs) {
|
|
182
|
+
const failed = (jobs ?? []).filter((job) => job.conclusion === 'failure')
|
|
183
|
+
if (failed.length === 0) return false
|
|
184
|
+
const steps = failed.flatMap((job) => job.steps ?? [])
|
|
185
|
+
if (steps.length === 0) return false
|
|
186
|
+
if (steps.some((step) => step.conclusion === 'failure')) return false
|
|
187
|
+
return steps.some((step) => STOPPED_SHORT.has(step.conclusion))
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Longest gap between pushes that still counts as a red branch blocking someone
|
|
194
|
+
* (#921). Past this, nobody is waiting — see {@link integrationHealth}.
|
|
195
|
+
*
|
|
196
|
+
* One hour, chosen against the estate's own push cadence: `biffo-template` merged
|
|
197
|
+
* 37 a day at its peak and `tabsii-platform` 33 in the window that motivated this,
|
|
198
|
+
* so an hour of total silence on an integration branch is a lull, not a queue.
|
|
199
|
+
*/
|
|
200
|
+
export const IDLE_CEILING_MINUTES = 60
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* The workflow whose success means "this code is now running" (#767).
|
|
204
|
+
*
|
|
205
|
+
* Named identically across every deploying repo in the estate — checked on
|
|
206
|
+
* biffo-template, biffo-platform and tabsii-platform. A repo without it yields
|
|
207
|
+
* `null` for the running stop rather than 0: "does not deploy" and "deployed
|
|
208
|
+
* instantly" are different claims and only one is good news.
|
|
209
|
+
*/
|
|
210
|
+
export const DEPLOY_WORKFLOW = 'Deploy Application'
|
|
211
|
+
|
|
212
|
+
/** Marker `biffo core upgrade` writes into its PR body (#767). */
|
|
213
|
+
export const CARRIED_PRS_MARKER = 'biffo:carries-template-prs:'
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Branch prefix `biffo core upgrade` always creates. Mirrors
|
|
217
|
+
* `UPGRADE_BRANCH_PREFIX` in `cli/src/lib/core-upgrade.ts`.
|
|
218
|
+
*/
|
|
219
|
+
export const UPGRADE_BRANCH_PREFIX = 'biffo/core-upgrade-'
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Is this PR actually an upgrade, rather than one that merely *mentions* the
|
|
223
|
+
* marker?
|
|
224
|
+
*
|
|
225
|
+
* Not paranoia — this fired on the first real run. `biffo-template` reported an
|
|
226
|
+
* upgrade PR carrying four template PRs, which is impossible: the template does
|
|
227
|
+
* not upgrade itself. The parser had matched the marker inside PR #772's own
|
|
228
|
+
* body, where it appears as **documentation of the format**. A PR describing the
|
|
229
|
+
* mechanism was counted as one emitting it.
|
|
230
|
+
*
|
|
231
|
+
* The branch name is the discriminator because the CLI controls it absolutely:
|
|
232
|
+
* `upgradeBranchName()` is the only thing that opens these PRs. Body text is
|
|
233
|
+
* written by whoever is describing the feature.
|
|
234
|
+
*
|
|
235
|
+
* @param {Record<string, any>} pr
|
|
236
|
+
*/
|
|
237
|
+
export function isUpgradePr(pr) {
|
|
238
|
+
return typeof pr.headRefName === 'string' && pr.headRefName.startsWith(UPGRADE_BRANCH_PREFIX)
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Extracts the marker's PR list from one piece of text. Shared by the PR body
|
|
243
|
+
* and commit-message paths in {@link parseCarriedPrs} — same regex, same
|
|
244
|
+
* validation, so the two can never quietly diverge.
|
|
245
|
+
*
|
|
246
|
+
* @param {string | null | undefined} text
|
|
247
|
+
*/
|
|
248
|
+
function extractCarriedPrs(text) {
|
|
249
|
+
if (typeof text !== 'string') return []
|
|
250
|
+
// `[\s0-9,]`, not `[0-9,]`: since #1198 the writer WRAPS a long list across
|
|
251
|
+
// lines so the commit message can satisfy commitlint's 100-character body
|
|
252
|
+
// limit. A parser that stopped at the first newline would still match, still
|
|
253
|
+
// return numbers, and silently return only the first line of them — under-
|
|
254
|
+
// reporting provenance while looking like it worked. It stops at the `-` of
|
|
255
|
+
// the closing `-->` either way, so the wrapped form is unambiguous.
|
|
256
|
+
const match = new RegExp(`${CARRIED_PRS_MARKER}([\\s0-9,]+)`).exec(text)
|
|
257
|
+
if (!match?.[1]) return []
|
|
258
|
+
const numbers = match[1]
|
|
259
|
+
.split(/[,\s]+/)
|
|
260
|
+
.map((n) => Number(n.trim()))
|
|
261
|
+
.filter((n) => Number.isInteger(n) && n > 0)
|
|
262
|
+
return [...new Set(numbers)].sort((a, b) => a - b)
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Template PR numbers an instance's upgrade PR carries (#767).
|
|
267
|
+
*
|
|
268
|
+
* Reads the marker from the PR body — where `biffo core upgrade` has always
|
|
269
|
+
* put it — or, failing that, from any of the PR's commit messages (#1011).
|
|
270
|
+
* `--apply` writes the same marker into the upgrade commit before the push
|
|
271
|
+
* step that can fail and abort the run before a PR ever gets opened. When the
|
|
272
|
+
* operator then pushes and opens the PR by hand, the body never gets written,
|
|
273
|
+
* but the commit — and its marker — survived. Checking commits is a pure
|
|
274
|
+
* fallback: a tool-created PR always matches on the body first and never
|
|
275
|
+
* touches this path.
|
|
276
|
+
*
|
|
277
|
+
* Returns `[]` when neither place has the marker, which is every PR except an
|
|
278
|
+
* upgrade — and every upgrade opened before the marker shipped. That is a
|
|
279
|
+
* *coverage* fact, not an error: the metric simply has nothing to say about
|
|
280
|
+
* those, and says nothing rather than guessing.
|
|
281
|
+
*
|
|
282
|
+
* @param {string | null | undefined} body
|
|
283
|
+
* @param {Array<{messageBody?: string, messageHeadline?: string}> | null | undefined} [commits]
|
|
284
|
+
*/
|
|
285
|
+
export function parseCarriedPrs(body, commits) {
|
|
286
|
+
const fromBody = extractCarriedPrs(body)
|
|
287
|
+
if (fromBody.length > 0) return fromBody
|
|
288
|
+
for (const commit of commits ?? []) {
|
|
289
|
+
const fromCommit = extractCarriedPrs(commit?.messageBody ?? commit?.messageHeadline)
|
|
290
|
+
if (fromCommit.length > 0) return fromCommit
|
|
291
|
+
}
|
|
292
|
+
return []
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Template PR number → the issue keys it closed.
|
|
297
|
+
*
|
|
298
|
+
* @param {string} templateSlug
|
|
299
|
+
* @param {Array<Record<string, any>>} templatePrs
|
|
300
|
+
* @returns {Map<number, string[]>}
|
|
301
|
+
*/
|
|
302
|
+
export function indexClosingIssues(templateSlug, templatePrs) {
|
|
303
|
+
/** @type {Map<number, string[]>} */
|
|
304
|
+
const index = new Map()
|
|
305
|
+
for (const pr of templatePrs) {
|
|
306
|
+
const keys = (pr.closingIssuesReferences ?? [])
|
|
307
|
+
.map((ref) => {
|
|
308
|
+
const owner = ref.repository?.owner?.login
|
|
309
|
+
const name = ref.repository?.name
|
|
310
|
+
return owner && name ? `${owner}/${name}#${ref.number}` : null
|
|
311
|
+
})
|
|
312
|
+
.filter((key) => key !== null)
|
|
313
|
+
if (keys.length > 0) index.set(pr.number, keys)
|
|
314
|
+
}
|
|
315
|
+
return index
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Time from a **template** issue being opened to it running in an instance —
|
|
320
|
+
* the whole six-hop distribution, measured rather than described (#767).
|
|
321
|
+
*
|
|
322
|
+
* A template feature is not usable when its template PR merges. It becomes
|
|
323
|
+
* usable when an instance deploys it, five hops later: tag → npm publish →
|
|
324
|
+
* `core upgrade` → instance PR → deploy. `development-practices.md` prices that
|
|
325
|
+
* at "~40 min minimum" and once at three full release cycles for one feature,
|
|
326
|
+
* but only ever from anecdote, because nothing recorded which issues an upgrade
|
|
327
|
+
* carried. The marker does; this reads it.
|
|
328
|
+
*
|
|
329
|
+
* `carriedWithoutIssue` is reported deliberately. The first marker ever emitted
|
|
330
|
+
* carried twelve template PRs and **none of them closed an issue** — every one
|
|
331
|
+
* used `Refs #N`, correctly, because the issues were not finished. That is the
|
|
332
|
+
* binding constraint on this metric and it must be visible, not inferred from a
|
|
333
|
+
* small `measured`.
|
|
334
|
+
*
|
|
335
|
+
* `templateIndexEmpty` exists because this metric reported `measured: 0` for a
|
|
336
|
+
* reason that had nothing to do with the data. `closingIssues` is built from the
|
|
337
|
+
* TEMPLATE repo's PRs, and `--repo <instance>` filters the fetch to that instance
|
|
338
|
+
* — so the template's PRs were never fetched, the map was empty, and every
|
|
339
|
+
* carried PR fell through to `carriedWithoutIssue`. Nothing distinguished "these
|
|
340
|
+
* PRs closed no issue" from "I never loaded the side of the join that would
|
|
341
|
+
* know". #776's own verification command was the one that could not work, and
|
|
342
|
+
* two separate triages concluded the metric was unproven on the strength of it.
|
|
343
|
+
*
|
|
344
|
+
* A zero that means "I could not see the input" must never be reported as a zero
|
|
345
|
+
* that means "I looked and there was nothing".
|
|
346
|
+
*
|
|
347
|
+
* @param {Array<Record<string, any>>} instancePrs merged, with `body` and `commits`
|
|
348
|
+
* @param {Map<number, string[]>} closingIssues template PR → issue keys
|
|
349
|
+
* @param {Map<string, string>} issueOpenedAt issue key → ISO createdAt
|
|
350
|
+
* @param {Array<{startedAt: number, finishedAt: number}>} deploys instance deploys
|
|
351
|
+
*/
|
|
352
|
+
export function crossRepoTimeToFeature(instancePrs, closingIssues, issueOpenedAt, deploys) {
|
|
353
|
+
const hours = []
|
|
354
|
+
let upgradePrs = 0
|
|
355
|
+
let carriedPrs = 0
|
|
356
|
+
let carriedWithoutIssue = 0
|
|
357
|
+
let awaitingDeploy = 0
|
|
358
|
+
// Captured before the loop: an empty index makes every count below meaningless,
|
|
359
|
+
// and the caller must be able to tell that from a genuine absence of issues.
|
|
360
|
+
const templateIndexEmpty = closingIssues.size === 0
|
|
361
|
+
for (const pr of instancePrs) {
|
|
362
|
+
// Branch name first: a PR that merely documents the marker is not an
|
|
363
|
+
// upgrade, and counting one as such is how the template reported carrying
|
|
364
|
+
// its own PRs on the first real run.
|
|
365
|
+
if (!isUpgradePr(pr)) continue
|
|
366
|
+
const carried = parseCarriedPrs(pr.body, pr.commits)
|
|
367
|
+
if (carried.length === 0) continue
|
|
368
|
+
upgradePrs += 1
|
|
369
|
+
carriedPrs += carried.length
|
|
370
|
+
// One deploy carries every issue in the upgrade, so it is resolved once
|
|
371
|
+
// rather than per issue.
|
|
372
|
+
const ranAt = firstDeployAfter(deploys, pr.mergedAt)
|
|
373
|
+
for (const number of carried) {
|
|
374
|
+
const keys = closingIssues.get(number)
|
|
375
|
+
if (!keys || keys.length === 0) {
|
|
376
|
+
carriedWithoutIssue += 1
|
|
377
|
+
continue
|
|
378
|
+
}
|
|
379
|
+
for (const key of keys) {
|
|
380
|
+
const openedAt = issueOpenedAt.get(key)
|
|
381
|
+
if (!openedAt) continue
|
|
382
|
+
if (ranAt === null) {
|
|
383
|
+
awaitingDeploy += 1
|
|
384
|
+
continue
|
|
385
|
+
}
|
|
386
|
+
const delta = ranAt - Date.parse(openedAt)
|
|
387
|
+
if (!Number.isFinite(delta) || delta < 0) continue
|
|
388
|
+
hours.push(delta / 3_600_000)
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
// Unattributable is NOT a subset of carriedWithoutIssue — it replaces it. With
|
|
393
|
+
// no template index there is no basis to claim a carried PR closed nothing, so
|
|
394
|
+
// the honest report is "this many could not be attributed", and the latency
|
|
395
|
+
// fields stay null rather than reading as a measured zero.
|
|
396
|
+
const unattributable = templateIndexEmpty ? carriedPrs : 0
|
|
397
|
+
return {
|
|
398
|
+
upgradePrs,
|
|
399
|
+
carriedPrs,
|
|
400
|
+
carriedWithoutIssue: templateIndexEmpty ? 0 : carriedWithoutIssue,
|
|
401
|
+
unattributable,
|
|
402
|
+
awaitingDeploy,
|
|
403
|
+
measured: hours.length,
|
|
404
|
+
hoursP50: round1(percentile(hours, 50)),
|
|
405
|
+
hoursP90: round1(percentile(hours, 90)),
|
|
406
|
+
hoursMax: hours.length ? round1(Math.max(...hours)) : null,
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/** Conclusions that mean the gate never evaluated the change. Never counted as a pass. */
|
|
411
|
+
export const INCONCLUSIVE_CONCLUSIONS = new Set(['skipped', 'neutral', 'stale', null])
|
|
412
|
+
|
|
413
|
+
/** Conventional-commit types whose merge implies an earlier change was wrong. */
|
|
414
|
+
export const REWORK_TYPES = ['fix', 'revert']
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Paths blame must not attribute through.
|
|
418
|
+
*
|
|
419
|
+
* A lockfile is rewritten by nearly every change, so blaming a line in one
|
|
420
|
+
* answers "who last touched the lockfile", never "which change is being
|
|
421
|
+
* corrected". Including them pulls every lag toward the last merge.
|
|
422
|
+
*/
|
|
423
|
+
export const OPAQUE_PATHS =
|
|
424
|
+
/(^|\/)(pnpm-lock\.yaml|uv\.lock|package-lock\.json|poetry\.lock|yarn\.lock|Cargo\.lock)$/
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* What a conventional-commit type says the work *was*.
|
|
428
|
+
*
|
|
429
|
+
* The type is a declared intent, written before the outcome was known, which
|
|
430
|
+
* makes it a cheap and reasonably honest classifier. It is the only signal
|
|
431
|
+
* available that separates "built the product" from "fought the toolchain"
|
|
432
|
+
* without anyone filling in a timesheet.
|
|
433
|
+
*/
|
|
434
|
+
/**
|
|
435
|
+
* A merge that carries the template into an instance. Platform work wherever it
|
|
436
|
+
* lands, because it maintains the machine rather than advancing the product.
|
|
437
|
+
*/
|
|
438
|
+
export const CORE_UPGRADE_SUBJECT = /upgrade biffo core|core[- ]upgrade/i
|
|
439
|
+
|
|
440
|
+
export const WORK_CLASS = {
|
|
441
|
+
feat: 'delivery',
|
|
442
|
+
fix: 'rework',
|
|
443
|
+
revert: 'rework',
|
|
444
|
+
ci: 'toil',
|
|
445
|
+
chore: 'toil',
|
|
446
|
+
infra: 'toil',
|
|
447
|
+
build: 'toil',
|
|
448
|
+
test: 'quality',
|
|
449
|
+
refactor: 'quality',
|
|
450
|
+
perf: 'quality',
|
|
451
|
+
security: 'quality',
|
|
452
|
+
docs: 'docs',
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
// ---------------------------------------------------------------------------
|
|
456
|
+
// Pure helpers — everything below this line is deterministic and unit-tested.
|
|
457
|
+
// ---------------------------------------------------------------------------
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* Classify one merge by its declared intent.
|
|
461
|
+
*
|
|
462
|
+
* `unconventional` is its own bucket rather than being folded into "other":
|
|
463
|
+
* ~8% of merges carry no parseable type, and quietly assigning them anywhere
|
|
464
|
+
* would move the headline ratio by more than most experiments will.
|
|
465
|
+
*
|
|
466
|
+
* @param {string} subject
|
|
467
|
+
*/
|
|
468
|
+
export function classifyWork(subject) {
|
|
469
|
+
const match = /^([a-z]+)(\(.+\))?!?:/.exec(subject)
|
|
470
|
+
if (!match) return 'unconventional'
|
|
471
|
+
return WORK_CLASS[match[1]] ?? 'other'
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* Which side of the house does *this merge* serve?
|
|
476
|
+
*
|
|
477
|
+
* The repo is the default answer, but it is not always the right one. An
|
|
478
|
+
* instance repo like `tabsii-platform` is simultaneously the product's backend
|
|
479
|
+
* **and** a Biffo instance, so maintenance of the machine lands inside a product
|
|
480
|
+
* repo: 30 of its 230 merges in the first 90-day window were core upgrades —
|
|
481
|
+
* 7.8% of all product-repo merges — every one of them counted as product
|
|
482
|
+
* delivery by the repo-level cut.
|
|
483
|
+
*
|
|
484
|
+
* That blur is not only a measurement artefact. A boundary where platform churn
|
|
485
|
+
* structurally lands in product repos is a candidate root cause in its own
|
|
486
|
+
* right, and is filed as such rather than merely corrected for here.
|
|
487
|
+
*
|
|
488
|
+
* @param {string} subject
|
|
489
|
+
* @param {string | undefined} repoSide
|
|
490
|
+
*/
|
|
491
|
+
export function classifyMergeSide(subject, repoSide) {
|
|
492
|
+
if (CORE_UPGRADE_SUBJECT.test(subject)) return 'platform'
|
|
493
|
+
return repoSide ?? null
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
/**
|
|
497
|
+
* The work-mix of a set of merges — the "are we building or maintaining?" view.
|
|
498
|
+
*
|
|
499
|
+
* `toilRatio` is the SRE framing: toil plus rework is effort that did not add
|
|
500
|
+
* product value. Google's SRE practice caps toil at 50%; the first measurement
|
|
501
|
+
* here across 1,023 merges put this estate at **43.5%**, which independently
|
|
502
|
+
* reproduced the practices corpus's hand-estimate of "~40% toolchain" from a
|
|
503
|
+
* single day's work.
|
|
504
|
+
*
|
|
505
|
+
* @param {Array<{subject: string}>} commits
|
|
506
|
+
*/
|
|
507
|
+
export function summariseWorkMix(commits, repoSide) {
|
|
508
|
+
const empty = {
|
|
509
|
+
merges: 0,
|
|
510
|
+
delivery: null, rework: null, toil: null, quality: null, docs: null, unconventional: null,
|
|
511
|
+
toilRatio: null,
|
|
512
|
+
counts: { delivery: 0, rework: 0, toil: 0, quality: 0, docs: 0, unconventional: 0, other: 0 },
|
|
513
|
+
sideCounts: { platform: 0, product: 0 },
|
|
514
|
+
productDelivery: 0,
|
|
515
|
+
}
|
|
516
|
+
if (commits.length === 0) return empty
|
|
517
|
+
|
|
518
|
+
const counts = { delivery: 0, rework: 0, toil: 0, quality: 0, docs: 0, unconventional: 0, other: 0 }
|
|
519
|
+
const sideCounts = { platform: 0, product: 0 }
|
|
520
|
+
let productDelivery = 0
|
|
521
|
+
|
|
522
|
+
for (const commit of commits) {
|
|
523
|
+
const kind = classifyWork(commit.subject)
|
|
524
|
+
counts[kind] = (counts[kind] ?? 0) + 1
|
|
525
|
+
const side = classifyMergeSide(commit.subject, repoSide)
|
|
526
|
+
if (side) sideCounts[side] += 1
|
|
527
|
+
if (side === 'product' && kind === 'delivery') productDelivery += 1
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
const n = commits.length
|
|
531
|
+
return {
|
|
532
|
+
merges: n,
|
|
533
|
+
delivery: rate(counts.delivery, n),
|
|
534
|
+
rework: rate(counts.rework, n),
|
|
535
|
+
toil: rate(counts.toil, n),
|
|
536
|
+
quality: rate(counts.quality, n),
|
|
537
|
+
docs: rate(counts.docs, n),
|
|
538
|
+
unconventional: rate(counts.unconventional, n),
|
|
539
|
+
toilRatio: rate(counts.toil + counts.rework, n),
|
|
540
|
+
// Absolute counts so the estate rollup can sum rather than reconstruct
|
|
541
|
+
// totals from percentages — that reconstruction was lossy and let a repo
|
|
542
|
+
// with three merges pull as hard as one with four hundred.
|
|
543
|
+
counts,
|
|
544
|
+
sideCounts,
|
|
545
|
+
productDelivery,
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* Narrow a repo's raw history to one observation window.
|
|
551
|
+
*
|
|
552
|
+
* Exists so a daily dashboard can show 24h, 7d and 90d from a **single** fetch.
|
|
553
|
+
* Collecting three times would triple the API calls and the blame work, and —
|
|
554
|
+
* worse — the three windows could then disagree because they were taken at
|
|
555
|
+
* different moments.
|
|
556
|
+
*
|
|
557
|
+
* `until` is what makes a **non-overlapping** baseline possible (#835). The 90d
|
|
558
|
+
* window contains the 7d window, so on 2026-07-29 exactly half the "90-day
|
|
559
|
+
* baseline" — 616 of 1232 merges — *was* the week being compared against it.
|
|
560
|
+
* A reference line that moves with the thing it is measuring cannot tell you
|
|
561
|
+
* the thing moved.
|
|
562
|
+
*
|
|
563
|
+
* @param {{prs: Array<any>, runs: Array<any>, defaultBranch: string, rework: {fixes: Array<any>, commits: Array<any>} | null}} data
|
|
564
|
+
* @param {string} since ISO timestamp
|
|
565
|
+
* @param {string | null} until ISO timestamp, exclusive; open-ended when null
|
|
566
|
+
*/
|
|
567
|
+
export function filterToWindow(data, since, until = null) {
|
|
568
|
+
const from = Date.parse(since)
|
|
569
|
+
const to = until === null ? Infinity : Date.parse(until)
|
|
570
|
+
return {
|
|
571
|
+
// The bundle knows its own window, so a metric with a narrower fetch than
|
|
572
|
+
// the window can say so rather than quietly reporting partial data (#914).
|
|
573
|
+
windowSince: since,
|
|
574
|
+
defaultBranch: data.defaultBranch,
|
|
575
|
+
prs: data.prs.filter(
|
|
576
|
+
(pr) => pr.mergedAt && Date.parse(pr.mergedAt) >= from && Date.parse(pr.mergedAt) < to,
|
|
577
|
+
),
|
|
578
|
+
runs: data.runs.filter(
|
|
579
|
+
(run) => Date.parse(run.created_at) >= from && Date.parse(run.created_at) < to,
|
|
580
|
+
),
|
|
581
|
+
rework: data.rework
|
|
582
|
+
? {
|
|
583
|
+
fixes: data.rework.fixes.filter((fix) => fix.at >= from && fix.at < to),
|
|
584
|
+
commits: data.rework.commits.filter(
|
|
585
|
+
(commit) => commit.at >= from && commit.at < to,
|
|
586
|
+
),
|
|
587
|
+
}
|
|
588
|
+
: null,
|
|
589
|
+
// Failing CI steps (#914), carrying the cutoff they were fetched from.
|
|
590
|
+
// `coveredSince` travels with the data because it is the difference between
|
|
591
|
+
// "no steps failed" and "we did not look" — a window wider than the fetch
|
|
592
|
+
// would otherwise report a flattering share over partial data.
|
|
593
|
+
steps: data.steps
|
|
594
|
+
? {
|
|
595
|
+
coveredSince: data.steps.coveredSince,
|
|
596
|
+
failing: data.steps.failing.filter((step) => step.at >= from && step.at < to),
|
|
597
|
+
// Carried whole, deliberately NOT filtered to the window: it is a
|
|
598
|
+
// lookup keyed by run id, and `integrationHealth` does its own window
|
|
599
|
+
// filtering. Narrowing it here would silently un-classify runs and
|
|
600
|
+
// reinstate the very failures this exists to re-attribute.
|
|
601
|
+
killedRunIds: data.steps.killedRunIds,
|
|
602
|
+
}
|
|
603
|
+
: null,
|
|
604
|
+
// Deliberately NOT filtered by the window. An issue opened long before the
|
|
605
|
+
// PR that closed it is the long-latency case time-to-feature exists to find;
|
|
606
|
+
// windowing it away would discard the worst results and flatter the median.
|
|
607
|
+
// The window applies to the *merge*, which is the event being measured.
|
|
608
|
+
issues: data.issues ?? [],
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/**
|
|
613
|
+
* Work out the **independent baseline** window from the configured lookbacks.
|
|
614
|
+
*
|
|
615
|
+
* Every window here is a lookback from now, so the long one contains the short
|
|
616
|
+
* one and "7d vs the 90d baseline" is partly a comparison of the week with
|
|
617
|
+
* itself. On 2026-07-29 the overlap was total enough to make the baseline
|
|
618
|
+
* useless as a reference: 616 of the 1232 merges in the 90-day window — exactly
|
|
619
|
+
* half — had happened in the 7 days being compared against it. Merge rate that
|
|
620
|
+
* week was 88/day against the 90-day average of 13.7/day, so the "baseline" was
|
|
621
|
+
* dominated by the very regime it was supposed to give perspective on. A
|
|
622
|
+
* baseline that moves with the reading always looks reassuringly close to it.
|
|
623
|
+
*
|
|
624
|
+
* The fix is a span, not a lookback: **the equal-length period immediately
|
|
625
|
+
* before the rate window**. Last week, against this week.
|
|
626
|
+
*
|
|
627
|
+
* ## Why equal-length, and why it changed (#850)
|
|
628
|
+
*
|
|
629
|
+
* The first version cut the rate window out of the long one — 90d minus 7d, an
|
|
630
|
+
* 83-day baseline. Independent, but not *matched*: a 7-day reading against an
|
|
631
|
+
* 83-day average compares a week to a quarter, and the quarter is dominated by
|
|
632
|
+
* whatever regime happened to prevail in it. That is the same units mismatch
|
|
633
|
+
* the green-wait tile had, one level up.
|
|
634
|
+
*
|
|
635
|
+
* Equal length also makes the feedback loop short, which is the point: this
|
|
636
|
+
* estate merged 616 PRs in seven days. A 30- or 90-day reference is not a
|
|
637
|
+
* reference for a codebase moving that fast, it is history. Confirmed before
|
|
638
|
+
* changing it — the last 7 days carry 144 failed CI runs and 199 failing steps
|
|
639
|
+
* estate-wide, ample to classify, and the locally-catchable share reads 66% on
|
|
640
|
+
* 7 days against 62% on 30, so the shorter window costs no comparability.
|
|
641
|
+
*
|
|
642
|
+
* Returns `null` when there is no second window to derive a rate from.
|
|
643
|
+
*
|
|
644
|
+
* @param {number[]} windowDays
|
|
645
|
+
* @returns {{ since: string, until: string, days: number } | null}
|
|
646
|
+
*/
|
|
647
|
+
export function priorWindow(windowDays, now = Date.now()) {
|
|
648
|
+
const sorted = [...new Set(windowDays)].sort((a, b) => a - b)
|
|
649
|
+
if (sorted.length < 2) return null
|
|
650
|
+
// The rate window is the one the dashboard reads percentages from — the
|
|
651
|
+
// largest window that is not the long-term context window.
|
|
652
|
+
const rate = sorted[sorted.length - 2]
|
|
653
|
+
return {
|
|
654
|
+
// [2×rate ago, rate ago) — the same span, immediately before, sharing no
|
|
655
|
+
// merge with the reading it anchors.
|
|
656
|
+
since: new Date(now - 2 * rate * 864e5).toISOString(),
|
|
657
|
+
until: new Date(now - rate * 864e5).toISOString(),
|
|
658
|
+
days: rate,
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/**
|
|
663
|
+
* Nearest-rank percentile.
|
|
664
|
+
*
|
|
665
|
+
* Returns `null` for an empty set rather than 0: no observations and "every
|
|
666
|
+
* observation was zero" are different claims, and only one of them is good news.
|
|
667
|
+
*
|
|
668
|
+
* @param {number[]} values
|
|
669
|
+
* @param {number} p percentile in 0..100
|
|
670
|
+
* @returns {number | null}
|
|
671
|
+
*/
|
|
672
|
+
/**
|
|
673
|
+
* Time-to-feature, stop A: issue opened → the PR that closed it merged (#767).
|
|
674
|
+
*
|
|
675
|
+
* ## Why the clock starts at the issue and stops at the merge
|
|
676
|
+
*
|
|
677
|
+
* Start: `issue.createdAt`. Thinking time is deliberately out of scope — the
|
|
678
|
+
* clock starts when an intention is written down, which is the first moment the
|
|
679
|
+
* tooling can see.
|
|
680
|
+
*
|
|
681
|
+
* Stop: **not** `closedAt`. This estate has twice shipped a "fixed" issue that
|
|
682
|
+
* was not fixed — #275 was diagnosed, closed and shipped on a wrong cause with a
|
|
683
|
+
* green suite throughout, and #726 was auto-closed by `Closes #N` before
|
|
684
|
+
* anything had run against a deployed instance. A metric that stops at closure
|
|
685
|
+
* therefore *improves the more carelessly issues are closed*, rewarding the
|
|
686
|
+
* exact failure it should expose. The merge of the closing PR is the earliest
|
|
687
|
+
* moment supported by evidence rather than by someone's belief.
|
|
688
|
+
*
|
|
689
|
+
* Stop B — first successful deploy after that merge — is Phase 2, and needs the
|
|
690
|
+
* template→instance hop to become machine-readable first.
|
|
691
|
+
*
|
|
692
|
+
* ## What `unlinked` counts, and why it is not zero
|
|
693
|
+
*
|
|
694
|
+
* A merged PR with no resolvable closing issue is counted, not dropped. Two
|
|
695
|
+
* different things produce one: a PR that legitimately closes nothing, and a PR
|
|
696
|
+
* whose closing reference is malformed. The latter is already on this project's
|
|
697
|
+
* scoreboard — `closes tabsii-crm#100` is repo-qualified but owner-less, which
|
|
698
|
+
* GitHub does not recognise, and it left a shipped issue open for two days
|
|
699
|
+
* looking like unstarted work. Folding those into the denominator would report
|
|
700
|
+
* a sample as if it were the whole, so they are reported alongside it instead.
|
|
701
|
+
*
|
|
702
|
+
* @param {Array<Record<string, any>>} mergedPrs PRs with `mergedAt` and `closingIssuesReferences`
|
|
703
|
+
* @param {Map<string, string>} issueOpenedAt keyed `owner/repo#number` → ISO createdAt
|
|
704
|
+
*/
|
|
705
|
+
export function timeToFeature(mergedPrs, issueOpenedAt, deploys = []) {
|
|
706
|
+
const hours = []
|
|
707
|
+
const runningHours = []
|
|
708
|
+
const deployGapHours = []
|
|
709
|
+
let linked = 0
|
|
710
|
+
let unresolved = 0
|
|
711
|
+
let awaitingDeploy = 0
|
|
712
|
+
for (const pr of mergedPrs) {
|
|
713
|
+
const refs = pr.closingIssuesReferences ?? []
|
|
714
|
+
for (const ref of refs) {
|
|
715
|
+
const owner = ref.repository?.owner?.login
|
|
716
|
+
const name = ref.repository?.name
|
|
717
|
+
const key = owner && name ? `${owner}/${name}#${ref.number}` : null
|
|
718
|
+
const openedAt = key ? issueOpenedAt.get(key) : undefined
|
|
719
|
+
if (!openedAt) {
|
|
720
|
+
// Referenced an issue we could not resolve — outside the fetched set, or
|
|
721
|
+
// in a repo not collected. Counted, never silently treated as instant.
|
|
722
|
+
unresolved += 1
|
|
723
|
+
continue
|
|
724
|
+
}
|
|
725
|
+
const delta = Date.parse(pr.mergedAt) - Date.parse(openedAt)
|
|
726
|
+
// A closing PR that merged *before* its issue was opened is not a fast
|
|
727
|
+
// feature — it is a mislinked reference. Excluded from the distribution
|
|
728
|
+
// and surfaced as unresolved rather than dragging the median toward zero.
|
|
729
|
+
if (!Number.isFinite(delta) || delta < 0) {
|
|
730
|
+
unresolved += 1
|
|
731
|
+
continue
|
|
732
|
+
}
|
|
733
|
+
hours.push(delta / 3_600_000)
|
|
734
|
+
linked += 1
|
|
735
|
+
|
|
736
|
+
// Stop B — running, not merely merged (#767).
|
|
737
|
+
const ranAt = firstDeployAfter(deploys, pr.mergedAt)
|
|
738
|
+
if (ranAt === null) {
|
|
739
|
+
// No successful deploy yet, or the deploy fell outside the fetched
|
|
740
|
+
// window. Either way it is *not* zero and not "instant" — the issue is
|
|
741
|
+
// merged and not yet known to be running.
|
|
742
|
+
awaitingDeploy += 1
|
|
743
|
+
continue
|
|
744
|
+
}
|
|
745
|
+
runningHours.push((ranAt - Date.parse(openedAt)) / 3_600_000)
|
|
746
|
+
deployGapHours.push((ranAt - Date.parse(pr.mergedAt)) / 3_600_000)
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
const withNoClosingRef = mergedPrs.filter(
|
|
750
|
+
(pr) => (pr.closingIssuesReferences ?? []).length === 0,
|
|
751
|
+
).length
|
|
752
|
+
return {
|
|
753
|
+
linked,
|
|
754
|
+
unresolved,
|
|
755
|
+
prsWithNoClosingIssue: withNoClosingRef,
|
|
756
|
+
// Coverage is the honesty check: a p50 over 4 of 143 merges is a statement
|
|
757
|
+
// about 4 merges. Reported next to the number so it cannot be read as the
|
|
758
|
+
// estate's feature latency.
|
|
759
|
+
coverage: rate(linked, mergedPrs.length),
|
|
760
|
+
hoursP50: round1(percentile(hours, 50)),
|
|
761
|
+
hoursP90: round1(percentile(hours, 90)),
|
|
762
|
+
hoursMax: hours.length ? round1(Math.max(...hours)) : null,
|
|
763
|
+
// Stop B: issue opened → deployed and running.
|
|
764
|
+
running: {
|
|
765
|
+
measured: runningHours.length,
|
|
766
|
+
awaitingDeploy,
|
|
767
|
+
hoursP50: round1(percentile(runningHours, 50)),
|
|
768
|
+
hoursP90: round1(percentile(runningHours, 90)),
|
|
769
|
+
// B − A: merged, but not yet usable. This is the distribution cost the
|
|
770
|
+
// practices page has only ever been able to describe anecdotally.
|
|
771
|
+
deployGapP50: round1(percentile(deployGapHours, 50)),
|
|
772
|
+
deployGapP90: round1(percentile(deployGapHours, 90)),
|
|
773
|
+
},
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
/**
|
|
778
|
+
* Successful deploy runs on the integration branch, oldest first (#767).
|
|
779
|
+
*
|
|
780
|
+
* @param {Array<Record<string, any>>} runs
|
|
781
|
+
* @param {string} branch
|
|
782
|
+
* @param {string} workflow
|
|
783
|
+
*/
|
|
784
|
+
export function successfulDeploys(runs, branch, workflow = DEPLOY_WORKFLOW) {
|
|
785
|
+
return runs
|
|
786
|
+
.filter(
|
|
787
|
+
(run) =>
|
|
788
|
+
run.name === workflow &&
|
|
789
|
+
run.head_branch === branch &&
|
|
790
|
+
run.event === 'push' &&
|
|
791
|
+
run.conclusion === 'success',
|
|
792
|
+
)
|
|
793
|
+
.map((run) => ({
|
|
794
|
+
// Two different instants, and conflating them makes deploys look free.
|
|
795
|
+
// `startedAt` decides *which* merges a run contains; `finishedAt` is when
|
|
796
|
+
// the code is actually running. A push-triggered run starts within
|
|
797
|
+
// seconds of the merge, so measuring the gap from `startedAt` reports ~0
|
|
798
|
+
// for every deploy no matter how long it took.
|
|
799
|
+
startedAt: Date.parse(String(run.created_at)),
|
|
800
|
+
finishedAt: Date.parse(String(run.updated_at ?? run.created_at)),
|
|
801
|
+
}))
|
|
802
|
+
.filter((d) => Number.isFinite(d.startedAt) && Number.isFinite(d.finishedAt))
|
|
803
|
+
.sort((a, b) => a.startedAt - b.startedAt)
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
/**
|
|
807
|
+
* When the first deploy that *necessarily contains* `mergedAt` finished.
|
|
808
|
+
*
|
|
809
|
+
* The rule is `created_at >= mergedAt`, and it is exact rather than a heuristic:
|
|
810
|
+
* a push-triggered run builds the branch tip at the moment it was created, so a
|
|
811
|
+
* run created after a merge necessarily includes that merge. A run created
|
|
812
|
+
* *before* it cannot. No commit-sha matching is needed, and none would be more
|
|
813
|
+
* correct — sha matching would additionally report `null` whenever deploys
|
|
814
|
+
* coalesce, which is common here and would look like "never shipped".
|
|
815
|
+
*
|
|
816
|
+
* Returns `null` when no successful deploy has happened yet, which is a
|
|
817
|
+
* different claim from "shipped instantly".
|
|
818
|
+
*
|
|
819
|
+
* Matches on `startedAt` and returns `finishedAt` — the code is running when the
|
|
820
|
+
* deploy *completes*, not when it is triggered.
|
|
821
|
+
*
|
|
822
|
+
* @param {Array<{startedAt: number, finishedAt: number}>} deploys ascending by startedAt
|
|
823
|
+
* @param {string} mergedAt
|
|
824
|
+
*/
|
|
825
|
+
export function firstDeployAfter(deploys, mergedAt) {
|
|
826
|
+
const from = Date.parse(mergedAt)
|
|
827
|
+
if (!Number.isFinite(from)) return null
|
|
828
|
+
for (const d of deploys) if (d.startedAt >= from) return d.finishedAt
|
|
829
|
+
return null
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
export function percentile(values, p) {
|
|
833
|
+
if (!Array.isArray(values) || values.length === 0) return null
|
|
834
|
+
const sorted = [...values].sort((a, b) => a - b)
|
|
835
|
+
const rank = Math.ceil((p / 100) * sorted.length)
|
|
836
|
+
const index = Math.min(Math.max(rank - 1, 0), sorted.length - 1)
|
|
837
|
+
return sorted[index]
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
/**
|
|
841
|
+
* Round to one decimal place, preserving `null`.
|
|
842
|
+
*
|
|
843
|
+
* @param {number | null} value
|
|
844
|
+
* @returns {number | null}
|
|
845
|
+
*/
|
|
846
|
+
export function round1(value) {
|
|
847
|
+
return value === null || value === undefined ? null : Math.round(value * 10) / 10
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* Percentage of `numerator` in `denominator`, or `null` when there is nothing to
|
|
852
|
+
* divide. Guards the case that makes a metric lie: 0/0 is not 0%.
|
|
853
|
+
*
|
|
854
|
+
* @param {number} numerator
|
|
855
|
+
* @param {number} denominator
|
|
856
|
+
* @returns {number | null}
|
|
857
|
+
*/
|
|
858
|
+
export function rate(numerator, denominator) {
|
|
859
|
+
if (!denominator) return null
|
|
860
|
+
return round1((numerator / denominator) * 100)
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
/**
|
|
864
|
+
* Group workflow runs by the branch they ran on, so a pull request can find its
|
|
865
|
+
* own runs without a per-PR API call.
|
|
866
|
+
*
|
|
867
|
+
* @param {Array<Record<string, unknown>>} runs
|
|
868
|
+
* @returns {Map<string, Array<Record<string, unknown>>>}
|
|
869
|
+
*/
|
|
870
|
+
export function indexRunsByBranch(runs) {
|
|
871
|
+
/** @type {Map<string, Array<Record<string, unknown>>>} */
|
|
872
|
+
const index = new Map()
|
|
873
|
+
for (const run of runs) {
|
|
874
|
+
const branch = /** @type {string} */ (run.head_branch)
|
|
875
|
+
if (!branch) continue
|
|
876
|
+
const bucket = index.get(branch)
|
|
877
|
+
if (bucket) bucket.push(run)
|
|
878
|
+
else index.set(branch, [run])
|
|
879
|
+
}
|
|
880
|
+
return index
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
/**
|
|
884
|
+
* The runs that belong to one pull request.
|
|
885
|
+
*
|
|
886
|
+
* Matched by branch name and bounded by the PR's own lifetime. Branch names are
|
|
887
|
+
* reused across pull requests here (`fix/…` names repeat), so the time window is
|
|
888
|
+
* what stops one PR claiming another's runs. A day of slack after the merge
|
|
889
|
+
* catches runs still finishing as the merge lands.
|
|
890
|
+
*
|
|
891
|
+
* @param {{createdAt: string, mergedAt: string | null, headRefName: string}} pr
|
|
892
|
+
* @param {Map<string, Array<Record<string, unknown>>>} runsByBranch
|
|
893
|
+
* @returns {Array<Record<string, unknown>>}
|
|
894
|
+
*/
|
|
895
|
+
export function runsForPr(pr, runsByBranch) {
|
|
896
|
+
const candidates = runsByBranch.get(pr.headRefName) ?? []
|
|
897
|
+
const opened = Date.parse(pr.createdAt)
|
|
898
|
+
const closed = pr.mergedAt ? Date.parse(pr.mergedAt) : Date.now()
|
|
899
|
+
const until = closed + 24 * 60 * 60 * 1000
|
|
900
|
+
return candidates.filter((run) => {
|
|
901
|
+
const created = Date.parse(/** @type {string} */ (run.created_at))
|
|
902
|
+
return created >= opened && created <= until
|
|
903
|
+
})
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
/**
|
|
907
|
+
* How much iteration one pull request cost.
|
|
908
|
+
*
|
|
909
|
+
* `revisions` counts distinct head SHAs that were pushed and tested — 0 means
|
|
910
|
+
* the branch landed exactly as first pushed. This is deliberately separate from
|
|
911
|
+
* `ciFailed`, because the two came apart the moment they were first measured:
|
|
912
|
+
* PR #691 pushed three SHAs and every CI run on all three was green. A single
|
|
913
|
+
* "first-pass green" metric scores that a perfect 100% while hiding three
|
|
914
|
+
* revisions and 29 minutes of churn. Gates rejecting work and humans guessing
|
|
915
|
+
* are different problems with different fixes.
|
|
916
|
+
*
|
|
917
|
+
* Returns `null` fields when no runs were found at all: that is an unmeasured
|
|
918
|
+
* PR, not a clean one.
|
|
919
|
+
*
|
|
920
|
+
* @param {{createdAt: string, mergedAt: string | null, headRefName: string}} pr
|
|
921
|
+
* @param {Map<string, Array<Record<string, unknown>>>} runsByBranch
|
|
922
|
+
*/
|
|
923
|
+
export function prChurn(pr, runsByBranch) {
|
|
924
|
+
const runs = runsForPr(pr, runsByBranch)
|
|
925
|
+
if (runs.length === 0) {
|
|
926
|
+
return { revisions: null, ciFailed: null, failedRuns: null, cancelledRuns: null, runs: 0 }
|
|
927
|
+
}
|
|
928
|
+
const shas = new Set(runs.map((run) => run.head_sha))
|
|
929
|
+
const failedRuns = runs.filter((run) =>
|
|
930
|
+
FAILING_CONCLUSIONS.has(/** @type {string} */ (run.conclusion)),
|
|
931
|
+
).length
|
|
932
|
+
const cancelledRuns = runs.filter((run) => run.conclusion === 'cancelled').length
|
|
933
|
+
return {
|
|
934
|
+
revisions: shas.size - 1,
|
|
935
|
+
ciFailed: failedRuns > 0,
|
|
936
|
+
failedRuns,
|
|
937
|
+
cancelledRuns,
|
|
938
|
+
runs: runs.length,
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/** Minutes a PR must sit green before its wait counts as losing the merge race. */
|
|
943
|
+
export const RACE_THRESHOLD_MINUTES = 10
|
|
944
|
+
|
|
945
|
+
/**
|
|
946
|
+
* Merge contention — work that was *correct* and still could not land.
|
|
947
|
+
*
|
|
948
|
+
* ## Why this is a separate axis from churn
|
|
949
|
+
*
|
|
950
|
+
* Churn means the code was wrong. Contention means the code was right and lost
|
|
951
|
+
* the up-to-date race against a fast-moving integration branch: green, then
|
|
952
|
+
* behind, then rebased, then green again, then behind again. The two need
|
|
953
|
+
* different fixes — one is about verifying before merging, the other is about
|
|
954
|
+
* how merges are sequenced — so collapsing them into one number would point at
|
|
955
|
+
* the wrong remedy.
|
|
956
|
+
*
|
|
957
|
+
* ## Why the median is the wrong statistic here
|
|
958
|
+
*
|
|
959
|
+
* The first attempt at measuring contention used runner pickup latency
|
|
960
|
+
* (`run_started_at - created_at`) and reported ~0, concluding there was none.
|
|
961
|
+
* That was contradicted by PR #659, which went green three minutes after
|
|
962
|
+
* opening and merged **46 minutes later** across five head SHAs — four rebases
|
|
963
|
+
* lost to the race, exactly as the practices corpus recorded.
|
|
964
|
+
*
|
|
965
|
+
* Measured properly, the median green-to-merge lag really is ~1 minute — and
|
|
966
|
+
* p90 is 25.8 minutes, the max is 15.7 hours, and the total green-but-unmerged
|
|
967
|
+
* time is **163 hours across 453 PRs**. The cost lives entirely in the tail, so
|
|
968
|
+
* this function reports p90, the max and the total, and never the median alone.
|
|
969
|
+
*
|
|
970
|
+
* ## The counter-metric: `staleMergeShare`
|
|
971
|
+
*
|
|
972
|
+
* Everything above prices the cost of `strict: true`. Relaxing it buys that
|
|
973
|
+
* cost back and sells something else, and until #977 nothing measured what:
|
|
974
|
+
* the practices corpus recorded the trade as *"a stale whole-file rewrite is
|
|
975
|
+
* now more likely to land silently — the experiment's falsification criteria
|
|
976
|
+
* measure merge friction and say nothing about content loss"* and left it
|
|
977
|
+
* **open**. An experiment that can only observe the benefit of its own
|
|
978
|
+
* intervention will conclude the intervention worked.
|
|
979
|
+
*
|
|
980
|
+
* A merge is **stale** when the base branch moved between the PR's first green
|
|
981
|
+
* run and its merge — so the combination that actually landed is one no CI run
|
|
982
|
+
* ever tested. That is precisely the exposure `strict` exists to prevent, which
|
|
983
|
+
* makes it the honest counter to `racedShare`: the two should move in opposite
|
|
984
|
+
* directions, and H3 is only decidable if both are on the page.
|
|
985
|
+
*
|
|
986
|
+
* It costs **no extra API calls**. `baseRefName` already rides along on the PR
|
|
987
|
+
* fetch, so the other merges to the same base are already in hand — measured by
|
|
988
|
+
* counting them, not by asking GitHub what the base tip was at merge time.
|
|
989
|
+
*
|
|
990
|
+
* Two honest limits, neither of which a reader can infer from the number:
|
|
991
|
+
*
|
|
992
|
+
* - **It counts exposure, not damage.** Most stale merges are harmless — the
|
|
993
|
+
* two changes touched nothing in common. This is the population content loss
|
|
994
|
+
* is drawn *from*, an upper bound on the risk rather than a count of
|
|
995
|
+
* incidents, and it must not be quoted as "N things broke".
|
|
996
|
+
* - **It undercounts at the window edge**, because a base merge that happened
|
|
997
|
+
* just before the window opened is not in `prs` to be counted. The bias is
|
|
998
|
+
* one-directional and toward zero, so a rise is always real.
|
|
999
|
+
*
|
|
1000
|
+
* @param {Array<{createdAt: string, mergedAt: string | null, headRefName: string, baseRefName?: string}>} prs
|
|
1001
|
+
* @param {Map<string, Array<Record<string, unknown>>>} runsByBranch
|
|
1002
|
+
*/
|
|
1003
|
+
export function mergeContention(prs, runsByBranch) {
|
|
1004
|
+
const merged = prs.filter((pr) => pr.mergedAt)
|
|
1005
|
+
/** Minutes each PR spent green but unmerged. */
|
|
1006
|
+
const greenToMerge = []
|
|
1007
|
+
let repushed = 0
|
|
1008
|
+
let raced = 0
|
|
1009
|
+
let measured = 0
|
|
1010
|
+
let stale = 0
|
|
1011
|
+
|
|
1012
|
+
// Merge times bucketed by base branch. Keyed by base ref because a repo's
|
|
1013
|
+
// `dev` and `staging` are separate races — a merge to `staging` does not make
|
|
1014
|
+
// a `dev` PR stale. Bucketing narrows the scan below; it does not make it a
|
|
1015
|
+
// lookup, and at this n (~1k merges) a linear scan per PR is not worth a
|
|
1016
|
+
// binary search.
|
|
1017
|
+
/** @type {Map<string, number[]>} */
|
|
1018
|
+
const baseMergeTimes = new Map()
|
|
1019
|
+
for (const pr of merged) {
|
|
1020
|
+
const base = pr.baseRefName
|
|
1021
|
+
if (!base) continue
|
|
1022
|
+
const at = Date.parse(/** @type {string} */ (pr.mergedAt))
|
|
1023
|
+
const times = baseMergeTimes.get(base)
|
|
1024
|
+
if (times) times.push(at)
|
|
1025
|
+
else baseMergeTimes.set(base, [at])
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
for (const pr of merged) {
|
|
1029
|
+
const churn = prChurn(pr, runsByBranch)
|
|
1030
|
+
if (churn.revisions === null) continue
|
|
1031
|
+
measured += 1
|
|
1032
|
+
if (churn.revisions > 0) repushed += 1
|
|
1033
|
+
|
|
1034
|
+
const runs = runsForPr(pr, runsByBranch)
|
|
1035
|
+
const greens = runs
|
|
1036
|
+
.filter((run) => run.conclusion === 'success')
|
|
1037
|
+
.map((run) => Date.parse(/** @type {string} */ (run.updated_at ?? run.created_at)))
|
|
1038
|
+
.sort((a, b) => a - b)
|
|
1039
|
+
if (greens.length === 0) continue
|
|
1040
|
+
|
|
1041
|
+
const lag = (Date.parse(/** @type {string} */ (pr.mergedAt)) - greens[0]) / 60000
|
|
1042
|
+
// A negative lag means the merge landed before any run completed — the PR
|
|
1043
|
+
// was merged without waiting, which is not contention.
|
|
1044
|
+
if (lag <= 0) continue
|
|
1045
|
+
greenToMerge.push(lag)
|
|
1046
|
+
if (lag > RACE_THRESHOLD_MINUTES && churn.revisions > 0) raced += 1
|
|
1047
|
+
|
|
1048
|
+
// Did the base move underneath this PR between the run that validated what
|
|
1049
|
+
// actually merged and the merge itself?
|
|
1050
|
+
//
|
|
1051
|
+
// **The anchor is the LAST green, not the first**, and the difference is
|
|
1052
|
+
// the whole metric. `greens[0]` is right for the wait above — it is when
|
|
1053
|
+
// the work first became correct — but wrong here: a PR that goes green,
|
|
1054
|
+
// falls behind, rebases and goes green again *was* tested against the base
|
|
1055
|
+
// it landed on, and anchoring to its first green would call that stale.
|
|
1056
|
+
// Doing so measured re-greened rebases, i.e. `racedShare` under another
|
|
1057
|
+
// name, and moved with the primary instead of against it — a counter-metric
|
|
1058
|
+
// that agrees with the thing it is supposed to check is worse than none.
|
|
1059
|
+
// Caught by running the collector on live data, where `strict: true` repos
|
|
1060
|
+
// scored 44% "stale" — a value the gate makes impossible by construction.
|
|
1061
|
+
//
|
|
1062
|
+
// Strictly between: a merge at exactly the green was already in what CI
|
|
1063
|
+
// tested, and this PR's own merge is excluded by the half-open upper bound
|
|
1064
|
+
// rather than by comparing PR numbers, which the shape of this data does
|
|
1065
|
+
// not guarantee are unique across repos.
|
|
1066
|
+
// Clamped to greens that completed **before the merge**. `runsForPr`
|
|
1067
|
+
// deliberately admits runs created up to 24h after `mergedAt`, so the last
|
|
1068
|
+
// green overall can postdate the merge — and an unclamped anchor would put
|
|
1069
|
+
// the window's start after its end, making the PR unstaleable. That is a
|
|
1070
|
+
// silent false negative: it suppresses exactly the merges this is looking
|
|
1071
|
+
// for, on the branches busy enough to still be running CI after they land.
|
|
1072
|
+
// `lag > 0` above guarantees at least one green precedes the merge.
|
|
1073
|
+
const mergedAt = Date.parse(/** @type {string} */ (pr.mergedAt))
|
|
1074
|
+
const greensBeforeMerge = greens.filter((at) => at <= mergedAt)
|
|
1075
|
+
const lastGreen = greensBeforeMerge[greensBeforeMerge.length - 1]
|
|
1076
|
+
const baseMerges = pr.baseRefName ? (baseMergeTimes.get(pr.baseRefName) ?? []) : []
|
|
1077
|
+
if (baseMerges.some((at) => at > lastGreen && at < mergedAt)) stale += 1
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
return {
|
|
1081
|
+
// The headline: total time correct work spent unable to land.
|
|
1082
|
+
greenButUnmergedHours: round1(greenToMerge.reduce((sum, m) => sum + m, 0) / 60),
|
|
1083
|
+
greenToMergeP50Minutes: round1(percentile(greenToMerge, 50)),
|
|
1084
|
+
greenToMergeP90Minutes: round1(percentile(greenToMerge, 90)),
|
|
1085
|
+
greenToMergeMaxMinutes: greenToMerge.length ? round1(Math.max(...greenToMerge)) : null,
|
|
1086
|
+
// Rebase pressure: a repush on an already-correct branch is pure race cost.
|
|
1087
|
+
repushRate: rate(repushed, measured),
|
|
1088
|
+
// The cleanest single indicator — green for longer than the threshold *and*
|
|
1089
|
+
// forced to repush. tabsii-crm scores 0% here and biffo-template 13.9%,
|
|
1090
|
+
// which is the difference a busy shared integration branch makes.
|
|
1091
|
+
racedShare: rate(raced, measured),
|
|
1092
|
+
// H3's counter-metric (#977): the share of merges whose base moved between
|
|
1093
|
+
// first green and merge, so the landed combination was never tested. Reads
|
|
1094
|
+
// as exposure, not damage — see the note above before quoting it.
|
|
1095
|
+
staleMergeShare: rate(stale, greenToMerge.length),
|
|
1096
|
+
staleMerges: stale,
|
|
1097
|
+
prsMeasured: measured,
|
|
1098
|
+
prsWithGreen: greenToMerge.length,
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
/**
|
|
1103
|
+
* Minutes from opening a pull request to merging it.
|
|
1104
|
+
*
|
|
1105
|
+
* Honest about what it excludes: everything before the PR existed. A change that
|
|
1106
|
+
* took three hours to write and two minutes to merge reads as two minutes here.
|
|
1107
|
+
* It measures the landing, not the work.
|
|
1108
|
+
*
|
|
1109
|
+
* @param {{createdAt: string, mergedAt: string | null}} pr
|
|
1110
|
+
* @returns {number | null}
|
|
1111
|
+
*/
|
|
1112
|
+
export function cycleTimeMinutes(pr) {
|
|
1113
|
+
if (!pr.mergedAt) return null
|
|
1114
|
+
return (Date.parse(pr.mergedAt) - Date.parse(pr.createdAt)) / 60000
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
/**
|
|
1118
|
+
* Runs that reached two different verdicts on the identical commit.
|
|
1119
|
+
*
|
|
1120
|
+
* A workflow that both passed and failed on one SHA cannot have been reacting to
|
|
1121
|
+
* the code. This is the number that decides whether a green check is evidence of
|
|
1122
|
+
* anything, so it is worth knowing even when it is small.
|
|
1123
|
+
*
|
|
1124
|
+
* @param {Array<Record<string, unknown>>} runs
|
|
1125
|
+
* @returns {{ pairs: number, shas: string[] }}
|
|
1126
|
+
*/
|
|
1127
|
+
export function detectFlakes(runs) {
|
|
1128
|
+
/** @type {Map<string, Set<string>>} */
|
|
1129
|
+
const verdicts = new Map()
|
|
1130
|
+
for (const run of runs) {
|
|
1131
|
+
const conclusion = /** @type {string} */ (run.conclusion)
|
|
1132
|
+
if (conclusion !== 'success' && !FAILING_CONCLUSIONS.has(conclusion)) continue
|
|
1133
|
+
const key = `${run.head_sha}::${run.name}`
|
|
1134
|
+
const seen = verdicts.get(key)
|
|
1135
|
+
if (seen) seen.add(conclusion === 'success' ? 'success' : 'failure')
|
|
1136
|
+
else verdicts.set(key, new Set([conclusion === 'success' ? 'success' : 'failure']))
|
|
1137
|
+
}
|
|
1138
|
+
const flaky = [...verdicts.entries()].filter(([, outcomes]) => outcomes.size > 1)
|
|
1139
|
+
return { pairs: flaky.length, shas: flaky.map(([key]) => key.split('::')[0]) }
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
/**
|
|
1143
|
+
* How long the integration branch spent red **while anyone was there to be
|
|
1144
|
+
* blocked by it**.
|
|
1145
|
+
*
|
|
1146
|
+
* A red `dev` blocks every agent at once, so its cost is multiplied by however
|
|
1147
|
+
* many are working. A failure never followed by a success is left open and
|
|
1148
|
+
* reported separately rather than being silently treated as instantly recovered.
|
|
1149
|
+
*
|
|
1150
|
+
* ## Why this is not simply failure-to-recovery (#921)
|
|
1151
|
+
*
|
|
1152
|
+
* It used to be, and the first real run of `practices-standup.mjs` ranked the
|
|
1153
|
+
* resulting number **first out of five findings** — 21.1 hours of red on
|
|
1154
|
+
* `biffo-plugin-ideation`, the most expensive thing in the estate that day.
|
|
1155
|
+
*
|
|
1156
|
+
* The run timeline: four failures between 09:28 and 11:47, then **nothing until
|
|
1157
|
+
* 06:36 the next morning**, when a push went green. Failure-to-recovery spanned
|
|
1158
|
+
* that entire overnight, so **18.8 of the 21.1 hours — 89% — was a branch sitting
|
|
1159
|
+
* red with zero pushes against it.** Nobody was blocked; everyone was asleep. The
|
|
1160
|
+
* genuinely blocked window was 2.3 hours, which would have ranked the finding
|
|
1161
|
+
* *last*.
|
|
1162
|
+
*
|
|
1163
|
+
* Worse, `docs/practices/metrics.md` instructs the reader to *multiply by
|
|
1164
|
+
* concurrency* — so the guidance was to scale up a figure that was 89% idle.
|
|
1165
|
+
*
|
|
1166
|
+
* The estate had already established the distinction one metric over, for the
|
|
1167
|
+
* runner fleet: *"Flat queue under an idle fleet is not contention."* It was
|
|
1168
|
+
* simply never applied here — the same "fix written for one caller, not the
|
|
1169
|
+
* class" shape this scoreboard keeps recording.
|
|
1170
|
+
*
|
|
1171
|
+
* ## The rule
|
|
1172
|
+
*
|
|
1173
|
+
* Red time accrues **between consecutive runs**, and each inter-run gap is capped
|
|
1174
|
+
* at `idleCeilingMinutes`. Past an hour with no new push, a red branch has stopped
|
|
1175
|
+
* costing anyone anything. A recovery run's own duration is counted uncapped — a
|
|
1176
|
+
* run that is executing is never idle.
|
|
1177
|
+
*
|
|
1178
|
+
* The ceiling is a judgement and is therefore declared, overridable and tested,
|
|
1179
|
+
* and `redMinutesUncapped` plus `idleGapsCapped` ship beside the headline so the
|
|
1180
|
+
* correction is always visible rather than folded silently into a smaller number.
|
|
1181
|
+
*
|
|
1182
|
+
* @param {Array<Record<string, unknown>>} runs
|
|
1183
|
+
* @param {string} branch
|
|
1184
|
+
* @param {number} idleCeilingMinutes longest gap that still counts as blocking
|
|
1185
|
+
*/
|
|
1186
|
+
export function integrationHealth(
|
|
1187
|
+
runs,
|
|
1188
|
+
branch,
|
|
1189
|
+
idleCeilingMinutes = IDLE_CEILING_MINUTES,
|
|
1190
|
+
kills = null,
|
|
1191
|
+
) {
|
|
1192
|
+
const onBranch = runs
|
|
1193
|
+
.filter((run) => run.head_branch === branch && run.event === 'push')
|
|
1194
|
+
.sort((a, b) => Date.parse(String(a.created_at)) - Date.parse(String(b.created_at)))
|
|
1195
|
+
if (onBranch.length === 0) {
|
|
1196
|
+
return {
|
|
1197
|
+
runs: 0,
|
|
1198
|
+
failures: null,
|
|
1199
|
+
redMinutes: null,
|
|
1200
|
+
redMinutesUncapped: null,
|
|
1201
|
+
idleGapsCapped: null,
|
|
1202
|
+
unresolvedFailures: null,
|
|
1203
|
+
runnerKills: null,
|
|
1204
|
+
failuresUnclassified: null,
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
// The jobs fetch is capped (#914), so a window can reach back further than
|
|
1209
|
+
// the classification does. Those failures are reported as
|
|
1210
|
+
// `failuresUnclassified` and left counted as failures rather than quietly
|
|
1211
|
+
// assumed innocent: an unclassified failure is not a proven kill, and the
|
|
1212
|
+
// whole point of this field is that a counter-metric must be able to refute.
|
|
1213
|
+
const killedIds = kills?.ids ?? null
|
|
1214
|
+
const classifiedFrom = kills?.coveredSince ? Date.parse(kills.coveredSince) : null
|
|
1215
|
+
let runnerKills = 0
|
|
1216
|
+
let failuresUnclassified = 0
|
|
1217
|
+
|
|
1218
|
+
/** @type {Map<string, number>} */
|
|
1219
|
+
const openedAt = new Map()
|
|
1220
|
+
// Where the clock was last stopped for this workflow — the end of the most
|
|
1221
|
+
// recent run while red. Red time accrues between *consecutive runs*, so this is
|
|
1222
|
+
// what makes an idle stretch visible as one long gap rather than being buried
|
|
1223
|
+
// inside a single failure-to-recovery span.
|
|
1224
|
+
/** @type {Map<string, number>} */
|
|
1225
|
+
const lastSeenAt = new Map()
|
|
1226
|
+
let redMinutes = 0
|
|
1227
|
+
let redMinutesUncapped = 0
|
|
1228
|
+
let idleGapsCapped = 0
|
|
1229
|
+
let failures = 0
|
|
1230
|
+
const ceiling = idleCeilingMinutes * 60000
|
|
1231
|
+
|
|
1232
|
+
for (const run of onBranch) {
|
|
1233
|
+
const workflow = /** @type {string} */ (run.name)
|
|
1234
|
+
const startedAt = Date.parse(String(run.created_at))
|
|
1235
|
+
const endedAt = Date.parse(String(run.updated_at ?? run.created_at))
|
|
1236
|
+
let failing = FAILING_CONCLUSIONS.has(/** @type {string} */ (run.conclusion))
|
|
1237
|
+
if (failing && killedIds) {
|
|
1238
|
+
if (classifiedFrom !== null && startedAt < classifiedFrom) {
|
|
1239
|
+
failuresUnclassified += 1
|
|
1240
|
+
} else if (killedIds.has(run.id)) {
|
|
1241
|
+
// A dead runner is not a gate rejecting a change, so it neither counts
|
|
1242
|
+
// as a failure nor opens a red span.
|
|
1243
|
+
runnerKills += 1
|
|
1244
|
+
failing = false
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
const wasRed = openedAt.has(workflow)
|
|
1249
|
+
const since = lastSeenAt.get(workflow)
|
|
1250
|
+
|
|
1251
|
+
if (wasRed) {
|
|
1252
|
+
// The waiting gap since the previous run. Capped: past the ceiling nobody is
|
|
1253
|
+
// waiting, so the branch is idle rather than blocking, and counting it makes
|
|
1254
|
+
// an overnight look like an outage.
|
|
1255
|
+
if (since !== undefined) {
|
|
1256
|
+
const gap = Math.max(0, startedAt - since)
|
|
1257
|
+
redMinutesUncapped += gap / 60000
|
|
1258
|
+
if (gap > ceiling) idleGapsCapped += 1
|
|
1259
|
+
redMinutes += Math.min(gap, ceiling) / 60000
|
|
1260
|
+
}
|
|
1261
|
+
// This run executed against a red branch, so its own duration is red time
|
|
1262
|
+
// whatever its verdict — and a run that is executing is never idle, so it is
|
|
1263
|
+
// never capped. Summing gaps *and* durations this way makes
|
|
1264
|
+
// `redMinutesUncapped` reproduce the pre-#921 figure exactly, which is what
|
|
1265
|
+
// lets the correction be audited rather than taken on trust.
|
|
1266
|
+
const ownDuration = Math.max(0, endedAt - startedAt) / 60000
|
|
1267
|
+
redMinutes += ownDuration
|
|
1268
|
+
redMinutesUncapped += ownDuration
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
if (failing) {
|
|
1272
|
+
failures += 1
|
|
1273
|
+
// The branch goes red at the *end* of the first failing run: until it
|
|
1274
|
+
// finished, nothing was known to be broken.
|
|
1275
|
+
if (!wasRed) openedAt.set(workflow, endedAt)
|
|
1276
|
+
} else if (run.conclusion === 'success' && wasRed) {
|
|
1277
|
+
openedAt.delete(workflow)
|
|
1278
|
+
}
|
|
1279
|
+
// Every run is activity and stops the waiting clock, including a cancelled or
|
|
1280
|
+
// skipped one — those are not verdicts, but somebody was plainly there.
|
|
1281
|
+
lastSeenAt.set(workflow, endedAt)
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
return {
|
|
1285
|
+
runs: onBranch.length,
|
|
1286
|
+
failures,
|
|
1287
|
+
redMinutes: round1(redMinutes),
|
|
1288
|
+
// Always reported beside the capped figure. A correction that cannot be seen
|
|
1289
|
+
// is indistinguishable from a metric that was always this way, and this one
|
|
1290
|
+
// moved a headline by 89%.
|
|
1291
|
+
redMinutesUncapped: round1(redMinutesUncapped),
|
|
1292
|
+
idleGapsCapped,
|
|
1293
|
+
unresolvedFailures: openedAt.size,
|
|
1294
|
+
// Failures re-attributed to a dead runner (#982), and failures the jobs
|
|
1295
|
+
// fetch did not reach. `null` on both when no classification was supplied,
|
|
1296
|
+
// so "not asked" stays distinguishable from "asked, found none".
|
|
1297
|
+
runnerKills: killedIds ? runnerKills : null,
|
|
1298
|
+
failuresUnclassified: killedIds ? failuresUnclassified : null,
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
/**
|
|
1303
|
+
* Check kinds a **local** gate could catch, and the ones it could not (#914).
|
|
1304
|
+
*
|
|
1305
|
+
* This is H4's primary outcome metric — the share of failing CI steps that a
|
|
1306
|
+
* deterministic, offline check on a developer's machine would have caught first.
|
|
1307
|
+
* Until #914 it existed only as a hand-reconstruction: the collector fetched
|
|
1308
|
+
* `/actions/runs` and never jobs or steps, so the 66% baseline in
|
|
1309
|
+
* `docs/practices/experiments/H4-shift-left-gates.md` was classified by hand and
|
|
1310
|
+
* the classification was never written down. This is that classification, in
|
|
1311
|
+
* code, where it can be re-run and disagreed with.
|
|
1312
|
+
*
|
|
1313
|
+
* ## Why this does NOT reuse `gate-coverage.sh`'s `EXCLUDED_KINDS`
|
|
1314
|
+
*
|
|
1315
|
+
* That list exists and it is tempting, and reusing it would be wrong. The two
|
|
1316
|
+
* answer different questions:
|
|
1317
|
+
*
|
|
1318
|
+
* - `EXCLUDED_KINDS` — kinds deliberately absent from `verify.sh`, the pre-push
|
|
1319
|
+
* gate, each with a reason.
|
|
1320
|
+
* - **locally catchable** — kinds a local check *could* catch, whether or not one
|
|
1321
|
+
* runs today.
|
|
1322
|
+
*
|
|
1323
|
+
* `ownership` is the case that proves they differ: it is in `EXCLUDED_KINDS`
|
|
1324
|
+
* because a **`commit-msg` hook** covers it rather than `verify.sh` — and H4
|
|
1325
|
+
* names its eleven CI failures as a headline cost, i.e. as catchable. Deriving
|
|
1326
|
+
* catchability from the exclusion list would have silently dropped those, plus
|
|
1327
|
+
* `build`, `gitleaks` (#897 — excluded on a rationale that turned out to be
|
|
1328
|
+
* false) and `pytest`. **The gap between "could be caught locally" and "is
|
|
1329
|
+
* caught locally" is the quantity H4 exists to measure**, so collapsing the two
|
|
1330
|
+
* would leave the metric unable to fail.
|
|
1331
|
+
*
|
|
1332
|
+
* ## Grounded in real step names, not invented ones
|
|
1333
|
+
*
|
|
1334
|
+
* Every pattern below was derived from the 25 distinct failing step names the
|
|
1335
|
+
* estate actually produced over the 7 days to 2026-07-30 (154 failing steps
|
|
1336
|
+
* across 12 repos). Unnamed steps arrive from the API as `Run <command>`, so
|
|
1337
|
+
* both the human `name:` form and the raw command form are matched.
|
|
1338
|
+
*
|
|
1339
|
+
* Order matters: `Build and deploy plugin frontends` is a deploy, not a build,
|
|
1340
|
+
* and `Sync and audit core-v<version>` is a release step, not a dependency
|
|
1341
|
+
* audit. The specific patterns come first for exactly this reason.
|
|
1342
|
+
*/
|
|
1343
|
+
const STEP_KINDS = [
|
|
1344
|
+
// --- not locally catchable: needs cloud credentials, a live environment, or
|
|
1345
|
+
// state that does not exist until merge/release time.
|
|
1346
|
+
{
|
|
1347
|
+
kind: 'deploy',
|
|
1348
|
+
catchable: false,
|
|
1349
|
+
match: /deploy|terraform (apply|init)|apply the .* schema|sync .+ to s3/,
|
|
1350
|
+
},
|
|
1351
|
+
// Schema applied to a LIVE database, not a local one: both steps this matches
|
|
1352
|
+
// are `aws lambda invoke` against the deployed function (`biffo:db-init`, and
|
|
1353
|
+
// the `db/imports/*/` loop). The `rls-test` entry below is the contrast worth
|
|
1354
|
+
// holding onto — that one needs a Postgres, which docker gives you in 3s;
|
|
1355
|
+
// this one needs *the* Postgres, which nothing local can stand in for.
|
|
1356
|
+
{
|
|
1357
|
+
kind: 'schema-apply',
|
|
1358
|
+
catchable: false,
|
|
1359
|
+
match: /ddl import|initiali[sz]e database|init database schema/,
|
|
1360
|
+
},
|
|
1361
|
+
// Before the `coverage` check below, which it contains. Measuring coverage is
|
|
1362
|
+
// offline and fast; shipping the report needs a token and the network.
|
|
1363
|
+
{ kind: 'coverage-upload', catchable: false, match: /coverage upload|upload .*coverage|codecov/ },
|
|
1364
|
+
{ kind: 'publish', catchable: false, match: /publish|sync and audit core-v|tag core version/ },
|
|
1365
|
+
// Not offline, and not deterministic over time: a newly-published advisory
|
|
1366
|
+
// reddens CI with no code change, so a developer running it an hour earlier
|
|
1367
|
+
// would legitimately have seen green.
|
|
1368
|
+
{ kind: 'dependency-audit', catchable: false, match: /dependency audit|pnpm audit|pip-audit/ },
|
|
1369
|
+
// The squash-merge subject does not exist until the merge, so nothing local
|
|
1370
|
+
// can check it. This is the one exclusion that is a fact rather than a choice.
|
|
1371
|
+
{ kind: 'release-subject', catchable: false, match: /release-subject/ },
|
|
1372
|
+
{ kind: 'setup', catchable: false, match: /^(set up|install) |^run actions\// },
|
|
1373
|
+
// Deterministic and offline, but **not seconds** — a full Next build is minutes,
|
|
1374
|
+
// which is why `local-gates.md` excludes it by name. H4's criterion is all three
|
|
1375
|
+
// ("deterministic, offline checks that reproduce locally in seconds"), so this
|
|
1376
|
+
// fails it on the third. Before `build`, so it wins over the generic test below.
|
|
1377
|
+
{ kind: 'build', catchable: false, match: /build/ },
|
|
1378
|
+
|
|
1379
|
+
// --- locally catchable. Specific guards before the generic kinds they contain.
|
|
1380
|
+
{ kind: 'ownership', catchable: true, match: /ownership/ },
|
|
1381
|
+
{ kind: 'corpus-guard', catchable: true, match: /corpus is append-only|practices-monotonic/ },
|
|
1382
|
+
// `sh scripts/biffo.sh check adr-numbering` — offline, seconds, no state.
|
|
1383
|
+
{ kind: 'adr-guard', catchable: true, match: /adr[- ]numbering|adr .*guard/ },
|
|
1384
|
+
// `uv run python scripts/error_branch_coverage.py --check`. Before `test`,
|
|
1385
|
+
// which "Test coverage" would otherwise take.
|
|
1386
|
+
{ kind: 'coverage', catchable: true, match: /coverage/ },
|
|
1387
|
+
{ kind: 'destructive-plan', catchable: true, match: /destructive-plan/ },
|
|
1388
|
+
{ kind: 'plugin-terraform', catchable: true, match: /plugin[- ]terraform/ },
|
|
1389
|
+
{ kind: 'plugin-collisions', catchable: true, match: /plugin[- ]collisions?/ },
|
|
1390
|
+
// Needs a real Postgres, which is why it looks un-local — but the lane runs in
|
|
1391
|
+
// ~3s against docker and is opt-in via TABSII_TEST_PG_DSN.
|
|
1392
|
+
{ kind: 'rls-test', catchable: true, match: /rls/ },
|
|
1393
|
+
{ kind: 'format', catchable: true, match: /format/ },
|
|
1394
|
+
{ kind: 'typecheck', catchable: true, match: /type ?check|pyright|tsc\b/ },
|
|
1395
|
+
{ kind: 'lint', catchable: true, match: /lint|ruff check|eslint/ },
|
|
1396
|
+
{ kind: 'sast', catchable: true, match: /sast|bandit/ },
|
|
1397
|
+
{ kind: 'gitleaks', catchable: true, match: /gitleaks/ },
|
|
1398
|
+
{ kind: 'terraform-fmt', catchable: true, match: /terraform.*fmt/ },
|
|
1399
|
+
{ kind: 'terraform-validate', catchable: true, match: /validate modules|terraform validate/ },
|
|
1400
|
+
{ kind: 'test', catchable: true, match: /test|pytest|vitest/ },
|
|
1401
|
+
]
|
|
1402
|
+
|
|
1403
|
+
/**
|
|
1404
|
+
* Classify one failing CI step.
|
|
1405
|
+
*
|
|
1406
|
+
* Returns `catchable: null` for a step no pattern matches, rather than guessing.
|
|
1407
|
+
* An unclassified step is counted and reported but contributes to **neither**
|
|
1408
|
+
* side of the share — the alternative is a silent default, and a default of
|
|
1409
|
+
* `false` would let the headline improve every time CI grew a step this file has
|
|
1410
|
+
* never seen. `unclassified > 0` on the dashboard is a prompt to extend the list.
|
|
1411
|
+
*
|
|
1412
|
+
* @param {string} stepName as reported by the jobs API
|
|
1413
|
+
* @returns {{ kind: string, catchable: boolean | null }}
|
|
1414
|
+
*/
|
|
1415
|
+
export function classifyFailingStep(stepName) {
|
|
1416
|
+
const name = String(stepName ?? '')
|
|
1417
|
+
.toLowerCase()
|
|
1418
|
+
.trim()
|
|
1419
|
+
for (const entry of STEP_KINDS) {
|
|
1420
|
+
if (entry.match.test(name)) return { kind: entry.kind, catchable: entry.catchable }
|
|
1421
|
+
}
|
|
1422
|
+
return { kind: 'unclassified', catchable: null }
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
/**
|
|
1426
|
+
* H4's primary metric: the locally-catchable share of failing CI steps (#914).
|
|
1427
|
+
*
|
|
1428
|
+
* `share` is the percentage of **classified** steps that a local gate could have
|
|
1429
|
+
* caught. Unclassified steps sit outside the ratio and are reported alongside it
|
|
1430
|
+
* so the denominator is always visible — a share quoted without its
|
|
1431
|
+
* `unclassified` count is not auditable.
|
|
1432
|
+
*
|
|
1433
|
+
* `byKind` exists so the headline can be argued with. A single percentage that
|
|
1434
|
+
* moved is not evidence about a gate; "format fell from 19 to 2" is.
|
|
1435
|
+
*
|
|
1436
|
+
* @param {Array<{name: string}>} steps failing steps, in any order
|
|
1437
|
+
*/
|
|
1438
|
+
export function summariseGates(steps) {
|
|
1439
|
+
/** @type {Record<string, number>} */
|
|
1440
|
+
const byKind = {}
|
|
1441
|
+
/** @type {Record<string, number>} */
|
|
1442
|
+
const unseen = {}
|
|
1443
|
+
let catchable = 0
|
|
1444
|
+
let notCatchable = 0
|
|
1445
|
+
let unclassified = 0
|
|
1446
|
+
for (const step of steps) {
|
|
1447
|
+
const { kind, catchable: isCatchable } = classifyFailingStep(step.name)
|
|
1448
|
+
byKind[kind] = (byKind[kind] ?? 0) + 1
|
|
1449
|
+
if (isCatchable === null) {
|
|
1450
|
+
unclassified += 1
|
|
1451
|
+
unseen[step.name] = (unseen[step.name] ?? 0) + 1
|
|
1452
|
+
} else if (isCatchable) catchable += 1
|
|
1453
|
+
else notCatchable += 1
|
|
1454
|
+
}
|
|
1455
|
+
return {
|
|
1456
|
+
failingSteps: steps.length,
|
|
1457
|
+
locallyCatchable: catchable,
|
|
1458
|
+
notLocallyCatchable: notCatchable,
|
|
1459
|
+
unclassified,
|
|
1460
|
+
unclassifiedNames: rankNames(unseen),
|
|
1461
|
+
share: rate(catchable, catchable + notCatchable),
|
|
1462
|
+
byKind,
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
/**
|
|
1467
|
+
* `{name: count}` as a most-frequent-first list, ties broken alphabetically so
|
|
1468
|
+
* the output is stable across runs and a diff of two snapshots is readable.
|
|
1469
|
+
*
|
|
1470
|
+
* @param {Record<string, number>} counts
|
|
1471
|
+
* @returns {Array<{name: string, count: number}>}
|
|
1472
|
+
*/
|
|
1473
|
+
function rankNames(counts) {
|
|
1474
|
+
return Object.entries(counts)
|
|
1475
|
+
.map(([name, count]) => ({ name, count }))
|
|
1476
|
+
.sort((a, b) => b.count - a.count || a.name.localeCompare(b.name))
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
/**
|
|
1480
|
+
* Is the locally-catchable metric still measuring what it claims to? (#1167)
|
|
1481
|
+
*
|
|
1482
|
+
* `share` is computed over **classified** steps, so a step name no pattern
|
|
1483
|
+
* matches leaves the denominator rather than reddening anything. That is the
|
|
1484
|
+
* correct behaviour for the ratio — guessing would let the headline improve
|
|
1485
|
+
* every time CI grew a step — but it means the metric degrades silently, and
|
|
1486
|
+
* `unclassified` was reported next to it for weeks with nothing asserting on it.
|
|
1487
|
+
*
|
|
1488
|
+
* On 2026-08-03 that cost a real misreading: 12 of 17 estate failing steps were
|
|
1489
|
+
* unclassified, the dashboard read **80%** locally-catchable, and over all 17
|
|
1490
|
+
* steps the honest figure was **47%** — with H4's review two days away and this
|
|
1491
|
+
* as its primary outcome metric. All 12 were six ordinary step names.
|
|
1492
|
+
*
|
|
1493
|
+
* This is the estate's own recurring shape, one level in: a check that cannot
|
|
1494
|
+
* see an input silently shrinks its scope and reports the remainder as the
|
|
1495
|
+
* whole (`AGENTS.md` §2, on `protection-audit.sh` skipping repos with no `dev`).
|
|
1496
|
+
* The fix there and here is the same — make the blind spot fail rather than
|
|
1497
|
+
* abstain.
|
|
1498
|
+
*
|
|
1499
|
+
* @param {{failingSteps?: number, unclassified?: number, unclassifiedNames?: Array<{name: string, count: number}>, error?: string}} gates
|
|
1500
|
+
* @returns {{ok: boolean, summary: string}}
|
|
1501
|
+
*/
|
|
1502
|
+
export function classificationBlindness(gates) {
|
|
1503
|
+
// Not measuring and measuring zero blindness are different claims — the same
|
|
1504
|
+
// distinction gatesForWindow draws, and the one this whole file keeps finding.
|
|
1505
|
+
if (gates?.error) {
|
|
1506
|
+
return { ok: false, summary: `gates ${gates.error} — classification cannot be audited` }
|
|
1507
|
+
}
|
|
1508
|
+
const failing = gates.failingSteps ?? 0
|
|
1509
|
+
const unclassified = gates.unclassified ?? 0
|
|
1510
|
+
// No failures is no denominator. There is nothing to be blind to, and firing
|
|
1511
|
+
// here would make a good day look like a broken metric.
|
|
1512
|
+
if (failing === 0) return { ok: true, summary: 'no failing steps — nothing to classify' }
|
|
1513
|
+
const share = (unclassified / failing) * 100
|
|
1514
|
+
// A snapshot collected before names were recorded has the count but not the
|
|
1515
|
+
// names. Say that, rather than printing an empty list that reads as though
|
|
1516
|
+
// the names were looked for and there were none.
|
|
1517
|
+
const names = (gates.unclassifiedNames ?? []).length
|
|
1518
|
+
? gates.unclassifiedNames.map((n) => `${n.name} (${n.count})`).join(', ')
|
|
1519
|
+
: 'names not recorded in this snapshot — re-collect to get them'
|
|
1520
|
+
if (share <= BLINDNESS_THRESHOLD) {
|
|
1521
|
+
return {
|
|
1522
|
+
ok: true,
|
|
1523
|
+
summary: `${unclassified} of ${failing} failing steps unclassified (${share.toFixed(1)}%)`,
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
return {
|
|
1527
|
+
ok: false,
|
|
1528
|
+
summary:
|
|
1529
|
+
`${unclassified} of ${failing} failing steps unclassified (${share.toFixed(1)}%) — ` +
|
|
1530
|
+
`add patterns to STEP_KINDS for: ${names}`,
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
/**
|
|
1535
|
+
* The share of unclassified steps this tolerates before failing.
|
|
1536
|
+
*
|
|
1537
|
+
* Not zero, deliberately. A new step name is normal estate churn, and a guard
|
|
1538
|
+
* that is red every morning trains people to stop reading it — the argument
|
|
1539
|
+
* `scripts/protection-audit.sh` makes at length, and the reason
|
|
1540
|
+
* `mustBeUniform` ratchets from a baseline rather than demanding day-one purity.
|
|
1541
|
+
* At 20% the 2026-08-03 snapshot (70.6%) fails loudly and a single unseen name
|
|
1542
|
+
* in a normal day's failures does not.
|
|
1543
|
+
*/
|
|
1544
|
+
export const BLINDNESS_THRESHOLD = 20
|
|
1545
|
+
|
|
1546
|
+
/**
|
|
1547
|
+
* {@link summariseGates} for one window, or an explicit `unmeasured` when the
|
|
1548
|
+
* window predates the fetch (#914).
|
|
1549
|
+
*
|
|
1550
|
+
* The distinction this preserves: **"nothing failed" and "we did not look" are
|
|
1551
|
+
* different claims, and only one of them is good news.** The same rule
|
|
1552
|
+
* {@link percentile} follows for an empty set.
|
|
1553
|
+
*
|
|
1554
|
+
* @param {{coveredSince: string, failing: Array<{name: string, at: number}>} | null} steps
|
|
1555
|
+
* @param {string | undefined} windowSince
|
|
1556
|
+
*/
|
|
1557
|
+
export function gatesForWindow(steps, windowSince) {
|
|
1558
|
+
if (!steps) return { error: 'unmeasured', reason: 'no jobs fetched' }
|
|
1559
|
+
if (windowSince && Date.parse(windowSince) < Date.parse(steps.coveredSince)) {
|
|
1560
|
+
return {
|
|
1561
|
+
error: 'unmeasured',
|
|
1562
|
+
reason: `window starts ${windowSince} but jobs were fetched only from ${steps.coveredSince}`,
|
|
1563
|
+
coveredSince: steps.coveredSince,
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
return summariseGates(steps.failing)
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
/**
|
|
1570
|
+
* Parse `git log` output into commits carrying the files they touched.
|
|
1571
|
+
*
|
|
1572
|
+
* Expects the format written by {@link gitLogCommand}: a header line of
|
|
1573
|
+
* `<sha>\x1f<unix-ts>\x1f<subject>` followed by one path per line.
|
|
1574
|
+
*
|
|
1575
|
+
* @param {string} stdout
|
|
1576
|
+
*/
|
|
1577
|
+
export function parseGitLog(stdout) {
|
|
1578
|
+
/** @type {Array<{sha: string, at: number, subject: string, files: string[]}>} */
|
|
1579
|
+
const commits = []
|
|
1580
|
+
for (const line of stdout.split('\n')) {
|
|
1581
|
+
if (line.includes('\x1f')) {
|
|
1582
|
+
const [sha, ts, ...rest] = line.split('\x1f')
|
|
1583
|
+
commits.push({ sha, at: Number(ts) * 1000, subject: rest.join('\x1f'), files: [] })
|
|
1584
|
+
} else if (line.trim() && commits.length > 0) {
|
|
1585
|
+
commits[commits.length - 1].files.push(line.trim())
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
return commits
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
/**
|
|
1592
|
+
* Is this commit subject a correction of earlier work?
|
|
1593
|
+
*
|
|
1594
|
+
* @param {string} subject
|
|
1595
|
+
*/
|
|
1596
|
+
export function isReworkSubject(subject) {
|
|
1597
|
+
return REWORK_TYPES.some((type) => new RegExp(`^${type}(\\(.+\\))?!?:`).test(subject))
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/**
|
|
1601
|
+
* Parse `git diff -U0` output into the *pre-image* line ranges a commit changed.
|
|
1602
|
+
*
|
|
1603
|
+
* `-U0` matters: with context lines the ranges spill into untouched code and
|
|
1604
|
+
* blame then attributes the fix to whoever last edited the neighbourhood.
|
|
1605
|
+
*
|
|
1606
|
+
* Pure insertions (`count === 0`) are dropped — a hunk that only adds lines
|
|
1607
|
+
* corrects no existing line, so there is nothing to attribute. New files
|
|
1608
|
+
* (`--- /dev/null`) are dropped for the same reason.
|
|
1609
|
+
*
|
|
1610
|
+
* @param {string} diff
|
|
1611
|
+
* @returns {Array<{file: string, start: number, count: number}>}
|
|
1612
|
+
*/
|
|
1613
|
+
export function parseDiffHunks(diff) {
|
|
1614
|
+
/** @type {Array<{file: string, start: number, count: number}>} */
|
|
1615
|
+
const hunks = []
|
|
1616
|
+
let file = null
|
|
1617
|
+
for (const line of diff.split('\n')) {
|
|
1618
|
+
if (line.startsWith('--- /dev/null')) file = null
|
|
1619
|
+
else if (line.startsWith('--- a/')) file = line.slice(6)
|
|
1620
|
+
else if (line.startsWith('@@') && file && !OPAQUE_PATHS.test(file)) {
|
|
1621
|
+
const match = /^@@ -(\d+)(?:,(\d+))? /.exec(line)
|
|
1622
|
+
if (!match) continue
|
|
1623
|
+
const count = match[2] === undefined ? 1 : Number(match[2])
|
|
1624
|
+
if (count === 0) continue
|
|
1625
|
+
hunks.push({ file, start: Number(match[1]), count })
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
return hunks
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
/**
|
|
1632
|
+
* Roll line-attributed fixes into the rework metrics.
|
|
1633
|
+
*
|
|
1634
|
+
* ## Why this is blame-based and not file-based
|
|
1635
|
+
*
|
|
1636
|
+
* The first implementation counted a fix as rework when it touched *any file*
|
|
1637
|
+
* another commit had touched in the previous seven days. Measured against real
|
|
1638
|
+
* history that filter removed 4 of 195 commits — on a repo merging every ~12
|
|
1639
|
+
* minutes, essentially every file has been touched recently, so the metric was
|
|
1640
|
+
* `fixShare` in a disguise and its "lag" tracked merge cadence rather than
|
|
1641
|
+
* correction latency. Spot-checking showed the top pairs were nonsense:
|
|
1642
|
+
* `fix(networking): sweep the VPC flow-logs group` "correcting"
|
|
1643
|
+
* `security(deps): override brace-expansion`, two unrelated changes that shared
|
|
1644
|
+
* a file.
|
|
1645
|
+
*
|
|
1646
|
+
* Attributing at line level moved the median from 0.8h to 2.4h and p90 from
|
|
1647
|
+
* 18.2h to 63.6h — the file-level version understated lag roughly threefold.
|
|
1648
|
+
*
|
|
1649
|
+
* @param {Array<{at: number, correctedAt: number | null}>} fixes
|
|
1650
|
+
* @param {number} merges total first-parent merges in the window
|
|
1651
|
+
*/
|
|
1652
|
+
export function summariseRework(fixes, merges) {
|
|
1653
|
+
if (merges === 0) {
|
|
1654
|
+
return {
|
|
1655
|
+
merges: 0,
|
|
1656
|
+
fixMerges: null,
|
|
1657
|
+
fixShare: null,
|
|
1658
|
+
attributed: null,
|
|
1659
|
+
medianHoursToRework: null,
|
|
1660
|
+
p90HoursToRework: null,
|
|
1661
|
+
correctedWithin1hShare: null,
|
|
1662
|
+
correctedWithin24hShare: null,
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
const lags = fixes
|
|
1667
|
+
.filter((fix) => fix.correctedAt !== null)
|
|
1668
|
+
.map((fix) => (fix.at - fix.correctedAt) / 3600000)
|
|
1669
|
+
|
|
1670
|
+
return {
|
|
1671
|
+
merges,
|
|
1672
|
+
fixMerges: fixes.length,
|
|
1673
|
+
// Robust and cheap: no attribution required, so it cannot be wrong, only
|
|
1674
|
+
// coarse. Reported beside the lag so a shifting lag can be checked against
|
|
1675
|
+
// a stable denominator.
|
|
1676
|
+
fixShare: rate(fixes.length, merges),
|
|
1677
|
+
// Coverage. Fixes that only insert lines are unattributable by construction,
|
|
1678
|
+
// so this is always below fixMerges and that is not a defect.
|
|
1679
|
+
attributed: lags.length,
|
|
1680
|
+
// The discriminating measurements. A fix correcting code written an hour
|
|
1681
|
+
// ago is a guess that shipped; one correcting code from last week is
|
|
1682
|
+
// ordinary defect discovery. Only these separate them.
|
|
1683
|
+
medianHoursToRework: round1(percentile(lags, 50)),
|
|
1684
|
+
p90HoursToRework: round1(percentile(lags, 90)),
|
|
1685
|
+
correctedWithin1hShare: rate(lags.filter((h) => h < 1).length, lags.length),
|
|
1686
|
+
correctedWithin24hShare: rate(lags.filter((h) => h < 24).length, lags.length),
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
/**
|
|
1691
|
+
* Roll one repo's raw GitHub data into the metric set.
|
|
1692
|
+
*
|
|
1693
|
+
* @param {{slug: string, role: string}} repo
|
|
1694
|
+
* @param {{prs: Array<Record<string, any>>, runs: Array<Record<string, any>>, defaultBranch: string, rework: {fixes: Array<{at: number, correctedAt: number | null}>, merges: number} | null}} data
|
|
1695
|
+
*/
|
|
1696
|
+
export function summariseRepo(repo, data, issueOpenedAt = new Map(), templateClosingIssues = new Map()) {
|
|
1697
|
+
const { prs, runs, defaultBranch, rework, steps, windowSince } = data
|
|
1698
|
+
const runsByBranch = indexRunsByBranch(runs)
|
|
1699
|
+
const merged = prs.filter((pr) => pr.mergedAt)
|
|
1700
|
+
|
|
1701
|
+
const churns = merged.map((pr) => prChurn(pr, runsByBranch))
|
|
1702
|
+
const measured = churns.filter((c) => c.revisions !== null)
|
|
1703
|
+
const cycleTimes = merged.map(cycleTimeMinutes).filter((minutes) => minutes !== null)
|
|
1704
|
+
|
|
1705
|
+
return {
|
|
1706
|
+
role: repo.role,
|
|
1707
|
+
side: repo.side,
|
|
1708
|
+
defaultBranch,
|
|
1709
|
+
mergedPrs: merged.length,
|
|
1710
|
+
// Are we building the product or maintaining the machine?
|
|
1711
|
+
workMix: rework
|
|
1712
|
+
? summariseWorkMix(rework.commits, repo.side)
|
|
1713
|
+
: {
|
|
1714
|
+
merges: null, delivery: null, rework: null, toil: null, quality: null,
|
|
1715
|
+
docs: null, unconventional: null, toilRatio: null,
|
|
1716
|
+
counts: null, sideCounts: null, productDelivery: null,
|
|
1717
|
+
},
|
|
1718
|
+
// How long from wanting a capability to having it (#767). Stop A only:
|
|
1719
|
+
// issue opened → closing PR merged. Stop B (running in an instance) needs
|
|
1720
|
+
// the template→instance hop to be machine-readable first.
|
|
1721
|
+
timeToFeature: {
|
|
1722
|
+
...timeToFeature(merged, issueOpenedAt, successfulDeploys(runs, defaultBranch)),
|
|
1723
|
+
// Template issue opened -> running here. Empty for the template itself and
|
|
1724
|
+
// for any repo that takes no core upgrades.
|
|
1725
|
+
crossRepo: crossRepoTimeToFeature(
|
|
1726
|
+
merged,
|
|
1727
|
+
templateClosingIssues,
|
|
1728
|
+
issueOpenedAt,
|
|
1729
|
+
successfulDeploys(runs, defaultBranch),
|
|
1730
|
+
),
|
|
1731
|
+
},
|
|
1732
|
+
// H4's primary metric (#914). `unmeasured` rather than a number when the
|
|
1733
|
+
// window reaches back further than the jobs fetch: the per-run jobs call is
|
|
1734
|
+
// O(failed runs), so it is capped, and a 90-day window over a 14-day fetch
|
|
1735
|
+
// would report a share of the fortnight as if it were the quarter.
|
|
1736
|
+
gates: gatesForWindow(steps, windowSince),
|
|
1737
|
+
// Consistency — two metrics, never one. See prChurn().
|
|
1738
|
+
ciFailureRate: rate(measured.filter((c) => c.ciFailed).length, measured.length),
|
|
1739
|
+
revisionsP50: percentile(
|
|
1740
|
+
measured.map((c) => c.revisions),
|
|
1741
|
+
50,
|
|
1742
|
+
),
|
|
1743
|
+
revisionsP90: percentile(
|
|
1744
|
+
measured.map((c) => c.revisions),
|
|
1745
|
+
90,
|
|
1746
|
+
),
|
|
1747
|
+
landedFirstPushRate: rate(measured.filter((c) => c.revisions === 0).length, measured.length),
|
|
1748
|
+
// Speed.
|
|
1749
|
+
cycleTimeP50Minutes: round1(percentile(cycleTimes, 50)),
|
|
1750
|
+
cycleTimeP90Minutes: round1(percentile(cycleTimes, 90)),
|
|
1751
|
+
// The anti-goal.
|
|
1752
|
+
rework: rework
|
|
1753
|
+
? summariseRework(rework.fixes, rework.commits.length)
|
|
1754
|
+
: {
|
|
1755
|
+
merges: null,
|
|
1756
|
+
fixMerges: null,
|
|
1757
|
+
fixShare: null,
|
|
1758
|
+
attributed: null,
|
|
1759
|
+
medianHoursToRework: null,
|
|
1760
|
+
p90HoursToRework: null,
|
|
1761
|
+
correctedWithin1hShare: null,
|
|
1762
|
+
correctedWithin24hShare: null,
|
|
1763
|
+
},
|
|
1764
|
+
// Correct work that could not land. Separate axis from churn — see
|
|
1765
|
+
// mergeContention() for why collapsing them points at the wrong fix.
|
|
1766
|
+
contention: mergeContention(prs, runsByBranch),
|
|
1767
|
+
// Was anything read by a second pass before it landed?
|
|
1768
|
+
review: reviewCoverage(merged),
|
|
1769
|
+
// Trust in the gates themselves.
|
|
1770
|
+
flakes: detectFlakes(runs),
|
|
1771
|
+
integration: integrationHealth(
|
|
1772
|
+
runs,
|
|
1773
|
+
defaultBranch,
|
|
1774
|
+
IDLE_CEILING_MINUTES,
|
|
1775
|
+
steps ? { ids: steps.killedRunIds, coveredSince: steps.coveredSince } : null,
|
|
1776
|
+
),
|
|
1777
|
+
// Honesty about coverage: the denominator every rate above was computed on.
|
|
1778
|
+
coverage: {
|
|
1779
|
+
prsMeasured: measured.length,
|
|
1780
|
+
prsUnmeasured: merged.length - measured.length,
|
|
1781
|
+
workflowRuns: runs.length,
|
|
1782
|
+
reworkSource: rework ? 'git-blame' : 'unavailable',
|
|
1783
|
+
},
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
|
|
1787
|
+
/**
|
|
1788
|
+
* Roll every repo up into the estate-level view the daily page leads with.
|
|
1789
|
+
*
|
|
1790
|
+
* The question this answers is the one that decides where effort goes: **how
|
|
1791
|
+
* much of what we do is building a capability at all?**
|
|
1792
|
+
*
|
|
1793
|
+
* ## Why the headline changed (#768)
|
|
1794
|
+
*
|
|
1795
|
+
* This used to lead with `productFeatureShare` — delivery merges in `tabsii-*`
|
|
1796
|
+
* as a share of everything — on the framing that "Biffo is the machine, Tabsii
|
|
1797
|
+
* is the product". **The north star set on 2026-07-27 inverts that: Biffo is
|
|
1798
|
+
* the fundable product and Tabsii is the proving ground that exercises it.**
|
|
1799
|
+
*
|
|
1800
|
+
* Under the old label the same 152 merges read **5.9%**, and it was quoted as
|
|
1801
|
+
* "we are barely shipping features". Re-cut on the Biffo/Tabsii axis the answer
|
|
1802
|
+
* is **35.5% capability** — Biffo 29.6%, Tabsii 5.9%. Same day, same merges,
|
|
1803
|
+
* **6× difference**, purely from which repo family is called "the product". The
|
|
1804
|
+
* arithmetic was never wrong; the denominator was the wrong product.
|
|
1805
|
+
*
|
|
1806
|
+
* That number had already been identified as measuring the wrong thing the day
|
|
1807
|
+
* before and stayed on the dashboard, so it was read as a headline again. The
|
|
1808
|
+
* old figure survives as `tabsiiCapabilityShare`, which is what it always was —
|
|
1809
|
+
* a legitimate number about the proving ground.
|
|
1810
|
+
*
|
|
1811
|
+
* Caveat carried in the output rather than left to memory: **merges are not
|
|
1812
|
+
* time.** A one-line `chore:` and a week-long `feat:` count the same. This is a
|
|
1813
|
+
* directional proxy that costs nothing, not a timesheet.
|
|
1814
|
+
*
|
|
1815
|
+
* @param {Record<string, any>} repos
|
|
1816
|
+
*/
|
|
1817
|
+
export function summariseEstate(repos) {
|
|
1818
|
+
const usable = Object.values(repos).filter((r) => r && !r.error && r.workMix?.counts)
|
|
1819
|
+
if (usable.length === 0) {
|
|
1820
|
+
return {
|
|
1821
|
+
merges: 0,
|
|
1822
|
+
platformShare: null,
|
|
1823
|
+
productShare: null,
|
|
1824
|
+
toilRatio: null,
|
|
1825
|
+
capabilityShare: null,
|
|
1826
|
+
capabilityBySide: {},
|
|
1827
|
+
tabsiiCapabilityShare: null,
|
|
1828
|
+
contentionHours: null,
|
|
1829
|
+
bySide: {},
|
|
1830
|
+
gates: aggregateGates(repos),
|
|
1831
|
+
note: 'merges are a proxy for effort, not a measure of time',
|
|
1832
|
+
}
|
|
1833
|
+
}
|
|
1834
|
+
|
|
1835
|
+
const sum = (fn) => usable.reduce((total, r) => total + fn(r), 0)
|
|
1836
|
+
const merges = sum((r) => r.workMix.merges)
|
|
1837
|
+
const platform = sum((r) => r.workMix.sideCounts.platform)
|
|
1838
|
+
const product = sum((r) => r.workMix.sideCounts.product)
|
|
1839
|
+
const toil = sum((r) => r.workMix.counts.toil)
|
|
1840
|
+
const rework = sum((r) => r.workMix.counts.rework)
|
|
1841
|
+
// Capability = a merge that built something, wherever it landed. Split by
|
|
1842
|
+
// family, because "which product" is the question the old headline got wrong.
|
|
1843
|
+
const capability = sum((r) => r.workMix.counts.delivery)
|
|
1844
|
+
const tabsiiCapability = sum((r) => r.workMix.productDelivery)
|
|
1845
|
+
const biffoCapability = capability - tabsiiCapability
|
|
1846
|
+
|
|
1847
|
+
/**
|
|
1848
|
+
* Per-side rollup. A repo contributes to *both* sides when its merges do —
|
|
1849
|
+
* an instance repo carries core upgrades (platform) alongside features
|
|
1850
|
+
* (product), and attributing the whole repo to one side is the error this
|
|
1851
|
+
* function exists to remove.
|
|
1852
|
+
*/
|
|
1853
|
+
const bySide = {}
|
|
1854
|
+
for (const side of ['platform', 'product']) {
|
|
1855
|
+
const n = sum((r) => r.workMix.sideCounts[side])
|
|
1856
|
+
if (!n) continue
|
|
1857
|
+
// Kind counts are not split by side (a merge has one kind and one side, but
|
|
1858
|
+
// the cross-tab is only tracked for the product-delivery cell that the
|
|
1859
|
+
// headline needs). Shares here are of that side's merges.
|
|
1860
|
+
bySide[side] = {
|
|
1861
|
+
merges: n,
|
|
1862
|
+
share: rate(n, merges),
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
return {
|
|
1867
|
+
merges,
|
|
1868
|
+
platformShare: rate(platform, merges),
|
|
1869
|
+
productShare: rate(product, merges),
|
|
1870
|
+
// SRE framing: toil + rework is effort that added no product value.
|
|
1871
|
+
toilRatio: rate(toil + rework, merges),
|
|
1872
|
+
// The headline: capability built anywhere, as a share of all merges.
|
|
1873
|
+
capabilityShare: rate(capability, merges),
|
|
1874
|
+
capabilityBySide: {
|
|
1875
|
+
// Biffo is the fundable product; Tabsii is the proving ground.
|
|
1876
|
+
platform: { merges: biffoCapability, share: rate(biffoCapability, merges) },
|
|
1877
|
+
product: { merges: tabsiiCapability, share: rate(tabsiiCapability, merges) },
|
|
1878
|
+
},
|
|
1879
|
+
// Formerly `productFeatureShare`, renamed rather than dropped: it is a real
|
|
1880
|
+
// number about the proving ground, and only its label was wrong (#768).
|
|
1881
|
+
tabsiiCapabilityShare: rate(tabsiiCapability, merges),
|
|
1882
|
+
contentionHours: round1(sum((r) => r.contention?.greenButUnmergedHours ?? 0)),
|
|
1883
|
+
bySide,
|
|
1884
|
+
// H4's headline, estate-wide (#914). Aggregated over its own set of repos,
|
|
1885
|
+
// not `usable`: a repo can fail CI steps in a window it merged nothing in,
|
|
1886
|
+
// and requiring a workMix would drop exactly those.
|
|
1887
|
+
gates: aggregateGates(repos),
|
|
1888
|
+
note: 'merges are a proxy for effort, not a measure of time',
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
|
|
1892
|
+
/**
|
|
1893
|
+
* Estate-wide roll-up of the per-repo gate metric (#914).
|
|
1894
|
+
*
|
|
1895
|
+
* Sums the raw counts and recomputes the share from them rather than averaging
|
|
1896
|
+
* the per-repo percentages: a repo with one failing step would otherwise weigh
|
|
1897
|
+
* the same as one with eighty. `repos` counts how many contributed, so a share
|
|
1898
|
+
* computed over two repos cannot be mistaken for one covering the estate.
|
|
1899
|
+
*
|
|
1900
|
+
* @param {Record<string, any>} repos
|
|
1901
|
+
*/
|
|
1902
|
+
export function aggregateGates(repos) {
|
|
1903
|
+
const measured = Object.values(repos).filter((r) => r?.gates && !r.gates.error)
|
|
1904
|
+
if (measured.length === 0) {
|
|
1905
|
+
return {
|
|
1906
|
+
repos: 0,
|
|
1907
|
+
failingSteps: 0,
|
|
1908
|
+
locallyCatchable: 0,
|
|
1909
|
+
unclassified: 0,
|
|
1910
|
+
unclassifiedNames: [],
|
|
1911
|
+
share: null,
|
|
1912
|
+
byKind: {},
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
/** @type {Record<string, number>} */
|
|
1916
|
+
const byKind = {}
|
|
1917
|
+
/** @type {Record<string, number>} */
|
|
1918
|
+
const unseen = {}
|
|
1919
|
+
let failingSteps = 0
|
|
1920
|
+
let locallyCatchable = 0
|
|
1921
|
+
let notLocallyCatchable = 0
|
|
1922
|
+
let unclassified = 0
|
|
1923
|
+
for (const repo of measured) {
|
|
1924
|
+
failingSteps += repo.gates.failingSteps
|
|
1925
|
+
locallyCatchable += repo.gates.locallyCatchable
|
|
1926
|
+
notLocallyCatchable += repo.gates.notLocallyCatchable
|
|
1927
|
+
unclassified += repo.gates.unclassified
|
|
1928
|
+
for (const [kind, n] of Object.entries(repo.gates.byKind)) {
|
|
1929
|
+
byKind[kind] = (byKind[kind] ?? 0) + /** @type {number} */ (n)
|
|
1930
|
+
}
|
|
1931
|
+
// A step name that goes unmatched in three repos is one pattern to write,
|
|
1932
|
+
// not three findings — merge by name so the audit says so.
|
|
1933
|
+
for (const { name, count } of repo.gates.unclassifiedNames ?? []) {
|
|
1934
|
+
unseen[name] = (unseen[name] ?? 0) + count
|
|
1935
|
+
}
|
|
1936
|
+
}
|
|
1937
|
+
return {
|
|
1938
|
+
repos: measured.length,
|
|
1939
|
+
failingSteps,
|
|
1940
|
+
locallyCatchable,
|
|
1941
|
+
notLocallyCatchable,
|
|
1942
|
+
unclassified,
|
|
1943
|
+
unclassifiedNames: rankNames(unseen),
|
|
1944
|
+
share: rate(locallyCatchable, locallyCatchable + notLocallyCatchable),
|
|
1945
|
+
byKind,
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
// ---------------------------------------------------------------------------
|
|
1950
|
+
// I/O — everything below shells out; none of it is unit-tested.
|
|
1951
|
+
// ---------------------------------------------------------------------------
|
|
1952
|
+
|
|
1953
|
+
/** @param {string[]} args */
|
|
1954
|
+
function gh(args) {
|
|
1955
|
+
const stdout = execFileSync('gh', args, { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 })
|
|
1956
|
+
return JSON.parse(stdout)
|
|
1957
|
+
}
|
|
1958
|
+
|
|
1959
|
+
/**
|
|
1960
|
+
* The `git log` invocation {@link parseGitLog} expects.
|
|
1961
|
+
*
|
|
1962
|
+
* `--first-parent` keeps this to what actually landed on the integration branch,
|
|
1963
|
+
* so a PR's internal commits do not each count as a merge.
|
|
1964
|
+
*
|
|
1965
|
+
* @param {string} since ISO date
|
|
1966
|
+
*/
|
|
1967
|
+
export function gitLogCommand(since) {
|
|
1968
|
+
return ['log', '--first-parent', `--since=${since}`, '--format=%H%x1f%ct%x1f%s', '--name-only']
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
/**
|
|
1972
|
+
* Attribute each fix to the change it corrects, by blaming the lines it altered.
|
|
1973
|
+
*
|
|
1974
|
+
* Returns `null` when the repo has no usable local clone — never an empty
|
|
1975
|
+
* result, which would score as "no rework".
|
|
1976
|
+
*
|
|
1977
|
+
* @param {string} repoPath @param {string} since @param {string} branch
|
|
1978
|
+
*/
|
|
1979
|
+
function fetchRework(repoPath, since, branch) {
|
|
1980
|
+
/** @param {string[]} args */
|
|
1981
|
+
const git = (args) =>
|
|
1982
|
+
execFileSync('git', ['-C', repoPath, ...args], {
|
|
1983
|
+
encoding: 'utf8',
|
|
1984
|
+
maxBuffer: 256 * 1024 * 1024,
|
|
1985
|
+
})
|
|
1986
|
+
|
|
1987
|
+
let commits
|
|
1988
|
+
try {
|
|
1989
|
+
execFileSync('git', ['-C', repoPath, 'fetch', 'origin', branch, '--quiet'], { stdio: 'ignore' })
|
|
1990
|
+
commits = parseGitLog(git([...gitLogCommand(since), `origin/${branch}`]))
|
|
1991
|
+
} catch {
|
|
1992
|
+
return null
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
const fixes = []
|
|
1996
|
+
for (const commit of commits.filter((c) => isReworkSubject(c.subject))) {
|
|
1997
|
+
let correctedAt = null
|
|
1998
|
+
let hunks = []
|
|
1999
|
+
try {
|
|
2000
|
+
hunks = parseDiffHunks(git(['diff', '-U0', `${commit.sha}^`, commit.sha]))
|
|
2001
|
+
} catch {
|
|
2002
|
+
// A root commit has no parent to diff against; nothing to attribute.
|
|
2003
|
+
}
|
|
2004
|
+
for (const hunk of hunks) {
|
|
2005
|
+
try {
|
|
2006
|
+
const porcelain = git([
|
|
2007
|
+
'blame',
|
|
2008
|
+
`${commit.sha}^`,
|
|
2009
|
+
'-L',
|
|
2010
|
+
`${hunk.start},+${hunk.count}`,
|
|
2011
|
+
'--porcelain',
|
|
2012
|
+
'--',
|
|
2013
|
+
hunk.file,
|
|
2014
|
+
])
|
|
2015
|
+
for (const line of porcelain.split('\n')) {
|
|
2016
|
+
const match = /^committer-time (\d+)$/.exec(line)
|
|
2017
|
+
if (!match) continue
|
|
2018
|
+
const at = Number(match[1]) * 1000
|
|
2019
|
+
// The most recent prior authorship is the change being corrected.
|
|
2020
|
+
if (at < commit.at && (correctedAt === null || at > correctedAt)) correctedAt = at
|
|
2021
|
+
}
|
|
2022
|
+
} catch {
|
|
2023
|
+
// File renamed away, deleted, or binary — unattributable, not clean.
|
|
2024
|
+
}
|
|
2025
|
+
}
|
|
2026
|
+
fixes.push({ at: commit.at, correctedAt })
|
|
2027
|
+
}
|
|
2028
|
+
|
|
2029
|
+
return { fixes, commits: commits.map((c) => ({ at: c.at, subject: c.subject })) }
|
|
2030
|
+
}
|
|
2031
|
+
|
|
2032
|
+
/**
|
|
2033
|
+
* `commits` cannot ride along on the bulk PR list (2026-08-02).
|
|
2034
|
+
*
|
|
2035
|
+
* GitHub's GraphQL node budget rejects it outright: `gh` expands `commits` to
|
|
2036
|
+
* `commits(first: 100) { nodes { authors(first: 100) } }`, and against a page of
|
|
2037
|
+
* 100 PRs that is 100 × 100 × 100 = **1,000,000** possible nodes, twice the
|
|
2038
|
+
* 500,000 ceiling. The estimate is static — it is computed from the page size,
|
|
2039
|
+
* not from what a repo actually holds — so lowering `--limit` does not help:
|
|
2040
|
+
* 1000, 500, 250 and 100 all fail with the identical message, and every one of
|
|
2041
|
+
* the 15 repos failed at once. `gh` itself is unchanged (2.96.0, Jul 2), and the
|
|
2042
|
+
* same query minus `commits` still succeeds, so this was a server-side change
|
|
2043
|
+
* rather than anything the estate did.
|
|
2044
|
+
*
|
|
2045
|
+
* The preflight guard (#917) did its job — it refused to write an empty snapshot
|
|
2046
|
+
* rather than reporting a day of zeros — but the day still had no data at all.
|
|
2047
|
+
*
|
|
2048
|
+
* So the field is fetched **per PR**, where the same expansion is 1 × 100 × 100 =
|
|
2049
|
+
* 10,000 nodes and well inside the budget. {@link parseCarriedPrs} reads the
|
|
2050
|
+
* marker from the body first and only consults commits when it is absent, and
|
|
2051
|
+
* the sole call site is already gated on {@link isUpgradePr}, so a normal PR and
|
|
2052
|
+
* a tool-created upgrade both cost nothing extra.
|
|
2053
|
+
*
|
|
2054
|
+
* **The residual cost is not negligible, and the first draft of this comment
|
|
2055
|
+
* guessed that it was.** Measured over 90 days: 107 extra requests across 1242
|
|
2056
|
+
* PRs (8.6%) — `biffo-platform` 43 of 119, `tabsii-platform` 64 of 380,
|
|
2057
|
+
* `biffo-template` 0 of 743. The template pays nothing because it is never the
|
|
2058
|
+
* *instance* side of an upgrade; the instances pay because the marker only
|
|
2059
|
+
* shipped with #767, so every upgrade PR predating it has the marker in neither
|
|
2060
|
+
* place and buys a request to discover that. That tail is finite and ages out of
|
|
2061
|
+
* the 90-day window. If it ever stops ageing out, cache the negative result
|
|
2062
|
+
* rather than widening the bulk query back into the node budget.
|
|
2063
|
+
*/
|
|
2064
|
+
const PR_LIST_FIELDS =
|
|
2065
|
+
// closingIssuesReferences is what makes time-to-feature (#767) cost nothing
|
|
2066
|
+
// extra: it rides along on a fetch that already happens. The alternative —
|
|
2067
|
+
// one timeline API call per closed issue — is O(issues) requests for the
|
|
2068
|
+
// same answer.
|
|
2069
|
+
// `body` carries the core-upgrade marker (#767) when the PR was opened by
|
|
2070
|
+
// the tool. Its commit-message fallback (#1011) is fetched separately below.
|
|
2071
|
+
// `reviews` rides along too (#952). Review coverage is otherwise unknowable:
|
|
2072
|
+
// nothing anywhere records whether a merged change was read by a second
|
|
2073
|
+
// pass, so "do we review?" had no answer but memory — and memory said yes
|
|
2074
|
+
// while a session shipping ~20 PRs reviewed almost none of them.
|
|
2075
|
+
'number,title,createdAt,mergedAt,headRefName,baseRefName,closingIssuesReferences,body,reviews'
|
|
2076
|
+
|
|
2077
|
+
/**
|
|
2078
|
+
* The commit-message fallback for one PR, or `[]` if it cannot be read.
|
|
2079
|
+
*
|
|
2080
|
+
* A failure here degrades to exactly what an upgrade PR with no marker anywhere
|
|
2081
|
+
* already produces — no carried PRs, which {@link parseCarriedPrs} documents as
|
|
2082
|
+
* a coverage fact rather than an error. It is warned about on stderr rather than
|
|
2083
|
+
* thrown, because throwing would mark the *whole repo* unmeasured over one
|
|
2084
|
+
* unreadable PR, which is a far larger blind spot than the one it reports.
|
|
2085
|
+
*
|
|
2086
|
+
* @param {string} slug @param {number} number
|
|
2087
|
+
*/
|
|
2088
|
+
function fetchPrCommits(slug, number) {
|
|
2089
|
+
try {
|
|
2090
|
+
return gh(['pr', 'view', String(number), '-R', slug, '--json', 'commits']).commits ?? []
|
|
2091
|
+
} catch (error) {
|
|
2092
|
+
process.stderr.write(
|
|
2093
|
+
` warn: ${slug}#${number} commits unreadable, carried-PR fallback skipped — ${String(error).split('\n')[0]}\n`,
|
|
2094
|
+
)
|
|
2095
|
+
return []
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
/** @param {string} slug @param {string} since */
|
|
2100
|
+
function fetchPrs(slug, since) {
|
|
2101
|
+
const prs = gh(['pr', 'list', '-R', slug, '--state', 'merged', '--limit', '1000', '--json', PR_LIST_FIELDS])
|
|
2102
|
+
const recent = prs.filter((pr) => pr.mergedAt >= since)
|
|
2103
|
+
for (const pr of recent) {
|
|
2104
|
+
// Body first, so a tool-created upgrade PR never costs a request.
|
|
2105
|
+
if (isUpgradePr(pr) && parseCarriedPrs(pr.body).length === 0) {
|
|
2106
|
+
pr.commits = fetchPrCommits(slug, pr.number)
|
|
2107
|
+
}
|
|
2108
|
+
}
|
|
2109
|
+
return recent
|
|
2110
|
+
}
|
|
2111
|
+
|
|
2112
|
+
/**
|
|
2113
|
+
* What share of merged PRs was read by anyone before it landed (#952).
|
|
2114
|
+
*
|
|
2115
|
+
* ## Why this exists
|
|
2116
|
+
*
|
|
2117
|
+
* Nothing in this estate recorded whether a change was reviewed, so "do we
|
|
2118
|
+
* review?" could only be answered from memory — and memory was wrong. A single
|
|
2119
|
+
* session shipped ~20 PRs, self-reviewed almost all of them, and two carried
|
|
2120
|
+
* reasoning that was internally consistent and externally false: a model slug
|
|
2121
|
+
* declared dead while the billing table showed 26 charged runs on it, and a
|
|
2122
|
+
* startup hook built on a premise checked against the wrong repo. Neither is the
|
|
2123
|
+
* kind of defect a test catches. Both are the kind a second reader catches.
|
|
2124
|
+
*
|
|
2125
|
+
* ## What counts
|
|
2126
|
+
*
|
|
2127
|
+
* A review event of any kind — APPROVED, CHANGES_REQUESTED or COMMENTED. The bar
|
|
2128
|
+
* is deliberately "someone looked", not "someone approved": on a solo-operator
|
|
2129
|
+
* estate an approval requirement would block every merge, and the thing worth
|
|
2130
|
+
* measuring is whether a diff was read at all.
|
|
2131
|
+
*
|
|
2132
|
+
* ## What this deliberately does NOT claim
|
|
2133
|
+
*
|
|
2134
|
+
* That a review event means the diff was read carefully, or that its absence
|
|
2135
|
+
* means nobody looked — an author reading their own combined diff (which
|
|
2136
|
+
* `build-plugin-feature` step 3.5 requires, and which demonstrably works) leaves
|
|
2137
|
+
* no trace here. So this measures a *recorded* second pass, and it is a floor
|
|
2138
|
+
* rather than the truth. It can still fall, which is the property that matters:
|
|
2139
|
+
* a metric that cannot get worse is not measuring anything.
|
|
2140
|
+
*/
|
|
2141
|
+
export function reviewCoverage(prs) {
|
|
2142
|
+
const measured = prs.length
|
|
2143
|
+
const reviewed = prs.filter((pr) => (pr.reviews ?? []).length > 0).length
|
|
2144
|
+
return {
|
|
2145
|
+
prsMeasured: measured,
|
|
2146
|
+
reviewed,
|
|
2147
|
+
unreviewed: measured - reviewed,
|
|
2148
|
+
reviewedShare: rate(reviewed, measured),
|
|
2149
|
+
}
|
|
2150
|
+
}
|
|
2151
|
+
|
|
2152
|
+
/**
|
|
2153
|
+
* Every closed issue's `createdAt`, in one request per repo (#767).
|
|
2154
|
+
*
|
|
2155
|
+
* Deliberately not filtered by the window: an issue opened months before the PR
|
|
2156
|
+
* that closed it is exactly the long-latency case this metric exists to find, so
|
|
2157
|
+
* filtering by open date would systematically discard the worst results and make
|
|
2158
|
+
* the median look good. The cap is the API's, and an issue beyond it resolves to
|
|
2159
|
+
* `unresolved` rather than being counted as fast.
|
|
2160
|
+
*
|
|
2161
|
+
* @param {string} slug
|
|
2162
|
+
* @returns {Array<{number: number, createdAt: string}>}
|
|
2163
|
+
*/
|
|
2164
|
+
function fetchClosedIssues(slug) {
|
|
2165
|
+
return gh(['issue', 'list', '-R', slug, '--state', 'closed', '--limit', '1000', '--json', 'number,createdAt'])
|
|
2166
|
+
}
|
|
2167
|
+
|
|
2168
|
+
/**
|
|
2169
|
+
* Every workflow run in the window, walked page by page.
|
|
2170
|
+
*
|
|
2171
|
+
* Stops as soon as a page's oldest run predates the window, so a repo with years
|
|
2172
|
+
* of history costs the same as one with a month.
|
|
2173
|
+
*
|
|
2174
|
+
* @param {string} slug @param {string} since
|
|
2175
|
+
*/
|
|
2176
|
+
function fetchRuns(slug, since) {
|
|
2177
|
+
/** @type {Array<Record<string, any>>} */
|
|
2178
|
+
const all = []
|
|
2179
|
+
for (let page = 1; page <= 40; page += 1) {
|
|
2180
|
+
const body = gh(['api', `repos/${slug}/actions/runs?per_page=100&page=${page}`])
|
|
2181
|
+
const runs = body.workflow_runs ?? []
|
|
2182
|
+
if (runs.length === 0) break
|
|
2183
|
+
all.push(...runs.filter((run) => run.created_at >= since))
|
|
2184
|
+
if (runs[runs.length - 1].created_at < since) break
|
|
2185
|
+
}
|
|
2186
|
+
return all
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
/**
|
|
2190
|
+
* Every failing step of every failing run in the window (#914).
|
|
2191
|
+
*
|
|
2192
|
+
* This is the only metric here that costs a request **per run**: there is no bulk
|
|
2193
|
+
* endpoint for job steps, so it is O(failed runs). That is why the caller caps
|
|
2194
|
+
* the lookback rather than reusing the widest window — 90 days of failed runs
|
|
2195
|
+
* across the estate is several thousand requests for a metric both experiments
|
|
2196
|
+
* define on 7 days.
|
|
2197
|
+
*
|
|
2198
|
+
* Timestamped by the *run*, not the job: the window is about when the failure was
|
|
2199
|
+
* discovered, and a job's own timing is an artefact of runner scheduling.
|
|
2200
|
+
*
|
|
2201
|
+
* @param {string} slug
|
|
2202
|
+
* @param {Array<Record<string, any>>} runs already-fetched runs for this repo
|
|
2203
|
+
* It also classifies each failed run as a **runner kill** or a real failure
|
|
2204
|
+
* (#982) — free, because the jobs payload it already fetches is exactly what
|
|
2205
|
+
* `isRunnerKill` needs. Doing it anywhere else would mean paying the per-run
|
|
2206
|
+
* request twice.
|
|
2207
|
+
*
|
|
2208
|
+
* @param {string} slug
|
|
2209
|
+
* @param {Array<Record<string, any>>} runs already-fetched runs for this repo
|
|
2210
|
+
* @param {string} since ISO cutoff — runs older than this are not walked
|
|
2211
|
+
* @returns {{coveredSince: string, failing: Array<{name: string, at: number}>, killedRunIds: Set<number>}}
|
|
2212
|
+
*/
|
|
2213
|
+
function fetchFailingSteps(slug, runs, since) {
|
|
2214
|
+
const failed = runs.filter(
|
|
2215
|
+
(run) => FAILING_CONCLUSIONS.has(run.conclusion) && run.created_at >= since,
|
|
2216
|
+
)
|
|
2217
|
+
/** @type {Array<{name: string, at: number}>} */
|
|
2218
|
+
const failing = []
|
|
2219
|
+
/** @type {Set<number>} */
|
|
2220
|
+
const killedRunIds = new Set()
|
|
2221
|
+
for (const run of failed) {
|
|
2222
|
+
const at = Date.parse(run.created_at)
|
|
2223
|
+
let body
|
|
2224
|
+
try {
|
|
2225
|
+
body = gh(['api', `repos/${slug}/actions/runs/${run.id}/jobs`])
|
|
2226
|
+
} catch {
|
|
2227
|
+
// A single unreadable run must not cost the whole repo its metric — the
|
|
2228
|
+
// jobs of a run GitHub has expired are gone for good, and refusing to
|
|
2229
|
+
// measure the other 150 steps because of one would be the fail-closed
|
|
2230
|
+
// mirror of the fail-open shape this scoreboard keeps finding.
|
|
2231
|
+
continue
|
|
2232
|
+
}
|
|
2233
|
+
if (isRunnerKill(body.jobs ?? [])) killedRunIds.add(run.id)
|
|
2234
|
+
for (const job of body.jobs ?? []) {
|
|
2235
|
+
if (job.conclusion !== 'failure') continue
|
|
2236
|
+
for (const step of job.steps ?? []) {
|
|
2237
|
+
if (step.conclusion !== 'failure') continue
|
|
2238
|
+
failing.push({ name: step.name, at })
|
|
2239
|
+
}
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
return { coveredSince: since, failing, killedRunIds }
|
|
2243
|
+
}
|
|
2244
|
+
|
|
2245
|
+
/**
|
|
2246
|
+
* Strip a merge subject down to the *change* it carries (#918).
|
|
2247
|
+
*
|
|
2248
|
+
* `chore(shared): sync template-shared files (#30)` and `… (#19)` in another repo
|
|
2249
|
+
* are the same upstream action arriving twice. The PR number is what makes them
|
|
2250
|
+
* look distinct, so it goes; case and whitespace follow for the same reason.
|
|
2251
|
+
*
|
|
2252
|
+
* @param {string} subject
|
|
2253
|
+
*/
|
|
2254
|
+
export function normaliseSubject(subject) {
|
|
2255
|
+
return String(subject ?? '')
|
|
2256
|
+
.replace(/\s*\(#\d+\)\s*$/, '')
|
|
2257
|
+
.replace(/\s+/g, ' ')
|
|
2258
|
+
.trim()
|
|
2259
|
+
.toLowerCase()
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
/**
|
|
2263
|
+
* **Mechanism amplification**: one upstream action multiplied across the estate
|
|
2264
|
+
* (#918).
|
|
2265
|
+
*
|
|
2266
|
+
* ## The measurement this exists to make possible
|
|
2267
|
+
*
|
|
2268
|
+
* On 2026-07-29, **81 of 226 merges — 35% of everything that landed** — were
|
|
2269
|
+
* `chore(shared): sync template-shared files`: roughly seven rounds across twelve
|
|
2270
|
+
* repos, six of the seven touching `scripts/verify.sh`. One gate being iterated
|
|
2271
|
+
* upstream and redistributed to the whole estate after *every* iteration instead
|
|
2272
|
+
* of once when it settled.
|
|
2273
|
+
*
|
|
2274
|
+
* Every existing metric reported that day as behaviour: toil 57.4% against 32%
|
|
2275
|
+
* recorded, the platform/product split inverting to 54.5% product, small-repo
|
|
2276
|
+
* repush rates of 66-77%. **None of them could show the cause**, because they are
|
|
2277
|
+
* all shares and rates — and amplification is invisible to a share. It looks
|
|
2278
|
+
* exactly like a busy day.
|
|
2279
|
+
*
|
|
2280
|
+
* ## `avoidableMerges` is the number, and why it is that number
|
|
2281
|
+
*
|
|
2282
|
+
* A change that must reach twelve repos costs twelve merges; that is the
|
|
2283
|
+
* mechanism working. What is avoidable is the *rounds* — distributing seven times
|
|
2284
|
+
* instead of once. So `avoidableMerges = merges - repos`: the floor is one round,
|
|
2285
|
+
* and everything above it is a batching decision.
|
|
2286
|
+
*
|
|
2287
|
+
* It is deliberately not `merges`, which would indict distribution itself, nor
|
|
2288
|
+
* `rounds`, which ignores how wide each round was.
|
|
2289
|
+
*
|
|
2290
|
+
* @param {Record<string, Array<{subject: string}>>} commitsByRepo repo slug → commits
|
|
2291
|
+
* @param {number} minRepos ignore a subject that reached fewer repos than this
|
|
2292
|
+
*/
|
|
2293
|
+
export function summariseAmplification(commitsByRepo, minRepos = 3) {
|
|
2294
|
+
/** @type {Map<string, {subject: string, repos: Set<string>, merges: number}>} */
|
|
2295
|
+
const groups = new Map()
|
|
2296
|
+
let totalMerges = 0
|
|
2297
|
+
for (const [repo, commits] of Object.entries(commitsByRepo)) {
|
|
2298
|
+
for (const commit of commits ?? []) {
|
|
2299
|
+
totalMerges += 1
|
|
2300
|
+
const key = normaliseSubject(commit.subject)
|
|
2301
|
+
if (!key) continue
|
|
2302
|
+
const group = groups.get(key) ?? { subject: key, repos: new Set(), merges: 0 }
|
|
2303
|
+
group.repos.add(repo)
|
|
2304
|
+
group.merges += 1
|
|
2305
|
+
groups.set(key, group)
|
|
2306
|
+
}
|
|
2307
|
+
}
|
|
2308
|
+
|
|
2309
|
+
const amplified = [...groups.values()]
|
|
2310
|
+
.filter((g) => g.repos.size >= minRepos && g.merges > g.repos.size)
|
|
2311
|
+
.map((g) => ({
|
|
2312
|
+
subject: g.subject,
|
|
2313
|
+
repos: g.repos.size,
|
|
2314
|
+
merges: g.merges,
|
|
2315
|
+
// Rounds of distribution: how many times this reached the average repo.
|
|
2316
|
+
rounds: round1(g.merges / g.repos.size),
|
|
2317
|
+
avoidableMerges: g.merges - g.repos.size,
|
|
2318
|
+
}))
|
|
2319
|
+
.sort((a, b) => b.avoidableMerges - a.avoidableMerges)
|
|
2320
|
+
|
|
2321
|
+
const avoidableMerges = amplified.reduce((total, g) => total + g.avoidableMerges, 0)
|
|
2322
|
+
return {
|
|
2323
|
+
totalMerges,
|
|
2324
|
+
avoidableMerges,
|
|
2325
|
+
// The headline: what share of everything that landed did not need to.
|
|
2326
|
+
avoidableShare: rate(avoidableMerges, totalMerges),
|
|
2327
|
+
minRepos,
|
|
2328
|
+
top: amplified.slice(0, 10),
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
|
|
2332
|
+
/**
|
|
2333
|
+
* Would this snapshot contain any data at all?
|
|
2334
|
+
*
|
|
2335
|
+
* Every metric here degrades gracefully on purpose: an unreadable repo becomes
|
|
2336
|
+
* `unmeasured` and leaves the aggregates rather than contributing a zero. That is
|
|
2337
|
+
* correct for *one* repo and wrong for *all* of them. On 2026-07-30 a cron run
|
|
2338
|
+
* whose credential had gone missing 401'd on all fifteen repos and still wrote a
|
|
2339
|
+
* well-formed 9.5KB snapshot — `estate.merges: 0`, every repo `unmeasured` —
|
|
2340
|
+
* which is indistinguishable at a glance from a quiet day.
|
|
2341
|
+
*
|
|
2342
|
+
* "Nothing could be measured" is not an observation, it is a broken job. Total
|
|
2343
|
+
* failure is therefore fatal, while partial failure stays graceful.
|
|
2344
|
+
*
|
|
2345
|
+
* @param {number} failed repos whose fetch threw
|
|
2346
|
+
* @param {number} attempted repos targeted
|
|
2347
|
+
*/
|
|
2348
|
+
export function isTotalFetchFailure(failed, attempted) {
|
|
2349
|
+
return attempted > 0 && failed === attempted
|
|
2350
|
+
}
|
|
2351
|
+
|
|
2352
|
+
/**
|
|
2353
|
+
* Name the likely cause of a total fetch failure from the error itself.
|
|
2354
|
+
*
|
|
2355
|
+
* This used to assert, unconditionally, that the cause was credentials — true of
|
|
2356
|
+
* the 401 that motivated the guard (#917), and confidently wrong on 2026-08-02,
|
|
2357
|
+
* when GitHub's GraphQL node budget started rejecting the bulk PR fetch and the
|
|
2358
|
+
* message sent the reader after a keyring that was working fine.
|
|
2359
|
+
*
|
|
2360
|
+
* A diagnostic that only ever prints one cause is not a diagnosis, and a
|
|
2361
|
+
* *confident* wrong one is worse than none: it costs the reader the time it takes
|
|
2362
|
+
* to disprove. So each branch is claimed only when the error says so, and the
|
|
2363
|
+
* fallback admits it does not know.
|
|
2364
|
+
*
|
|
2365
|
+
* @param {string | undefined} error the first failure's message
|
|
2366
|
+
*/
|
|
2367
|
+
export function diagnoseTotalFetchFailure(error) {
|
|
2368
|
+
const text = String(error ?? '')
|
|
2369
|
+
if (/exceeds the maximum limit|node limit|too complex/i.test(text)) {
|
|
2370
|
+
return (
|
|
2371
|
+
'The cause is a GitHub GraphQL node-budget rejection, NOT credentials — the query asks for more\n' +
|
|
2372
|
+
'nodes than the API allows, so it fails identically at every --limit and on every repo. Narrow the\n' +
|
|
2373
|
+
'requested fields (a sub-connection like `commits` multiplies the estimate) rather than retrying.'
|
|
2374
|
+
)
|
|
2375
|
+
}
|
|
2376
|
+
if (/HTTP 401|Bad credentials|authentication|gh auth login|not logged/i.test(text)) {
|
|
2377
|
+
return 'The cause is credentials: gh stores its token in the keyring, which cron cannot read.'
|
|
2378
|
+
}
|
|
2379
|
+
if (/HTTP 403|rate limit|abuse detection|secondary rate/i.test(text)) {
|
|
2380
|
+
return 'The cause is a GitHub rate limit or permissions refusal — check `gh api rate_limit` before retrying.'
|
|
2381
|
+
}
|
|
2382
|
+
return 'The cause is not one this script recognises; read the first failure below rather than assuming credentials.'
|
|
2383
|
+
}
|
|
2384
|
+
|
|
2385
|
+
/**
|
|
2386
|
+
* How far back the per-run jobs fetch walks, in days (#914).
|
|
2387
|
+
*
|
|
2388
|
+
* 14 = the 7-day window H4 and H5 are both defined on, plus the equal-length
|
|
2389
|
+
* prior period they are compared against. Wider costs a request per additional
|
|
2390
|
+
* failed run for a number no experiment reads; narrower leaves the rate window
|
|
2391
|
+
* `unmeasured`.
|
|
2392
|
+
*/
|
|
2393
|
+
export const GATE_LOOKBACK_DAYS = 14
|
|
2394
|
+
|
|
2395
|
+
function parseArgs(argv) {
|
|
2396
|
+
const args = {
|
|
2397
|
+
windows: DEFAULT_WINDOWS,
|
|
2398
|
+
out: 'docs/practices/data',
|
|
2399
|
+
repo: null,
|
|
2400
|
+
reposRoot: null,
|
|
2401
|
+
gateLookbackDays: GATE_LOOKBACK_DAYS,
|
|
2402
|
+
// Relative to cwd, like `out`. Overridable so the daily worktree can point
|
|
2403
|
+
// at the corpus it actually carries rather than whichever it is run beside.
|
|
2404
|
+
corpus: 'docs/practices/evidence.jsonl',
|
|
2405
|
+
}
|
|
2406
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
2407
|
+
if (argv[i] === '--window') args.windows = [Number(argv[++i])]
|
|
2408
|
+
else if (argv[i] === '--windows') args.windows = argv[++i].split(',').map(Number)
|
|
2409
|
+
else if (argv[i] === '--out') args.out = argv[++i]
|
|
2410
|
+
else if (argv[i] === '--repo') args.repo = argv[++i]
|
|
2411
|
+
else if (argv[i] === '--repos-root') args.reposRoot = argv[++i]
|
|
2412
|
+
else if (argv[i] === '--gate-lookback') args.gateLookbackDays = Number(argv[++i])
|
|
2413
|
+
else if (argv[i] === '--corpus') args.corpus = argv[++i]
|
|
2414
|
+
}
|
|
2415
|
+
args.windows.sort((a, b) => a - b)
|
|
2416
|
+
return args
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
/**
|
|
2420
|
+
* Where the sibling clones live.
|
|
2421
|
+
*
|
|
2422
|
+
* Derived from git's *common* dir, which points at the primary checkout even
|
|
2423
|
+
* when this runs from a worktree — so the answer is the same from
|
|
2424
|
+
* `biffo-template/` and from `biffo-template/.worktrees/anything/`. Guessing
|
|
2425
|
+
* with a fixed `../../..` silently resolves to `/home` from the primary
|
|
2426
|
+
* checkout, and every rework metric would then report `unavailable` while
|
|
2427
|
+
* looking like it ran.
|
|
2428
|
+
*/
|
|
2429
|
+
function resolveReposRoot() {
|
|
2430
|
+
const commonDir = execFileSync('git', ['rev-parse', '--path-format=absolute', '--git-common-dir'], {
|
|
2431
|
+
encoding: 'utf8',
|
|
2432
|
+
}).trim()
|
|
2433
|
+
return join(commonDir, '..', '..')
|
|
2434
|
+
}
|
|
2435
|
+
|
|
2436
|
+
/** Statuses that mean the finding is dealt with. Everything else is outstanding. */
|
|
2437
|
+
const FAIL_OPEN_DONE = new Set(['fixed', 'closed', 'fixed downstream'])
|
|
2438
|
+
|
|
2439
|
+
/**
|
|
2440
|
+
* The fail-open backlog, read from the corpus (#956).
|
|
2441
|
+
*
|
|
2442
|
+
* ## Why this is top-level and not under a window
|
|
2443
|
+
*
|
|
2444
|
+
* It is a point-in-time count of a single file, identical whether you ask about
|
|
2445
|
+
* one day or ninety. Filing it under `windows.90.estate` would render one number
|
|
2446
|
+
* three times and invite reading it as a trend, which is the drift the page's own
|
|
2447
|
+
* headline suffered.
|
|
2448
|
+
*
|
|
2449
|
+
* ## Which of these may be used as a target, and which may not
|
|
2450
|
+
*
|
|
2451
|
+
* **`unfiled` and `oldestUnfixedDays` are targets.** `unfiled` means recorded and
|
|
2452
|
+
* never converted into an issue — 8 of 90 when this was written — and nothing
|
|
2453
|
+
* except neglect makes it rise. `oldestUnfixedDays` rises only by leaving things.
|
|
2454
|
+
*
|
|
2455
|
+
* **`unfixed` is NOT a target, and must not be quoted as health.** It rises when
|
|
2456
|
+
* findings are *discovered*, and discovering them is the work. A day that finds
|
|
2457
|
+
* four and fixes three moves it the "wrong" way while being an excellent day.
|
|
2458
|
+
* Driving that number down rewards not looking, which is precisely the failure
|
|
2459
|
+
* this whole corpus exists to record.
|
|
2460
|
+
*
|
|
2461
|
+
* Returns `{ error: 'unmeasured' }` rather than zeros when the corpus cannot be
|
|
2462
|
+
* read — this file's own rule (see the header): "could not measure" is never
|
|
2463
|
+
* reported as zero, because a zero here would read as a clean estate.
|
|
2464
|
+
*/
|
|
2465
|
+
export function summariseFailOpenBacklog(corpusPath, now = new Date()) {
|
|
2466
|
+
let rows
|
|
2467
|
+
try {
|
|
2468
|
+
rows = readCorpusStrict(corpusPath)
|
|
2469
|
+
} catch {
|
|
2470
|
+
return { error: 'unmeasured' }
|
|
2471
|
+
}
|
|
2472
|
+
|
|
2473
|
+
const failOpen = rows.filter(
|
|
2474
|
+
(r) => r.class === 'fail-open' || (r.alsoClass ?? []).includes('fail-open'),
|
|
2475
|
+
)
|
|
2476
|
+
const outstanding = failOpen.filter((r) => !FAIL_OPEN_DONE.has(r.status))
|
|
2477
|
+
|
|
2478
|
+
const ages = outstanding
|
|
2479
|
+
.map((r) => (r.date ? Math.round((now - Date.parse(`${r.date}T00:00:00Z`)) / 864e5) : null))
|
|
2480
|
+
.filter((d) => typeof d === 'number' && Number.isFinite(d) && d >= 0)
|
|
2481
|
+
|
|
2482
|
+
return {
|
|
2483
|
+
total: failOpen.length,
|
|
2484
|
+
// Not a target. See the note above before quoting this anywhere.
|
|
2485
|
+
unfixed: outstanding.length,
|
|
2486
|
+
unfiled: outstanding.filter((r) => r.status === 'unfiled').length,
|
|
2487
|
+
oldestUnfixedDays: ages.length > 0 ? Math.max(...ages) : null,
|
|
2488
|
+
byStatus: Object.fromEntries(
|
|
2489
|
+
[...new Set(outstanding.map((r) => r.status ?? 'unset'))]
|
|
2490
|
+
.sort()
|
|
2491
|
+
.map((k) => [k, outstanding.filter((r) => (r.status ?? 'unset') === k).length]),
|
|
2492
|
+
),
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2496
|
+
function main() {
|
|
2497
|
+
const args = parseArgs(process.argv.slice(2))
|
|
2498
|
+
const reposRoot = args.reposRoot ?? resolveReposRoot()
|
|
2499
|
+
const maxWindow = Math.max(...args.windows)
|
|
2500
|
+
// One fetch, at the widest window; every narrower window is a filter over the
|
|
2501
|
+
// same data. Collecting per-window would triple the API and blame cost and —
|
|
2502
|
+
// worse — let the windows disagree because they were taken at different times.
|
|
2503
|
+
const fetchSince = new Date(Date.now() - maxWindow * 864e5).toISOString()
|
|
2504
|
+
const prior = priorWindow(args.windows)
|
|
2505
|
+
// The gate metric's lookback is capped separately and **explicitly**, because
|
|
2506
|
+
// it alone costs a request per failed run (see fetchFailingSteps).
|
|
2507
|
+
//
|
|
2508
|
+
// An earlier version derived this as `2 × priorWindow().days`, which was too
|
|
2509
|
+
// clever: `priorWindow` calls the *second-largest* window the rate window, so
|
|
2510
|
+
// running `--windows 1,7` made the cap 2 days and left the 7-day window — the
|
|
2511
|
+
// one both H4 and H5 are defined on — `unmeasured`. The guard reported that
|
|
2512
|
+
// honestly rather than hiding it, but a cost cap should not move when an
|
|
2513
|
+
// unrelated window list changes. A constant says what it is.
|
|
2514
|
+
const gatesSince = new Date(Date.now() - args.gateLookbackDays * 864e5).toISOString()
|
|
2515
|
+
const targets = args.repo ? REPOS.filter((r) => r.slug === args.repo) : REPOS
|
|
2516
|
+
|
|
2517
|
+
/** @type {Record<string, any>} */
|
|
2518
|
+
const raw = {}
|
|
2519
|
+
/** @type {Array<{repo: string, error: string}>} */
|
|
2520
|
+
const failures = []
|
|
2521
|
+
|
|
2522
|
+
for (const repo of targets) {
|
|
2523
|
+
process.stderr.write(` ${repo.slug} … `)
|
|
2524
|
+
try {
|
|
2525
|
+
const meta = gh(['repo', 'view', repo.slug, '--json', 'defaultBranchRef'])
|
|
2526
|
+
const defaultBranch = meta.defaultBranchRef?.name ?? 'dev'
|
|
2527
|
+
const prs = fetchPrs(repo.slug, fetchSince)
|
|
2528
|
+
const runs = fetchRuns(repo.slug, fetchSince)
|
|
2529
|
+
const rework = fetchRework(join(reposRoot, repo.path), fetchSince, defaultBranch)
|
|
2530
|
+
const issues = fetchClosedIssues(repo.slug)
|
|
2531
|
+
const steps = fetchFailingSteps(repo.slug, runs, gatesSince)
|
|
2532
|
+
raw[repo.slug] = { prs, runs, defaultBranch, rework, issues, steps }
|
|
2533
|
+
process.stderr.write(
|
|
2534
|
+
`${prs.length} PRs, ${runs.length} runs, ${issues.length} closed issues, ${steps.failing.length} failing steps\n`,
|
|
2535
|
+
)
|
|
2536
|
+
} catch (error) {
|
|
2537
|
+
// A repo that could not be read is recorded as such and excluded from
|
|
2538
|
+
// every aggregate. It is never allowed to contribute a zero.
|
|
2539
|
+
failures.push({ repo: repo.slug, error: String(error).split('\n')[0] })
|
|
2540
|
+
raw[repo.slug] = null
|
|
2541
|
+
process.stderr.write('FAILED\n')
|
|
2542
|
+
}
|
|
2543
|
+
}
|
|
2544
|
+
|
|
2545
|
+
// Refuse to write a snapshot with nothing in it (see isTotalFetchFailure).
|
|
2546
|
+
// Placed before any aggregation so the failure is attributed to the fetch,
|
|
2547
|
+
// where it happened, rather than surfacing later as a page full of dashes.
|
|
2548
|
+
if (isTotalFetchFailure(failures.length, targets.length)) {
|
|
2549
|
+
process.stderr.write(
|
|
2550
|
+
`\nFATAL: all ${targets.length} repos failed to fetch — refusing to write an empty snapshot.\n` +
|
|
2551
|
+
`${diagnoseTotalFetchFailure(failures[0]?.error)}\n` +
|
|
2552
|
+
`First failure: ${failures[0]?.error}\n`,
|
|
2553
|
+
)
|
|
2554
|
+
process.exit(1)
|
|
2555
|
+
}
|
|
2556
|
+
|
|
2557
|
+
// The cross-repo join's LEFT side is the template's PRs and issues, and it is
|
|
2558
|
+
// needed even when `--repo <instance>` narrows the targets to one instance —
|
|
2559
|
+
// otherwise the join has nothing to resolve against and reports every carried
|
|
2560
|
+
// PR as closing no issue. Fetch it supplementarily: PRs and closed issues only
|
|
2561
|
+
// (no runs, rework or steps), and deliberately NOT added to `raw`, so it feeds
|
|
2562
|
+
// the indexes without appearing as a reported repo in the snapshot.
|
|
2563
|
+
const templateRepo = REPOS.find((r) => r.role === 'template')
|
|
2564
|
+
const templateIsTarget = targets.some((r) => r.slug === templateRepo?.slug)
|
|
2565
|
+
/** @type {{prs: Array<Record<string, any>>, issues: Array<Record<string, any>>}} */
|
|
2566
|
+
let templateJoinData = { prs: [], issues: [] }
|
|
2567
|
+
if (templateRepo && !templateIsTarget) {
|
|
2568
|
+
process.stderr.write(` ${templateRepo.slug} (join only) … `)
|
|
2569
|
+
try {
|
|
2570
|
+
templateJoinData = {
|
|
2571
|
+
prs: fetchPrs(templateRepo.slug, fetchSince),
|
|
2572
|
+
issues: fetchClosedIssues(templateRepo.slug),
|
|
2573
|
+
}
|
|
2574
|
+
process.stderr.write(
|
|
2575
|
+
`${templateJoinData.prs.length} PRs, ${templateJoinData.issues.length} closed issues\n`,
|
|
2576
|
+
)
|
|
2577
|
+
} catch (error) {
|
|
2578
|
+
// Non-fatal: the rest of the snapshot is still valid. The crossRepo block
|
|
2579
|
+
// reports `unattributable` rather than a misleading zero.
|
|
2580
|
+
process.stderr.write(`FAILED — crossRepo will report unattributable\n`)
|
|
2581
|
+
failures.push({ repo: `${templateRepo.slug} (join)`, error: String(error).split('\n')[0] })
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
|
|
2585
|
+
// One index across every collected repo, not per-repo: an instance PR routinely
|
|
2586
|
+
// closes a template issue, and a per-repo map would report every one of those
|
|
2587
|
+
// as unresolved — losing exactly the cross-repo distribution cases this metric
|
|
2588
|
+
// is most useful for.
|
|
2589
|
+
/** @type {Map<string, string>} */
|
|
2590
|
+
const issueOpenedAt = new Map()
|
|
2591
|
+
for (const repo of targets) {
|
|
2592
|
+
for (const issue of raw[repo.slug]?.issues ?? []) {
|
|
2593
|
+
issueOpenedAt.set(`${repo.slug}#${issue.number}`, issue.createdAt)
|
|
2594
|
+
}
|
|
2595
|
+
}
|
|
2596
|
+
if (templateRepo && !templateIsTarget) {
|
|
2597
|
+
for (const issue of templateJoinData.issues) {
|
|
2598
|
+
issueOpenedAt.set(`${templateRepo.slug}#${issue.number}`, issue.createdAt)
|
|
2599
|
+
}
|
|
2600
|
+
}
|
|
2601
|
+
|
|
2602
|
+
// Template PR -> the issues it closed, built once. An instance upgrade PR
|
|
2603
|
+
// names template PR numbers; this is what turns those into issues, and hence
|
|
2604
|
+
// into a start time. Sourced from `raw` when the template is a target, and
|
|
2605
|
+
// from the supplementary join fetch above when `--repo` narrowed it out —
|
|
2606
|
+
// without that fallback the map is empty and the join silently resolves
|
|
2607
|
+
// nothing (see `crossRepoTimeToFeature`'s `unattributable`).
|
|
2608
|
+
const templateClosingIssues = indexClosingIssues(
|
|
2609
|
+
templateRepo?.slug ?? '',
|
|
2610
|
+
raw[templateRepo?.slug ?? '']?.prs ?? templateJoinData.prs,
|
|
2611
|
+
)
|
|
2612
|
+
|
|
2613
|
+
/** @type {Record<string, any>} */
|
|
2614
|
+
const windows = {}
|
|
2615
|
+
for (const days of args.windows) {
|
|
2616
|
+
const since = new Date(Date.now() - days * 864e5).toISOString()
|
|
2617
|
+
/** @type {Record<string, any>} */
|
|
2618
|
+
const repos = {}
|
|
2619
|
+
for (const repo of targets) {
|
|
2620
|
+
repos[repo.slug] = raw[repo.slug]
|
|
2621
|
+
? summariseRepo(repo, filterToWindow(raw[repo.slug], since), issueOpenedAt, templateClosingIssues)
|
|
2622
|
+
: { error: 'unmeasured' }
|
|
2623
|
+
}
|
|
2624
|
+
// Amplification is estate-level by nature — it is the *cross-repo* repeat of
|
|
2625
|
+
// one subject — so it is computed here from every repo's commits rather than
|
|
2626
|
+
// inside summariseRepo, which can only ever see one repo.
|
|
2627
|
+
/** @type {Record<string, Array<{subject: string}>>} */
|
|
2628
|
+
const commitsByRepo = {}
|
|
2629
|
+
for (const repo of targets) {
|
|
2630
|
+
const windowed = raw[repo.slug] ? filterToWindow(raw[repo.slug], since) : null
|
|
2631
|
+
if (windowed?.rework) commitsByRepo[repo.slug] = windowed.rework.commits
|
|
2632
|
+
}
|
|
2633
|
+
windows[days] = {
|
|
2634
|
+
since,
|
|
2635
|
+
repos,
|
|
2636
|
+
estate: summariseEstate(repos),
|
|
2637
|
+
amplification: summariseAmplification(commitsByRepo),
|
|
2638
|
+
}
|
|
2639
|
+
}
|
|
2640
|
+
|
|
2641
|
+
// The independent baseline (#835): the long window with the rate window cut
|
|
2642
|
+
// out of it, so "vs baseline" compares two disjoint sets of merges. Keyed by
|
|
2643
|
+
// name rather than a day count because it is a *span*, not a lookback, and
|
|
2644
|
+
// reading it as one would put its start date 83 days ago instead of 90.
|
|
2645
|
+
// Computed once, above, because the gate lookback is derived from it too.
|
|
2646
|
+
if (prior) {
|
|
2647
|
+
/** @type {Record<string, any>} */
|
|
2648
|
+
const repos = {}
|
|
2649
|
+
for (const repo of targets) {
|
|
2650
|
+
repos[repo.slug] = raw[repo.slug]
|
|
2651
|
+
? summariseRepo(
|
|
2652
|
+
repo,
|
|
2653
|
+
filterToWindow(raw[repo.slug], prior.since, prior.until),
|
|
2654
|
+
issueOpenedAt,
|
|
2655
|
+
templateClosingIssues,
|
|
2656
|
+
)
|
|
2657
|
+
: { error: 'unmeasured' }
|
|
2658
|
+
}
|
|
2659
|
+
windows.prior = { ...prior, repos, estate: summariseEstate(repos) }
|
|
2660
|
+
}
|
|
2661
|
+
|
|
2662
|
+
|
|
2663
|
+
|
|
2664
|
+
const snapshot = {
|
|
2665
|
+
schema: SCHEMA_VERSION,
|
|
2666
|
+
collectedAt: new Date().toISOString(),
|
|
2667
|
+
windowDays: args.windows,
|
|
2668
|
+
windows,
|
|
2669
|
+
// Point-in-time, deliberately outside `windows` — see summariseFailOpenBacklog.
|
|
2670
|
+
failOpenBacklog: summariseFailOpenBacklog(args.corpus),
|
|
2671
|
+
unmeasured: failures,
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
mkdirSync(args.out, { recursive: true })
|
|
2675
|
+
const file = join(args.out, `${new Date().toISOString().slice(0, 10)}.json`)
|
|
2676
|
+
writeFileSync(file, `${JSON.stringify(snapshot, null, 2)}\n`)
|
|
2677
|
+
process.stderr.write(`\nwrote ${file}\n`)
|
|
2678
|
+
}
|
|
2679
|
+
|
|
2680
|
+
if (process.argv[1] && process.argv[1].endsWith('practices-metrics.mjs')) {
|
|
2681
|
+
main()
|
|
2682
|
+
}
|