@futurelastic/colab-handbook 1.13.0 → 1.14.0-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/README.vi.md +2 -2
- package/audit/audit.mjs +218 -53
- package/package.json +1 -1
- package/templates/README.md +18 -3
- package/templates/ci-laravel.yml +360 -40
- package/templates/ci-node.yml +491 -35
- package/templates/ci-python.yml +301 -32
- package/templates/deploy-container-run.mjs +17 -3
- package/templates/deploy-xserver.yml +12 -0
- package/templates/instruction-files.yml +16 -0
- package/templates/pre-commit-instruction-files +55 -0
- package/templates/release-auto.yml +16 -3
- package/tools/colab +884 -98
- package/tools/lib/adopt.js +1 -1
- package/tools/lib/audit-cache.js +277 -0
- package/tools/lib/batch-stats.js +66 -43
- package/tools/lib/ci-cure.js +10 -1
- package/tools/lib/claim-comments.js +3 -2
- package/tools/lib/codec/decision.d.ts +7 -0
- package/tools/lib/codec/decision.js +54 -0
- package/tools/lib/codec/hold.d.ts +8 -1
- package/tools/lib/codec/hold.js +42 -8
- package/tools/lib/codec/release.js +6 -3
- package/tools/lib/codec/samples.json +45 -0
- package/tools/lib/config-set.js +4 -3
- package/tools/lib/cure-diff.js +27 -1
- package/tools/lib/decision-record.js +42 -0
- package/tools/lib/docs-only.js +47 -0
- package/tools/lib/git.js +57 -8
- package/tools/lib/grant-policy.js +133 -0
- package/tools/lib/instruction-file.js +75 -0
- package/tools/lib/instruction-gate.js +179 -0
- package/tools/lib/landed-prs.js +58 -0
- package/tools/lib/migration-grant.js +29 -5
- package/tools/lib/owner-policy.js +110 -0
- package/tools/lib/release-cut.js +127 -10
- package/tools/lib/release-finalize.js +66 -5
- package/tools/lib/ship-batch.js +188 -14
- package/tools/lib/ship-handoff.js +212 -0
- package/tools/lib/stamp.js +6 -2
- package/tools/lib/template-trunk.js +80 -0
- package/tools/lib/tree-green.js +21 -4
- package/tools/lib/workflow-triggers.js +57 -0
package/tools/lib/adopt.js
CHANGED
|
@@ -397,7 +397,7 @@ function remainingSteps(ctx = {}) {
|
|
|
397
397
|
{ n: 4, text: 'Optional: add the tier topic to the GitHub repo — `gh repo edit --add-topic tier-<b|c|a>`, step 4. Needs repo ADMIN (push access alone gets `HTTP 404 …/topics`); nothing in colab or the audit reads it, so skip it when you are not one (#522)' },
|
|
398
398
|
);
|
|
399
399
|
if (ctx.fork) {
|
|
400
|
-
steps.push({ n: 5, text: `Fork of an upstream (${ctx.fork.source === 'flag' ? '--fork' : `remote "${ctx.fork.remote}"`}): do NOT make CLAUDE.md the thin shell — append templates/repo-CLAUDE-block.md at the END of the upstream's CLAUDE.md (create it only if upstream has none), record that append as a fork patch, leave AGENTS.md and the upstream prose untouched; CONVENTIONS.md §9 "A fork of an upstream", step 5` });
|
|
400
|
+
steps.push({ n: 5, text: `Fork of an upstream (${ctx.fork.source === 'flag' ? '--fork' : `remote "${ctx.fork.remote}"`}): do NOT make CLAUDE.md the thin shell — append templates/repo-CLAUDE-block.md at the END of the upstream's CLAUDE.md (create it only if upstream has none), record that append as a fork patch, leave AGENTS.md and the upstream prose untouched, and keep \`instruction-files: upstream\` in .github/project.yml (adopt writes it with the descriptor; add it by hand to one that predates #602) so the instruction-file check leaves the upstream's files unjudged; CONVENTIONS.md §9 "A fork of an upstream", step 5` });
|
|
401
401
|
const agentFiles = ctx.upstreamAgentFiles || [];
|
|
402
402
|
if (agentFiles.length) {
|
|
403
403
|
steps.push({ n: 5, text: `Upstream ships its own agent workflow (${listShort(agentFiles)}): leave it in place, and say in the appended block that fork work runs code-start → code-wrap → code-ship, not that workflow — §9 "A fork of an upstream", step 5` });
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* The audit's GitHub REST reads, cached across runs (#593).
|
|
4
|
+
*
|
|
5
|
+
* Why: the audit reads GitHub for every repo on every run (`repos/<r>`, `repos/<r>/labels`, and
|
|
6
|
+
* for a slug target its files, branches and tags). A host that runs it often — a dashboard
|
|
7
|
+
* rebuilding its overview — made it the largest consumer of the account's REST quota (measured
|
|
8
|
+
* ≈10,000 calls/h on one machine against a 5,000/h limit), stalling everything else that machine
|
|
9
|
+
* does with GitHub. Repo settings and label sets change rarely.
|
|
10
|
+
*
|
|
11
|
+
* How:
|
|
12
|
+
* - one JSON file per repo, keyed by API path (+ Accept), holding each PAGE's body and ETag;
|
|
13
|
+
* - within the TTL (default 600 s) an entry is served with no request at all;
|
|
14
|
+
* - past it, every page is re-asked with `If-None-Match` — a 304 costs no quota and re-arms the
|
|
15
|
+
* TTL; a 200 replaces the page;
|
|
16
|
+
* - `fresh` skips the TTL (every read goes to the server, still conditional — a 304 is the
|
|
17
|
+
* server saying "unchanged", which is as fresh as a 200);
|
|
18
|
+
* - a 404 is cached as "missing" for the TTL (it has no ETag, so re-asking it past the TTL costs one call);
|
|
19
|
+
* - on a rate limit or no response at all, an existing entry is served and labelled `stale`;
|
|
20
|
+
* any other error (404, 403, unparseable) is an error, never a stale answer — a repo that
|
|
21
|
+
* was deleted must not keep reporting its old label set.
|
|
22
|
+
*
|
|
23
|
+
* Where: `COLAB_AUDIT_CACHE_DIR`, else `<COLAB_HOME>/cache/audit` — but only when COLAB_HOME
|
|
24
|
+
* already EXISTS. The audit never creates the operator's colab home; without it the cache lives
|
|
25
|
+
* in memory for one run (each read still made at most once per run).
|
|
26
|
+
*
|
|
27
|
+
* Partitioned by WHO answers: the `gh` binary's resolved path, GH_HOST and a hash of any token in
|
|
28
|
+
* the environment. Two tokens can see different repos, and a test's fake `gh` must never read —
|
|
29
|
+
* or write — the operator's real entries.
|
|
30
|
+
*
|
|
31
|
+
* Provenance: every read is logged against the repo currently being audited (`begin()`), and
|
|
32
|
+
* `summary()` reports how many reads came from cache, were revalidated, fetched, or stale, and
|
|
33
|
+
* the age of the oldest data served — what `--json` shows per repo.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
const fs = require('fs');
|
|
37
|
+
const path = require('path');
|
|
38
|
+
const crypto = require('crypto');
|
|
39
|
+
const { spawnSync } = require('child_process');
|
|
40
|
+
const { parseHttp } = require('./ci-wait');
|
|
41
|
+
|
|
42
|
+
const DEFAULT_TTL_SECONDS = 600;
|
|
43
|
+
const PER_PAGE = 100;
|
|
44
|
+
const MAX_PAGES = 50; // 5,000 labels/tags/branches — a runaway Link chain must not loop forever
|
|
45
|
+
const PRUNE_AFTER_MS = 7 * 24 * 3600 * 1000;
|
|
46
|
+
const FORMAT = 1;
|
|
47
|
+
|
|
48
|
+
/** TTL in seconds from a flag value, else COLAB_AUDIT_CACHE_TTL, else the default. null = invalid. */
|
|
49
|
+
function resolveTtlSeconds(flag, env = process.env) {
|
|
50
|
+
const raw = flag !== undefined && flag !== null ? flag : env.COLAB_AUDIT_CACHE_TTL;
|
|
51
|
+
if (raw === undefined || raw === null || raw === '') return DEFAULT_TTL_SECONDS;
|
|
52
|
+
const s = String(raw).trim();
|
|
53
|
+
if (!/^\d+$/.test(s)) return null;
|
|
54
|
+
return Number(s);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** The cache directory, or null when there is nowhere we are allowed to write. */
|
|
58
|
+
function resolveDir(env = process.env, colabHome) {
|
|
59
|
+
if (env.COLAB_AUDIT_CACHE_DIR) return path.resolve(env.COLAB_AUDIT_CACHE_DIR);
|
|
60
|
+
if (colabHome && fs.existsSync(colabHome)) return path.join(colabHome, 'cache', 'audit');
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** First executable `gh` on PATH, resolved — the transport half of the partition key. */
|
|
65
|
+
function ghOnPath(env = process.env) {
|
|
66
|
+
for (const d of String(env.PATH || '').split(path.delimiter)) {
|
|
67
|
+
if (!d) continue;
|
|
68
|
+
const p = path.join(d, 'gh');
|
|
69
|
+
try {
|
|
70
|
+
fs.accessSync(p, fs.constants.X_OK);
|
|
71
|
+
return fs.realpathSync(p);
|
|
72
|
+
} catch (_) { /* next */ }
|
|
73
|
+
}
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function partitionKey(env = process.env) {
|
|
78
|
+
const token = env.GH_TOKEN || env.GITHUB_TOKEN || env.GH_ENTERPRISE_TOKEN || '';
|
|
79
|
+
const parts = [ghOnPath(env) || 'no-gh', env.GH_HOST || 'github.com',
|
|
80
|
+
token ? crypto.createHash('sha256').update(token).digest('hex') : 'ambient'];
|
|
81
|
+
return crypto.createHash('sha256').update(parts.join('\n')).digest('hex').slice(0, 16);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* One GET through `gh api -i`. The path stays the first argument after `api` (callers and fakes
|
|
86
|
+
* match on it). Returns { kind: 'ok'|'not-modified'|'rate-limited'|'unreachable'|'error',
|
|
87
|
+
* etag, next, body }.
|
|
88
|
+
*/
|
|
89
|
+
function ghFetch(apiPath, { etag, accept } = {}) {
|
|
90
|
+
const args = ['api', apiPath, '-i'];
|
|
91
|
+
if (etag) args.push('-H', `If-None-Match: ${etag}`);
|
|
92
|
+
if (accept) args.push('-H', `Accept: ${accept}`);
|
|
93
|
+
const r = spawnSync('gh', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 64 * 1024 * 1024 });
|
|
94
|
+
if (r.error) return { kind: 'unreachable' };
|
|
95
|
+
return classify(r.stdout, r.stderr, r.status);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function classify(stdout, stderr, exitCode) {
|
|
99
|
+
const http = parseHttp(stdout);
|
|
100
|
+
const status = http.status;
|
|
101
|
+
const h = http.headers || {};
|
|
102
|
+
const rateText = /rate limit/i.test(String(stderr || '') + (status && status >= 400 ? http.body : ''));
|
|
103
|
+
if (status === 429 || (status === 403 && (h['x-ratelimit-remaining'] === '0' || rateText)) || (status === null && rateText)) {
|
|
104
|
+
return { kind: 'rate-limited' };
|
|
105
|
+
}
|
|
106
|
+
if (status === 304) return { kind: 'not-modified', etag: h.etag || null };
|
|
107
|
+
const next = /rel="next"/.test(h.link || '');
|
|
108
|
+
if (status !== null && status >= 200 && status < 300) return { kind: 'ok', etag: h.etag || null, next, linkKnown: true, body: http.body };
|
|
109
|
+
// No status line: a `gh` that does not print headers. Exit 0 is an answer without an ETag.
|
|
110
|
+
if (status === null && exitCode === 0) return { kind: 'ok', etag: null, next: false, linkKnown: false, body: String(stdout || '') };
|
|
111
|
+
if (status === null) return { kind: 'unreachable' };
|
|
112
|
+
return { kind: 'error', status };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function withPage(apiPath, page) {
|
|
116
|
+
return `${apiPath}${apiPath.includes('?') ? '&' : '?'}per_page=${PER_PAGE}&page=${page}`;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* create({ dir, ttlSeconds, fresh, offline, now, fetch, env })
|
|
121
|
+
* get(slug, apiPath, { accept, paginate }) → { status: 'ok', text } | { status: 'ok', items }
|
|
122
|
+
* | { status: 'offline' } | { status: 'error' }
|
|
123
|
+
* begin() — start logging reads for the next repo; summary() — that repo's provenance.
|
|
124
|
+
*/
|
|
125
|
+
function create(opts = {}) {
|
|
126
|
+
const env = opts.env || process.env;
|
|
127
|
+
const now = opts.now || Date.now;
|
|
128
|
+
const fetch = opts.fetch || ghFetch;
|
|
129
|
+
const ttlMs = (opts.ttlSeconds === undefined ? DEFAULT_TTL_SECONDS : opts.ttlSeconds) * 1000;
|
|
130
|
+
const fresh = !!opts.fresh;
|
|
131
|
+
const offline = !!opts.offline;
|
|
132
|
+
const dir = opts.dir ? path.join(opts.dir, opts.partition || partitionKey(env)) : null;
|
|
133
|
+
|
|
134
|
+
const files = new Map(); // slug → { entries }
|
|
135
|
+
const doneThisRun = new Set(); // slug|key validated or fetched in this process
|
|
136
|
+
let log = [];
|
|
137
|
+
let pruned = false;
|
|
138
|
+
|
|
139
|
+
const fileFor = (slug) => (dir ? path.join(dir, `${slug.replace(/[^A-Za-z0-9._-]/g, '__')}.json`) : null);
|
|
140
|
+
|
|
141
|
+
function load(slug) {
|
|
142
|
+
if (files.has(slug)) return files.get(slug);
|
|
143
|
+
let data = { format: FORMAT, slug, entries: {} };
|
|
144
|
+
const f = fileFor(slug);
|
|
145
|
+
if (f) {
|
|
146
|
+
try {
|
|
147
|
+
const parsed = JSON.parse(fs.readFileSync(f, 'utf8'));
|
|
148
|
+
if (parsed && parsed.format === FORMAT && parsed.entries && typeof parsed.entries === 'object') data = parsed;
|
|
149
|
+
} catch (_) { /* absent or corrupt: start empty, the next write repairs it */ }
|
|
150
|
+
}
|
|
151
|
+
files.set(slug, data);
|
|
152
|
+
return data;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function save(slug) {
|
|
156
|
+
const f = fileFor(slug);
|
|
157
|
+
if (!f) return;
|
|
158
|
+
try {
|
|
159
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
160
|
+
const tmp = `${f}.${process.pid}.tmp`;
|
|
161
|
+
fs.writeFileSync(tmp, JSON.stringify(files.get(slug)));
|
|
162
|
+
fs.renameSync(tmp, f); // atomic: a concurrent audit reads the old file or the new one
|
|
163
|
+
} catch (_) { /* a cache that cannot be written is a slower run, never a failed one */ }
|
|
164
|
+
if (!pruned) { pruned = true; prune(); }
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function prune() {
|
|
168
|
+
const root = path.dirname(dir);
|
|
169
|
+
const cutoff = now() - PRUNE_AFTER_MS;
|
|
170
|
+
let parts;
|
|
171
|
+
try { parts = fs.readdirSync(root); } catch (_) { return; }
|
|
172
|
+
for (const p of parts) {
|
|
173
|
+
const d = path.join(root, p);
|
|
174
|
+
let names;
|
|
175
|
+
try { names = fs.readdirSync(d); } catch (_) { continue; }
|
|
176
|
+
for (const n of names) {
|
|
177
|
+
const fp = path.join(d, n);
|
|
178
|
+
try { if (fs.statSync(fp).mtimeMs < cutoff) fs.unlinkSync(fp); } catch (_) { /* raced */ }
|
|
179
|
+
}
|
|
180
|
+
try { if (!fs.readdirSync(d).length) fs.rmdirSync(d); } catch (_) { /* not empty / raced */ }
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function record(source, ageMs) { log.push({ source, ageSeconds: Math.max(0, Math.round(ageMs / 1000)) }); }
|
|
185
|
+
|
|
186
|
+
function answer(entry, paginate) {
|
|
187
|
+
if (entry.missing) return { status: 'error' };
|
|
188
|
+
if (!paginate) return { status: 'ok', text: entry.pages[0].body };
|
|
189
|
+
const items = [];
|
|
190
|
+
for (const p of entry.pages) {
|
|
191
|
+
let arr;
|
|
192
|
+
try { arr = JSON.parse(p.body); } catch (_) { return { status: 'error' }; }
|
|
193
|
+
if (!Array.isArray(arr)) return { status: 'error' };
|
|
194
|
+
items.push(...arr);
|
|
195
|
+
}
|
|
196
|
+
return { status: 'ok', items };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function get(slug, apiPath, { accept = null, paginate = false } = {}) {
|
|
200
|
+
if (offline) { record('offline', 0); return { status: 'offline' }; }
|
|
201
|
+
const key = accept ? `${apiPath}|${accept}` : apiPath;
|
|
202
|
+
const data = load(slug);
|
|
203
|
+
const entry = data.entries[key];
|
|
204
|
+
const runKey = `${slug}|${key}`;
|
|
205
|
+
|
|
206
|
+
if (entry && (doneThisRun.has(runKey) || (!fresh && now() - entry.at < ttlMs))) {
|
|
207
|
+
record(doneThisRun.has(runKey) ? 'run' : 'cache', now() - entry.at);
|
|
208
|
+
return answer(entry, paginate);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const pages = [];
|
|
212
|
+
let allNotModified = true;
|
|
213
|
+
for (let i = 0; i < (paginate ? MAX_PAGES : 1); i++) {
|
|
214
|
+
const cached = entry && !entry.missing && entry.pages[i];
|
|
215
|
+
const r = fetch(paginate ? withPage(apiPath, i + 1) : apiPath, { etag: cached ? cached.etag : null, accept });
|
|
216
|
+
let page;
|
|
217
|
+
if (r.kind === 'not-modified' && cached) page = cached;
|
|
218
|
+
else if (r.kind === 'ok') {
|
|
219
|
+
allNotModified = false;
|
|
220
|
+
let next = !!r.next;
|
|
221
|
+
if (paginate && !r.linkKnown) { // no headers to follow: a full page means there may be more
|
|
222
|
+
try { const a = JSON.parse(r.body); next = Array.isArray(a) && a.length >= PER_PAGE; } catch (_) { next = false; }
|
|
223
|
+
}
|
|
224
|
+
page = { etag: r.etag || null, next, body: r.body };
|
|
225
|
+
} else {
|
|
226
|
+
if (entry && (r.kind === 'rate-limited' || r.kind === 'unreachable')) {
|
|
227
|
+
record('stale', now() - entry.at);
|
|
228
|
+
return { ...answer(entry, paginate), stale: true };
|
|
229
|
+
}
|
|
230
|
+
if (r.kind === 'error' && r.status === 404 && i === 0) {
|
|
231
|
+
// An absent file is an answer too, and the audit asks for several on every run (a
|
|
232
|
+
// runbook, a tool, a workflow dir). Cached for the TTL like any other; past it, asked
|
|
233
|
+
// again — a 404 carries no ETag, so that one costs a call.
|
|
234
|
+
data.entries[key] = { at: now(), missing: true, pages: [] };
|
|
235
|
+
doneThisRun.add(runKey);
|
|
236
|
+
save(slug);
|
|
237
|
+
record('fetched', 0);
|
|
238
|
+
return { status: 'error' };
|
|
239
|
+
}
|
|
240
|
+
record('error', 0);
|
|
241
|
+
return { status: 'error' };
|
|
242
|
+
}
|
|
243
|
+
pages.push(page);
|
|
244
|
+
if (!paginate || !page.next) break;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
data.entries[key] = { at: now(), pages };
|
|
248
|
+
doneThisRun.add(runKey);
|
|
249
|
+
save(slug);
|
|
250
|
+
record(allNotModified ? 'revalidated' : 'fetched', 0);
|
|
251
|
+
return answer(data.entries[key], paginate);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function begin() { log = []; }
|
|
255
|
+
|
|
256
|
+
/** This repo's remote provenance since begin(); null when it made no GitHub read at all. */
|
|
257
|
+
function summary() {
|
|
258
|
+
if (offline) return { mode: 'offline' };
|
|
259
|
+
if (!log.length) return null;
|
|
260
|
+
const count = (s) => log.filter((e) => e.source === s).length;
|
|
261
|
+
const served = log.filter((e) => e.source !== 'error');
|
|
262
|
+
return {
|
|
263
|
+
mode: 'live',
|
|
264
|
+
reads: log.length,
|
|
265
|
+
cache: count('cache') + count('run'),
|
|
266
|
+
revalidated: count('revalidated'),
|
|
267
|
+
fetched: count('fetched'),
|
|
268
|
+
stale: count('stale'),
|
|
269
|
+
errors: count('error'),
|
|
270
|
+
oldestAgeSeconds: served.length ? Math.max(...served.map((e) => e.ageSeconds)) : null,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
return { get, begin, summary, dir, ttlSeconds: ttlMs / 1000, fresh, offline };
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
module.exports = { create, ghFetch, classify, resolveTtlSeconds, resolveDir, partitionKey, DEFAULT_TTL_SECONDS, PER_PAGE };
|
package/tools/lib/batch-stats.js
CHANGED
|
@@ -20,12 +20,18 @@
|
|
|
20
20
|
* What it measures, per window:
|
|
21
21
|
* batches landed and their fill; every combined-run build, first-attempt green, green after
|
|
22
22
|
* the one re-run, red, still pending
|
|
23
|
-
* red red builds and how many of their members then landed serially
|
|
23
|
+
* red red builds and how many of their members then landed alone — serially, or as a
|
|
24
|
+
* batch of one (#580) — the fallback
|
|
24
25
|
* dropped members dropped at build, by class, and the eviction rate among attempted members
|
|
25
26
|
* serial serial landings, and MISSED PARTNERS: another candidate that landed later but was
|
|
26
27
|
* already green when this one landed alone (could have joined), or turned green
|
|
27
28
|
* within this landing's trunk-CI cycle (a partner wait of `waitSec` would have caught
|
|
28
|
-
* it — the number `ship-batch-wait` is tuned from)
|
|
29
|
+
* it — the number `ship-batch-wait` is tuned from). `serial` means landed through the
|
|
30
|
+
* SERIAL PATH (no Ship-Batch trailer), and only that
|
|
31
|
+
* alone the same partner scan over every change that LANDED ALONE: a serial landing, or a
|
|
32
|
+
* batch of one (#580). Since #562 a lone ready candidate on a ship-batch repo lands as
|
|
33
|
+
* a fill-1 batch, so on such a repo `serial` reads zero whether partners were missed or
|
|
34
|
+
* not; `alone` is the signal `ship-batch-wait` is tuned from there
|
|
29
35
|
* queueWait per landed change, green-at-head → landing
|
|
30
36
|
*/
|
|
31
37
|
|
|
@@ -175,16 +181,21 @@ function derive({ commits = [], runs = [], builds = {}, trunk = 'main', config =
|
|
|
175
181
|
const rerunGreen = buildRows.filter((b) => b.state === 'green' && b.attempt > 1).length;
|
|
176
182
|
const redBuilds = buildRows.filter((b) => b.state === 'red');
|
|
177
183
|
|
|
178
|
-
// ---- red builds → did their members then land serially? ----
|
|
184
|
+
// ---- red builds → did their members then land alone (serially, or as a batch of one)? ----
|
|
179
185
|
const serialChanges = changes.filter((c) => c.landing.kind === 'serial');
|
|
180
|
-
|
|
186
|
+
// #580: a fill-1 batch landed alone exactly as a serial landing did — only its path differs.
|
|
187
|
+
const isBatchOfOne = (c) => c.landing.kind === 'batch' && c.landing.members.length === 1;
|
|
188
|
+
const aloneChanges = changes.filter((c) => c.landing.kind === 'serial' || isBatchOfOne(c));
|
|
189
|
+
let redMembers = 0, relanded = 0, relandedAlone = 0, redMembersUnread = 0;
|
|
181
190
|
for (const b of redBuilds) {
|
|
182
191
|
const members = b.messages && b.messages.length ? shipBatch.parseMemberTrailers(b.messages) : [];
|
|
183
192
|
if (!members.length) { redMembersUnread++; continue; }
|
|
184
193
|
for (const m of members) {
|
|
185
194
|
redMembers++;
|
|
186
195
|
const nums = branchName.branchIssueNumbers(m.branch);
|
|
187
|
-
|
|
196
|
+
const isMember = (c) => c.atMs >= b.createdMs && (c.branch === m.branch || c.issues.some((n) => nums.includes(n)));
|
|
197
|
+
if (serialChanges.some(isMember)) relanded++;
|
|
198
|
+
if (aloneChanges.some(isMember)) relandedAlone++;
|
|
188
199
|
}
|
|
189
200
|
}
|
|
190
201
|
if (redMembersUnread) notes.push(`${redMembersUnread} red build(s) whose members could not be read (the ref is gone and its commits were not readable)`);
|
|
@@ -205,42 +216,57 @@ function derive({ commits = [], runs = [], builds = {}, trunk = 'main', config =
|
|
|
205
216
|
const byClass = {};
|
|
206
217
|
for (const d of dropped) byClass[d.cls] = (byClass[d.cls] || 0) + 1;
|
|
207
218
|
|
|
208
|
-
// ---- missed partners:
|
|
209
|
-
const pairs = [];
|
|
210
|
-
let withMissed = 0, withNear = 0, cyclesUnread = 0, overlapped = 0;
|
|
219
|
+
// ---- missed partners: landings that went alone while a partner was (nearly) ready ----
|
|
211
220
|
// A partner is judged by ITS state at this landing's moment, never by the head it finally landed
|
|
212
221
|
// with: a serial ship syncs the other branches, so their final heads all post-date it. The cycle
|
|
213
222
|
// ends when this landing's trunk run finishes — or at the next landing, whichever is first (a
|
|
214
223
|
// re-run moves a run's updated_at hours later, and the next landing moves trunk anyway).
|
|
215
224
|
const landingTimes = landings.map((l) => l.atMs).sort((a, b) => a - b);
|
|
216
|
-
|
|
217
|
-
const
|
|
218
|
-
let
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
const
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
225
|
+
const scanPartners = (lone, tagVia) => {
|
|
226
|
+
const pairs = [];
|
|
227
|
+
let withMissed = 0, withNear = 0, cyclesUnread = 0, overlapped = 0;
|
|
228
|
+
for (const c of lone) {
|
|
229
|
+
const next = landingTimes.find((t) => t > c.atMs);
|
|
230
|
+
let end = cycleEnd(c.landing.sha);
|
|
231
|
+
if (end === null) cyclesUnread++;
|
|
232
|
+
else if (next !== undefined) end = Math.min(end, next);
|
|
233
|
+
let missed = false, near = false;
|
|
234
|
+
const label = c.branch || c.landing.sha.slice(0, 7);
|
|
235
|
+
const via = tagVia ? { via: c.landing.kind === 'serial' ? 'serial' : 'batch-of-one' } : {};
|
|
236
|
+
for (const o of changes) {
|
|
237
|
+
if (o === c || o.landing === c.landing || o.atMs <= c.atMs || !o.branch || o.branch === c.branch) continue;
|
|
238
|
+
const gNow = greenAt(runsByBranch, o.branch, c.atMs);
|
|
239
|
+
if (gNow !== null) {
|
|
240
|
+
missed = true;
|
|
241
|
+
pairs.push({ kind: 'missed', landed: label, partner: o.branch, ...via });
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
if (end === null) continue;
|
|
245
|
+
const gEnd = greenAt(runsByBranch, o.branch, end);
|
|
246
|
+
if (gEnd !== null && gEnd > c.atMs) {
|
|
247
|
+
near = true;
|
|
248
|
+
const from = c.greenMs !== null ? c.greenMs : c.atMs;
|
|
249
|
+
pairs.push({ kind: 'near', landed: label, partner: o.branch, waitSec: Math.round((gEnd - from) / 1000), ...via });
|
|
250
|
+
}
|
|
237
251
|
}
|
|
252
|
+
if (missed) withMissed++;
|
|
253
|
+
else if (near) withNear++;
|
|
254
|
+
if (end !== null && (missed || near)) overlapped++;
|
|
238
255
|
}
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
256
|
+
return {
|
|
257
|
+
landings: lone.length, withMissedPartner: withMissed, withNearMiss: withNear,
|
|
258
|
+
// #556: the overlap rate's honest denominator — only a landing whose trunk-CI cycle was read
|
|
259
|
+
// could have been judged for a partner inside it; `overlapped` counts those that had one.
|
|
260
|
+
cyclesRead: lone.length - cyclesUnread, overlapped,
|
|
261
|
+
nearMissWait: summary(pairs.filter((p) => p.kind === 'near').map((p) => p.waitSec)),
|
|
262
|
+
pairs,
|
|
263
|
+
};
|
|
264
|
+
};
|
|
265
|
+
const serial = scanPartners(serialChanges, false);
|
|
266
|
+
const alone = scanPartners(aloneChanges, true);
|
|
267
|
+
// The alone set contains the serial one, so its unread count is the one note (never two).
|
|
268
|
+
const aloneUnread = alone.landings - alone.cyclesRead;
|
|
269
|
+
if (aloneUnread) notes.push(`${aloneUnread} lone landing(s) (serial or a batch of one) with no finished trunk run in the window — near misses not judged for them`);
|
|
244
270
|
const unknownGreen = changes.filter((c) => c.greenMs === null).length;
|
|
245
271
|
if (unknownGreen) notes.push(`${unknownGreen} landed change(s) with no readable green-at-head run — left out of queue wait and partner matching`);
|
|
246
272
|
|
|
@@ -259,16 +285,13 @@ function derive({ commits = [], runs = [], builds = {}, trunk = 'main', config =
|
|
|
259
285
|
pending: buildRows.length - finished.length,
|
|
260
286
|
firstAttemptGreenRate: rate(firstGreen, finished.length),
|
|
261
287
|
},
|
|
262
|
-
|
|
288
|
+
// relandedSerially: through the serial path only; relandedAlone: serially OR as a batch of one (#580).
|
|
289
|
+
red: { builds: redBuilds.length, members: redMembers, relandedSerially: relanded, relandedAlone },
|
|
263
290
|
dropped: { count: dropped.length, attempted: builtMembers + dropped.length, rate: rate(dropped.length, builtMembers + dropped.length), byClass, members: dropped },
|
|
264
|
-
serial
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
cyclesRead: serialChanges.length - cyclesUnread, overlapped,
|
|
269
|
-
nearMissWait: summary(pairs.filter((p) => p.kind === 'near').map((p) => p.waitSec)),
|
|
270
|
-
pairs,
|
|
271
|
-
},
|
|
291
|
+
// `serial`: landed through the serial path. `alone`: landed alone by either path — a serial
|
|
292
|
+
// landing or a batch of one (#580); its pairs carry `via: 'serial' | 'batch-of-one'`.
|
|
293
|
+
serial,
|
|
294
|
+
alone: { ...alone, batchOfOne: aloneChanges.filter(isBatchOfOne).length },
|
|
272
295
|
queueWait,
|
|
273
296
|
notes,
|
|
274
297
|
};
|
package/tools/lib/ci-cure.js
CHANGED
|
@@ -820,6 +820,8 @@ function stackingWithProgress(stacking, priorRedJobs, jobEvidence) {
|
|
|
820
820
|
* `containsRedSha` — bool, whether the branch's tree contains the red trunk sha as an ancestor
|
|
821
821
|
* (the caller's `git merge-base --is-ancestor <redSha> <branch>`, translated to a bool — never
|
|
822
822
|
* re-derived here, this module does not touch git).
|
|
823
|
+
* `null` (#589) means the caller could NOT measure it — the ref did not resolve, or git failed —
|
|
824
|
+
* and refuses with `containmentReason` ("could not resolve …"), never as "does not contain".
|
|
823
825
|
*
|
|
824
826
|
* `evidence` — `{ok, sha}` for a completed, successful CI run measured on the branch's CURRENT
|
|
825
827
|
* head, or `null` when the caller's own read of it FAILED. Passing `evidence: null` means "the
|
|
@@ -886,7 +888,14 @@ function stackingWithProgress(stacking, priorRedJobs, jobEvidence) {
|
|
|
886
888
|
*/
|
|
887
889
|
function cureVerdict({ containsRedSha, evidence, redSha, stacking, workflowsTouched, jobEvidence,
|
|
888
890
|
manifestScriptsTouched, manifestPaths, manifestScriptsAddOnly, pythonManifestTouched, pythonManifestPaths,
|
|
889
|
-
pythonPinOnly, dryRunCapable, dispatchable, branchRuns, priorRedJobs }) {
|
|
891
|
+
pythonPinOnly, dryRunCapable, dispatchable, branchRuns, priorRedJobs, containmentReason }) {
|
|
892
|
+
// #589: `null` = containment could not be MEASURED (an unresolvable ref, a failed git read) — a
|
|
893
|
+
// different fact from "not an ancestor", whose remedy (rebase onto the red) would be wrong advice.
|
|
894
|
+
if (containsRedSha === null) {
|
|
895
|
+
return { ok: false,
|
|
896
|
+
reason: `${containmentReason || `could not resolve the branch or trunk's red head \`${redSha}\``} — ` +
|
|
897
|
+
'containment was not measured, so this cannot prove a cure (an unmeasured read is never a cure)' };
|
|
898
|
+
}
|
|
890
899
|
if (!containsRedSha) {
|
|
891
900
|
return { ok: false,
|
|
892
901
|
reason: `branch does not contain trunk's current red head \`${redSha}\` as an ancestor — ` +
|
|
@@ -55,7 +55,7 @@ function releaseComments(sorted) {
|
|
|
55
55
|
|
|
56
56
|
/**
|
|
57
57
|
* Live claim comments on an issue. Returns [{login, host, machine, session, sessionName, at,
|
|
58
|
-
* identity}], `at` = the comment's real GitHub createdAt (authoritative, sub-second). `identity`
|
|
58
|
+
* branch, worktree, identity}], `at` = the comment's real GitHub createdAt (authoritative, sub-second). `identity`
|
|
59
59
|
* is built from `comps` (claim-identity.js `components()`, #267); the raw fields are returned too
|
|
60
60
|
* for callers that need `sameClaimant`'s degrade-on-missing comparison.
|
|
61
61
|
*
|
|
@@ -91,7 +91,8 @@ function liveClaimComments(comments, comps) {
|
|
|
91
91
|
const { host, session, sessionName } = claim;
|
|
92
92
|
const machineTok = claim.machine; // #327 — '' on a legacy comment: compared by host
|
|
93
93
|
const at = c.createdAt;
|
|
94
|
-
|
|
94
|
+
// #578: branch/worktree ride along so a reader can ask WHICH branch a live claim holds.
|
|
95
|
+
const entry = { login, host, machine: machineTok, session, sessionName, at, branch: claim.branch, worktree: claim.worktree };
|
|
95
96
|
const cancelled = releases.some((r) => r.at > at && (r.kind === 'release'
|
|
96
97
|
|| (r.login === login && !isNamedWinner(entry, r.winner))));
|
|
97
98
|
if (!cancelled) live.push({ ...entry, identity: claimIdentity.identityString({ login, host, session }, comps) });
|
|
@@ -27,3 +27,10 @@ export const OPTIONS_RE: RegExp;
|
|
|
27
27
|
export const MOCKUP_RE: RegExp;
|
|
28
28
|
export function encodeMockup(url: string): string;
|
|
29
29
|
export function mockupUrls(body: unknown): string[];
|
|
30
|
+
|
|
31
|
+
/** An ask record a dashboard that tracks asks wrote (#582) — read-only to colab. */
|
|
32
|
+
export interface AskRecord { ns: string; id: string; version: number; index: number; }
|
|
33
|
+
/** An answer record naming an ask by id (#582). */
|
|
34
|
+
export interface AnswerRecord { ns: string; ask: string; version: number | null; index: number; }
|
|
35
|
+
export function askRecords(text: unknown): AskRecord[];
|
|
36
|
+
export function answerRecords(text: unknown): AnswerRecord[];
|
|
@@ -74,8 +74,62 @@ function mockupUrls(body) {
|
|
|
74
74
|
return out;
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
/**
|
|
78
|
+
* ASK / ANSWER RECORDS (#582) — the structured question a dashboard that tracks asks writes onto an
|
|
79
|
+
* issue, and the answer that settles it. READ-ONLY here: colab never writes either, it only needs to
|
|
80
|
+
* know which asks are still open so `colab decision --record` can name the one a ruling settles.
|
|
81
|
+
*
|
|
82
|
+
* Wire shape, namespaced by whatever tool wrote it (the namespace is not interpreted — any tool
|
|
83
|
+
* writing this shape is read the same way):
|
|
84
|
+
* `<!-- <tool>:ask id=<id> v=<n> [key=value …] -->` + the question below it
|
|
85
|
+
* `<!-- <tool>:answer ask=<id> [v=<n>] [key=value …] -->` (+ optional note)
|
|
86
|
+
* and the same heads as a bare line form `<tool>:ask id=…` at the start of a line. Attribute values
|
|
87
|
+
* are percent-encoded and space-separated. Other kinds naming an ask (a wait, a delegation) are not
|
|
88
|
+
* answers and are not read.
|
|
89
|
+
*/
|
|
90
|
+
const RECORD_NS = '[a-z][a-z0-9_-]*';
|
|
91
|
+
function recordHeads(text, kind) {
|
|
92
|
+
if (typeof text !== 'string' || text === '') return [];
|
|
93
|
+
const heads = [];
|
|
94
|
+
const markerRe = new RegExp(`<!--\\s*(${RECORD_NS}):${kind}\\s+([^>]*?)\\s*-->`, 'g');
|
|
95
|
+
const lineRe = new RegExp(`^(${RECORD_NS}):${kind}\\s+([^\\n]*)$`, 'gm');
|
|
96
|
+
for (const re of [markerRe, lineRe]) {
|
|
97
|
+
let m;
|
|
98
|
+
while ((m = re.exec(text))) {
|
|
99
|
+
const attrs = parseRecordAttrs(m[2]);
|
|
100
|
+
if (attrs) heads.push({ index: m.index, ns: m[1], attrs });
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return heads.sort((a, b) => a.index - b.index);
|
|
104
|
+
}
|
|
105
|
+
/** `k=v k2=v2` → `{k: v, k2: v2}` (values percent-decoded), or null when any token is not `k=v`. */
|
|
106
|
+
function parseRecordAttrs(raw) {
|
|
107
|
+
const out = {};
|
|
108
|
+
const toks = String(raw).trim().split(/\s+/).filter(Boolean);
|
|
109
|
+
if (toks.length === 0) return null;
|
|
110
|
+
for (const t of toks) {
|
|
111
|
+
const m = /^([a-z][a-z0-9_-]*)=(\S*)$/i.exec(t);
|
|
112
|
+
if (!m) return null;
|
|
113
|
+
try { out[m[1]] = decodeURIComponent(m[2]); } catch (_) { out[m[1]] = m[2]; }
|
|
114
|
+
}
|
|
115
|
+
return out;
|
|
116
|
+
}
|
|
117
|
+
/** Every ask record in `text`, in order: `{ns, id, version, index}`. Requires `id` and an integer `v >= 1`. */
|
|
118
|
+
function askRecords(text) {
|
|
119
|
+
return recordHeads(text, 'ask')
|
|
120
|
+
.filter((h) => h.attrs.id && /^\d+$/.test(h.attrs.v || '') && Number(h.attrs.v) >= 1)
|
|
121
|
+
.map((h) => ({ ns: h.ns, id: h.attrs.id, version: Number(h.attrs.v), index: h.index }));
|
|
122
|
+
}
|
|
123
|
+
/** Every answer record in `text`, in order: `{ns, ask, version, index}` (`version` null when absent). */
|
|
124
|
+
function answerRecords(text) {
|
|
125
|
+
return recordHeads(text, 'answer')
|
|
126
|
+
.filter((h) => h.attrs.ask)
|
|
127
|
+
.map((h) => ({ ns: h.ns, ask: h.attrs.ask, version: /^\d+$/.test(h.attrs.v || '') ? Number(h.attrs.v) : null, index: h.index }));
|
|
128
|
+
}
|
|
129
|
+
|
|
77
130
|
module.exports = {
|
|
78
131
|
DECISION_MARK, REOPEN_MARK, DECISION_RE, REOPEN_RE,
|
|
79
132
|
encodeDecision, decodeDecision, encodeReopen, decodeReopen,
|
|
80
133
|
OPTIONS_RE, MOCKUP_RE, encodeMockup, mockupUrls,
|
|
134
|
+
askRecords, answerRecords,
|
|
81
135
|
};
|
|
@@ -14,13 +14,20 @@ export function parseQualifiedIssueRef(arg: unknown): { slug: string; number: nu
|
|
|
14
14
|
|
|
15
15
|
export const HOLD_RE: RegExp;
|
|
16
16
|
export const BECAUSE_RE: RegExp;
|
|
17
|
+
export type HoldShape = 'ask' | 'task' | 'wait';
|
|
18
|
+
export const HOLD_SHAPES: ReadonlyArray<HoldShape>;
|
|
17
19
|
export interface DecodedHold {
|
|
18
20
|
label: string;
|
|
19
21
|
owner: string;
|
|
22
|
+
/** The declared `shape:` as written (#569) — present only when the line carries one. */
|
|
23
|
+
shape?: string;
|
|
20
24
|
/** The `wake:` field as written — `parseWakeLine` decides whether it names a wake. */
|
|
21
25
|
wake: string;
|
|
22
26
|
/** The `Because:` line's text, or null when the comment is the `Hold:` line alone. */
|
|
23
27
|
because: string | null;
|
|
24
28
|
}
|
|
25
|
-
export function encodeHold(h?: { label: string; owner: string; wake: string | string[]; because?: string | null }): string;
|
|
29
|
+
export function encodeHold(h?: { label: string; owner: string; shape?: HoldShape | null; wake: string | string[]; because?: string | null }): string;
|
|
26
30
|
export function decodeHold(body: string | null | undefined): DecodedHold | null;
|
|
31
|
+
/** Declared shape when in the closed set, else inferred from the wake (`ruling` → ask, else wait). */
|
|
32
|
+
export function holdShape(decoded: Partial<DecodedHold> | null | undefined):
|
|
33
|
+
{ shape: HoldShape; declared: boolean; invalid: string | null };
|