@futurelastic/colab-handbook 1.13.0-rc.28 → 1.13.0-rc.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.vi.md +1 -1
- package/audit/audit.mjs +65 -1
- package/package.json +1 -1
- package/templates/ci-laravel.yml +7 -4
- package/templates/ci-node.yml +7 -4
- package/templates/ci-python.yml +7 -4
- package/tools/colab +10 -3
- package/tools/lib/batch-history.js +104 -0
- package/tools/lib/batch-stats.js +5 -1
- package/tools/lib/thresholds.js +20 -8
package/README.md
CHANGED
|
@@ -244,7 +244,7 @@ migration hold with or without it.
|
|
|
244
244
|
| [`project.schema.md`](project.schema.md) | Field reference for `.github/project.yml`. |
|
|
245
245
|
| [`templates/`](templates/) | **Copy-and-own** starting points: CI, release, git hooks (a secret scan and an identity scan), and the `CLAUDE.md` block for adopting repos. Nothing is called remotely — copy it, edit it, own it. |
|
|
246
246
|
| [`tools/`](tools/) | `colab` — a small CLI (optional): adopt a repo, claim issues, allocate ports, manage worktrees, and merge a finished branch to trunk when the repo grants it. JSON state, zero dependencies. Command reference: [`tools/README.md`](tools/README.md). |
|
|
247
|
-
| [`audit/`](audit/) | An external conformance checker. Reads all your repos — every owner, including local-only ones — and reports drift in a single run. Advisory only. `--identity` also scans public repository descriptions and topics, which no git hook can see. What each check means: [`audit/README.md`](audit/README.md). |
|
|
247
|
+
| [`audit/`](audit/) | An external conformance checker. Reads all your repos — every owner, including local-only ones — and reports drift in a single run. Advisory only. `--identity` also scans public repository descriptions and topics, which no git hook can see. `--batch-history` shows each repo's measured batch landing picture, flagged only against thresholds the repo declares. What each check means: [`audit/README.md`](audit/README.md). |
|
|
248
248
|
| [`skills/`](skills/) | The session flow — see [*Your first session*](#your-first-session). Plain `SKILL.md` folders; `install.sh` links them into each coding agent you choose. |
|
|
249
249
|
| [`engines/`](engines/) | One small data file per coding agent: where its user-level skills go, how a skill is invoked, what it supports. Adding an engine is adding a file — [`engines/README.md`](engines/README.md). |
|
|
250
250
|
| [`install.sh`](install.sh) | Sets up **your machine**: skills, the `colab` CLI, the pre-commit hook, the fleet list. Idempotent; `--dry` shows everything first. |
|
package/README.vi.md
CHANGED
|
@@ -243,7 +243,7 @@ quyền cấp trước khi chạy migration vẫn đứng vững dù có nó hay
|
|
|
243
243
|
| [`project.schema.md`](project.schema.md) | Tham chiếu field của `.github/project.yml`. |
|
|
244
244
|
| [`templates/`](templates/) | Điểm khởi đầu **copy-về-là-của-bạn**: CI, release, git hook (một bản quét secret và một bản quét danh tính), và block `CLAUDE.md` cho repo adopt. Không có gì được gọi từ xa — copy, sửa, sở hữu. |
|
|
245
245
|
| [`tools/`](tools/) | `colab` — một CLI nhỏ (tùy chọn): adopt một repo, claim issue, cấp port, quản lý worktree, và merge nhánh đã xong vào trunk khi repo cho phép. State JSON, không dependency. Tham chiếu các lệnh: [`tools/README.md`](tools/README.md). |
|
|
246
|
-
| [`audit/`](audit/) | Trình kiểm tra conformance từ bên ngoài. Đọc mọi repo của bạn — mọi owner, kể cả repo local-only — và báo drift trong một lần chạy. Chỉ cảnh báo. Thêm `--identity` thì quét cả description và topic của repo public — thứ mà không git hook nào nhìn thấy được. Ý nghĩa từng check: [`audit/README.md`](audit/README.md). |
|
|
246
|
+
| [`audit/`](audit/) | Trình kiểm tra conformance từ bên ngoài. Đọc mọi repo của bạn — mọi owner, kể cả repo local-only — và báo drift trong một lần chạy. Chỉ cảnh báo. Thêm `--identity` thì quét cả description và topic của repo public — thứ mà không git hook nào nhìn thấy được. Thêm `--batch-history` thì hiện số liệu batch landing đo được của từng repo, chỉ cảnh báo theo ngưỡng repo tự khai báo. Ý nghĩa từng check: [`audit/README.md`](audit/README.md). |
|
|
247
247
|
| [`skills/`](skills/) | Luồng phiên làm việc — xem [*Phiên làm việc đầu tiên*](#phiên-làm-việc-đầu-tiên). Là các thư mục `SKILL.md` thuần; `install.sh` link chúng vào từng coding agent bạn chọn. |
|
|
248
248
|
| [`engines/`](engines/) | Mỗi coding agent một file dữ liệu nhỏ: skill cấp user đặt ở đâu, gọi skill thế nào, agent hỗ trợ gì. Thêm engine là thêm một file — [`engines/README.md`](engines/README.md). |
|
|
249
249
|
| [`install.sh`](install.sh) | Cài đặt cho **máy của bạn**: skills, CLI `colab`, hook pre-commit, danh sách fleet. Idempotent; `--dry` cho xem trước mọi thứ. |
|
package/audit/audit.mjs
CHANGED
|
@@ -37,6 +37,8 @@
|
|
|
37
37
|
// node audit.mjs --identity # also scan PUBLIC repository METADATA (description,
|
|
38
38
|
// # topics, homepage, name) against the operator's
|
|
39
39
|
// # private identity vocabulary — see "Identity" below
|
|
40
|
+
// node audit.mjs --batch-history # also read each local repo's batch landing history
|
|
41
|
+
// # (colab batch-stats --json) — see "Batch history" below
|
|
40
42
|
//
|
|
41
43
|
// Identity (--identity): metadata never passes through git, so the pre-commit identity scan
|
|
42
44
|
// (templates/pre-commit-identity) structurally cannot see it, and it is the first thing a
|
|
@@ -45,6 +47,13 @@
|
|
|
45
47
|
// vocabulary — and is never shipped, printed, or written into a report. Off unless asked
|
|
46
48
|
// for: it needs the network, and a fleet sweep must stay runnable offline.
|
|
47
49
|
//
|
|
50
|
+
// Batch history (--batch-history, #556): per LOCAL repo, the measured batch picture from
|
|
51
|
+
// `colab batch-stats --json` — overlap of serial landings with a partner inside their trunk-CI
|
|
52
|
+
// cycle, batch fill, first-attempt green rate, eviction rate. Shown, never judged on its own:
|
|
53
|
+
// an advisory is raised ONLY against a `batch-*` value the repo declares under `thresholds:`
|
|
54
|
+
// in project.yml, and none of those has a default. Off unless asked for, like --identity: it
|
|
55
|
+
// reads the CI history over the network (one API listing per day of the window).
|
|
56
|
+
//
|
|
48
57
|
// Exit code: 0 when every repo passes, 1 when any repo has a finding, 2 on a usage
|
|
49
58
|
// error. Findings never crash the run — a repo missing project.yml is a result, not
|
|
50
59
|
// an exception.
|
|
@@ -100,6 +109,7 @@ const { parseWorkflowOn, workflowFiresOnTag, prereleaseTagTriggers, workflowsFir
|
|
|
100
109
|
const shipBatch = require("../tools/lib/ship-batch.js");
|
|
101
110
|
const ciProfile = require("../tools/lib/ci-profile.js"); // #559
|
|
102
111
|
const thresholdsLib = require("../tools/lib/thresholds.js"); // #560
|
|
112
|
+
const batchHistory = require("../tools/lib/batch-history.js"); // #556
|
|
103
113
|
// #383: where migrations live — the one rule `colab ship`'s gate and `release cut` read; the audit
|
|
104
114
|
// validates the `migrations:` declaration against it and reports a `*/migrations/` dir it misses.
|
|
105
115
|
const migrationPaths = require("../tools/lib/migration-paths.js");
|
|
@@ -140,12 +150,13 @@ const COLAB_HOME = process.env.COLAB_HOME || join(homedir(), ".colab");
|
|
|
140
150
|
|
|
141
151
|
function parseArgs(argv) {
|
|
142
152
|
// config === null means "resolve from the precedence chain"; a string means explicit.
|
|
143
|
-
const opts = { config: null, locals: [], slugs: [], json: false, quiet: false, help: false, identity: false };
|
|
153
|
+
const opts = { config: null, locals: [], slugs: [], json: false, quiet: false, help: false, identity: false, batchHistory: false };
|
|
144
154
|
for (let i = 0; i < argv.length; i++) {
|
|
145
155
|
const a = argv[i];
|
|
146
156
|
if (a === "--json") opts.json = true;
|
|
147
157
|
else if (a === "--quiet" || a === "-q") opts.quiet = true;
|
|
148
158
|
else if (a === "--identity") opts.identity = true;
|
|
159
|
+
else if (a === "--batch-history") opts.batchHistory = true;
|
|
149
160
|
else if (a === "--local") {
|
|
150
161
|
const p = argv[++i];
|
|
151
162
|
if (!p) die("--local needs a path");
|
|
@@ -1997,6 +2008,9 @@ function auditRepo(target, ctx) {
|
|
|
1997
2008
|
// read by every visitor, and it is subject to the rule it publishes.
|
|
1998
2009
|
if (ctx.identity) checkIdentityMetadata(src, ctx.identity, info, fail);
|
|
1999
2010
|
|
|
2011
|
+
// ---- batch history (#556) ------------------------------------------------------
|
|
2012
|
+
checkBatchHistory(src, target, cfg, ctx, info, warn);
|
|
2013
|
+
|
|
2000
2014
|
function finish() {
|
|
2001
2015
|
info.findings = findings;
|
|
2002
2016
|
info.ok = !findings.some((f) => f.level === "fail");
|
|
@@ -2006,6 +2020,43 @@ function auditRepo(target, ctx) {
|
|
|
2006
2020
|
return finish();
|
|
2007
2021
|
}
|
|
2008
2022
|
|
|
2023
|
+
/**
|
|
2024
|
+
* #556 — the repo's measured batch picture, from `colab batch-stats --json` (this handbook's own
|
|
2025
|
+
* CLI, so the audit and the command read history the same way). Writes `info.batchHistory` on every
|
|
2026
|
+
* path. The picture is never a finding; an advisory is raised only for a `batch-*` threshold the
|
|
2027
|
+
* repo declares (tools/lib/batch-history.js judge — no defaults, by the owner's ruling). A run that
|
|
2028
|
+
* was asked for and could not read the history warns: a quiet row must not read as "checked".
|
|
2029
|
+
*/
|
|
2030
|
+
function checkBatchHistory(src, target, cfg, ctx, info, warn) {
|
|
2031
|
+
const declared = batchHistory.declaredBatchKeys(cfg);
|
|
2032
|
+
if (!ctx.batchHistory) {
|
|
2033
|
+
info.batchHistory = { status: "not-requested", declared };
|
|
2034
|
+
return;
|
|
2035
|
+
}
|
|
2036
|
+
if (src.kind !== "local") {
|
|
2037
|
+
info.batchHistory = { status: "not-applicable", reason: "batch-stats reads a clone's trunk log — audit a local path", declared };
|
|
2038
|
+
return;
|
|
2039
|
+
}
|
|
2040
|
+
let rep;
|
|
2041
|
+
try {
|
|
2042
|
+
const out = execFileSync(process.execPath, [join(HANDBOOK_ROOT, "tools", "colab"), "batch-stats", "--repo", resolve(target.path), "--json"],
|
|
2043
|
+
{ encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 600_000, maxBuffer: 64 * 1024 * 1024 });
|
|
2044
|
+
rep = JSON.parse(out);
|
|
2045
|
+
} catch (err) {
|
|
2046
|
+
const why = String((err.stderr || err.message || "")).trim().split("\n").pop() || "no output";
|
|
2047
|
+
info.batchHistory = { status: "unreadable", reason: why, declared };
|
|
2048
|
+
warn(`batch history: colab batch-stats could not read it (${why}) — nothing was measured or judged`);
|
|
2049
|
+
return;
|
|
2050
|
+
}
|
|
2051
|
+
const measured = batchHistory.measure(rep);
|
|
2052
|
+
const verdict = batchHistory.judge(rep, cfg, measured);
|
|
2053
|
+
info.batchHistory = {
|
|
2054
|
+
status: "read", since: rep.since, until: rep.until, landings: rep.landings, measured,
|
|
2055
|
+
picture: batchHistory.picture(rep, measured), declared, judged: verdict.judged, unjudged: verdict.unjudged, notes: rep.notes || [],
|
|
2056
|
+
};
|
|
2057
|
+
verdict.advisories.forEach(warn);
|
|
2058
|
+
}
|
|
2059
|
+
|
|
2009
2060
|
/**
|
|
2010
2061
|
* #228 part 4 — the identity scan of REPOSITORY METADATA: name, description, homepage and
|
|
2011
2062
|
* topics. None of it passes through git, so the pre-commit scan cannot see any of it, and it
|
|
@@ -3110,6 +3161,7 @@ function report(results, opts, ctx) {
|
|
|
3110
3161
|
identity: ctx.identity
|
|
3111
3162
|
? { scanned: true, vocabulary: ctx.identity.path, source: ctx.identity.source, terms: ctx.identity.count }
|
|
3112
3163
|
: { scanned: false, reason: "not requested (--identity)" },
|
|
3164
|
+
batchHistory: ctx.batchHistory ? { read: true } : { read: false, reason: "not requested (--batch-history)" },
|
|
3113
3165
|
results,
|
|
3114
3166
|
}, null, 2));
|
|
3115
3167
|
return;
|
|
@@ -3126,6 +3178,11 @@ function report(results, opts, ctx) {
|
|
|
3126
3178
|
console.log(ctx.identity
|
|
3127
3179
|
? `identity: ${ctx.identity.count} term(s) from ${ctx.identity.path} (${ctx.identity.source}) — public repository metadata scanned`
|
|
3128
3180
|
: "identity: repository metadata NOT scanned (pass --identity)");
|
|
3181
|
+
// #556: same once-per-run statement for batch history — a per-repo line on every offline run
|
|
3182
|
+
// would make every row non-clean for a read nobody asked for.
|
|
3183
|
+
console.log(ctx.batchHistory
|
|
3184
|
+
? "batch: batch landing history read per local repo (colab batch-stats); advisories only against declared thresholds"
|
|
3185
|
+
: "batch: batch landing history NOT read (pass --batch-history)");
|
|
3129
3186
|
console.log("");
|
|
3130
3187
|
|
|
3131
3188
|
const shown = opts.quiet ? results.filter((r) => !r.clean) : results;
|
|
@@ -3145,6 +3202,12 @@ function report(results, opts, ctx) {
|
|
|
3145
3202
|
else if (r.adoption === "local") lines.push(`⌂ adopted locally, not committed — descriptor hidden by .git/info/exclude; a repo the fleet does not own (CONVENTIONS.md §9)${r.clean ? " ✓" : ""}`);
|
|
3146
3203
|
else if (r.clean) lines.push("✓");
|
|
3147
3204
|
r.findings.forEach((f) => lines.push(`${f.level === "fail" ? "⚠" : "·"} ${f.text}`));
|
|
3205
|
+
// #556: the measured picture — information, not a finding, so it never changes clean/ok.
|
|
3206
|
+
if (r.batchHistory?.status === "read") {
|
|
3207
|
+
r.batchHistory.picture.forEach((l) => lines.push(`▸ ${l}`));
|
|
3208
|
+
if (!r.batchHistory.declared.length) lines.push("▸ no batch-* thresholds declared — shown, not judged");
|
|
3209
|
+
r.batchHistory.unjudged.forEach((u) => lines.push(`▸ thresholds.${u.key} not judged: ${u.why}`));
|
|
3210
|
+
} else if (r.batchHistory?.status === "not-applicable") lines.push(`▸ batch history: ${r.batchHistory.reason}`);
|
|
3148
3211
|
|
|
3149
3212
|
// Repeat the name on every line so the output stays greppable.
|
|
3150
3213
|
lines.forEach((l, i) => console.log(`${i === 0 ? head : cont} ${l}`));
|
|
@@ -3203,6 +3266,7 @@ function runAudit(opts) {
|
|
|
3203
3266
|
|
|
3204
3267
|
const ctx = { handbook: handbookInfo(HANDBOOK_ROOT), templateNames: templateNames(HANDBOOK_ROOT) };
|
|
3205
3268
|
ctx.identity = opts.identity ? loadIdentityVocabulary() : null;
|
|
3269
|
+
ctx.batchHistory = !!opts.batchHistory;
|
|
3206
3270
|
|
|
3207
3271
|
const results = [];
|
|
3208
3272
|
for (const t of targets) {
|
package/package.json
CHANGED
package/templates/ci-laravel.yml
CHANGED
|
@@ -297,7 +297,8 @@ jobs:
|
|
|
297
297
|
fi ;;
|
|
298
298
|
esac
|
|
299
299
|
# thresholds (#560): one level of `name: whole number`, each name known and at or above its
|
|
300
|
-
# floor — the table in tools/lib/thresholds.js (SPEC),
|
|
300
|
+
# floor (and, for a percent, at most 100) — the table in tools/lib/thresholds.js (SPEC),
|
|
301
|
+
# which a test holds this list to.
|
|
301
302
|
# A bad entry falls back to the default everywhere, silently, so it fails here.
|
|
302
303
|
bad=0
|
|
303
304
|
while IFS= read -r t; do
|
|
@@ -307,18 +308,20 @@ jobs:
|
|
|
307
308
|
esac
|
|
308
309
|
k="${t%%:*}"
|
|
309
310
|
n="$(printf '%s' "${t#*:}" | sed -E -e 's/^[[:space:]]+//' -e 's/^"(.*)"$/\1/' -e "s/^'(.*)'\$/\\1/")"
|
|
311
|
+
max=999999999
|
|
310
312
|
case "$k" in
|
|
311
313
|
hot-file-count|claude-md-line-multiple) min=2 ;;
|
|
312
|
-
dependents-count|hold-stale-days|smoke-minutes|claude-md-kb|transitional-days) min=1 ;;
|
|
314
|
+
dependents-count|hold-stale-days|smoke-minutes|claude-md-kb|transitional-days|batch-min-samples) min=1 ;;
|
|
313
315
|
claude-md-line-floor-bytes|doc-budget-slack) min=0 ;;
|
|
316
|
+
batch-first-green-pct-min|batch-eviction-pct-max|batch-overlap-pct) min=0 max=100 ;;
|
|
314
317
|
*) echo "::error file=.github/project.yml::thresholds.$k is not a known threshold (colab thresholds lists them). See project.schema.md, thresholds."
|
|
315
318
|
bad=1; continue ;;
|
|
316
319
|
esac
|
|
317
320
|
case "$n" in ''|'~'|null) continue ;; esac
|
|
318
|
-
if printf '%s\n' "$n" | grep -Eq '^[0-9]{1,9}$' && [ "$n" -ge "$min" ]; then
|
|
321
|
+
if printf '%s\n' "$n" | grep -Eq '^[0-9]{1,9}$' && [ "$n" -ge "$min" ] && [ "$n" -le "$max" ]; then
|
|
319
322
|
echo "thresholds.$k: $n — ok"
|
|
320
323
|
else
|
|
321
|
-
echo "::error file=.github/project.yml::thresholds.$k is '$n', expected a whole number
|
|
324
|
+
echo "::error file=.github/project.yml::thresholds.$k is '$n', expected a whole number from $min to $max. Every reader ignores it (a batch-* key is then not judged; any other uses its default). See project.schema.md, thresholds."
|
|
322
325
|
bad=1
|
|
323
326
|
fi
|
|
324
327
|
done < <(awk '
|
package/templates/ci-node.yml
CHANGED
|
@@ -292,7 +292,8 @@ jobs:
|
|
|
292
292
|
fi ;;
|
|
293
293
|
esac
|
|
294
294
|
# thresholds (#560): one level of `name: whole number`, each name known and at or above its
|
|
295
|
-
# floor — the table in tools/lib/thresholds.js (SPEC),
|
|
295
|
+
# floor (and, for a percent, at most 100) — the table in tools/lib/thresholds.js (SPEC),
|
|
296
|
+
# which a test holds this list to.
|
|
296
297
|
# A bad entry falls back to the default everywhere, silently, so it fails here.
|
|
297
298
|
bad=0
|
|
298
299
|
while IFS= read -r t; do
|
|
@@ -302,18 +303,20 @@ jobs:
|
|
|
302
303
|
esac
|
|
303
304
|
k="${t%%:*}"
|
|
304
305
|
n="$(printf '%s' "${t#*:}" | sed -E -e 's/^[[:space:]]+//' -e 's/^"(.*)"$/\1/' -e "s/^'(.*)'\$/\\1/")"
|
|
306
|
+
max=999999999
|
|
305
307
|
case "$k" in
|
|
306
308
|
hot-file-count|claude-md-line-multiple) min=2 ;;
|
|
307
|
-
dependents-count|hold-stale-days|smoke-minutes|claude-md-kb|transitional-days) min=1 ;;
|
|
309
|
+
dependents-count|hold-stale-days|smoke-minutes|claude-md-kb|transitional-days|batch-min-samples) min=1 ;;
|
|
308
310
|
claude-md-line-floor-bytes|doc-budget-slack) min=0 ;;
|
|
311
|
+
batch-first-green-pct-min|batch-eviction-pct-max|batch-overlap-pct) min=0 max=100 ;;
|
|
309
312
|
*) echo "::error file=.github/project.yml::thresholds.$k is not a known threshold (colab thresholds lists them). See project.schema.md, thresholds."
|
|
310
313
|
bad=1; continue ;;
|
|
311
314
|
esac
|
|
312
315
|
case "$n" in ''|'~'|null) continue ;; esac
|
|
313
|
-
if printf '%s\n' "$n" | grep -Eq '^[0-9]{1,9}$' && [ "$n" -ge "$min" ]; then
|
|
316
|
+
if printf '%s\n' "$n" | grep -Eq '^[0-9]{1,9}$' && [ "$n" -ge "$min" ] && [ "$n" -le "$max" ]; then
|
|
314
317
|
echo "thresholds.$k: $n — ok"
|
|
315
318
|
else
|
|
316
|
-
echo "::error file=.github/project.yml::thresholds.$k is '$n', expected a whole number
|
|
319
|
+
echo "::error file=.github/project.yml::thresholds.$k is '$n', expected a whole number from $min to $max. Every reader ignores it (a batch-* key is then not judged; any other uses its default). See project.schema.md, thresholds."
|
|
317
320
|
bad=1
|
|
318
321
|
fi
|
|
319
322
|
done < <(awk '
|
package/templates/ci-python.yml
CHANGED
|
@@ -283,7 +283,8 @@ jobs:
|
|
|
283
283
|
fi ;;
|
|
284
284
|
esac
|
|
285
285
|
# thresholds (#560): one level of `name: whole number`, each name known and at or above its
|
|
286
|
-
# floor — the table in tools/lib/thresholds.js (SPEC),
|
|
286
|
+
# floor (and, for a percent, at most 100) — the table in tools/lib/thresholds.js (SPEC),
|
|
287
|
+
# which a test holds this list to.
|
|
287
288
|
# A bad entry falls back to the default everywhere, silently, so it fails here.
|
|
288
289
|
bad=0
|
|
289
290
|
while IFS= read -r t; do
|
|
@@ -293,18 +294,20 @@ jobs:
|
|
|
293
294
|
esac
|
|
294
295
|
k="${t%%:*}"
|
|
295
296
|
n="$(printf '%s' "${t#*:}" | sed -E -e 's/^[[:space:]]+//' -e 's/^"(.*)"$/\1/' -e "s/^'(.*)'\$/\\1/")"
|
|
297
|
+
max=999999999
|
|
296
298
|
case "$k" in
|
|
297
299
|
hot-file-count|claude-md-line-multiple) min=2 ;;
|
|
298
|
-
dependents-count|hold-stale-days|smoke-minutes|claude-md-kb|transitional-days) min=1 ;;
|
|
300
|
+
dependents-count|hold-stale-days|smoke-minutes|claude-md-kb|transitional-days|batch-min-samples) min=1 ;;
|
|
299
301
|
claude-md-line-floor-bytes|doc-budget-slack) min=0 ;;
|
|
302
|
+
batch-first-green-pct-min|batch-eviction-pct-max|batch-overlap-pct) min=0 max=100 ;;
|
|
300
303
|
*) echo "::error file=.github/project.yml::thresholds.$k is not a known threshold (colab thresholds lists them). See project.schema.md, thresholds."
|
|
301
304
|
bad=1; continue ;;
|
|
302
305
|
esac
|
|
303
306
|
case "$n" in ''|'~'|null) continue ;; esac
|
|
304
|
-
if printf '%s\n' "$n" | grep -Eq '^[0-9]{1,9}$' && [ "$n" -ge "$min" ]; then
|
|
307
|
+
if printf '%s\n' "$n" | grep -Eq '^[0-9]{1,9}$' && [ "$n" -ge "$min" ] && [ "$n" -le "$max" ]; then
|
|
305
308
|
echo "thresholds.$k: $n — ok"
|
|
306
309
|
else
|
|
307
|
-
echo "::error file=.github/project.yml::thresholds.$k is '$n', expected a whole number
|
|
310
|
+
echo "::error file=.github/project.yml::thresholds.$k is '$n', expected a whole number from $min to $max. Every reader ignores it (a batch-* key is then not judged; any other uses its default). See project.schema.md, thresholds."
|
|
308
311
|
bad=1
|
|
309
312
|
fi
|
|
310
313
|
done < <(awk '
|
package/tools/colab
CHANGED
|
@@ -15131,7 +15131,11 @@ hot-file count and dependents band, code-wrap's gate.smoke target — and read t
|
|
|
15131
15131
|
skill hard-codes a number a repo may have changed. A malformed entry falls back to the default
|
|
15132
15132
|
and is listed under problems; the audit and the CI templates' descriptor check fail it.
|
|
15133
15133
|
|
|
15134
|
-
|
|
15134
|
+
The batch-* names (#556) have NO default: undeclared, they read "—" and the audit's
|
|
15135
|
+
--batch-history shows the measurement without judging it. Percents are bounded at 100.
|
|
15136
|
+
|
|
15137
|
+
<name> print only that threshold's value (a bare number, for a script or a skill;
|
|
15138
|
+
an empty line for an undeclared no-default name)
|
|
15135
15139
|
--json { repo, thresholds: { <name>: { value, default, declared, what, where } }, problems }`;
|
|
15136
15140
|
|
|
15137
15141
|
function cmdThresholds(argv) {
|
|
@@ -15146,7 +15150,7 @@ function cmdThresholds(argv) {
|
|
|
15146
15150
|
if (name !== undefined) {
|
|
15147
15151
|
if (!thresholdsLib.SPEC[name]) throw new UserError(`unknown threshold ${name} — expected one of: ${Object.keys(thresholdsLib.SPEC).join(', ')} (run \`colab thresholds\` to list them)`);
|
|
15148
15152
|
for (const p of r.problems) process.stderr.write(`! ${p}\n`);
|
|
15149
|
-
print(String(r.values[name]));
|
|
15153
|
+
print(r.values[name] === null ? '' : String(r.values[name])); // #556: empty = undeclared, no default
|
|
15150
15154
|
return 0;
|
|
15151
15155
|
}
|
|
15152
15156
|
const rows = Object.fromEntries(Object.entries(thresholdsLib.SPEC).map(([k, s]) => [k, {
|
|
@@ -15159,7 +15163,10 @@ function cmdThresholds(argv) {
|
|
|
15159
15163
|
print(`thresholds ${shortRepo(repoAbs)}`);
|
|
15160
15164
|
const w = Math.max(...Object.keys(rows).map((k) => k.length));
|
|
15161
15165
|
for (const [k, row] of Object.entries(rows)) {
|
|
15162
|
-
|
|
15166
|
+
// #556: a no-default key (batch-*) shows `—` and "not declared" — nothing judges it until it is.
|
|
15167
|
+
const shown = row.value === null ? '—' : String(row.value);
|
|
15168
|
+
const src = row.declared ? `declared (default ${row.default === null ? 'none' : row.default})` : (row.default === null ? 'not declared' : 'default');
|
|
15169
|
+
print(` ${k.padEnd(w)} ${shown.padStart(5)} ${src.padEnd(20)} ${row.what}`);
|
|
15163
15170
|
}
|
|
15164
15171
|
for (const p of r.problems) print(` ! ${p}`);
|
|
15165
15172
|
return 0;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* tools/lib/batch-history.js — the audit's reading of `colab batch-stats --json` (#556).
|
|
4
|
+
*
|
|
5
|
+
* Two halves, deliberately separate:
|
|
6
|
+
* measure the repo's batch picture as four rates — overlap of serial landings with a partner
|
|
7
|
+
* inside their trunk-CI cycle, batch fill, first-attempt green rate of combined runs,
|
|
8
|
+
* eviction rate of members at build. Always shown; never a finding by itself.
|
|
9
|
+
* judge an advisory ONLY against a threshold the repo declares in `thresholds:` (the four
|
|
10
|
+
* `batch-*` keys in tools/lib/thresholds.js, none of which has a default). The owner's
|
|
11
|
+
* ruling on #556 — "should not set any hard code number" — is why: nothing declared ⇒
|
|
12
|
+
* the numbers are shown and nothing is flagged. Replaces #506's proposed fixed
|
|
13
|
+
* "suggest ship-batch: 3".
|
|
14
|
+
*
|
|
15
|
+
* A rate with no samples is unmeasured (null), and an unmeasured rate is never judged — the same
|
|
16
|
+
* rule as `batch-min-samples`, which a repo may declare to demand more than one sample.
|
|
17
|
+
*
|
|
18
|
+
* Pure: a batch-stats report + the parsed descriptor in, `{ measured, picture, advisories }` out.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
const thresholds = require('./thresholds');
|
|
22
|
+
const shipBatch = require('./ship-batch');
|
|
23
|
+
|
|
24
|
+
const pctOf = (num, den) => (den > 0 ? Math.round((num / den) * 100) : null);
|
|
25
|
+
const pctRate = (r) => (r === null || r === undefined ? null : Math.round(r * 100));
|
|
26
|
+
|
|
27
|
+
/** The four rates and their sample counts, from a batch-stats report. */
|
|
28
|
+
function measure(rep) {
|
|
29
|
+
const s = rep.serial || {};
|
|
30
|
+
const b = rep.batches || {};
|
|
31
|
+
const d = rep.dropped || {};
|
|
32
|
+
const finished = (b.builds || 0) - (b.pending || 0);
|
|
33
|
+
return {
|
|
34
|
+
overlap: { pct: pctOf(s.overlapped || 0, s.cyclesRead || 0), samples: s.cyclesRead || 0, overlapped: s.overlapped || 0 },
|
|
35
|
+
fill: { batches: b.landed || 0, byMembers: b.fill || {} },
|
|
36
|
+
firstGreen: { pct: pctRate(b.firstAttemptGreenRate), samples: finished, green: b.firstAttemptGreen || 0 },
|
|
37
|
+
eviction: { pct: pctRate(d.rate), samples: d.attempted || 0, dropped: d.count || 0 },
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** One line per rate, for the human report. `—` is unmeasured, never zero. */
|
|
42
|
+
function picture(rep, m = measure(rep)) {
|
|
43
|
+
const p = (x) => (x.pct === null ? '—' : `${x.pct}%`);
|
|
44
|
+
const fill = Object.keys(m.fill.byMembers).sort().map((k) => `${m.fill.byMembers[k]}×${k}`).join(' · ') || '—';
|
|
45
|
+
const head = `batch history ${String(rep.since || '').slice(0, 10)} … ${String(rep.until || '').slice(0, 10)}: ${(rep.landings || {}).total || 0} landing(s), ${(rep.landings || {}).batch || 0} batch · ${(rep.landings || {}).serial || 0} serial (ship-batch: ${(rep.config || {}).n || 1})`;
|
|
46
|
+
return [
|
|
47
|
+
head,
|
|
48
|
+
`overlap ${p(m.overlap)} of ${m.overlap.samples} serial landing(s) had a partner green inside their trunk-CI cycle · fill ${fill} · first-attempt green ${p(m.firstGreen)} of ${m.firstGreen.samples} combined run(s) · eviction ${p(m.eviction)} of ${m.eviction.samples} member(s)`,
|
|
49
|
+
];
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Advisories for the rates that cross a DECLARED threshold. `doc` is the parsed descriptor.
|
|
54
|
+
* Returns `{ advisories: [text], judged: [key], unjudged: [{ key, why }] }` — a declared key that
|
|
55
|
+
* could not be judged (no samples, under batch-min-samples, not applicable) says why, so a quiet
|
|
56
|
+
* report never reads as "checked and fine" when it was not checked.
|
|
57
|
+
*/
|
|
58
|
+
function judge(rep, doc, m = measure(rep)) {
|
|
59
|
+
const t = thresholds.parseThresholds(doc).values;
|
|
60
|
+
const n = shipBatch.parseShipBatch(doc || {}).n || 1;
|
|
61
|
+
const minN = t['batch-min-samples'] === null ? 1 : t['batch-min-samples'];
|
|
62
|
+
const advisories = [];
|
|
63
|
+
const judged = [];
|
|
64
|
+
const unjudged = [];
|
|
65
|
+
const enough = (key, rate) => {
|
|
66
|
+
if (rate.pct === null || rate.samples < minN) {
|
|
67
|
+
unjudged.push({ key, why: `${rate.samples} sample(s)${minN > 1 ? `, batch-min-samples is ${minN}` : ''}` });
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
judged.push(key);
|
|
71
|
+
return true;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
const ov = t['batch-overlap-pct'];
|
|
75
|
+
if (ov !== null) {
|
|
76
|
+
if (n > 1) unjudged.push({ key: 'batch-overlap-pct', why: `already opted in (ship-batch: ${n})` });
|
|
77
|
+
else if (enough('batch-overlap-pct', m.overlap) && m.overlap.pct >= ov) {
|
|
78
|
+
advisories.push(`batch history: ${m.overlap.pct}% of ${m.overlap.samples} serial landings overlapped a partner's green inside their trunk-CI cycle (thresholds.batch-overlap-pct: ${ov}) — consider opting in to ship-batch; colab batch-stats has the pairs`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
const fg = t['batch-first-green-pct-min'];
|
|
82
|
+
if (fg !== null) {
|
|
83
|
+
if (n <= 1 && !m.firstGreen.samples) unjudged.push({ key: 'batch-first-green-pct-min', why: 'not batching (ship-batch absent or 1)' });
|
|
84
|
+
else if (enough('batch-first-green-pct-min', m.firstGreen) && m.firstGreen.pct < fg) {
|
|
85
|
+
advisories.push(`batch history: ${m.firstGreen.pct}% of ${m.firstGreen.samples} combined runs went green on the first attempt (thresholds.batch-first-green-pct-min: ${fg}) — consider lowering ship-batch${n > 1 ? ` below ${n}` : ''}`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
const ev = t['batch-eviction-pct-max'];
|
|
89
|
+
if (ev !== null) {
|
|
90
|
+
if (n <= 1 && !m.eviction.samples) unjudged.push({ key: 'batch-eviction-pct-max', why: 'not batching (ship-batch absent or 1)' });
|
|
91
|
+
else if (enough('batch-eviction-pct-max', m.eviction) && m.eviction.pct > ev) {
|
|
92
|
+
advisories.push(`batch history: ${m.eviction.pct}% of ${m.eviction.samples} batch members were dropped at build (thresholds.batch-eviction-pct-max: ${ev}) — colab batch-stats lists them by class`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return { advisories, judged, unjudged };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** The batch-* threshold names a descriptor declares — what makes a no-flag audit run worth a note. */
|
|
99
|
+
function declaredBatchKeys(doc) {
|
|
100
|
+
const d = thresholds.parseThresholds(doc).declared;
|
|
101
|
+
return Object.keys(d).filter((k) => k.startsWith('batch-'));
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
module.exports = { measure, picture, judge, declaredBatchKeys };
|
package/tools/lib/batch-stats.js
CHANGED
|
@@ -207,7 +207,7 @@ function derive({ commits = [], runs = [], builds = {}, trunk = 'main', config =
|
|
|
207
207
|
|
|
208
208
|
// ---- missed partners: serial landings that landed alone while a partner was (nearly) ready ----
|
|
209
209
|
const pairs = [];
|
|
210
|
-
let withMissed = 0, withNear = 0, cyclesUnread = 0;
|
|
210
|
+
let withMissed = 0, withNear = 0, cyclesUnread = 0, overlapped = 0;
|
|
211
211
|
// A partner is judged by ITS state at this landing's moment, never by the head it finally landed
|
|
212
212
|
// with: a serial ship syncs the other branches, so their final heads all post-date it. The cycle
|
|
213
213
|
// ends when this landing's trunk run finishes — or at the next landing, whichever is first (a
|
|
@@ -238,6 +238,7 @@ function derive({ commits = [], runs = [], builds = {}, trunk = 'main', config =
|
|
|
238
238
|
}
|
|
239
239
|
if (missed) withMissed++;
|
|
240
240
|
else if (near) withNear++;
|
|
241
|
+
if (end !== null && (missed || near)) overlapped++;
|
|
241
242
|
}
|
|
242
243
|
if (cyclesUnread) notes.push(`${cyclesUnread} serial landing(s) with no finished trunk run in the window — near misses not judged for them`);
|
|
243
244
|
const unknownGreen = changes.filter((c) => c.greenMs === null).length;
|
|
@@ -262,6 +263,9 @@ function derive({ commits = [], runs = [], builds = {}, trunk = 'main', config =
|
|
|
262
263
|
dropped: { count: dropped.length, attempted: builtMembers + dropped.length, rate: rate(dropped.length, builtMembers + dropped.length), byClass, members: dropped },
|
|
263
264
|
serial: {
|
|
264
265
|
landings: serialChanges.length, withMissedPartner: withMissed, withNearMiss: withNear,
|
|
266
|
+
// #556: the overlap rate's honest denominator — only a landing whose trunk-CI cycle was read
|
|
267
|
+
// could have been judged for a partner inside it; `overlapped` counts those that had one.
|
|
268
|
+
cyclesRead: serialChanges.length - cyclesUnread, overlapped,
|
|
265
269
|
nearMissWait: summary(pairs.filter((p) => p.kind === 'near').map((p) => p.waitSec)),
|
|
266
270
|
pairs,
|
|
267
271
|
},
|
package/tools/lib/thresholds.js
CHANGED
|
@@ -14,9 +14,13 @@
|
|
|
14
14
|
* that decides what gets flagged, ranked or warned about. Safety limits (a cap a repo may only
|
|
15
15
|
* tighten) and protocol counts are out of scope (epic #558) and keep their own fields.
|
|
16
16
|
*
|
|
17
|
-
* Every value is a whole number with a per-key floor
|
|
18
|
-
* the CI templates' shell copy of this rule (the descriptor
|
|
19
|
-
* this table by tools/lib/ci-template-descriptor-check.test.js.
|
|
17
|
+
* Every value is a whole number with a per-key floor (and, for a percent, a ceiling of 100), and the
|
|
18
|
+
* unit is in the key's name. That keeps the CI templates' shell copy of this rule (the descriptor
|
|
19
|
+
* check, #416) small enough to be held to this table by tools/lib/ci-template-descriptor-check.test.js.
|
|
20
|
+
*
|
|
21
|
+
* NO-DEFAULT keys (#556): `def: null`. The batch-history judgement has no fleet-wide number to fall
|
|
22
|
+
* back on — the owner's ruling is that none be hard-coded — so an undeclared one means "do not judge":
|
|
23
|
+
* its value is null, and its reader shows the measurement without an advisory.
|
|
20
24
|
*
|
|
21
25
|
* A malformed value — a typo'd key, a non-number, a number under the floor — falls back to the
|
|
22
26
|
* default and is reported; the audit and the descriptor check fail it. A typo'd key is a problem,
|
|
@@ -29,7 +33,8 @@
|
|
|
29
33
|
const KEY = 'thresholds';
|
|
30
34
|
|
|
31
35
|
/**
|
|
32
|
-
* key → { def, min, what, where }. `where` names the one place that reads the value.
|
|
36
|
+
* key → { def, min, max?, what, where }. `where` names the one place that reads the value. `def: null`
|
|
37
|
+
* = no default (#556): undeclared means its reader does not judge. `max` absent = MAX_VALUE.
|
|
33
38
|
* Order is the order `colab thresholds` prints.
|
|
34
39
|
*/
|
|
35
40
|
const SPEC = Object.freeze({
|
|
@@ -42,6 +47,11 @@ const SPEC = Object.freeze({
|
|
|
42
47
|
'claude-md-line-floor-bytes': { def: 2048, min: 0, what: 'a line under this many bytes is never flagged on the multiple alone', where: 'audit/audit.mjs' },
|
|
43
48
|
'transitional-days': { def: 180, min: 1, what: 'days a transitional descriptor value holds before the audit says how long', where: 'audit/audit.mjs' },
|
|
44
49
|
'doc-budget-slack': { def: 100, min: 0, what: 'lines under a doc budget before check-doc-budget asks to lower it', where: 'scripts/check-doc-budget.mjs' },
|
|
50
|
+
// #556 — the audit's batch-history advisories (tools/lib/batch-history.js). No defaults, by ruling.
|
|
51
|
+
'batch-min-samples': { def: null, min: 1, what: 'measured events a batch rate needs before the audit judges it (undeclared: any)', where: 'audit/audit.mjs --batch-history' },
|
|
52
|
+
'batch-first-green-pct-min': { def: null, min: 0, max: 100, what: 'first-attempt green % of combined runs below which the audit suggests lowering ship-batch', where: 'audit/audit.mjs --batch-history' },
|
|
53
|
+
'batch-eviction-pct-max': { def: null, min: 0, max: 100, what: '% of batch members dropped at build above which the audit flags eviction', where: 'audit/audit.mjs --batch-history' },
|
|
54
|
+
'batch-overlap-pct': { def: null, min: 0, max: 100, what: '% of serial landings with a missed or near-miss partner at which the audit suggests opting in to ship-batch', where: 'audit/audit.mjs --batch-history' },
|
|
45
55
|
});
|
|
46
56
|
|
|
47
57
|
const DEFAULTS = Object.freeze(Object.fromEntries(Object.entries(SPEC).map(([k, s]) => [k, s.def])));
|
|
@@ -57,7 +67,7 @@ function wholeNumber(raw) {
|
|
|
57
67
|
|
|
58
68
|
/**
|
|
59
69
|
* `doc` → `{ values, declared, problems }`. `values` always carries every key (the default where
|
|
60
|
-
* nothing valid was declared); `declared` names the keys whose declared value is in force;
|
|
70
|
+
* nothing valid was declared — null for a no-default key); `declared` names the keys whose declared value is in force;
|
|
61
71
|
* `problems` is one sentence per malformed entry, each ending with the default it fell back to.
|
|
62
72
|
*/
|
|
63
73
|
function parseThresholds(doc) {
|
|
@@ -79,8 +89,10 @@ function parseThresholds(doc) {
|
|
|
79
89
|
}
|
|
80
90
|
if (v === null || v === undefined || v === '') continue; // an empty entry declares nothing
|
|
81
91
|
const n = wholeNumber(v);
|
|
82
|
-
|
|
83
|
-
|
|
92
|
+
const max = spec.max === undefined ? MAX_VALUE : spec.max;
|
|
93
|
+
if (!Number.isFinite(n) || n < spec.min || n > max) {
|
|
94
|
+
const range = spec.max === undefined ? `≥ ${spec.min}` : `${spec.min}–${spec.max}`;
|
|
95
|
+
problems.push(`${KEY}.${k} must be a whole number ${range}, got ${JSON.stringify(v)} — ${spec.def === null ? 'not declared, so not judged' : `using the default ${spec.def}`}`);
|
|
84
96
|
continue;
|
|
85
97
|
}
|
|
86
98
|
values[k] = n;
|
|
@@ -89,7 +101,7 @@ function parseThresholds(doc) {
|
|
|
89
101
|
return { values, declared, problems };
|
|
90
102
|
}
|
|
91
103
|
|
|
92
|
-
/** The resolved value of one key — the convenience a single consumer wants. */
|
|
104
|
+
/** The resolved value of one key — the convenience a single consumer wants. Null = undeclared, no default. */
|
|
93
105
|
function thresholdValue(doc, key) {
|
|
94
106
|
if (!SPEC[key]) throw new Error(`unknown threshold ${key}`);
|
|
95
107
|
return parseThresholds(doc).values[key];
|