@feigi/fleet-ctl 3.50.1 → 3.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/member-outcomes.mjs +59 -15
- package/scripts/reclaim-scratch.mjs +57 -3
- package/scripts/report-files.mjs +94 -0
- package/scripts/tier-outcomes.mjs +59 -4
- package/skills/run-team/SKILL.md +49 -41
package/package.json
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// Scraper for per-member model/effort facts. Pure over omp's own subagent
|
|
2
|
-
// transcripts: no clock, no network, no gh.
|
|
2
|
+
// transcripts: no clock, no network, no gh. The one column that looks beside
|
|
3
|
+
// them is `report`, which asks whether the member's report file is in the
|
|
4
|
+
// scratch root (`--scratch <dir>`, default the root scratch-root.mjs prints).
|
|
3
5
|
//
|
|
4
6
|
// The transcript parsing (fold, model/effort extraction, ticket/pr naming,
|
|
5
7
|
// and the `gh pr create` result an implementer's `pr` comes from)
|
|
@@ -11,6 +13,8 @@ import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
|
11
13
|
import { join, basename } from "node:path";
|
|
12
14
|
|
|
13
15
|
import { readOmpSession, normalizeModel, parseMemberName, isOmpSessionDirName } from "./member-record.mjs";
|
|
16
|
+
import { reportFamily, reportStatus, defaultScratch } from "./report-files.mjs";
|
|
17
|
+
import { FleetDirUnresolvable } from "./fleet-dir.mjs";
|
|
14
18
|
|
|
15
19
|
export { normalizeModel, parseMemberName };
|
|
16
20
|
|
|
@@ -84,19 +88,34 @@ export function rowsForSession(sessionDir, stats = {}) {
|
|
|
84
88
|
}));
|
|
85
89
|
}
|
|
86
90
|
|
|
87
|
-
// APPENDED
|
|
88
|
-
// in the
|
|
89
|
-
//
|
|
90
|
-
//
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
//
|
|
91
|
+
// A column is APPENDED, never inserted, as a rule: a read-out that indexes by
|
|
92
|
+
// position (`$5`, in the file's header) silently repoints when one is added
|
|
93
|
+
// ahead of it. `report` is placed after `subagent_type` regardless, which
|
|
94
|
+
// moves the four priced columns one place right; every script that reads this
|
|
95
|
+
// file through parseTsv takes those four by name, and the header's
|
|
96
|
+
// position-indexed read-outs name columns ahead of them. `subagent_type`
|
|
97
|
+
// follows `harness`. `cost` is the provider's own per-turn `usage.cost.total`,
|
|
98
|
+
// summed to the micro-dollar — the $ of record; `tokens_cache_write_1h` and
|
|
99
|
+
// `cost` are blank where the transcript recorded no figure, never 0.
|
|
94
100
|
export const COLUMNS = [
|
|
95
101
|
"session", "run_date", "role", "member", "model", "effort", "ticket", "pr",
|
|
96
102
|
"tokens_cache_create", "tokens_out", "wall_s", "turns", "agent", "harness",
|
|
97
|
-
"subagent_type", "tokens_in", "tokens_cache_read", "tokens_cache_write_1h", "cost",
|
|
103
|
+
"subagent_type", "report", "tokens_in", "tokens_cache_read", "tokens_cache_write_1h", "cost",
|
|
98
104
|
];
|
|
99
105
|
|
|
106
|
+
// `report`: what reportStatus() says of the row's member under `scratch` — a
|
|
107
|
+
// pure function of the member name and the file system, never of the
|
|
108
|
+
// transcript, so it is stamped here on the rows rowsForSession() returned.
|
|
109
|
+
// With no `scratch` (null: none could be resolved) a member a schema covers
|
|
110
|
+
// reads blank — unknown, the file's own spelling of it — and one none covers
|
|
111
|
+
// still reads `-`, which needs no file to look at.
|
|
112
|
+
export function withReports(rows, scratch) {
|
|
113
|
+
return rows.map((r) => ({
|
|
114
|
+
...r,
|
|
115
|
+
report: scratch === null && reportFamily(r.member) !== null ? "" : reportStatus(scratch, r.member),
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
118
|
+
|
|
100
119
|
// Row objects use camelCase; the file uses snake_case. One map, one direction
|
|
101
120
|
// each, so a rename cannot silently drop a column.
|
|
102
121
|
const FIELD = {
|
|
@@ -164,21 +183,27 @@ if (isCLI(import.meta.url)) {
|
|
|
164
183
|
const argv = process.argv.slice(2);
|
|
165
184
|
const fileIdx = argv.indexOf("--file");
|
|
166
185
|
const file = fileIdx >= 0 ? argv[fileIdx + 1] : "docs/metrics/member-outcomes.tsv";
|
|
186
|
+
const scratchIdx = argv.indexOf("--scratch");
|
|
187
|
+
const scratchArg = scratchIdx >= 0 ? argv[scratchIdx + 1] : null;
|
|
167
188
|
// An unrecognised `--flag` is silently dropped by the dirs filter below
|
|
168
189
|
// rather than refused — worst case `--file=/tmp/x.tsv` (no such flag;
|
|
169
190
|
// --file wants a SPACE, not `=`) never sets fileIdx, so this would silently
|
|
170
191
|
// write the production metrics file instead of the path the operator asked
|
|
171
192
|
// for.
|
|
172
|
-
const bad = argv.find((a) => a.startsWith("--") && a !== "--file");
|
|
173
|
-
if (bad) die(`unknown option ${bad} (did you mean --file <path>?)`);
|
|
193
|
+
const bad = argv.find((a) => a.startsWith("--") && a !== "--file" && a !== "--scratch");
|
|
194
|
+
if (bad) die(`unknown option ${bad} (did you mean --file <path> or --scratch <dir>?)`);
|
|
174
195
|
|
|
175
196
|
// fileIdx is -1 when --file is absent, which makes fileIdx + 1 equal 0 — the
|
|
176
197
|
// FIRST positional argument, not a real index into argv. Without the
|
|
177
198
|
// fileIdx < 0 guard this filters out the session dir itself, and the bare
|
|
178
199
|
// form (the one the spec, the backfill loop and run-team's phase-3
|
|
179
|
-
// instruction all use) always exits 2.
|
|
180
|
-
|
|
181
|
-
|
|
200
|
+
// instruction all use) always exits 2. `--scratch`'s value is skipped the
|
|
201
|
+
// same way.
|
|
202
|
+
const valueOf = (idx) => (i) => idx >= 0 && i === idx + 1;
|
|
203
|
+
const isFileValue = valueOf(fileIdx);
|
|
204
|
+
const isScratchValue = valueOf(scratchIdx);
|
|
205
|
+
const dirs = argv.filter((a, i) => !a.startsWith("--") && !isFileValue(i) && !isScratchValue(i));
|
|
206
|
+
if (dirs.length !== 1) die("usage: member-outcomes.mjs <session-dir> [--file <tsv>] [--scratch <dir>]");
|
|
182
207
|
// arg.mjs's rule, consumed rather than copied. This line used to
|
|
183
208
|
// hand-write `!file || file.startsWith("--")`, which was that rule with the
|
|
184
209
|
// blank spelling missing — so `--file " "` was accepted here and wrote the
|
|
@@ -187,6 +212,16 @@ if (isCLI(import.meta.url)) {
|
|
|
187
212
|
// is imported. The refusal WORDING stays this script's own: `--file` is the
|
|
188
213
|
// flag the operator typed.
|
|
189
214
|
if (isFlagLike(file)) die("--file needs a path");
|
|
215
|
+
if (scratchIdx >= 0 && isFlagLike(scratchArg)) die("--scratch needs a path");
|
|
216
|
+
// An explicit root that is not a directory is a mistyped path, and reading it
|
|
217
|
+
// as a root would stamp `absent` on every covered row and, by the replace-by-key
|
|
218
|
+
// merge, over a recorded `ok`/`invalid` — indistinguishable from "wrote no report".
|
|
219
|
+
if (scratchIdx >= 0) {
|
|
220
|
+
let isDir = false;
|
|
221
|
+
try { isDir = statSync(scratchArg).isDirectory(); }
|
|
222
|
+
catch (e) { die(`cannot read --scratch ${scratchArg}: ${e.code ?? e.message}`); }
|
|
223
|
+
if (!isDir) die(`--scratch ${scratchArg} is not a directory`);
|
|
224
|
+
}
|
|
190
225
|
|
|
191
226
|
const sessionDir = dirs[0].replace(/\/+$/, "");
|
|
192
227
|
// A wrong guess used to exit 0 having scraped and written nothing, because
|
|
@@ -223,8 +258,17 @@ if (isCLI(import.meta.url)) {
|
|
|
223
258
|
try { existing = parseTsv(prev); }
|
|
224
259
|
catch (e) { die(`${file} is not readable as this schema: ${e.message}`); }
|
|
225
260
|
|
|
261
|
+
let scratch = scratchArg;
|
|
262
|
+
if (scratch === null) {
|
|
263
|
+
try { scratch = defaultScratch(); }
|
|
264
|
+
catch (e) {
|
|
265
|
+
if (!(e instanceof FleetDirUnresolvable)) throw e;
|
|
266
|
+
process.stderr.write(`${NAME}: WARNING no scratch root to look for report files in (${e.message}) — \`report\` is left blank where a schema covers the member; pass --scratch <dir>\n`);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
226
270
|
const stats = {};
|
|
227
|
-
const merged = mergeRows(existing, rowsForSession(sessionDir, stats));
|
|
271
|
+
const merged = mergeRows(existing, withReports(rowsForSession(sessionDir, stats), scratch));
|
|
228
272
|
// Write-then-rename: writeFileSync truncates before it writes, so an interrupt
|
|
229
273
|
// over the committed corpus left a torn file that the NEXT scrape read, merged
|
|
230
274
|
// and rewrote — losing every other session's rows for good.
|
|
@@ -24,6 +24,17 @@
|
|
|
24
24
|
// removed when issue #N is CLOSED, or OPEN with no live
|
|
25
25
|
// impl-<N>[-x] token and idle past the retention, so a bailed or
|
|
26
26
|
// released ticket's partition is not inherited by its next claim.
|
|
27
|
+
// reports/<member>.json
|
|
28
|
+
// a member's report file. `reports/` itself is never removed, and
|
|
29
|
+
// is not aged out whole: each file is judged on its own, by the
|
|
30
|
+
// entry it goes with. A fix-pr-<M>[-x], finisher-pr-<M>[-x] or
|
|
31
|
+
// review-pr-<M>[-x] report goes with pr<M>/ — the same verdict,
|
|
32
|
+
// the same live-token holds, though no pr<M>/ need exist; an
|
|
33
|
+
// impl-<N>[-x] report goes with impl-<N>/ the same way; any other
|
|
34
|
+
// file in it (a merge-bot-<n> report, which names no PR and no
|
|
35
|
+
// ticket) is judged as "anything else" below, so it is removed
|
|
36
|
+
// only when idle past the retention and its member's token is not
|
|
37
|
+
// live.
|
|
27
38
|
// anything else, and a keyed name whose number GitHub resolves to the
|
|
28
39
|
// other type (a pr<N> that is an issue, an impl-<N> that is a PR) or to
|
|
29
40
|
// nothing
|
|
@@ -114,6 +125,11 @@ const REMOVE = [
|
|
|
114
125
|
const PR_ENTRY = /^pr([1-9][0-9]*)$/;
|
|
115
126
|
const IMPL_ENTRY = /^impl-([1-9][0-9]*)$/;
|
|
116
127
|
const DISPOSITIONS_ENTRY = /^dispositions-fix-pr-([1-9][0-9]*)(?:-[a-z])?\.json$/;
|
|
128
|
+
// A member's report file, `reports/<member>.json`: one that names a PR goes
|
|
129
|
+
// with that PR's entries, one that names a ticket with that ticket's.
|
|
130
|
+
const REPORTS_DIR = "reports";
|
|
131
|
+
const REPORT_PR_ENTRY = /^(?:fix-pr|finisher-pr|review-pr)-([1-9][0-9]*)(?:-[a-z])?\.json$/;
|
|
132
|
+
const REPORT_IMPL_ENTRY = /^impl-([1-9][0-9]*)(?:-[a-z])?\.json$/;
|
|
117
133
|
const RUN_ROOT = /^run-/;
|
|
118
134
|
|
|
119
135
|
function how(r) {
|
|
@@ -279,12 +295,50 @@ function judgeRoot(root, names, live, now) {
|
|
|
279
295
|
kept.push({ path, reason: young ? `${young}; modified within 7 days` : "modified within 7 days" });
|
|
280
296
|
};
|
|
281
297
|
|
|
282
|
-
|
|
283
|
-
|
|
298
|
+
// `reports/` is not an entry to age out whole: each file in it is judged as
|
|
299
|
+
// the member's own, so a live member's report is never swept along with an
|
|
300
|
+
// idle directory.
|
|
301
|
+
const entries = [];
|
|
284
302
|
for (const name of names) {
|
|
285
303
|
const path = join(root, name);
|
|
304
|
+
let isDir = false;
|
|
305
|
+
try {
|
|
306
|
+
isDir = name === REPORTS_DIR && lstatSync(path).isDirectory();
|
|
307
|
+
} catch (e) {
|
|
308
|
+
// raced away: judged below as any other entry. Anything else (an
|
|
309
|
+
// unsearchable root) leaves `reports` unclassified, and judging it as an
|
|
310
|
+
// ordinary entry would age the whole directory out.
|
|
311
|
+
if (e.code !== "ENOENT") {
|
|
312
|
+
kept.push({ path, reason: `its type cannot be read: ${e.message}` });
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
if (!isDir) {
|
|
317
|
+
entries.push({ path, name, report: false });
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
let inner;
|
|
321
|
+
try {
|
|
322
|
+
inner = readdirSync(path).sort();
|
|
323
|
+
} catch (e) {
|
|
324
|
+
kept.push({ path, reason: `its entries cannot be listed: ${e.message}` });
|
|
325
|
+
continue;
|
|
326
|
+
}
|
|
327
|
+
for (const f of inner) entries.push({ path: join(path, f), name: f, report: true });
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const keyed = [];
|
|
331
|
+
const numbers = new Set();
|
|
332
|
+
for (const { path, name, report } of entries) {
|
|
286
333
|
let m;
|
|
287
|
-
if (
|
|
334
|
+
if (report) {
|
|
335
|
+
if ((m = REPORT_PR_ENTRY.exec(name))) keyed.push({ path, name, kind: "pr", n: Number(m[1]) });
|
|
336
|
+
else if ((m = REPORT_IMPL_ENTRY.exec(name))) keyed.push({ path, name, kind: "impl", n: Number(m[1]) });
|
|
337
|
+
else {
|
|
338
|
+
byAge(path, name, null);
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
} else if ((m = PR_ENTRY.exec(name))) keyed.push({ path, name, kind: "pr", n: Number(m[1]) });
|
|
288
342
|
else if ((m = IMPL_ENTRY.exec(name))) keyed.push({ path, name, kind: "impl", n: Number(m[1]) });
|
|
289
343
|
else if ((m = DISPOSITIONS_ENTRY.exec(name))) keyed.push({ path, name, kind: "pr", n: Number(m[1]) });
|
|
290
344
|
else {
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// Where a member's report file lives and what state it is in: the reading
|
|
2
|
+
// side of report-schema.mjs, shared by the scripts that look a report up
|
|
3
|
+
// rather than write one.
|
|
4
|
+
//
|
|
5
|
+
// The readers here take a member's report at `<scratch>/reports/<member>.json`
|
|
6
|
+
// (flat: no per-run directory), `<member>` being its ledger name (`impl-412`,
|
|
7
|
+
// `impl-412-b`, `fix-pr-88`, `finisher-pr-88`, `merge-bot-3`). The file is
|
|
8
|
+
// checked against the schema of the member's family, report-schema.mjs's
|
|
9
|
+
// SCHEMAS. The review runner and every member outside those four families
|
|
10
|
+
// have no schema, so no file to look up.
|
|
11
|
+
//
|
|
12
|
+
// This module only looks files up: it writes none, and creates no scratch
|
|
13
|
+
// root.
|
|
14
|
+
|
|
15
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { SCHEMAS, validate } from "./report-schema.mjs";
|
|
18
|
+
import { parseMember } from "./ledger-grammar.mjs";
|
|
19
|
+
import { scratchRoot } from "./fleet-dir.mjs";
|
|
20
|
+
|
|
21
|
+
// The ledger's family name to report-schema.mjs's.
|
|
22
|
+
const SCHEMA_FAMILY = { impl: "impl", "fix-pr": "fix-pr", "finisher-pr": "finisher", "merge-bot": "merge-bot" };
|
|
23
|
+
|
|
24
|
+
/** The report-schema family of a ledger member name, or null for a name no
|
|
25
|
+
* schema covers (a specialist, a review runner, a memory member). */
|
|
26
|
+
export function reportFamily(member) {
|
|
27
|
+
const m = parseMember(member);
|
|
28
|
+
return m === null ? null : SCHEMA_FAMILY[m.family];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export const reportsDir = (scratch) => join(scratch, "reports");
|
|
32
|
+
|
|
33
|
+
export const reportPath = (scratch, member) => join(reportsDir(scratch), `${member}.json`);
|
|
34
|
+
|
|
35
|
+
/** The report file of `member` under `scratch`, read and checked against its
|
|
36
|
+
* family's schema: `{state: "ok", path, report}`, `{state: "invalid", path,
|
|
37
|
+
* violation}` for a file that exists and is unreadable, not JSON, or the
|
|
38
|
+
* wrong shape, or `{state: "absent", path}`. Throws on a member no schema
|
|
39
|
+
* covers: ask reportFamily() first. */
|
|
40
|
+
export function readReport(scratch, member) {
|
|
41
|
+
const family = reportFamily(member);
|
|
42
|
+
if (family === null) throw new Error(`no report schema for '${member}'`);
|
|
43
|
+
const path = reportPath(scratch, member);
|
|
44
|
+
let text;
|
|
45
|
+
try {
|
|
46
|
+
text = readFileSync(path, "utf8");
|
|
47
|
+
} catch (e) {
|
|
48
|
+
if (e.code === "ENOENT" || e.code === "ENOTDIR") return { state: "absent", path };
|
|
49
|
+
return { state: "invalid", path, violation: `$: unreadable (${e.code ?? e.message})` };
|
|
50
|
+
}
|
|
51
|
+
let report;
|
|
52
|
+
try {
|
|
53
|
+
report = JSON.parse(text);
|
|
54
|
+
} catch (e) {
|
|
55
|
+
return { state: "invalid", path, violation: `$: not JSON (${e.message})` };
|
|
56
|
+
}
|
|
57
|
+
const violation = validate(SCHEMAS[family], report);
|
|
58
|
+
return violation === null ? { state: "ok", path, report } : { state: "invalid", path, violation };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** `ok`, `invalid`, `absent`, or `-` when no schema covers the member. */
|
|
62
|
+
export function reportStatus(scratch, member) {
|
|
63
|
+
return reportFamily(member) === null ? "-" : readReport(scratch, member).state;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The implementer report of `ticket` that wins: `{member, path}` for
|
|
67
|
+
* `impl-<ticket>` or `impl-<ticket>-<letter>` with the highest retry letter
|
|
68
|
+
* (the name with no letter is the first attempt, so it ranks lowest), or null
|
|
69
|
+
* when none exists. */
|
|
70
|
+
export function latestImplReport(scratch, ticket) {
|
|
71
|
+
let names;
|
|
72
|
+
try {
|
|
73
|
+
names = readdirSync(reportsDir(scratch));
|
|
74
|
+
} catch (e) {
|
|
75
|
+
if (e.code === "ENOENT" || e.code === "ENOTDIR") return null;
|
|
76
|
+
throw e;
|
|
77
|
+
}
|
|
78
|
+
const mine = new RegExp(`^impl-${ticket}(-[a-z])?\\.json$`);
|
|
79
|
+
let best = null;
|
|
80
|
+
for (const name of names) {
|
|
81
|
+
const m = mine.exec(name);
|
|
82
|
+
if (m === null) continue;
|
|
83
|
+
const letter = m[1] ? m[1].slice(1) : "";
|
|
84
|
+
if (best === null || letter > best.letter) best = { letter, member: name.slice(0, -".json".length) };
|
|
85
|
+
}
|
|
86
|
+
return best === null ? null : { member: best.member, path: reportPath(scratch, best.member) };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** The scratch root this repository's members write under: the path
|
|
90
|
+
* scratch-root.mjs prints, computed the same way, but never created. Throws
|
|
91
|
+
* fleet-dir.mjs's FleetDirUnresolvable where no repository is there to key on. */
|
|
92
|
+
export function defaultScratch() {
|
|
93
|
+
return scratchRoot();
|
|
94
|
+
}
|
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
//
|
|
8
8
|
// append <pr> --closed-own-ticket yes|no --minted-false-claim yes|no
|
|
9
9
|
// --note <text> [--sizing light|heavy] [--profile <p>] [--loc <n>] [--files <n>]
|
|
10
|
+
// [--scratch <dir>]
|
|
10
11
|
// Fills `run_date` (today's UTC date, the ruling date), `pr`, `ticket` (off
|
|
11
|
-
// `gh pr view --json closingIssuesReferences`)
|
|
12
|
+
// `gh pr view --json closingIssuesReferences`), `tier` and, unless
|
|
13
|
+
// `--sizing` names it, `sizing` (see below). Idempotent by
|
|
12
14
|
// PR: a row already present is named and nothing is written, exit 0.
|
|
13
15
|
// check [--live]
|
|
14
16
|
// Post-switch rows only: a filled `tier` must equal the short name of the
|
|
@@ -43,6 +45,17 @@
|
|
|
43
45
|
// `opus`/`sonnet` spelling and are never checked; the header records the
|
|
44
46
|
// switch and how the old values map.
|
|
45
47
|
//
|
|
48
|
+
// `sizing` resolution, in order:
|
|
49
|
+
// 1. `--sizing`, as given — it wins, and stderr names it as the source;
|
|
50
|
+
// 2. else the `sizing.verdict` of the ticket's implementer report,
|
|
51
|
+
// `<scratch>/reports/impl-<ticket>[-x].json` (`--scratch`, default the
|
|
52
|
+
// root scratch-root.mjs prints), the highest retry letter's when several
|
|
53
|
+
// exist — and only when the report checks against its schema, its
|
|
54
|
+
// `sizing.ranBeforePr` is true and its `sizing.signal` is non-empty: a
|
|
55
|
+
// verdict typed after the PR was opened measured nothing, and one with no
|
|
56
|
+
// signal named is a guess;
|
|
57
|
+
// 3. else blank, with a WARNING naming each condition that failed.
|
|
58
|
+
//
|
|
46
59
|
// A row's position in the file is its order: rows are appended, never inserted
|
|
47
60
|
// mid-file or reordered. An in-place edit (a re-judged `class`, a backfilled
|
|
48
61
|
// verdict) keeps the row's position; a ruling meant to supersede an earlier one
|
|
@@ -60,6 +73,8 @@ import { makeDie, defineFlags, isDigits } from "./arg.mjs";
|
|
|
60
73
|
import { parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
|
|
61
74
|
import { parseMember, parseToken, memberTokens, rowPr } from "./ledger-grammar.mjs";
|
|
62
75
|
import { rowText } from "./tier-check.mjs";
|
|
76
|
+
import { latestImplReport, readReport, reportsDir, defaultScratch } from "./report-files.mjs";
|
|
77
|
+
import { FleetDirUnresolvable } from "./fleet-dir.mjs";
|
|
63
78
|
|
|
64
79
|
const NAME = "tier-outcomes";
|
|
65
80
|
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
|
|
@@ -274,6 +289,25 @@ export function resolveTier({ ledger, memberRows, ticket, pr }) {
|
|
|
274
289
|
return tierFromLedger(ledger, ticket, pr) ?? tierFromMembers(memberRows, ticket);
|
|
275
290
|
}
|
|
276
291
|
|
|
292
|
+
// The `sizing` covariate off the ticket's implementer report under `scratch`:
|
|
293
|
+
// `{sizing, source, warning: null}`, or `{sizing: "", source: null, warning}`
|
|
294
|
+
// naming every condition that failed. The report must check against its
|
|
295
|
+
// schema first: a verdict read off a malformed file is not a measurement.
|
|
296
|
+
function sizingFromReport(scratch, ticket) {
|
|
297
|
+
const blank = (warning) => ({ sizing: "", source: null, warning });
|
|
298
|
+
const found = latestImplReport(scratch, ticket);
|
|
299
|
+
const report = found === null ? null : readReport(scratch, found.member);
|
|
300
|
+
if (report === null || report.state === "absent") return blank(`no implementer report for ticket #${ticket} under ${reportsDir(scratch)}`);
|
|
301
|
+
if (report.state === "invalid") return blank(`${report.path} does not check against its schema: ${report.violation}`);
|
|
302
|
+
const s = report.report.sizing;
|
|
303
|
+
if (s === null) return blank(`${report.path} carries no sizing (sizing is null)`);
|
|
304
|
+
const failed = [];
|
|
305
|
+
if (s.ranBeforePr !== true) failed.push("sizing.ranBeforePr is not true");
|
|
306
|
+
if (s.signal.trim() === "") failed.push("sizing.signal is empty");
|
|
307
|
+
if (failed.length) return blank(`${report.path}: ${failed.join(" and ")}`);
|
|
308
|
+
return { sizing: s.verdict, source: report.path, warning: null };
|
|
309
|
+
}
|
|
310
|
+
|
|
277
311
|
// `check` over the committed files. Returns failure lines and skip counts.
|
|
278
312
|
export function checkRows(rows, memberRows) {
|
|
279
313
|
const failures = [];
|
|
@@ -338,13 +372,13 @@ export function unrecordedReviewedPrs(ledger, rows) {
|
|
|
338
372
|
// ---------------------------------------------------------------------------
|
|
339
373
|
|
|
340
374
|
const USAGE = "usage: tier-outcomes.mjs append <pr> --closed-own-ticket yes|no --minted-false-claim yes|no --note <text> "
|
|
341
|
-
+ "[--sizing light|heavy] [--profile <p>] [--loc <n>] [--files <n>] [--file <tsv>] [--member-outcomes <tsv>] [--ledger <path>]\n"
|
|
375
|
+
+ "[--sizing light|heavy] [--profile <p>] [--loc <n>] [--files <n>] [--scratch <dir>] [--file <tsv>] [--member-outcomes <tsv>] [--ledger <path>]\n"
|
|
342
376
|
+ " tier-outcomes.mjs check [--live] [--file <tsv>] [--member-outcomes <tsv>] [--ledger <path>]";
|
|
343
377
|
|
|
344
378
|
const die = makeDie(NAME);
|
|
345
379
|
const APPEND_FLAGS = {
|
|
346
380
|
"closed-own-ticket": "value", "minted-false-claim": "value", note: "value",
|
|
347
|
-
sizing: "value", profile: "value", loc: "value", files: "value",
|
|
381
|
+
sizing: "value", profile: "value", loc: "value", files: "value", scratch: "value",
|
|
348
382
|
};
|
|
349
383
|
const FLAGS = { ...APPEND_FLAGS, file: "value", "member-outcomes": "value", ledger: "value", live: "bool" };
|
|
350
384
|
const { arg, has, sweep } = defineFlags(die, { flags: FLAGS });
|
|
@@ -442,9 +476,10 @@ function append(pr, paths) {
|
|
|
442
476
|
if (!isDigits(pr) || Number(pr) === 0) die(`append needs a PR number, got '${pr}'`);
|
|
443
477
|
pr = String(Number(pr));
|
|
444
478
|
if (has("live")) die("--live is a check flag");
|
|
479
|
+
const sizingFlag = arg("sizing");
|
|
445
480
|
const fields = {
|
|
446
481
|
closed_own_ticket: arg("closed-own-ticket"), minted_false_claim: arg("minted-false-claim"),
|
|
447
|
-
note: arg("note"), sizing:
|
|
482
|
+
note: arg("note"), sizing: sizingFlag ?? "", profile: arg("profile") ?? "", loc: arg("loc") ?? "", files: arg("files") ?? "",
|
|
448
483
|
};
|
|
449
484
|
for (const col of ["closed_own_ticket", "minted_false_claim", "note"]) {
|
|
450
485
|
if (fields[col] === null) die(`append needs --${col.replaceAll("_", "-")}\n${USAGE}`);
|
|
@@ -474,6 +509,25 @@ function append(pr, paths) {
|
|
|
474
509
|
if (resolved.warning) console.error(`${NAME}: WARNING tier left blank for PR #${pr} (ticket #${ticket}): ${resolved.warning}`);
|
|
475
510
|
else console.error(`${NAME}: tier=${resolved.tier} from ${resolved.source}`);
|
|
476
511
|
|
|
512
|
+
if (sizingFlag !== null) {
|
|
513
|
+
console.error(`${NAME}: sizing=${fields.sizing} from --sizing`);
|
|
514
|
+
} else {
|
|
515
|
+
let scratch = paths.scratch;
|
|
516
|
+
let unresolvable = null;
|
|
517
|
+
if (scratch === null) {
|
|
518
|
+
try {
|
|
519
|
+
scratch = defaultScratch();
|
|
520
|
+
} catch (e) {
|
|
521
|
+
if (!(e instanceof FleetDirUnresolvable)) throw e;
|
|
522
|
+
unresolvable = e.message;
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
const sized = unresolvable === null ? sizingFromReport(scratch, ticket) : { sizing: "", source: null, warning: `no scratch root to look in: ${unresolvable}` };
|
|
526
|
+
fields.sizing = sized.sizing;
|
|
527
|
+
if (sized.warning) console.error(`${NAME}: WARNING sizing left blank for PR #${pr} (ticket #${ticket}): ${sized.warning}`);
|
|
528
|
+
else console.error(`${NAME}: sizing=${sized.sizing} from ${sized.source}`);
|
|
529
|
+
}
|
|
530
|
+
|
|
477
531
|
const line = formatRow({ ...fields, run_date: today(), pr, ticket, tier: resolved.tier });
|
|
478
532
|
if (text === null) writeFileSync(paths.file, `# ${COLUMNS.join("\t")}\n${line}\n`);
|
|
479
533
|
else appendFileSync(paths.file, `${text === "" || text.endsWith("\n") ? "" : "\n"}${line}\n`);
|
|
@@ -507,6 +561,7 @@ function main() {
|
|
|
507
561
|
file: arg("file") ?? "docs/metrics/tier-outcomes.tsv",
|
|
508
562
|
members: arg("member-outcomes") ?? "docs/metrics/member-outcomes.tsv",
|
|
509
563
|
ledger: arg("ledger"),
|
|
564
|
+
scratch: arg("scratch"),
|
|
510
565
|
};
|
|
511
566
|
// Below every flag read, per arg.mjs's ordering contract, and above the
|
|
512
567
|
// subcommand checks, so a stray flag is refused by name before anything
|
package/skills/run-team/SKILL.md
CHANGED
|
@@ -1462,32 +1462,37 @@ untouched, so re-running it is safe. That append is the whole duty.
|
|
|
1462
1462
|
The row's last four fields are the ticket's difficulty, and they are what lets a
|
|
1463
1463
|
tier comparison condition on the thing that swamps it. `sizing` is the
|
|
1464
1464
|
**member's** own `light`/`heavy` verdict from its phase-2 `sizing-a-ticket`
|
|
1465
|
-
run,
|
|
1466
|
-
|
|
1465
|
+
run, which `append` reads itself off the member's report file
|
|
1466
|
+
(`<scratch>/reports/impl-<ticket>[-x].json`, its `sizing.verdict`, the highest
|
|
1467
|
+
retry letter's when there are several), so you pass no `--sizing`; the member
|
|
1468
|
+
also puts it in the PR body's `Sizing:` line, with the signal it turned on;
|
|
1467
1469
|
`profile`, `loc` and `files` all come from `diff-stats.mjs` over the merged
|
|
1468
1470
|
diff. **Phase 0 does not size anything** — it shortlists, a Pull claims, and
|
|
1469
1471
|
the sizing run happens inside the member after both, which is why the verdict
|
|
1470
|
-
has to travel in the
|
|
1472
|
+
has to travel in the member's report rather than being something you already hold.
|
|
1471
1473
|
|
|
1472
1474
|
**A verdict is a measurement only if it post-dates the run that produced it.** A
|
|
1473
1475
|
`Sizing:` line typed in before the skill ever ran is byte-identical to one the
|
|
1474
|
-
skill produced, so the body cannot settle this alone and
|
|
1475
|
-
carries the second source:
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1476
|
+
skill produced, so the body cannot settle this alone, and neither can the
|
|
1477
|
+
member's prose. The report file carries the second source: `sizing.ranBeforePr`,
|
|
1478
|
+
whether `sizing-a-ticket` ran before the PR was opened (measured on a member
|
|
1479
|
+
that wrote the line first, caught itself, and corrected it), and
|
|
1480
|
+
`sizing.signal`, what the verdict turned on. `append` applies the rule itself:
|
|
1481
|
+
it records `sizing` only when `ranBeforePr` is true AND `signal` is non-empty.
|
|
1482
|
+
Any other state — no report file, one that fails its schema, `sizing` null,
|
|
1483
|
+
`ranBeforePr` not true, an empty `signal` — leaves the field BLANK with a
|
|
1484
|
+
WARNING naming which, and you say which in `note`. A `--sizing` you do pass
|
|
1485
|
+
wins over the file, and stderr names the flag as the source: do not use it to
|
|
1486
|
+
fill that gap, and do not re-derive the verdict from the diff to fill it
|
|
1487
|
+
either — `sizing-a-ticket` weighs more than file count, and a substituted proxy
|
|
1488
|
+
reads as measured too. **A value not in hand is left BLANK, never estimated** —
|
|
1489
|
+
blank reads as unknown and drops the row from a stratified comparison, while a
|
|
1490
|
+
guess reads as measured and poisons one. Blank still means the field is
|
|
1491
|
+
WRITTEN and empty: omit the flag (`--sizing`, `--profile`, `--loc`, `--files`)
|
|
1492
|
+
and the script writes that field empty at full width, because a row of some
|
|
1493
|
+
in-between width cannot be told apart from a shifted one. The `note` field is
|
|
1494
|
+
free text and sits before those four, so the script refuses a tab or newline in
|
|
1495
|
+
it: one tab would shift all four for that row alone.
|
|
1491
1496
|
|
|
1492
1497
|
**Then record the run's member facts — do not author them.** Scrape EVERY session
|
|
1493
1498
|
directory for this cwd, not one:
|
|
@@ -2183,27 +2188,18 @@ while :; do # one tick
|
|
|
2183
2188
|
echo "WATCHER DEGRADED: core REST budget=$rl — CI polling paused, silence is NOT green"
|
|
2184
2189
|
budget_out=1
|
|
2185
2190
|
fi
|
|
2186
|
-
else #
|
|
2191
|
+
else # every RECOVERED waits for the pass gates below
|
|
2187
2192
|
prs=$(gh pr list --state open --limit "$pr_cap" --json number --jq '.[].number' 2>/dev/null) || prs=ERR
|
|
2188
2193
|
if [ "$prs" = ERR ]; then
|
|
2189
2194
|
if [ -z "$list_out" ]; then # global latch: this cause is account-level too
|
|
2190
2195
|
echo "WATCHER DEGRADED: cannot list open PRs — CI polling paused, silence is NOT green"
|
|
2191
2196
|
list_out=1
|
|
2192
2197
|
fi
|
|
2193
|
-
else
|
|
2194
|
-
if [ -n "$list_out" ]; then
|
|
2195
|
-
echo "WATCHER RECOVERED: open-PR list readable again — CI polling resumed"
|
|
2196
|
-
list_out=
|
|
2197
|
-
fi
|
|
2198
|
+
else # a list or cap RECOVERED waits for the pass gates too
|
|
2198
2199
|
count=$(printf '%s\n' "$prs" | grep -c .) || : # grep -c exits 1 on zero matches; set -e must not take it
|
|
2199
|
-
if [ "$count" -ge "$pr_cap" ]; then
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
cap_out=1
|
|
2203
|
-
fi
|
|
2204
|
-
elif [ -n "$cap_out" ]; then
|
|
2205
|
-
echo "WATCHER RECOVERED: open-PR list under its --limit $pr_cap — every open PR watched again"
|
|
2206
|
-
cap_out=
|
|
2200
|
+
if [ "$count" -ge "$pr_cap" ] && [ -z "$cap_out" ]; then # global latch: a full page may have been cut there
|
|
2201
|
+
echo "WATCHER DEGRADED: open-PR list hit its --limit $pr_cap — PRs past it are NOT watched, silence on them is NOT green"
|
|
2202
|
+
cap_out=1
|
|
2207
2203
|
fi
|
|
2208
2204
|
# gh pr list is GraphQL, so $rl still holds. The pass is budgeted at 4 core
|
|
2209
2205
|
# REST units per listed PR on top of the 200 floor. The 4 is the measured
|
|
@@ -2220,9 +2216,9 @@ while :; do # one tick
|
|
|
2220
2216
|
# The workflow and run lookups bill a second core bucket that
|
|
2221
2217
|
# `rate_limit` does not report. Its count is only in the response
|
|
2222
2218
|
# headers of a lookup that bills it: one 1-unit read per tick, 4 units
|
|
2223
|
-
# per listed PR on the same 200 floor. Recovery for
|
|
2224
|
-
# until this gate clears too, so no line says polling resumed
|
|
2225
|
-
# this one still pauses it.
|
|
2219
|
+
# per listed PR on the same 200 floor. Recovery for every global latch
|
|
2220
|
+
# waits until this gate clears too, so no line says polling resumed
|
|
2221
|
+
# while this one still pauses it.
|
|
2226
2222
|
hrl=$(gh api -i 'repos/{owner}/{repo}/actions/workflows?per_page=1' 2>/dev/null | awk 'tolower($1)=="x-ratelimit-remaining:" {sub(/\r$/, "", $2); print $2; exit}')
|
|
2227
2223
|
case "$hrl" in ''|*[!0-9]*) hrl=ERR;; esac # a missing header is an outage, not a budget
|
|
2228
2224
|
if [ "$hrl" = ERR ] || [ "$hrl" -lt $((200 + 4 * count)) ]; then
|
|
@@ -2232,6 +2228,14 @@ while :; do # one tick
|
|
|
2232
2228
|
hidden_out=1
|
|
2233
2229
|
fi
|
|
2234
2230
|
else
|
|
2231
|
+
if [ -n "$list_out" ]; then
|
|
2232
|
+
echo "WATCHER RECOVERED: open-PR list readable again — CI polling resumed"
|
|
2233
|
+
list_out=
|
|
2234
|
+
fi
|
|
2235
|
+
if [ -n "$cap_out" ] && [ "$count" -lt "$pr_cap" ]; then
|
|
2236
|
+
echo "WATCHER RECOVERED: open-PR list under its --limit $pr_cap — every open PR watched again"
|
|
2237
|
+
cap_out=
|
|
2238
|
+
fi
|
|
2235
2239
|
if [ -n "$budget_out" ]; then
|
|
2236
2240
|
echo "WATCHER RECOVERED: core budget=$rl — CI polling resumed"
|
|
2237
2241
|
budget_out=
|
|
@@ -2361,10 +2365,14 @@ would run dry mid-pass and each emit a blind line. The DEGRADED line from each
|
|
|
2361
2365
|
pass gate names the amount it needed and the count it needed it for — except an
|
|
2362
2366
|
unreadable header, which has no amount to compare and prints `bucket=ERR` alone
|
|
2363
2367
|
— so a healthy-looking `budget=999` is not read as an outage with no cause. Only the
|
|
2364
|
-
pass gates emit RECOVERED lines
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
+
pass gates emit RECOVERED lines — the budget's and the unreported bucket's, and the
|
|
2369
|
+
open-PR list's and its cap's with them — and all of them only once the pass will
|
|
2370
|
+
actually run: announcing recovery at the floor, or for one bucket while the other
|
|
2371
|
+
still pauses the pass, would report resumption on a tick whose other gate then
|
|
2372
|
+
pauses it again — a DEGRADED/RECOVERED pair every tick — and announcing the list or
|
|
2373
|
+
its cap on the tick that reads it back would say polling resumed on a tick that
|
|
2374
|
+
polls nothing. A list or cap latch held across a paused tick stays latched and says
|
|
2375
|
+
nothing more.
|
|
2368
2376
|
|
|
2369
2377
|
**Judge `ci-state` on its payload, never its exit code** — the same rule as
|
|
2370
2378
|
`merge-gate.mjs` applies on the merge path, applied to the watcher. `not-green` is an
|