@feigi/fleet-ctl 3.50.2 → 3.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/member-outcomes.mjs +59 -15
- package/scripts/reclaim-scratch.mjs +57 -3
- package/scripts/report-files.mjs +94 -0
- package/scripts/tier-outcomes.mjs +59 -4
- package/skills/run-team/SKILL.md +25 -20
package/package.json
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// Scraper for per-member model/effort facts. Pure over omp's own subagent
|
|
2
|
-
// transcripts: no clock, no network, no gh.
|
|
2
|
+
// transcripts: no clock, no network, no gh. The one column that looks beside
|
|
3
|
+
// them is `report`, which asks whether the member's report file is in the
|
|
4
|
+
// scratch root (`--scratch <dir>`, default the root scratch-root.mjs prints).
|
|
3
5
|
//
|
|
4
6
|
// The transcript parsing (fold, model/effort extraction, ticket/pr naming,
|
|
5
7
|
// and the `gh pr create` result an implementer's `pr` comes from)
|
|
@@ -11,6 +13,8 @@ import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
|
11
13
|
import { join, basename } from "node:path";
|
|
12
14
|
|
|
13
15
|
import { readOmpSession, normalizeModel, parseMemberName, isOmpSessionDirName } from "./member-record.mjs";
|
|
16
|
+
import { reportFamily, reportStatus, defaultScratch } from "./report-files.mjs";
|
|
17
|
+
import { FleetDirUnresolvable } from "./fleet-dir.mjs";
|
|
14
18
|
|
|
15
19
|
export { normalizeModel, parseMemberName };
|
|
16
20
|
|
|
@@ -84,19 +88,34 @@ export function rowsForSession(sessionDir, stats = {}) {
|
|
|
84
88
|
}));
|
|
85
89
|
}
|
|
86
90
|
|
|
87
|
-
// APPENDED
|
|
88
|
-
// in the
|
|
89
|
-
//
|
|
90
|
-
//
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
//
|
|
91
|
+
// A column is APPENDED, never inserted, as a rule: a read-out that indexes by
|
|
92
|
+
// position (`$5`, in the file's header) silently repoints when one is added
|
|
93
|
+
// ahead of it. `report` is placed after `subagent_type` regardless, which
|
|
94
|
+
// moves the four priced columns one place right; every script that reads this
|
|
95
|
+
// file through parseTsv takes those four by name, and the header's
|
|
96
|
+
// position-indexed read-outs name columns ahead of them. `subagent_type`
|
|
97
|
+
// follows `harness`. `cost` is the provider's own per-turn `usage.cost.total`,
|
|
98
|
+
// summed to the micro-dollar — the $ of record; `tokens_cache_write_1h` and
|
|
99
|
+
// `cost` are blank where the transcript recorded no figure, never 0.
|
|
94
100
|
export const COLUMNS = [
|
|
95
101
|
"session", "run_date", "role", "member", "model", "effort", "ticket", "pr",
|
|
96
102
|
"tokens_cache_create", "tokens_out", "wall_s", "turns", "agent", "harness",
|
|
97
|
-
"subagent_type", "tokens_in", "tokens_cache_read", "tokens_cache_write_1h", "cost",
|
|
103
|
+
"subagent_type", "report", "tokens_in", "tokens_cache_read", "tokens_cache_write_1h", "cost",
|
|
98
104
|
];
|
|
99
105
|
|
|
106
|
+
// `report`: what reportStatus() says of the row's member under `scratch` — a
|
|
107
|
+
// pure function of the member name and the file system, never of the
|
|
108
|
+
// transcript, so it is stamped here on the rows rowsForSession() returned.
|
|
109
|
+
// With no `scratch` (null: none could be resolved) a member a schema covers
|
|
110
|
+
// reads blank — unknown, the file's own spelling of it — and one none covers
|
|
111
|
+
// still reads `-`, which needs no file to look at.
|
|
112
|
+
export function withReports(rows, scratch) {
|
|
113
|
+
return rows.map((r) => ({
|
|
114
|
+
...r,
|
|
115
|
+
report: scratch === null && reportFamily(r.member) !== null ? "" : reportStatus(scratch, r.member),
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
118
|
+
|
|
100
119
|
// Row objects use camelCase; the file uses snake_case. One map, one direction
|
|
101
120
|
// each, so a rename cannot silently drop a column.
|
|
102
121
|
const FIELD = {
|
|
@@ -164,21 +183,27 @@ if (isCLI(import.meta.url)) {
|
|
|
164
183
|
const argv = process.argv.slice(2);
|
|
165
184
|
const fileIdx = argv.indexOf("--file");
|
|
166
185
|
const file = fileIdx >= 0 ? argv[fileIdx + 1] : "docs/metrics/member-outcomes.tsv";
|
|
186
|
+
const scratchIdx = argv.indexOf("--scratch");
|
|
187
|
+
const scratchArg = scratchIdx >= 0 ? argv[scratchIdx + 1] : null;
|
|
167
188
|
// An unrecognised `--flag` is silently dropped by the dirs filter below
|
|
168
189
|
// rather than refused — worst case `--file=/tmp/x.tsv` (no such flag;
|
|
169
190
|
// --file wants a SPACE, not `=`) never sets fileIdx, so this would silently
|
|
170
191
|
// write the production metrics file instead of the path the operator asked
|
|
171
192
|
// for.
|
|
172
|
-
const bad = argv.find((a) => a.startsWith("--") && a !== "--file");
|
|
173
|
-
if (bad) die(`unknown option ${bad} (did you mean --file <path>?)`);
|
|
193
|
+
const bad = argv.find((a) => a.startsWith("--") && a !== "--file" && a !== "--scratch");
|
|
194
|
+
if (bad) die(`unknown option ${bad} (did you mean --file <path> or --scratch <dir>?)`);
|
|
174
195
|
|
|
175
196
|
// fileIdx is -1 when --file is absent, which makes fileIdx + 1 equal 0 — the
|
|
176
197
|
// FIRST positional argument, not a real index into argv. Without the
|
|
177
198
|
// fileIdx < 0 guard this filters out the session dir itself, and the bare
|
|
178
199
|
// form (the one the spec, the backfill loop and run-team's phase-3
|
|
179
|
-
// instruction all use) always exits 2.
|
|
180
|
-
|
|
181
|
-
|
|
200
|
+
// instruction all use) always exits 2. `--scratch`'s value is skipped the
|
|
201
|
+
// same way.
|
|
202
|
+
const valueOf = (idx) => (i) => idx >= 0 && i === idx + 1;
|
|
203
|
+
const isFileValue = valueOf(fileIdx);
|
|
204
|
+
const isScratchValue = valueOf(scratchIdx);
|
|
205
|
+
const dirs = argv.filter((a, i) => !a.startsWith("--") && !isFileValue(i) && !isScratchValue(i));
|
|
206
|
+
if (dirs.length !== 1) die("usage: member-outcomes.mjs <session-dir> [--file <tsv>] [--scratch <dir>]");
|
|
182
207
|
// arg.mjs's rule, consumed rather than copied. This line used to
|
|
183
208
|
// hand-write `!file || file.startsWith("--")`, which was that rule with the
|
|
184
209
|
// blank spelling missing — so `--file " "` was accepted here and wrote the
|
|
@@ -187,6 +212,16 @@ if (isCLI(import.meta.url)) {
|
|
|
187
212
|
// is imported. The refusal WORDING stays this script's own: `--file` is the
|
|
188
213
|
// flag the operator typed.
|
|
189
214
|
if (isFlagLike(file)) die("--file needs a path");
|
|
215
|
+
if (scratchIdx >= 0 && isFlagLike(scratchArg)) die("--scratch needs a path");
|
|
216
|
+
// An explicit root that is not a directory is a mistyped path, and reading it
|
|
217
|
+
// as a root would stamp `absent` on every covered row and, by the replace-by-key
|
|
218
|
+
// merge, over a recorded `ok`/`invalid` — indistinguishable from "wrote no report".
|
|
219
|
+
if (scratchIdx >= 0) {
|
|
220
|
+
let isDir = false;
|
|
221
|
+
try { isDir = statSync(scratchArg).isDirectory(); }
|
|
222
|
+
catch (e) { die(`cannot read --scratch ${scratchArg}: ${e.code ?? e.message}`); }
|
|
223
|
+
if (!isDir) die(`--scratch ${scratchArg} is not a directory`);
|
|
224
|
+
}
|
|
190
225
|
|
|
191
226
|
const sessionDir = dirs[0].replace(/\/+$/, "");
|
|
192
227
|
// A wrong guess used to exit 0 having scraped and written nothing, because
|
|
@@ -223,8 +258,17 @@ if (isCLI(import.meta.url)) {
|
|
|
223
258
|
try { existing = parseTsv(prev); }
|
|
224
259
|
catch (e) { die(`${file} is not readable as this schema: ${e.message}`); }
|
|
225
260
|
|
|
261
|
+
let scratch = scratchArg;
|
|
262
|
+
if (scratch === null) {
|
|
263
|
+
try { scratch = defaultScratch(); }
|
|
264
|
+
catch (e) {
|
|
265
|
+
if (!(e instanceof FleetDirUnresolvable)) throw e;
|
|
266
|
+
process.stderr.write(`${NAME}: WARNING no scratch root to look for report files in (${e.message}) — \`report\` is left blank where a schema covers the member; pass --scratch <dir>\n`);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
226
270
|
const stats = {};
|
|
227
|
-
const merged = mergeRows(existing, rowsForSession(sessionDir, stats));
|
|
271
|
+
const merged = mergeRows(existing, withReports(rowsForSession(sessionDir, stats), scratch));
|
|
228
272
|
// Write-then-rename: writeFileSync truncates before it writes, so an interrupt
|
|
229
273
|
// over the committed corpus left a torn file that the NEXT scrape read, merged
|
|
230
274
|
// and rewrote — losing every other session's rows for good.
|
|
@@ -24,6 +24,17 @@
|
|
|
24
24
|
// removed when issue #N is CLOSED, or OPEN with no live
|
|
25
25
|
// impl-<N>[-x] token and idle past the retention, so a bailed or
|
|
26
26
|
// released ticket's partition is not inherited by its next claim.
|
|
27
|
+
// reports/<member>.json
|
|
28
|
+
// a member's report file. `reports/` itself is never removed, and
|
|
29
|
+
// is not aged out whole: each file is judged on its own, by the
|
|
30
|
+
// entry it goes with. A fix-pr-<M>[-x], finisher-pr-<M>[-x] or
|
|
31
|
+
// review-pr-<M>[-x] report goes with pr<M>/ — the same verdict,
|
|
32
|
+
// the same live-token holds, though no pr<M>/ need exist; an
|
|
33
|
+
// impl-<N>[-x] report goes with impl-<N>/ the same way; any other
|
|
34
|
+
// file in it (a merge-bot-<n> report, which names no PR and no
|
|
35
|
+
// ticket) is judged as "anything else" below, so it is removed
|
|
36
|
+
// only when idle past the retention and its member's token is not
|
|
37
|
+
// live.
|
|
27
38
|
// anything else, and a keyed name whose number GitHub resolves to the
|
|
28
39
|
// other type (a pr<N> that is an issue, an impl-<N> that is a PR) or to
|
|
29
40
|
// nothing
|
|
@@ -114,6 +125,11 @@ const REMOVE = [
|
|
|
114
125
|
const PR_ENTRY = /^pr([1-9][0-9]*)$/;
|
|
115
126
|
const IMPL_ENTRY = /^impl-([1-9][0-9]*)$/;
|
|
116
127
|
const DISPOSITIONS_ENTRY = /^dispositions-fix-pr-([1-9][0-9]*)(?:-[a-z])?\.json$/;
|
|
128
|
+
// A member's report file, `reports/<member>.json`: one that names a PR goes
|
|
129
|
+
// with that PR's entries, one that names a ticket with that ticket's.
|
|
130
|
+
const REPORTS_DIR = "reports";
|
|
131
|
+
const REPORT_PR_ENTRY = /^(?:fix-pr|finisher-pr|review-pr)-([1-9][0-9]*)(?:-[a-z])?\.json$/;
|
|
132
|
+
const REPORT_IMPL_ENTRY = /^impl-([1-9][0-9]*)(?:-[a-z])?\.json$/;
|
|
117
133
|
const RUN_ROOT = /^run-/;
|
|
118
134
|
|
|
119
135
|
function how(r) {
|
|
@@ -279,12 +295,50 @@ function judgeRoot(root, names, live, now) {
|
|
|
279
295
|
kept.push({ path, reason: young ? `${young}; modified within 7 days` : "modified within 7 days" });
|
|
280
296
|
};
|
|
281
297
|
|
|
282
|
-
|
|
283
|
-
|
|
298
|
+
// `reports/` is not an entry to age out whole: each file in it is judged as
|
|
299
|
+
// the member's own, so a live member's report is never swept along with an
|
|
300
|
+
// idle directory.
|
|
301
|
+
const entries = [];
|
|
284
302
|
for (const name of names) {
|
|
285
303
|
const path = join(root, name);
|
|
304
|
+
let isDir = false;
|
|
305
|
+
try {
|
|
306
|
+
isDir = name === REPORTS_DIR && lstatSync(path).isDirectory();
|
|
307
|
+
} catch (e) {
|
|
308
|
+
// raced away: judged below as any other entry. Anything else (an
|
|
309
|
+
// unsearchable root) leaves `reports` unclassified, and judging it as an
|
|
310
|
+
// ordinary entry would age the whole directory out.
|
|
311
|
+
if (e.code !== "ENOENT") {
|
|
312
|
+
kept.push({ path, reason: `its type cannot be read: ${e.message}` });
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
if (!isDir) {
|
|
317
|
+
entries.push({ path, name, report: false });
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
let inner;
|
|
321
|
+
try {
|
|
322
|
+
inner = readdirSync(path).sort();
|
|
323
|
+
} catch (e) {
|
|
324
|
+
kept.push({ path, reason: `its entries cannot be listed: ${e.message}` });
|
|
325
|
+
continue;
|
|
326
|
+
}
|
|
327
|
+
for (const f of inner) entries.push({ path: join(path, f), name: f, report: true });
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const keyed = [];
|
|
331
|
+
const numbers = new Set();
|
|
332
|
+
for (const { path, name, report } of entries) {
|
|
286
333
|
let m;
|
|
287
|
-
if (
|
|
334
|
+
if (report) {
|
|
335
|
+
if ((m = REPORT_PR_ENTRY.exec(name))) keyed.push({ path, name, kind: "pr", n: Number(m[1]) });
|
|
336
|
+
else if ((m = REPORT_IMPL_ENTRY.exec(name))) keyed.push({ path, name, kind: "impl", n: Number(m[1]) });
|
|
337
|
+
else {
|
|
338
|
+
byAge(path, name, null);
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
} else if ((m = PR_ENTRY.exec(name))) keyed.push({ path, name, kind: "pr", n: Number(m[1]) });
|
|
288
342
|
else if ((m = IMPL_ENTRY.exec(name))) keyed.push({ path, name, kind: "impl", n: Number(m[1]) });
|
|
289
343
|
else if ((m = DISPOSITIONS_ENTRY.exec(name))) keyed.push({ path, name, kind: "pr", n: Number(m[1]) });
|
|
290
344
|
else {
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// Where a member's report file lives and what state it is in: the reading
|
|
2
|
+
// side of report-schema.mjs, shared by the scripts that look a report up
|
|
3
|
+
// rather than write one.
|
|
4
|
+
//
|
|
5
|
+
// The readers here take a member's report at `<scratch>/reports/<member>.json`
|
|
6
|
+
// (flat: no per-run directory), `<member>` being its ledger name (`impl-412`,
|
|
7
|
+
// `impl-412-b`, `fix-pr-88`, `finisher-pr-88`, `merge-bot-3`). The file is
|
|
8
|
+
// checked against the schema of the member's family, report-schema.mjs's
|
|
9
|
+
// SCHEMAS. The review runner and every member outside those four families
|
|
10
|
+
// have no schema, so no file to look up.
|
|
11
|
+
//
|
|
12
|
+
// This module only looks files up: it writes none, and creates no scratch
|
|
13
|
+
// root.
|
|
14
|
+
|
|
15
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { SCHEMAS, validate } from "./report-schema.mjs";
|
|
18
|
+
import { parseMember } from "./ledger-grammar.mjs";
|
|
19
|
+
import { scratchRoot } from "./fleet-dir.mjs";
|
|
20
|
+
|
|
21
|
+
// The ledger's family name to report-schema.mjs's.
|
|
22
|
+
const SCHEMA_FAMILY = { impl: "impl", "fix-pr": "fix-pr", "finisher-pr": "finisher", "merge-bot": "merge-bot" };
|
|
23
|
+
|
|
24
|
+
/** The report-schema family of a ledger member name, or null for a name no
|
|
25
|
+
* schema covers (a specialist, a review runner, a memory member). */
|
|
26
|
+
export function reportFamily(member) {
|
|
27
|
+
const m = parseMember(member);
|
|
28
|
+
return m === null ? null : SCHEMA_FAMILY[m.family];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export const reportsDir = (scratch) => join(scratch, "reports");
|
|
32
|
+
|
|
33
|
+
export const reportPath = (scratch, member) => join(reportsDir(scratch), `${member}.json`);
|
|
34
|
+
|
|
35
|
+
/** The report file of `member` under `scratch`, read and checked against its
|
|
36
|
+
* family's schema: `{state: "ok", path, report}`, `{state: "invalid", path,
|
|
37
|
+
* violation}` for a file that exists and is unreadable, not JSON, or the
|
|
38
|
+
* wrong shape, or `{state: "absent", path}`. Throws on a member no schema
|
|
39
|
+
* covers: ask reportFamily() first. */
|
|
40
|
+
export function readReport(scratch, member) {
|
|
41
|
+
const family = reportFamily(member);
|
|
42
|
+
if (family === null) throw new Error(`no report schema for '${member}'`);
|
|
43
|
+
const path = reportPath(scratch, member);
|
|
44
|
+
let text;
|
|
45
|
+
try {
|
|
46
|
+
text = readFileSync(path, "utf8");
|
|
47
|
+
} catch (e) {
|
|
48
|
+
if (e.code === "ENOENT" || e.code === "ENOTDIR") return { state: "absent", path };
|
|
49
|
+
return { state: "invalid", path, violation: `$: unreadable (${e.code ?? e.message})` };
|
|
50
|
+
}
|
|
51
|
+
let report;
|
|
52
|
+
try {
|
|
53
|
+
report = JSON.parse(text);
|
|
54
|
+
} catch (e) {
|
|
55
|
+
return { state: "invalid", path, violation: `$: not JSON (${e.message})` };
|
|
56
|
+
}
|
|
57
|
+
const violation = validate(SCHEMAS[family], report);
|
|
58
|
+
return violation === null ? { state: "ok", path, report } : { state: "invalid", path, violation };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** `ok`, `invalid`, `absent`, or `-` when no schema covers the member. */
|
|
62
|
+
export function reportStatus(scratch, member) {
|
|
63
|
+
return reportFamily(member) === null ? "-" : readReport(scratch, member).state;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The implementer report of `ticket` that wins: `{member, path}` for
|
|
67
|
+
* `impl-<ticket>` or `impl-<ticket>-<letter>` with the highest retry letter
|
|
68
|
+
* (the name with no letter is the first attempt, so it ranks lowest), or null
|
|
69
|
+
* when none exists. */
|
|
70
|
+
export function latestImplReport(scratch, ticket) {
|
|
71
|
+
let names;
|
|
72
|
+
try {
|
|
73
|
+
names = readdirSync(reportsDir(scratch));
|
|
74
|
+
} catch (e) {
|
|
75
|
+
if (e.code === "ENOENT" || e.code === "ENOTDIR") return null;
|
|
76
|
+
throw e;
|
|
77
|
+
}
|
|
78
|
+
const mine = new RegExp(`^impl-${ticket}(-[a-z])?\\.json$`);
|
|
79
|
+
let best = null;
|
|
80
|
+
for (const name of names) {
|
|
81
|
+
const m = mine.exec(name);
|
|
82
|
+
if (m === null) continue;
|
|
83
|
+
const letter = m[1] ? m[1].slice(1) : "";
|
|
84
|
+
if (best === null || letter > best.letter) best = { letter, member: name.slice(0, -".json".length) };
|
|
85
|
+
}
|
|
86
|
+
return best === null ? null : { member: best.member, path: reportPath(scratch, best.member) };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** The scratch root this repository's members write under: the path
|
|
90
|
+
* scratch-root.mjs prints, computed the same way, but never created. Throws
|
|
91
|
+
* fleet-dir.mjs's FleetDirUnresolvable where no repository is there to key on. */
|
|
92
|
+
export function defaultScratch() {
|
|
93
|
+
return scratchRoot();
|
|
94
|
+
}
|
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
//
|
|
8
8
|
// append <pr> --closed-own-ticket yes|no --minted-false-claim yes|no
|
|
9
9
|
// --note <text> [--sizing light|heavy] [--profile <p>] [--loc <n>] [--files <n>]
|
|
10
|
+
// [--scratch <dir>]
|
|
10
11
|
// Fills `run_date` (today's UTC date, the ruling date), `pr`, `ticket` (off
|
|
11
|
-
// `gh pr view --json closingIssuesReferences`)
|
|
12
|
+
// `gh pr view --json closingIssuesReferences`), `tier` and, unless
|
|
13
|
+
// `--sizing` names it, `sizing` (see below). Idempotent by
|
|
12
14
|
// PR: a row already present is named and nothing is written, exit 0.
|
|
13
15
|
// check [--live]
|
|
14
16
|
// Post-switch rows only: a filled `tier` must equal the short name of the
|
|
@@ -43,6 +45,17 @@
|
|
|
43
45
|
// `opus`/`sonnet` spelling and are never checked; the header records the
|
|
44
46
|
// switch and how the old values map.
|
|
45
47
|
//
|
|
48
|
+
// `sizing` resolution, in order:
|
|
49
|
+
// 1. `--sizing`, as given — it wins, and stderr names it as the source;
|
|
50
|
+
// 2. else the `sizing.verdict` of the ticket's implementer report,
|
|
51
|
+
// `<scratch>/reports/impl-<ticket>[-x].json` (`--scratch`, default the
|
|
52
|
+
// root scratch-root.mjs prints), the highest retry letter's when several
|
|
53
|
+
// exist — and only when the report checks against its schema, its
|
|
54
|
+
// `sizing.ranBeforePr` is true and its `sizing.signal` is non-empty: a
|
|
55
|
+
// verdict typed after the PR was opened measured nothing, and one with no
|
|
56
|
+
// signal named is a guess;
|
|
57
|
+
// 3. else blank, with a WARNING naming each condition that failed.
|
|
58
|
+
//
|
|
46
59
|
// A row's position in the file is its order: rows are appended, never inserted
|
|
47
60
|
// mid-file or reordered. An in-place edit (a re-judged `class`, a backfilled
|
|
48
61
|
// verdict) keeps the row's position; a ruling meant to supersede an earlier one
|
|
@@ -60,6 +73,8 @@ import { makeDie, defineFlags, isDigits } from "./arg.mjs";
|
|
|
60
73
|
import { parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
|
|
61
74
|
import { parseMember, parseToken, memberTokens, rowPr } from "./ledger-grammar.mjs";
|
|
62
75
|
import { rowText } from "./tier-check.mjs";
|
|
76
|
+
import { latestImplReport, readReport, reportsDir, defaultScratch } from "./report-files.mjs";
|
|
77
|
+
import { FleetDirUnresolvable } from "./fleet-dir.mjs";
|
|
63
78
|
|
|
64
79
|
const NAME = "tier-outcomes";
|
|
65
80
|
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
|
|
@@ -274,6 +289,25 @@ export function resolveTier({ ledger, memberRows, ticket, pr }) {
|
|
|
274
289
|
return tierFromLedger(ledger, ticket, pr) ?? tierFromMembers(memberRows, ticket);
|
|
275
290
|
}
|
|
276
291
|
|
|
292
|
+
// The `sizing` covariate off the ticket's implementer report under `scratch`:
|
|
293
|
+
// `{sizing, source, warning: null}`, or `{sizing: "", source: null, warning}`
|
|
294
|
+
// naming every condition that failed. The report must check against its
|
|
295
|
+
// schema first: a verdict read off a malformed file is not a measurement.
|
|
296
|
+
function sizingFromReport(scratch, ticket) {
|
|
297
|
+
const blank = (warning) => ({ sizing: "", source: null, warning });
|
|
298
|
+
const found = latestImplReport(scratch, ticket);
|
|
299
|
+
const report = found === null ? null : readReport(scratch, found.member);
|
|
300
|
+
if (report === null || report.state === "absent") return blank(`no implementer report for ticket #${ticket} under ${reportsDir(scratch)}`);
|
|
301
|
+
if (report.state === "invalid") return blank(`${report.path} does not check against its schema: ${report.violation}`);
|
|
302
|
+
const s = report.report.sizing;
|
|
303
|
+
if (s === null) return blank(`${report.path} carries no sizing (sizing is null)`);
|
|
304
|
+
const failed = [];
|
|
305
|
+
if (s.ranBeforePr !== true) failed.push("sizing.ranBeforePr is not true");
|
|
306
|
+
if (s.signal.trim() === "") failed.push("sizing.signal is empty");
|
|
307
|
+
if (failed.length) return blank(`${report.path}: ${failed.join(" and ")}`);
|
|
308
|
+
return { sizing: s.verdict, source: report.path, warning: null };
|
|
309
|
+
}
|
|
310
|
+
|
|
277
311
|
// `check` over the committed files. Returns failure lines and skip counts.
|
|
278
312
|
export function checkRows(rows, memberRows) {
|
|
279
313
|
const failures = [];
|
|
@@ -338,13 +372,13 @@ export function unrecordedReviewedPrs(ledger, rows) {
|
|
|
338
372
|
// ---------------------------------------------------------------------------
|
|
339
373
|
|
|
340
374
|
const USAGE = "usage: tier-outcomes.mjs append <pr> --closed-own-ticket yes|no --minted-false-claim yes|no --note <text> "
|
|
341
|
-
+ "[--sizing light|heavy] [--profile <p>] [--loc <n>] [--files <n>] [--file <tsv>] [--member-outcomes <tsv>] [--ledger <path>]\n"
|
|
375
|
+
+ "[--sizing light|heavy] [--profile <p>] [--loc <n>] [--files <n>] [--scratch <dir>] [--file <tsv>] [--member-outcomes <tsv>] [--ledger <path>]\n"
|
|
342
376
|
+ " tier-outcomes.mjs check [--live] [--file <tsv>] [--member-outcomes <tsv>] [--ledger <path>]";
|
|
343
377
|
|
|
344
378
|
const die = makeDie(NAME);
|
|
345
379
|
const APPEND_FLAGS = {
|
|
346
380
|
"closed-own-ticket": "value", "minted-false-claim": "value", note: "value",
|
|
347
|
-
sizing: "value", profile: "value", loc: "value", files: "value",
|
|
381
|
+
sizing: "value", profile: "value", loc: "value", files: "value", scratch: "value",
|
|
348
382
|
};
|
|
349
383
|
const FLAGS = { ...APPEND_FLAGS, file: "value", "member-outcomes": "value", ledger: "value", live: "bool" };
|
|
350
384
|
const { arg, has, sweep } = defineFlags(die, { flags: FLAGS });
|
|
@@ -442,9 +476,10 @@ function append(pr, paths) {
|
|
|
442
476
|
if (!isDigits(pr) || Number(pr) === 0) die(`append needs a PR number, got '${pr}'`);
|
|
443
477
|
pr = String(Number(pr));
|
|
444
478
|
if (has("live")) die("--live is a check flag");
|
|
479
|
+
const sizingFlag = arg("sizing");
|
|
445
480
|
const fields = {
|
|
446
481
|
closed_own_ticket: arg("closed-own-ticket"), minted_false_claim: arg("minted-false-claim"),
|
|
447
|
-
note: arg("note"), sizing:
|
|
482
|
+
note: arg("note"), sizing: sizingFlag ?? "", profile: arg("profile") ?? "", loc: arg("loc") ?? "", files: arg("files") ?? "",
|
|
448
483
|
};
|
|
449
484
|
for (const col of ["closed_own_ticket", "minted_false_claim", "note"]) {
|
|
450
485
|
if (fields[col] === null) die(`append needs --${col.replaceAll("_", "-")}\n${USAGE}`);
|
|
@@ -474,6 +509,25 @@ function append(pr, paths) {
|
|
|
474
509
|
if (resolved.warning) console.error(`${NAME}: WARNING tier left blank for PR #${pr} (ticket #${ticket}): ${resolved.warning}`);
|
|
475
510
|
else console.error(`${NAME}: tier=${resolved.tier} from ${resolved.source}`);
|
|
476
511
|
|
|
512
|
+
if (sizingFlag !== null) {
|
|
513
|
+
console.error(`${NAME}: sizing=${fields.sizing} from --sizing`);
|
|
514
|
+
} else {
|
|
515
|
+
let scratch = paths.scratch;
|
|
516
|
+
let unresolvable = null;
|
|
517
|
+
if (scratch === null) {
|
|
518
|
+
try {
|
|
519
|
+
scratch = defaultScratch();
|
|
520
|
+
} catch (e) {
|
|
521
|
+
if (!(e instanceof FleetDirUnresolvable)) throw e;
|
|
522
|
+
unresolvable = e.message;
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
const sized = unresolvable === null ? sizingFromReport(scratch, ticket) : { sizing: "", source: null, warning: `no scratch root to look in: ${unresolvable}` };
|
|
526
|
+
fields.sizing = sized.sizing;
|
|
527
|
+
if (sized.warning) console.error(`${NAME}: WARNING sizing left blank for PR #${pr} (ticket #${ticket}): ${sized.warning}`);
|
|
528
|
+
else console.error(`${NAME}: sizing=${sized.sizing} from ${sized.source}`);
|
|
529
|
+
}
|
|
530
|
+
|
|
477
531
|
const line = formatRow({ ...fields, run_date: today(), pr, ticket, tier: resolved.tier });
|
|
478
532
|
if (text === null) writeFileSync(paths.file, `# ${COLUMNS.join("\t")}\n${line}\n`);
|
|
479
533
|
else appendFileSync(paths.file, `${text === "" || text.endsWith("\n") ? "" : "\n"}${line}\n`);
|
|
@@ -507,6 +561,7 @@ function main() {
|
|
|
507
561
|
file: arg("file") ?? "docs/metrics/tier-outcomes.tsv",
|
|
508
562
|
members: arg("member-outcomes") ?? "docs/metrics/member-outcomes.tsv",
|
|
509
563
|
ledger: arg("ledger"),
|
|
564
|
+
scratch: arg("scratch"),
|
|
510
565
|
};
|
|
511
566
|
// Below every flag read, per arg.mjs's ordering contract, and above the
|
|
512
567
|
// subcommand checks, so a stray flag is refused by name before anything
|
package/skills/run-team/SKILL.md
CHANGED
|
@@ -1462,32 +1462,37 @@ untouched, so re-running it is safe. That append is the whole duty.
|
|
|
1462
1462
|
The row's last four fields are the ticket's difficulty, and they are what lets a
|
|
1463
1463
|
tier comparison condition on the thing that swamps it. `sizing` is the
|
|
1464
1464
|
**member's** own `light`/`heavy` verdict from its phase-2 `sizing-a-ticket`
|
|
1465
|
-
run,
|
|
1466
|
-
|
|
1465
|
+
run, which `append` reads itself off the member's report file
|
|
1466
|
+
(`<scratch>/reports/impl-<ticket>[-x].json`, its `sizing.verdict`, the highest
|
|
1467
|
+
retry letter's when there are several), so you pass no `--sizing`; the member
|
|
1468
|
+
also puts it in the PR body's `Sizing:` line, with the signal it turned on;
|
|
1467
1469
|
`profile`, `loc` and `files` all come from `diff-stats.mjs` over the merged
|
|
1468
1470
|
diff. **Phase 0 does not size anything** — it shortlists, a Pull claims, and
|
|
1469
1471
|
the sizing run happens inside the member after both, which is why the verdict
|
|
1470
|
-
has to travel in the
|
|
1472
|
+
has to travel in the member's report rather than being something you already hold.
|
|
1471
1473
|
|
|
1472
1474
|
**A verdict is a measurement only if it post-dates the run that produced it.** A
|
|
1473
1475
|
`Sizing:` line typed in before the skill ever ran is byte-identical to one the
|
|
1474
|
-
skill produced, so the body cannot settle this alone and
|
|
1475
|
-
carries the second source:
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1476
|
+
skill produced, so the body cannot settle this alone, and neither can the
|
|
1477
|
+
member's prose. The report file carries the second source: `sizing.ranBeforePr`,
|
|
1478
|
+
whether `sizing-a-ticket` ran before the PR was opened (measured on a member
|
|
1479
|
+
that wrote the line first, caught itself, and corrected it), and
|
|
1480
|
+
`sizing.signal`, what the verdict turned on. `append` applies the rule itself:
|
|
1481
|
+
it records `sizing` only when `ranBeforePr` is true AND `signal` is non-empty.
|
|
1482
|
+
Any other state — no report file, one that fails its schema, `sizing` null,
|
|
1483
|
+
`ranBeforePr` not true, an empty `signal` — leaves the field BLANK with a
|
|
1484
|
+
WARNING naming which, and you say which in `note`. A `--sizing` you do pass
|
|
1485
|
+
wins over the file, and stderr names the flag as the source: do not use it to
|
|
1486
|
+
fill that gap, and do not re-derive the verdict from the diff to fill it
|
|
1487
|
+
either — `sizing-a-ticket` weighs more than file count, and a substituted proxy
|
|
1488
|
+
reads as measured too. **A value not in hand is left BLANK, never estimated** —
|
|
1489
|
+
blank reads as unknown and drops the row from a stratified comparison, while a
|
|
1490
|
+
guess reads as measured and poisons one. Blank still means the field is
|
|
1491
|
+
WRITTEN and empty: omit the flag (`--sizing`, `--profile`, `--loc`, `--files`)
|
|
1492
|
+
and the script writes that field empty at full width, because a row of some
|
|
1493
|
+
in-between width cannot be told apart from a shifted one. The `note` field is
|
|
1494
|
+
free text and sits before those four, so the script refuses a tab or newline in
|
|
1495
|
+
it: one tab would shift all four for that row alone.
|
|
1491
1496
|
|
|
1492
1497
|
**Then record the run's member facts — do not author them.** Scrape EVERY session
|
|
1493
1498
|
directory for this cwd, not one:
|