@feigi/fleet-ctl 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +9 -0
- package/LICENSE +202 -0
- package/NOTICE +18 -0
- package/agents/fleet-finisher.agent.md +7 -0
- package/agents/fleet-implementer-alt.agent.md +210 -0
- package/agents/fleet-implementer.agent.md +210 -0
- package/agents/fleet-merge-bot.agent.md +7 -0
- package/agents/fleet-review-comments.agent.md +40 -0
- package/agents/fleet-review-correctness.agent.md +43 -0
- package/agents/fleet-review-runner.agent.md +73 -0
- package/agents/fleet-review-silent-failure.agent.md +49 -0
- package/agents/fleet-review-simplify.agent.md +46 -0
- package/agents/fleet-review-snapshot.agent.md +24 -0
- package/agents/fleet-review-tests.agent.md +46 -0
- package/agents/fleet-review-types.agent.md +41 -0
- package/agents/fleet-review-verifier.agent.md +29 -0
- package/commands/review-and-fix.md +98 -0
- package/commands/run-merge-bot.md +357 -0
- package/commands/run-team-help.md +17 -0
- package/package.json +23 -0
- package/scripts/agent-test-dir-prose.test.mjs +59 -0
- package/scripts/agent-test.test.mjs +1636 -0
- package/scripts/ambient-git-vars-mjs-prose.test.mjs +354 -0
- package/scripts/ambient-git-vars-prose.test.mjs +192 -0
- package/scripts/ancestry-check-position-prose.test.mjs +175 -0
- package/scripts/apply-ruleset.test.mjs +420 -0
- package/scripts/arg-header-probes-prose.test.mjs +393 -0
- package/scripts/arg.mjs +600 -0
- package/scripts/arg.test.mjs +1354 -0
- package/scripts/board-cli.test.mjs +694 -0
- package/scripts/board-identity.test.mjs +429 -0
- package/scripts/board-liveness-banner.test.mjs +119 -0
- package/scripts/board-prev-shape.test.mjs +229 -0
- package/scripts/board-tryparse-null.test.mjs +119 -0
- package/scripts/board.html +471 -0
- package/scripts/board.mjs +2037 -0
- package/scripts/board.test.mjs +3086 -0
- package/scripts/brief-after-cut-prose.test.mjs +53 -0
- package/scripts/candidates-exit3-prose.test.mjs +302 -0
- package/scripts/candidates.mjs +674 -0
- package/scripts/candidates.test.mjs +2101 -0
- package/scripts/caveat-unification-prose.test.mjs +98 -0
- package/scripts/ci-comment-rot-prose.test.mjs +2025 -0
- package/scripts/ci-completes-premise-prose.test.mjs +257 -0
- package/scripts/ci-state-prose.test.mjs +113 -0
- package/scripts/ci-state.mjs +761 -0
- package/scripts/ci-state.test.mjs +1640 -0
- package/scripts/ci-vacuous-green.test.mjs +368 -0
- package/scripts/citation-sweep-prose.test.mjs +1572 -0
- package/scripts/claim-lifecycle.test.mjs +185 -0
- package/scripts/claim-ticket.sh +417 -0
- package/scripts/claim-ticket.test.mjs +1121 -0
- package/scripts/closing-keyword-prose.test.mjs +193 -0
- package/scripts/cockpit-launch-idempotent-prose.test.mjs +165 -0
- package/scripts/cockpit-tick-nonblocking.test.mjs +210 -0
- package/scripts/compute-board.mjs +579 -0
- package/scripts/compute-board.test.mjs +1060 -0
- package/scripts/compute-spend.mjs +329 -0
- package/scripts/compute-spend.test.mjs +451 -0
- package/scripts/cross-repo-citation-prose.test.mjs +156 -0
- package/scripts/cwd-isolation-pins.mjs +83 -0
- package/scripts/dedupe-guard-prose.test.mjs +243 -0
- package/scripts/derive-testcmd.sh +239 -0
- package/scripts/derive-testcmd.test.mjs +380 -0
- package/scripts/diff-stats.mjs +231 -0
- package/scripts/diff-stats.test.mjs +408 -0
- package/scripts/dispatch-block-golden-prose.test.mjs +1024 -0
- package/scripts/dispatch-block-pins-prose.test.mjs +615 -0
- package/scripts/drop-merged-label.sh +173 -0
- package/scripts/drop-merged-label.test.mjs +480 -0
- package/scripts/expected-jobs-prose.test.mjs +80 -0
- package/scripts/filing-bar-seat-prose.test.mjs +242 -0
- package/scripts/filing-claim-bar-prose.test.mjs +166 -0
- package/scripts/finisher-dispatch-premise-prose.test.mjs +312 -0
- package/scripts/finisher-halt-prose.test.mjs +106 -0
- package/scripts/finisher-name-prose.test.mjs +134 -0
- package/scripts/finisher-own-tree-prose.test.mjs +285 -0
- package/scripts/finisher-pin-race-prose.test.mjs +339 -0
- package/scripts/fix-applier-correction-rules-prose.test.mjs +201 -0
- package/scripts/fix-applier-result-file-prose.test.mjs +100 -0
- package/scripts/fleet-bootstrap +155 -0
- package/scripts/fleet-heartbeat.mjs +272 -0
- package/scripts/fleet-heartbeat.test.mjs +450 -0
- package/scripts/fleet-provenance +207 -0
- package/scripts/fleet-provenance.test.mjs +64 -0
- package/scripts/fleet-run +148 -0
- package/scripts/fleet-run.test.mjs +90 -0
- package/scripts/fleet-state.mjs +400 -0
- package/scripts/fleet-state.test.mjs +373 -0
- package/scripts/fleet-tick-prose.test.mjs +299 -0
- package/scripts/fleet-tick.mjs +832 -0
- package/scripts/fleet-tick.test.mjs +1332 -0
- package/scripts/force-with-lease-reason-prose.test.mjs +184 -0
- package/scripts/frontmatter-allowlist.json +64 -0
- package/scripts/frontmatter-check.mjs +256 -0
- package/scripts/frontmatter-check.test.mjs +436 -0
- package/scripts/git-env.mjs +124 -0
- package/scripts/git-env.test.mjs +149 -0
- package/scripts/git-status-untracked-mode-sweep.test.mjs +216 -0
- package/scripts/gone-walk.test.mjs +213 -0
- package/scripts/gp-sep-invariant-prose.test.mjs +188 -0
- package/scripts/head-moved-after-label-prose.test.mjs +233 -0
- package/scripts/immutable-body-claim-prose.test.mjs +413 -0
- package/scripts/implementer-model-tier.test.mjs +643 -0
- package/scripts/inflight-citation-prose.test.mjs +144 -0
- package/scripts/inflight-exit2-prose.test.mjs +81 -0
- package/scripts/inflight-probe-guard-prose.test.mjs +109 -0
- package/scripts/inflight-scan-scope-prose.test.mjs +57 -0
- package/scripts/inflight.sh +1106 -0
- package/scripts/inflight.test.mjs +2984 -0
- package/scripts/injection-control-prose.test.mjs +404 -0
- package/scripts/install-root-audit.test.mjs +464 -0
- package/scripts/instrument-check-prose.test.mjs +235 -0
- package/scripts/instruments.sh +394 -0
- package/scripts/instruments.test.mjs +787 -0
- package/scripts/issue-tracker-prose.test.mjs +71 -0
- package/scripts/json.sh +248 -0
- package/scripts/json.test.mjs +364 -0
- package/scripts/ledger-dispatch.test.mjs +516 -0
- package/scripts/ledger-grammar.mjs +85 -0
- package/scripts/ledger-grammar.test.mjs +142 -0
- package/scripts/ledger-read-require-file.test.mjs +197 -0
- package/scripts/ledger.mjs +1753 -0
- package/scripts/ledger.test.mjs +3554 -0
- package/scripts/line-distance-prose.test.mjs +245 -0
- package/scripts/liveness-rationale-prose.test.mjs +83 -0
- package/scripts/locale-pin-prose.test.mjs +189 -0
- package/scripts/member-outcome-prose.test.mjs +167 -0
- package/scripts/member-outcomes-header.test.mjs +148 -0
- package/scripts/member-outcomes-prose.test.mjs +63 -0
- package/scripts/member-outcomes.mjs +231 -0
- package/scripts/member-outcomes.test.mjs +507 -0
- package/scripts/member-prompt-prose.test.mjs +161 -0
- package/scripts/member-record.mjs +605 -0
- package/scripts/member-record.test.mjs +516 -0
- package/scripts/member-write-guard.mjs +279 -0
- package/scripts/member-write-guard.test.mjs +211 -0
- package/scripts/merge-commit-probe-prose.test.mjs +122 -0
- package/scripts/merge-gate.mjs +262 -0
- package/scripts/merge-gate.test.mjs +613 -0
- package/scripts/muted-git-guard-sweep.test.mjs +493 -0
- package/scripts/net-ssh-precedence-prose.test.mjs +91 -0
- package/scripts/net-watchdog-prose.test.mjs +79 -0
- package/scripts/net.sh +409 -0
- package/scripts/net.test.mjs +211 -0
- package/scripts/no-ci-gate-prose.test.mjs +108 -0
- package/scripts/no-undo-audit.sh +1134 -0
- package/scripts/no-undo-audit.test.mjs +3965 -0
- package/scripts/no-wave.test.mjs +138 -0
- package/scripts/node-floor-sweep.test.mjs +611 -0
- package/scripts/pointer-target-prose.test.mjs +186 -0
- package/scripts/pr-overlap.mjs +417 -0
- package/scripts/pr-overlap.test.mjs +695 -0
- package/scripts/printf-die-sweep.test.mjs +384 -0
- package/scripts/prompt-renderer.mjs +39 -0
- package/scripts/prompt-renderer.test.mjs +68 -0
- package/scripts/prose-pin.mjs +808 -0
- package/scripts/prose-pin.test.mjs +1034 -0
- package/scripts/prove-merge.sh +228 -0
- package/scripts/prove-merge.test.mjs +995 -0
- package/scripts/pull-table-relabel-prose.test.mjs +60 -0
- package/scripts/quiet-payload-prose.test.mjs +222 -0
- package/scripts/reap.sh +1768 -0
- package/scripts/reap.test.mjs +4395 -0
- package/scripts/reaping-prose.test.mjs +159 -0
- package/scripts/refuter-scratch-prose.test.mjs +135 -0
- package/scripts/release-label-required-prose.test.mjs +71 -0
- package/scripts/release-label-split-prose.test.mjs +169 -0
- package/scripts/release-ticket.sh +1716 -0
- package/scripts/release-ticket.test.mjs +5106 -0
- package/scripts/renovate-release-contract.test.mjs +379 -0
- package/scripts/repo-root.mjs +544 -0
- package/scripts/repo-root.test.mjs +873 -0
- package/scripts/reporter-output-rule-prose.test.mjs +208 -0
- package/scripts/review-agent-report-only.test.mjs +49 -0
- package/scripts/review-core-cwd-isolation.test.mjs +374 -0
- package/scripts/review-core-reads.test.mjs +478 -0
- package/scripts/review-core-refuter-scratch.test.mjs +148 -0
- package/scripts/review-core-snapshot-path.test.mjs +777 -0
- package/scripts/review-core-specialist-scratch.test.mjs +189 -0
- package/scripts/review-core-testcmd.test.mjs +288 -0
- package/scripts/review-core-unrun.test.mjs +530 -0
- package/scripts/review-core-unverified-discriminant.test.mjs +208 -0
- package/scripts/review-core.mjs +1028 -0
- package/scripts/review-eval-resolver-path.test.mjs +55 -0
- package/scripts/review-eval.mjs +197 -0
- package/scripts/review-host-fixture.mjs +73 -0
- package/scripts/review-in-run-retry.test.mjs +187 -0
- package/scripts/review-no-vendor-dispatch.test.mjs +45 -0
- package/scripts/review-path-default.test.mjs +796 -0
- package/scripts/review-runner-spawns.test.mjs +93 -0
- package/scripts/review-runner.test.mjs +155 -0
- package/scripts/review-tier-audit.test.mjs +140 -0
- package/scripts/run-artifact-pr-prose.test.mjs +341 -0
- package/scripts/run-merge-bot-prose.test.mjs +1391 -0
- package/scripts/scratch-partition-invariant-prose.test.mjs +74 -0
- package/scripts/scripts-path-citation-sweep.test.mjs +73 -0
- package/scripts/section-between-guard-prose.test.mjs +378 -0
- package/scripts/select-dimensions.test.mjs +674 -0
- package/scripts/shared-refusal.test.mjs +388 -0
- package/scripts/shell-traps-prose.test.mjs +507 -0
- package/scripts/shortlist.mjs +355 -0
- package/scripts/shortlist.test.mjs +366 -0
- package/scripts/slow-transport.mjs +72 -0
- package/scripts/snapshot-repo.test.mjs +772 -0
- package/scripts/snapshot-runner-audience-prose.test.mjs +129 -0
- package/scripts/spend-view.test.mjs +312 -0
- package/scripts/staleness-qualifier-prose.test.mjs +251 -0
- package/scripts/staleness-verdict-prose.test.mjs +142 -0
- package/scripts/staleness.mjs +414 -0
- package/scripts/staleness.test.mjs +893 -0
- package/scripts/stalled-member-liveness-prose.test.mjs +219 -0
- package/scripts/stash-prohibition-prose.test.mjs +313 -0
- package/scripts/strip-comments.mjs +44 -0
- package/scripts/tier-check.mjs +468 -0
- package/scripts/tier-check.test.mjs +726 -0
- package/scripts/tier-guard-gate-prose.test.mjs +216 -0
- package/scripts/tier-outcomes-header.test.mjs +586 -0
- package/scripts/tier-outcomes.mjs +441 -0
- package/scripts/tier-outcomes.test.mjs +342 -0
- package/scripts/tier-roles.mjs +250 -0
- package/scripts/tier-roles.test.mjs +316 -0
- package/scripts/tracker-block-copy-prose.test.mjs +288 -0
- package/scripts/triage-labels-gloss-prose.test.mjs +97 -0
- package/scripts/unattended-git-sweep.test.mjs +142 -0
- package/scripts/verify-sha-exit2-prose.test.mjs +269 -0
- package/scripts/verify-sha.sh +117 -0
- package/scripts/verify-sha.test.mjs +693 -0
- package/scripts/watcher-degraded-block.test.mjs +372 -0
- package/scripts/watcher-key-prose.test.mjs +106 -0
- package/scripts/within-run-pair-prose.test.mjs +208 -0
- package/scripts/worktree-audit.sh +482 -0
- package/scripts/worktree-audit.test.mjs +1407 -0
- package/scripts/worktree-head-prose.test.mjs +276 -0
- package/scripts/worktree-listing-sweep.test.mjs +217 -0
- package/scripts/worktree-model-prose.test.mjs +216 -0
- package/scripts/worktree.sh +470 -0
- package/skills/next-ticket/SKILL.md +138 -0
- package/skills/run-team/SKILL.md +3603 -0
- package/skills/run-team/references/ci-and-staleness.md +75 -0
- package/skills/run-team/references/correction-tickets.md +37 -0
- package/skills/run-team/references/isolation.md +23 -0
- package/skills/run-team/references/member-lifecycle.md +73 -0
- package/skills/run-team/references/reaping.md +73 -0
- package/skills/sizing-a-ticket/SKILL.md +39 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
// #1591's outcome classification, retargeted by #1804 from the omp workpool
|
|
2
|
+
// block to the direct-dispatch Pull (spec 2026-09-24 § 2, ruled without a
|
|
3
|
+
// question: "#1591 outcome classification survives verbatim, keyed to the
|
|
4
|
+
// member's own job status … reworded per member, 'confirm death first'
|
|
5
|
+
// unchanged").
|
|
6
|
+
//
|
|
7
|
+
// The fact the classification rests on survives the pool: two very different
|
|
8
|
+
// facts produce the same silence. A member that DIED (its job settled `failed`
|
|
9
|
+
// or `cancelled` — CONTEXT.md § Coordination's Settle vocabulary) never got
|
|
10
|
+
// the chance to bail; a member that `completed` its turn with nothing to report
|
|
11
|
+
// is merely quiet. Collapsing either into "the member bailed" would demote a
|
|
12
|
+
// ticket its member never judged. Only an explicit bail report — the member
|
|
13
|
+
// naming the cause itself — may reach `Implementer bails before implementing`
|
|
14
|
+
// and its demote-by-cause table (phase 3). What changed is only where the
|
|
15
|
+
// status arrives from: under Pull each member's job status arrives as its own,
|
|
16
|
+
// where a pool settled once on its whole drain.
|
|
17
|
+
//
|
|
18
|
+
// The second half of #1591 closed a refill loop: a demotion drops
|
|
19
|
+
// `ready-for-agent`, and the ticket it demoted must never be the one handed
|
|
20
|
+
// back. Under Pull the same hazard has a new shape — the shortlist file can be
|
|
21
|
+
// older than a relabel (this run's own Pulls relabel by cause, ADR 0013
|
|
22
|
+
// Decision 4) — and phase 1 closes it by reading the ticket's labels in the
|
|
23
|
+
// Pull's own full read.
|
|
24
|
+
//
|
|
25
|
+
// TABLE-DRIVEN OVER OUTCOME SHAPES, per #1591's own acceptance criterion, so a
|
|
26
|
+
// future edit that quietly drops ONE shape reds on exactly that entry instead
|
|
27
|
+
// of surviving inside a still-green bundle.
|
|
28
|
+
//
|
|
29
|
+
// SELF-CHECK carried over from #1591's measurement: a presence-only check per
|
|
30
|
+
// settle state passed with `cancelled` stripped from the three-state
|
|
31
|
+
// enumeration, because a second `cancelled` occurs later in the paragraph — so
|
|
32
|
+
// the enumeration and the collapse sentence are each pinned by their own words.
|
|
33
|
+
import { test } from "node:test";
|
|
34
|
+
import assert from "node:assert/strict";
|
|
35
|
+
import { readFileSync } from "node:fs";
|
|
36
|
+
import { join } from "node:path";
|
|
37
|
+
import { between, paragraph, phrase, sentences } from "./prose-pin.mjs";
|
|
38
|
+
|
|
39
|
+
const REPO = join(import.meta.dirname, "..");
|
|
40
|
+
const RUN_TEAM = readFileSync(join(REPO, "skills", "run-team", "SKILL.md"), "utf8");
|
|
41
|
+
|
|
42
|
+
const phase = (from, to) => between(RUN_TEAM, from, to, `run-team/SKILL.md ${from.trim()}`);
|
|
43
|
+
|
|
44
|
+
const outcomeParagraph = () =>
|
|
45
|
+
paragraph(
|
|
46
|
+
phase("## Phase 2", "## Phase 3"),
|
|
47
|
+
"A member settling with nothing is not, by itself, a bail.",
|
|
48
|
+
"run-team/SKILL.md member outcome-classification paragraph",
|
|
49
|
+
);
|
|
50
|
+
|
|
51
|
+
const relabelParagraph = () =>
|
|
52
|
+
paragraph(
|
|
53
|
+
phase("## Phase 1", "## Phase 2"),
|
|
54
|
+
"A Pull never re-admits a ticket already relabelled.",
|
|
55
|
+
"run-team/SKILL.md Pull relabel-safety paragraph",
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
test("run-team/SKILL.md: a member's outcome arrives as its own job status", () => {
|
|
59
|
+
assert.match(outcomeParagraph(), phrase("Each member's outcome arrives as its own job status"));
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
// Settle (CONTEXT.md § Coordination) is three states on omp, and the paragraph
|
|
63
|
+
// must name all three — never fewer, which is exactly how a `failed`/
|
|
64
|
+
// `cancelled` reading could quietly re-collapse into `completed`.
|
|
65
|
+
for (const state of ["completed", "failed", "cancelled"]) {
|
|
66
|
+
test(`run-team/SKILL.md: the outcome paragraph names omp's \`${state}\` settle state`, () => {
|
|
67
|
+
assert.match(outcomeParagraph(), new RegExp("`" + state + "`"));
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
test("run-team/SKILL.md: the outcome paragraph enumerates all three omp settle states together, in one list", () => {
|
|
72
|
+
assert.match(outcomeParagraph(), phrase("Settle (CONTEXT.md § Coordination) is three states on omp — `completed`, `failed`, `cancelled` —"));
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("run-team/SKILL.md: the outcome paragraph names the failed/cancelled-vs-completed collapse as the exact failure it blocks", () => {
|
|
76
|
+
assert.match(outcomeParagraph(), phrase("and reading `failed`/`cancelled` the same as a `completed` turn that had nothing"));
|
|
77
|
+
assert.match(outcomeParagraph(), phrase("is the exact collapse this line blocks"));
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
// #1591's core acceptance, table-driven: each of the three outcome shapes gets
|
|
81
|
+
// its OWN reading, distinct text a reader cannot satisfy by pointing at either
|
|
82
|
+
// of the other two rows.
|
|
83
|
+
const OUTCOME_SHAPES = [
|
|
84
|
+
{ name: "died (failed/cancelled settle)", pin: phrase("a member that died never chose to bail") },
|
|
85
|
+
{ name: "completed with nothing (alive, merely quiet)", pin: phrase("a died or merely-quiet member's silence never does") },
|
|
86
|
+
{
|
|
87
|
+
name: "explicit bail report",
|
|
88
|
+
pin: phrase("Only an explicit bail report — the member itself naming the cause — reaches Implementer bails before implementing"),
|
|
89
|
+
},
|
|
90
|
+
];
|
|
91
|
+
for (const shape of OUTCOME_SHAPES) {
|
|
92
|
+
test(`run-team/SKILL.md: the outcome paragraph gives "${shape.name}" its own reading`, () => {
|
|
93
|
+
assert.match(outcomeParagraph(), shape.pin);
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
test("run-team/SKILL.md: a dead member recovers exactly as the Member-killed row does, never as a demotion", () => {
|
|
98
|
+
assert.match(outcomeParagraph(), phrase("Confirm death first, exactly as the Member-killed row does"));
|
|
99
|
+
assert.match(
|
|
100
|
+
outcomeParagraph(),
|
|
101
|
+
phrase("A confirmed-dead member recovers exactly as the Member-killed row says — new member, new name, the SAME ticket — never a demotion"),
|
|
102
|
+
);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
// The contradiction this paragraph exists to block: reading silence itself —
|
|
106
|
+
// with no confirmation step — as grounds to demote. Phrase-agnostic by design
|
|
107
|
+
// (inherited narrow idiom-only regex previously missed a colon-phrased
|
|
108
|
+
// restatement of the same forbidden meaning, e.g. "Silence alone: treat it as
|
|
109
|
+
// a bail." — caught live against the pinned paragraph): any short clause that
|
|
110
|
+
// co-locates "silence" with "bail" is blocked outright, and any clause that
|
|
111
|
+
// equates "silence" with a demotion via a copula/colon/"treat as" construction
|
|
112
|
+
// is blocked too — while the paragraph's own legitimate "so demoting its
|
|
113
|
+
// ticket on its silence demotes a ticket" (describing the collapse the line
|
|
114
|
+
// BLOCKS, not endorsing it) must keep passing, which is why the demote guard
|
|
115
|
+
// requires an equivalence construction rather than bare proximity.
|
|
116
|
+
//
|
|
117
|
+
// The co-location is read one sentence at a time, via `sentences()`, with a
|
|
118
|
+
// gap that tolerates a period — never a first-period `[^.]` window (#1983).
|
|
119
|
+
// This pin is NEGATIVE, so a window cut short is its silent direction: "Treat
|
|
120
|
+
// silence (i.e. no report) as a bail." ended the old window at "i.e." and
|
|
121
|
+
// passed. The sentence bound is load-bearing the other way too: this
|
|
122
|
+
// paragraph's own "never silence alone." sits 20 characters before the next
|
|
123
|
+
// sentence's "bail", so an unbounded period-tolerant gap reds the real text.
|
|
124
|
+
// RESIDUAL: sentences() still cuts short at a `.)`, `."`, a mid-sentence `?`,
|
|
125
|
+
// a capitalised "E.g." or an "etc." (silent here — none of those four is in
|
|
126
|
+
// this paragraph today), and still merges two real sentences across #1987's
|
|
127
|
+
// `**late**.[1]` and #1899's sentence-final lowercase "vs." (an over-fire
|
|
128
|
+
// here, also absent today). `.**` itself IS in this paragraph once (its own
|
|
129
|
+
// opening bold lead), but that boundary is a genuine sentence end, not one
|
|
130
|
+
// straddling the silence/bail co-location this pin reads.
|
|
131
|
+
const silenceBails = (p) => sentences(p).filter((s) => /\bsilence\b[\s\S]{0,30}\bbail\b/i.test(s));
|
|
132
|
+
|
|
133
|
+
test("run-team/SKILL.md: no text in the outcome paragraph lets silence alone stand in for a bail or a confirmed death", () => {
|
|
134
|
+
const p = outcomeParagraph();
|
|
135
|
+
assert.deepEqual(silenceBails(p), [], "a sentence of the outcome paragraph co-locates silence with a bail");
|
|
136
|
+
assert.doesNotMatch(
|
|
137
|
+
p,
|
|
138
|
+
/\bsilence\b(?:\s+alone)?\s*[:,]?\s*(?:is|means|counts\s+as|equates?\s+to|treat(?:s|ed)?(?:\s+it)?\s+as|reads?\s+as)\s+(?:an?\s+)?(?:clean\s+)?(?:grounds?\s+(?:for|to)\s+)?demot\w*/i,
|
|
139
|
+
);
|
|
140
|
+
assert.doesNotMatch(p, /(?:always|automatically)\s+demote/i);
|
|
141
|
+
assert.doesNotMatch(p, /\b(?:is|are)\s+demoted\b/i);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test("a period between silence and bail does not hide the co-location", () => {
|
|
145
|
+
// #1983's reproduction, copied from the ticket: green under the `[^.]` window.
|
|
146
|
+
const p = outcomeParagraph();
|
|
147
|
+
assert.deepEqual(silenceBails(`${p} Treat silence (i.e. no report) as a bail.`), ["Treat silence (i.e. no report) as a bail."]);
|
|
148
|
+
assert.deepEqual(silenceBails(`${p} Treat silence, per SKILL.md, as a bail.`), ["Treat silence, per SKILL.md, as a bail."]);
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
// #1591's second acceptance, in the Pull's shape: a relabel must hold across a
|
|
152
|
+
// shortlist built before it landed. Both the check (the Pull's own label read)
|
|
153
|
+
// and the failure it blocks are pinned — dropping either leaves a stale-list
|
|
154
|
+
// admission unguarded.
|
|
155
|
+
test("run-team/SKILL.md: a Pull reads the ticket's labels and drops one that lost `ready-for-agent`", () => {
|
|
156
|
+
assert.match(relabelParagraph(), phrase("The shortlist file can be older than a relabel"));
|
|
157
|
+
assert.match(relabelParagraph(), /`--json\s+title,body,comments,labels`\)\s+and\s+drop\s+it\s+when\s+`ready-for-agent`\s+is\s+gone/);
|
|
158
|
+
assert.match(relabelParagraph(), phrase("A Pull that trusted a list built before the relabel landed would hand a ticket the fleet has already judged unimplementable straight to the next member"));
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
// The label the paragraph reads has to be one the Pull's full read actually
|
|
162
|
+
// fetches — cross-checked against step 3's own invocation, the way #1591 once
|
|
163
|
+
// cross-checked the pool's borrowed `candidates.mjs` call against phase 0's.
|
|
164
|
+
test("run-team/SKILL.md: the Pull's full read fetches the labels the relabel check reads", () => {
|
|
165
|
+
const step3 = between(RUN_TEAM, "3. **Read the ticket in full, once**", "One read answers every question", "run-team/SKILL.md Pull step 3");
|
|
166
|
+
assert.match(step3, /gh\s+issue\s+view\s+<N>\s+--json\s+title,body,comments,labels/, "the Pull's full read no longer fetches `labels`, so the relabel check has nothing to read");
|
|
167
|
+
});
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { readFileSync, writeFileSync, mkdtempSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { spawnSync } from "node:child_process";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
import { COLUMNS } from "./member-outcomes.mjs";
|
|
8
|
+
|
|
9
|
+
const REPO = join(import.meta.dirname, "..", "..");
|
|
10
|
+
const TSV = readFileSync(join(REPO, "docs", "metrics", "member-outcomes.tsv"), "utf8");
|
|
11
|
+
// The header is the LEADING comment block, which is also exactly what the CLI
|
|
12
|
+
// preserves across a rewrite. Slicing it — rather than collecting `#` lines from
|
|
13
|
+
// anywhere — is what keeps the prose assertions below from being satisfied by a
|
|
14
|
+
// stray comment sitting among the data rows.
|
|
15
|
+
const headerLines = [];
|
|
16
|
+
for (const l of TSV.split("\n")) {
|
|
17
|
+
if (!l.startsWith("#") && l.trim()) break;
|
|
18
|
+
headerLines.push(l);
|
|
19
|
+
}
|
|
20
|
+
const HEADER = headerLines.join("\n");
|
|
21
|
+
|
|
22
|
+
test("the header's column line matches COLUMNS exactly, in order", () => {
|
|
23
|
+
// Drift here is silent and total: every awk one-liner in the header indexes
|
|
24
|
+
// by position, so a column inserted in the code shifts $5 for every reader.
|
|
25
|
+
//
|
|
26
|
+
// Anchored on the tab-separated shape, not on `includes("run_date")`: the
|
|
27
|
+
// loose lookup took the first top-down match, so a future header paragraph
|
|
28
|
+
// that merely mentioned run_date above this line would silently pin prose.
|
|
29
|
+
const line = headerLines.find((l) => l.startsWith("# session\t"));
|
|
30
|
+
assert.ok(line, "header has no `# session<TAB>...` column line");
|
|
31
|
+
assert.deepEqual(line.replace(/^#\s*/, "").split("\t"), COLUMNS);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("the header states the blank, hand-edit, and superseded-generation rules", () => {
|
|
35
|
+
// All three are load-bearing and none is enforceable in code, so prose is the
|
|
36
|
+
// only carrier — which is what makes them worth pinning. The third is the one
|
|
37
|
+
// most likely to be dropped as noise by a future editor, and dropping it is
|
|
38
|
+
// how an older, dearer generation gets read as a cheap tier.
|
|
39
|
+
//
|
|
40
|
+
// Matched against the HEADER SLICE, not the whole file: matching the file
|
|
41
|
+
// meant a pinned rule that had been moved down among the data rows still
|
|
42
|
+
// satisfied the pin.
|
|
43
|
+
assert.match(HEADER, /BLANK MEANS UNKNOWN/);
|
|
44
|
+
// …and the one column where it does not. Blank `subagent_type` means the
|
|
45
|
+
// dispatch named no agent definition — a closed category, not missing data.
|
|
46
|
+
// Read as "unknown" it turns thousands of pre-2026-08-28 rows into evidence
|
|
47
|
+
// someone believes a re-scrape could recover, which is how #1066's
|
|
48
|
+
// over-count got argued for in the first place.
|
|
49
|
+
assert.match(HEADER, /ITS BLANK IS NOT UNKNOWN/);
|
|
50
|
+
assert.match(HEADER, /NEVER hand-edit a row/);
|
|
51
|
+
assert.match(HEADER, /SUPERSEDED GENERATIONS ARE A SEPARATE POPULATION/);
|
|
52
|
+
assert.match(HEADER, /OPAQUE JOIN KEY/);
|
|
53
|
+
// The corpus spans both transcript depths. A reader who assumes the flat half
|
|
54
|
+
// only would under-count every review role by roughly half.
|
|
55
|
+
assert.match(HEADER, /workflows\/wf_/);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("the header's awk read-outs index the columns they name", () => {
|
|
59
|
+
// This test asserted COLUMNS[0]/[2]/[4] — a fact about the CODE, not about
|
|
60
|
+
// the queries it is named for. Mutation-proven: repointing the header's pair
|
|
61
|
+
// query from $1/$3/$5 to $2/$4/$6 left it GREEN, so the query the header
|
|
62
|
+
// calls "the only unconfounded comparison" could silently group on run_date
|
|
63
|
+
// and distinguish by effort with nothing to catch it.
|
|
64
|
+
//
|
|
65
|
+
// So read the `$n` references out of the header's own awk lines and resolve
|
|
66
|
+
// each against COLUMNS. Mutation this must survive: changing any `$n` in the
|
|
67
|
+
// header without changing COLUMNS.
|
|
68
|
+
const awkLines = headerLines.filter((l) => l.includes("awk -F"));
|
|
69
|
+
assert.ok(awkLines.length >= 2, "header lost its awk read-outs");
|
|
70
|
+
|
|
71
|
+
const pair = pairQuery();
|
|
72
|
+
// BOTH arms are identified by the dispatch record (#1066), never by the role
|
|
73
|
+
// classifier and never by "the models differ" — so both `$n ~ /…/` tests
|
|
74
|
+
// must land on subagent_type, and there must be two of them.
|
|
75
|
+
const arms = [...pair.matchAll(/\$(\d+) ~ \/\(\^\|:\)fleet-implementer(-alt)?\$\//g)];
|
|
76
|
+
assert.equal(arms.length, 2, "the pair query no longer tests BOTH implementer definitions");
|
|
77
|
+
assert.deepEqual(arms.map((m) => Boolean(m[2])), [true, false], "the -alt arm must be tested FIRST — /fleet-implementer$/ is checked in the else branch");
|
|
78
|
+
for (const [, n] of arms) assert.equal(COLUMNS[n - 1], "subagent_type", "deliberateness comes from the dispatch record");
|
|
79
|
+
|
|
80
|
+
const keys = [...pair.matchAll(/[at]\[\$(\d+) FS \$(\d+)\]/g)];
|
|
81
|
+
assert.equal(keys.length, 2, "the pair query no longer keys both arms");
|
|
82
|
+
for (const [, session, model] of keys) {
|
|
83
|
+
assert.equal(COLUMNS[session - 1], "session", "a pair is WITHIN one session");
|
|
84
|
+
assert.equal(COLUMNS[model - 1], "model", "the two arms must have run different MODELS");
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const [, dateKey, dateValue] = /d\[\$(\d+)\]=\$(\d+)/.exec(pair);
|
|
88
|
+
assert.equal(COLUMNS[dateKey - 1], "session");
|
|
89
|
+
assert.equal(COLUMNS[dateValue - 1], "run_date", "the gate's second number is distinct run_dateS");
|
|
90
|
+
|
|
91
|
+
const byModel = headerLines.find((l) => /\{n\[\$\d+\]\+\+\}/.test(l));
|
|
92
|
+
assert.ok(byModel, "header lost its rows-by-model read-out");
|
|
93
|
+
assert.equal(COLUMNS[/\{n\[\$(\d+)\]\+\+\}/.exec(byModel)[1] - 1], "model");
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
// The pair query as a runnable awk program: the header block from `awk -F` to
|
|
97
|
+
// the path it reads, `#` stripped, the shell quoting removed. Index pins alone
|
|
98
|
+
// cannot see a query that indexes the right columns and still counts the wrong
|
|
99
|
+
// thing — which is precisely the defect #1066 was filed for.
|
|
100
|
+
function pairQuery() {
|
|
101
|
+
const start = headerLines.findIndex((l) => l.includes("awk -F") && l.includes("fleet-implementer-alt"));
|
|
102
|
+
assert.ok(start >= 0, "header lost its within-run pair query");
|
|
103
|
+
let end = start;
|
|
104
|
+
while (end < headerLines.length && !headerLines[end].endsWith("docs/metrics/member-outcomes.tsv")) end++;
|
|
105
|
+
assert.ok(end < headerLines.length, "the pair query never reaches the file it reads");
|
|
106
|
+
return headerLines.slice(start, end + 1).map((l) => l.replace(/^#\s{0,3}/, "")).join("\n");
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
test("the header's pair query counts DELIBERATE pairs, run against a corpus where every near-miss is present", () => {
|
|
110
|
+
// #1066: the query this replaces counted any session+role carrying two
|
|
111
|
+
// models, reported 198 keys against 17 real pairs, and so reported run-team's
|
|
112
|
+
// ten-across-five gate MET while the controlled comparison did not exist.
|
|
113
|
+
//
|
|
114
|
+
// Mutations this must survive, all measured against this fixture and all of
|
|
115
|
+
// which leave the column indexes (and therefore the test above) intact:
|
|
116
|
+
// dropping the `x[2]!=y[2]` model test reads 4 2, dropping the `-alt` anchor
|
|
117
|
+
// so both arms match one definition reads 0 0, and going back to #1066's own
|
|
118
|
+
// query — arms identified by `$3=="implementer"` rather than by the dispatch
|
|
119
|
+
// — reads 4 2, counting the untyped session and missing the omp one.
|
|
120
|
+
const rows = [
|
|
121
|
+
// a real pair: alt at sonnet against the top definition at opus
|
|
122
|
+
["sA", "2026-09-01", "implementer", "impl-1", "claude-sonnet-5", "high", "1", "", "0", "0", "0", "1", "agent-a1", "claude", "fleet-implementer-alt"],
|
|
123
|
+
["sA", "2026-09-01", "implementer", "impl-2", "claude-opus-5", "high", "2", "", "0", "0", "0", "1", "agent-a2", "claude", "fleet-implementer"],
|
|
124
|
+
// deliberate dispatch, empty comparison: both arms resolved to one model
|
|
125
|
+
["sB", "2026-09-01", "other", "impl-3", "claude-sonnet-5", "high", "3", "", "0", "0", "0", "1", "agent-a3", "omp", "fleet-implementer-alt"],
|
|
126
|
+
["sB", "2026-09-01", "other", "impl-4", "claude-sonnet-5", "high", "4", "", "0", "0", "0", "1", "agent-a4", "omp", "fleet-implementer"],
|
|
127
|
+
// two models, no deliberate dispatch: the accidental pair #1066 counted
|
|
128
|
+
["sC", "2026-09-01", "implementer", "impl-5", "claude-sonnet-5", "high", "5", "", "0", "0", "0", "1", "agent-a5", "claude", ""],
|
|
129
|
+
["sC", "2026-09-01", "implementer", "impl-6", "claude-opus-5", "high", "6", "", "0", "0", "0", "1", "agent-a6", "claude", ""],
|
|
130
|
+
// a second pair on the SAME day — two pairs, one run_date
|
|
131
|
+
["sD", "2026-09-01", "implementer", "impl-7", "claude-sonnet-5", "high", "7", "", "0", "0", "0", "1", "agent-a7", "claude", "fleet-implementer-alt"],
|
|
132
|
+
["sD", "2026-09-01", "implementer", "impl-8", "claude-opus-5", "high", "8", "", "0", "0", "0", "1", "agent-a8", "claude", "fleet-implementer"],
|
|
133
|
+
// and one on another day, so the second number is not the first
|
|
134
|
+
["sE", "2026-09-02", "implementer", "impl-9", "claude-sonnet-5", "high", "9", "", "0", "0", "0", "1", "agent-a9", "claude", "fleet-implementer-alt"],
|
|
135
|
+
["sE", "2026-09-02", "implementer", "impl-10", "claude-opus-5", "high", "10", "", "0", "0", "0", "1", "agent-a10", "claude", "fleet-implementer"],
|
|
136
|
+
// a typed non-implementer at a third model inside a pair session: the
|
|
137
|
+
// review fan-out runs one on every session and must not inflate anything
|
|
138
|
+
["sE", "2026-09-02", "specialist", "rev", "claude-haiku-4-5", "", "", "", "0", "0", "0", "1", "agent-a11", "claude", "fleet-review-tests"],
|
|
139
|
+
];
|
|
140
|
+
const corpus = join(mkdtempSync(join(tmpdir(), "mo-hdr-")), "member-outcomes.tsv");
|
|
141
|
+
writeFileSync(corpus, "# header line, skipped by the query\n" + rows.map((r) => r.join("\t")).join("\n") + "\n");
|
|
142
|
+
|
|
143
|
+
const program = /awk -F'\\t' '([\s\S]*)' docs\/metrics\/member-outcomes\.tsv$/.exec(pairQuery());
|
|
144
|
+
assert.ok(program, "the pair query is no longer a runnable `awk -F'\\t' '<program>' <file>` line");
|
|
145
|
+
const r = spawnSync("awk", ["-F", "\t", program[1], corpus], { encoding: "utf8" });
|
|
146
|
+
assert.equal(r.status, 0, r.stderr);
|
|
147
|
+
assert.equal(r.stdout.trim(), "3 2", "three deliberate pairs across two distinct run_dates");
|
|
148
|
+
});
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { between as section } from "./prose-pin.mjs";
|
|
6
|
+
|
|
7
|
+
const REPO = join(import.meta.dirname, "..");
|
|
8
|
+
const RUN_TEAM = readFileSync(join(REPO, "skills", "run-team", "SKILL.md"), "utf8");
|
|
9
|
+
|
|
10
|
+
// Slice by named anchors and fail loudly when one moves; slice SIZE is what does
|
|
11
|
+
// the work. The end anchor is the paragraph immediately after the insertion
|
|
12
|
+
// point (not the next "## " heading, which is 174 lines away and would satisfy
|
|
13
|
+
// this pin on prose it never touches).
|
|
14
|
+
const SLICE = () => section(RUN_TEAM, "Append one row", "**Why not decide inside one run.**", "member-outcomes ruling step");
|
|
15
|
+
|
|
16
|
+
// Everything the instruction needs in order to RUN. Matching only the filename
|
|
17
|
+
// was mutation-proven vacuous: the pin stayed green both when the command was
|
|
18
|
+
// corrupted (wrong script path, unquoted variable) and when the entire fenced
|
|
19
|
+
// block was deleted and replaced with the prose "the controller should remember
|
|
20
|
+
// to run member-outcomes.mjs somehow". That is the exact defect class that
|
|
21
|
+
// already shipped once here — a phase-3 instruction whose `$SESSION_DIR` was
|
|
22
|
+
// defined nowhere, guarded by a pin that only grepped for the filename.
|
|
23
|
+
const RUNNABLE = [
|
|
24
|
+
// the fenced block itself, not prose describing one
|
|
25
|
+
/```bash\n[\s\S]*?```/,
|
|
26
|
+
// the derivation, from the helper rather than a hand-guessed path
|
|
27
|
+
/encodeProjectDir/,
|
|
28
|
+
// the invocation, at the path it actually lives at
|
|
29
|
+
/node scripts\/member-outcomes\.mjs/,
|
|
30
|
+
// every shell expansion quoted — an unquoted one splits on the spaces in a
|
|
31
|
+
// stringified error object and turns exit 2 into a usage error
|
|
32
|
+
/"\$HOME\/\.omp\/agent\/sessions\/\$PROJECT_DIR"/,
|
|
33
|
+
];
|
|
34
|
+
|
|
35
|
+
test("the ruling step carries a RUNNABLE scraper invocation, not a mention of one", () => {
|
|
36
|
+
const slice = SLICE();
|
|
37
|
+
for (const re of RUNNABLE) assert.match(slice, re);
|
|
38
|
+
// Every variable the block uses must be assigned inside the same block.
|
|
39
|
+
const block = /```bash\n([\s\S]*?)```/.exec(slice)[1];
|
|
40
|
+
for (const [, name] of block.matchAll(/\$\{?([A-Z_][A-Z0-9_]*)\}?/g)) {
|
|
41
|
+
if (name === "HOME") continue; // supplied by the shell
|
|
42
|
+
assert.match(block, new RegExp(`^${name}=`, "m"), `${name} is used but never assigned`);
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("the ruling step scrapes EVERY session for the cwd, not one guessed session", () => {
|
|
47
|
+
// findSubagentsDir answers "newest transcript for this cwd", which is not
|
|
48
|
+
// "this run" — any other session sharing this cwd can hold a newer
|
|
49
|
+
// transcript at the moment this step runs, so the wrong-session pick is
|
|
50
|
+
// routine, silent, and exits 0.
|
|
51
|
+
const slice = SLICE();
|
|
52
|
+
assert.match(slice, /for d in .*\*\/;/, "the scrape must loop every session dir");
|
|
53
|
+
// Banned from the COMMAND, not from the prose — the paragraph that explains
|
|
54
|
+
// why the single-session helper is wrong has to be able to name it.
|
|
55
|
+
const block = /```bash\n([\s\S]*?)```/.exec(slice)[1];
|
|
56
|
+
assert.doesNotMatch(block, /findSubagentsDir/, "the single-session helper is the defect, not the fix");
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("the ruling step does NOT ask the controller to hand-write member rows", () => {
|
|
60
|
+
// The whole invariant: member-outcomes.tsv is derived. An instruction to
|
|
61
|
+
// author a row there would make it un-regenerable.
|
|
62
|
+
assert.doesNotMatch(SLICE(), /append .{0,40}member-outcomes/i);
|
|
63
|
+
});
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
// Scraper for per-member model/effort facts. Pure over omp's own subagent
|
|
2
|
+
// transcripts: no clock, no network, no gh. See
|
|
3
|
+
// docs/specs/2026-08-27-fleet-member-outcomes-instrumentation-design.md.
|
|
4
|
+
//
|
|
5
|
+
// The transcript parsing (fold, model/effort extraction, ticket/pr naming,
|
|
6
|
+
// and the `gh pr create` result an implementer's `pr` comes from, #2209)
|
|
7
|
+
// lives in member-record.mjs (#1342), shared with board.mjs. `normalizeModel`
|
|
8
|
+
// and `parseMemberName` are re-exported here verbatim so nothing importing
|
|
9
|
+
// them from this file needs to change.
|
|
10
|
+
|
|
11
|
+
import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
12
|
+
import { join, basename } from "node:path";
|
|
13
|
+
|
|
14
|
+
import { readOmpSession, normalizeModel, parseMemberName, isOmpSessionDirName } from "./member-record.mjs";
|
|
15
|
+
|
|
16
|
+
export { normalizeModel, parseMemberName };
|
|
17
|
+
|
|
18
|
+
// One row per member, via member-record.mjs's readOmpSession(). This file's
|
|
19
|
+
// own row shape stays camelCase (`effort`, `tokensCacheCreate`, `tokensOut`,
|
|
20
|
+
// `wallS`) so the TSV/FIELD machinery below and every existing consumer are
|
|
21
|
+
// untouched, while the actual transcript parsing — the fold, model/effort
|
|
22
|
+
// last-wins, ticket/pr naming — lives in exactly one place and is the same
|
|
23
|
+
// primitive board.mjs's spend reader calls too, so the two can no longer
|
|
24
|
+
// drift.
|
|
25
|
+
//
|
|
26
|
+
// `effort` reads the record's `thinking` field: #1342 keeps the TSV COLUMN
|
|
27
|
+
// named `effort` rather than renaming it, because every awk one-liner in
|
|
28
|
+
// this file's header and in docs/specs indexes columns by position, and a
|
|
29
|
+
// rename buys nothing a comment does not already say. `thinking` is never
|
|
30
|
+
// blank on the record (`-` marks the hole so it stays visible there), but
|
|
31
|
+
// this TSV's own `effort` column predates that convention and already
|
|
32
|
+
// documents blank as ITS spelling of "unknown" (header: "BLANK MEANS
|
|
33
|
+
// UNKNOWN") — so `-` maps back to `""` here, at the boundary, rather than
|
|
34
|
+
// widening the legacy column's vocabulary.
|
|
35
|
+
//
|
|
36
|
+
// `subagentType` is the record's `subagent_type` unchanged — `""` stays
|
|
37
|
+
// `""` here, because unlike `effort` it does NOT mean unknown: it means the
|
|
38
|
+
// dispatch named no agent definition, which is the reading the pair query
|
|
39
|
+
// in this file's header depends on.
|
|
40
|
+
//
|
|
41
|
+
// A member is never torn in the sense this file historically tracked (no
|
|
42
|
+
// fold-back to tear mid-turn); `false` says so plainly rather than leaving
|
|
43
|
+
// the column blank, which would read as "unmeasured".
|
|
44
|
+
//
|
|
45
|
+
// The walk is RECURSIVE: a member can itself dispatch further members
|
|
46
|
+
// (measured on disk — a research session held seven more `.jsonl` files one
|
|
47
|
+
// level down), and `agent` is the path-relative stem, so a nested member's
|
|
48
|
+
// row reads `<parent>/<id>` while a flat one is unchanged — which is what
|
|
49
|
+
// lets this widening REPLACE the existing rows rather than duplicate them.
|
|
50
|
+
//
|
|
51
|
+
// `stats` is an optional out-param: callers that omit it behave exactly as
|
|
52
|
+
// before. The CLI needs it because a session where every member dropped was
|
|
53
|
+
// otherwise indistinguishable from a healthy one.
|
|
54
|
+
//
|
|
55
|
+
// run_date comes from the newest transcript's mtime rather than a clock
|
|
56
|
+
// read, so a backfill run in December still dates an August session in
|
|
57
|
+
// August.
|
|
58
|
+
export function rowsForSession(sessionDir, stats = {}) {
|
|
59
|
+
let names;
|
|
60
|
+
try { names = readdirSync(sessionDir, { recursive: true }); }
|
|
61
|
+
catch { names = []; }
|
|
62
|
+
const jsonlNames = names.filter((x) => x.endsWith(".jsonl"));
|
|
63
|
+
let newest = 0;
|
|
64
|
+
for (const f of jsonlNames) {
|
|
65
|
+
try { newest = Math.max(newest, statSync(join(sessionDir, f)).mtimeMs); } catch { /* raced away */ }
|
|
66
|
+
}
|
|
67
|
+
const recs = readOmpSession(sessionDir); // may throw — a wrong-root refusal, not a per-file fault
|
|
68
|
+
stats.seen = jsonlNames.length;
|
|
69
|
+
stats.dropped = jsonlNames.length - recs.length;
|
|
70
|
+
stats.torn = 0;
|
|
71
|
+
const session = basename(sessionDir);
|
|
72
|
+
const run_date = newest ? new Date(newest).toISOString().slice(0, 10) : "";
|
|
73
|
+
return recs.map((r) => ({
|
|
74
|
+
session, run_date,
|
|
75
|
+
harness: r.harness, role: r.role, member: r.member, model: r.model,
|
|
76
|
+
effort: r.thinking === "-" ? "" : r.thinking, ticket: r.ticket, pr: r.pr,
|
|
77
|
+
tokensCacheCreate: r.tokens_cache_create, tokensOut: r.tokens_out,
|
|
78
|
+
wallS: r.wall_s, turns: r.turns, agent: r.agent, torn: false,
|
|
79
|
+
subagentType: r.subagent_type,
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// APPENDED TO, never inserted into: every read-out in the file's header and
|
|
84
|
+
// in docs/specs indexes by position, so a column added anywhere but the end
|
|
85
|
+
// silently repoints every `$n` a reader already wrote down. `subagent_type`
|
|
86
|
+
// (#1066) is therefore last, after `harness`.
|
|
87
|
+
export const COLUMNS = [
|
|
88
|
+
"session", "run_date", "role", "member", "model", "effort", "ticket", "pr",
|
|
89
|
+
"tokens_cache_create", "tokens_out", "wall_s", "turns", "agent", "harness",
|
|
90
|
+
"subagent_type",
|
|
91
|
+
];
|
|
92
|
+
|
|
93
|
+
// Row objects use camelCase; the file uses snake_case. One map, one direction
|
|
94
|
+
// each, so a rename cannot silently drop a column.
|
|
95
|
+
const FIELD = {
|
|
96
|
+
tokens_cache_create: "tokensCacheCreate", tokens_out: "tokensOut", wall_s: "wallS",
|
|
97
|
+
subagent_type: "subagentType",
|
|
98
|
+
};
|
|
99
|
+
const field = (c) => FIELD[c] ?? c;
|
|
100
|
+
// Keyed on the transcript's path-relative stem, not the member name: `member`
|
|
101
|
+
// collides on the agentType fallback for every unnamed dispatch, collapsing
|
|
102
|
+
// 2,715 rows onto 1,497 keys (measured). `agent` is unique WITHIN A SESSION,
|
|
103
|
+
// which is all the key needs — it is NOT globally unique, because a NAMED
|
|
104
|
+
// member's stem is derived from its name and so repeats across sessions.
|
|
105
|
+
const key = (r) => `${r.session}\0${r.agent}`;
|
|
106
|
+
|
|
107
|
+
// Replace-by-key, not append. This is what makes a phase-3 re-run and a full
|
|
108
|
+
// regeneration both safe, and it is the reason no backfill-only code path is
|
|
109
|
+
// needed: the idempotency backfill wants is the idempotency a re-run wants.
|
|
110
|
+
export function mergeRows(existing, incoming) {
|
|
111
|
+
const by = new Map((existing ?? []).map((r) => [key(r), r]));
|
|
112
|
+
for (const r of incoming ?? []) by.set(key(r), r);
|
|
113
|
+
return [...by.values()].sort((a, b) => key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function formatTsv(rows) {
|
|
117
|
+
return rows.map((r) => COLUMNS.map((c) => String(r[field(c)] ?? "")).join("\t")).join("\n") + "\n";
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// REFUSES a row whose cell count is not exactly COLUMNS.length rather than
|
|
121
|
+
// padding it. Padding looked harmless and was not: `agent` is the LAST column,
|
|
122
|
+
// so a short row parsed to `agent: ""` — a key rowsForSession can never
|
|
123
|
+
// produce, which means mergeRows can never REPLACE it. Three ways to reach that
|
|
124
|
+
// were measured, all previously exit 0: a transcript torn mid-write became a
|
|
125
|
+
// permanent phantom that re-scraping could not heal; git conflict markers became
|
|
126
|
+
// three data rows; and adding one column ahead of `agent` collapsed the corpus
|
|
127
|
+
// onto one key per session, 2,702 rows to 156.
|
|
128
|
+
export function parseTsv(text) {
|
|
129
|
+
return String(text ?? "").split("\n")
|
|
130
|
+
.filter((l) => l.trim() && !l.startsWith("#"))
|
|
131
|
+
.map((l) => {
|
|
132
|
+
const cells = l.split("\t");
|
|
133
|
+
if (cells.length !== COLUMNS.length) {
|
|
134
|
+
throw new Error(`malformed row: ${cells.length} fields, expected ${COLUMNS.length} — ${l.slice(0, 60)}`);
|
|
135
|
+
}
|
|
136
|
+
const r = {};
|
|
137
|
+
COLUMNS.forEach((c, i) => { r[field(c)] = cells[i]; });
|
|
138
|
+
return r;
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
import { writeFileSync, existsSync, renameSync } from "node:fs";
|
|
143
|
+
import { pathToFileURL } from "node:url";
|
|
144
|
+
import { makeDie, isFlagLike } from "./arg.mjs";
|
|
145
|
+
|
|
146
|
+
const NAME = "member-outcomes";
|
|
147
|
+
|
|
148
|
+
// Only runs as a CLI, never on import — the test file and any wrapper import
|
|
149
|
+
// the pure helpers. Exact identity, not a suffix match: fleet-tick.test.mjs
|
|
150
|
+
// records a copy under an unresolved path silently never running main(), and a
|
|
151
|
+
// suffix test additionally fires for any file ending in this one's name.
|
|
152
|
+
if (import.meta.url === pathToFileURL(process.argv[1] || "").href) {
|
|
153
|
+
const die = makeDie(NAME);
|
|
154
|
+
const argv = process.argv.slice(2);
|
|
155
|
+
const fileIdx = argv.indexOf("--file");
|
|
156
|
+
const file = fileIdx >= 0 ? argv[fileIdx + 1] : "docs/metrics/member-outcomes.tsv";
|
|
157
|
+
// An unrecognised `--flag` is silently dropped by the dirs filter below
|
|
158
|
+
// rather than refused — worst case `--file=/tmp/x.tsv` (no such flag;
|
|
159
|
+
// --file wants a SPACE, not `=`) never sets fileIdx, so this would silently
|
|
160
|
+
// write the production metrics file instead of the path the operator asked
|
|
161
|
+
// for.
|
|
162
|
+
const bad = argv.find((a) => a.startsWith("--") && a !== "--file");
|
|
163
|
+
if (bad) die(`unknown option ${bad} (did you mean --file <path>?)`);
|
|
164
|
+
|
|
165
|
+
// fileIdx is -1 when --file is absent, which makes fileIdx + 1 equal 0 — the
|
|
166
|
+
// FIRST positional argument, not a real index into argv. Without the
|
|
167
|
+
// fileIdx < 0 guard this filters out the session dir itself, and the bare
|
|
168
|
+
// form (the one the spec, the backfill loop and run-team's phase-3
|
|
169
|
+
// instruction all use) always exits 2.
|
|
170
|
+
const dirs = argv.filter((a, i) => !a.startsWith("--") && (fileIdx < 0 || i !== fileIdx + 1));
|
|
171
|
+
if (dirs.length !== 1) die("usage: member-outcomes.mjs <session-dir> [--file <tsv>]");
|
|
172
|
+
// arg.mjs's rule, consumed rather than copied (#567). This line used to
|
|
173
|
+
// hand-write `!file || file.startsWith("--")`, which was that rule with the
|
|
174
|
+
// blank spelling missing — so `--file " "` was accepted here and wrote the
|
|
175
|
+
// metrics TSV to a whitespace-named path, while every script routing through
|
|
176
|
+
// arg() refused it. That gap is what a copy costs and is why the predicate
|
|
177
|
+
// is imported. The refusal WORDING stays this script's own: `--file` is the
|
|
178
|
+
// flag the operator typed.
|
|
179
|
+
if (isFlagLike(file)) die("--file needs a path");
|
|
180
|
+
|
|
181
|
+
const sessionDir = dirs[0].replace(/\/+$/, "");
|
|
182
|
+
// A wrong guess used to exit 0 having scraped and written nothing, because
|
|
183
|
+
// rowsForSession() catches the readdir failure and returns []. Probe with
|
|
184
|
+
// the SAME call it makes: an existsSync test closes only the ENOENT half,
|
|
185
|
+
// and EACCES (an unreadable dir) and ENOTDIR (a regular FILE, not a
|
|
186
|
+
// directory) both walked straight past it back into the silent exit 0.
|
|
187
|
+
try { readdirSync(sessionDir); }
|
|
188
|
+
catch (e) { die(`cannot read ${sessionDir}: ${e.code ?? e.message}`); }
|
|
189
|
+
// #1302's ruling already covers `readMembers`'s tree search (findOmpSessionDirs);
|
|
190
|
+
// this CLI hands rowsForSession one EXPLICIT directory instead of a tree to
|
|
191
|
+
// search, which skipped that check entirely — any readable directory (e.g. the
|
|
192
|
+
// encoded-cwd project dir one level up from a real session) passed the readdirSync
|
|
193
|
+
// probe above, then had every nested .jsonl folded in and every row stamped with
|
|
194
|
+
// the PROJECT dir's own name as `session`, never a real omp session id.
|
|
195
|
+
if (!isOmpSessionDirName(basename(sessionDir))) {
|
|
196
|
+
die(`${sessionDir} is not an omp <ISO>_<uuid> session directory`);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Header comments are preserved verbatim across the rewrite: they carry the
|
|
200
|
+
// read-out commands and the blank-means-unknown rule, and the rewrite is
|
|
201
|
+
// routine (every re-scrape), so losing them would be a slow, silent erasure.
|
|
202
|
+
//
|
|
203
|
+
// The LEADING block only. Collecting every `#` line from anywhere hoisted a
|
|
204
|
+
// below-data note to the top, away from the row it annotated, and dropped the
|
|
205
|
+
// blank lines between paragraphs — silently, while promising "verbatim".
|
|
206
|
+
const prev = existsSync(file) ? readFileSync(file, "utf8") : "";
|
|
207
|
+
const prevLines = prev ? prev.split("\n") : [];
|
|
208
|
+
let h = 0;
|
|
209
|
+
while (h < prevLines.length && (prevLines[h].startsWith("#") || !prevLines[h].trim())) h++;
|
|
210
|
+
const header = prevLines.slice(0, h).join("\n").replace(/\n+$/, "");
|
|
211
|
+
|
|
212
|
+
let existing;
|
|
213
|
+
try { existing = parseTsv(prev); }
|
|
214
|
+
catch (e) { die(`${file} is not readable as this schema: ${e.message}`); }
|
|
215
|
+
|
|
216
|
+
const stats = {};
|
|
217
|
+
const merged = mergeRows(existing, rowsForSession(sessionDir, stats));
|
|
218
|
+
// Write-then-rename: writeFileSync truncates before it writes, so an interrupt
|
|
219
|
+
// over the committed corpus left a torn file that the NEXT scrape read, merged
|
|
220
|
+
// and rewrote — losing every other session's rows for good.
|
|
221
|
+
writeFileSync(file + ".tmp", (header ? header + "\n" : "") + formatTsv(merged));
|
|
222
|
+
renameSync(file + ".tmp", file);
|
|
223
|
+
// Report the SCRAPE, not just the file. `merged.length` is the whole corpus,
|
|
224
|
+
// so a session where every member dropped printed the same reassuring line as
|
|
225
|
+
// a healthy one — 0-of-2 was indistinguishable from full success.
|
|
226
|
+
const torn = stats.torn ? `, ${stats.torn} torn mid-write` : "";
|
|
227
|
+
process.stderr.write(
|
|
228
|
+
`${NAME}: scraped ${stats.seen - stats.dropped} of ${stats.seen} members ` +
|
|
229
|
+
`(${stats.dropped} dropped${torn}); ${file} now holds ${merged.length} rows\n`,
|
|
230
|
+
);
|
|
231
|
+
}
|