@feigi/fleet-ctl 3.37.2 → 3.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/fleet-state.mjs +23 -3
- package/scripts/fleet-tick.mjs +95 -9
- package/skills/run-team/SKILL.md +20 -2
package/package.json
CHANGED
package/scripts/fleet-state.mjs
CHANGED
|
@@ -35,6 +35,12 @@
|
|
|
35
35
|
// when rotate wrote it — a mark older than `at` is the previous
|
|
36
36
|
// run's. Removed, not left stale, by a rotate that finds no omp
|
|
37
37
|
// ancestor.
|
|
38
|
+
// seen fleet-tick only. {member: firstSeenAtMs}: the first tick that saw
|
|
39
|
+
// each live ledger token, so a later tick can say how long a token
|
|
40
|
+
// has stayed live — the ledger records that a member was dispatched,
|
|
41
|
+
// never when. Pruned to the live set on every tick, so a settled
|
|
42
|
+
// member's stamp does not outlive it. Carried, never written, by
|
|
43
|
+
// the heartbeat and by rotate.
|
|
38
44
|
//
|
|
39
45
|
// One writer per key. A key two scripts wrote would need locking to be
|
|
40
46
|
// correct, and none of them is in a position to hold one. That holds per key,
|
|
@@ -105,7 +111,7 @@ export function readState(path, name) {
|
|
|
105
111
|
// to judge and nothing to report. A zeroed mark would date the beat to the
|
|
106
112
|
// epoch and read as decades overdue on a run that has simply not started
|
|
107
113
|
// one — the cry-wolf direction this key must never fail in.
|
|
108
|
-
const fresh = { quiet: 0, elapsed: 0, digest: "", beat: null, ticked: null, controller: null, rest: {} };
|
|
114
|
+
const fresh = { quiet: 0, elapsed: 0, digest: "", beat: null, ticked: null, controller: null, seen: {}, rest: {} };
|
|
109
115
|
// The same state, flagged: the file EXISTS and could not be used. A caller
|
|
110
116
|
// that acts on the absence of a record — ledger.mjs rotate — must tell this
|
|
111
117
|
// from `fresh`, which is a run that has simply not written one yet.
|
|
@@ -197,7 +203,17 @@ export function readState(path, name) {
|
|
|
197
203
|
at: Number.isInteger(v.at) && v.at > 0 ? v.at : null,
|
|
198
204
|
};
|
|
199
205
|
};
|
|
200
|
-
|
|
206
|
+
// `seen` is validated per entry, not as a unit: each stamp is a separate
|
|
207
|
+
// member's age, so one junk entry costs only that member its age. A stamp
|
|
208
|
+
// that cannot be read restarts that member's clock, which is the late
|
|
209
|
+
// direction — never the early one, where a healthy member reads as stuck.
|
|
210
|
+
const stamps = (v) => {
|
|
211
|
+
if (!isRecord(v)) return {};
|
|
212
|
+
const out = {};
|
|
213
|
+
for (const [member, at] of Object.entries(v)) if (Number.isInteger(at) && at > 0) out[member] = at;
|
|
214
|
+
return out;
|
|
215
|
+
};
|
|
216
|
+
const { quiet, elapsed, digest, beat, ticked, controller, seen, ...rest } = parsed;
|
|
201
217
|
return {
|
|
202
218
|
quiet: num(quiet),
|
|
203
219
|
elapsed: num(elapsed),
|
|
@@ -205,6 +221,7 @@ export function readState(path, name) {
|
|
|
205
221
|
beat: mark(beat),
|
|
206
222
|
ticked: tick(ticked),
|
|
207
223
|
controller: ctl(controller),
|
|
224
|
+
seen: stamps(seen),
|
|
208
225
|
rest,
|
|
209
226
|
};
|
|
210
227
|
}
|
|
@@ -257,6 +274,9 @@ export function writeState(path, name, prev, patch) {
|
|
|
257
274
|
// by naming it with the value `undefined`, which JSON.stringify drops —
|
|
258
275
|
// the way rotate retires a record no omp ancestor replaces.
|
|
259
276
|
...(prev.controller ? { controller: prev.controller } : {}),
|
|
277
|
+
// `seen` is fleet-tick's, and the heartbeat and rotate write this file
|
|
278
|
+
// too: carried the same way, so neither erases the ages the tick keeps.
|
|
279
|
+
...(prev.seen && Object.keys(prev.seen).length > 0 ? { seen: prev.seen } : {}),
|
|
260
280
|
...patch,
|
|
261
281
|
};
|
|
262
282
|
const tmp = `${path}.${process.pid}.tmp`;
|
|
@@ -269,7 +289,7 @@ export function writeState(path, name, prev, patch) {
|
|
|
269
289
|
// Best-effort and never fatal: a cleanup that threw would turn a reported,
|
|
270
290
|
// survived write failure into the crash the comment above rules out.
|
|
271
291
|
try { rmSync(tmp, { force: true }); } catch {}
|
|
272
|
-
console.error(`${name}: WARNING could not write ${path} (${e.message}) — back-off progress will not persist`);
|
|
292
|
+
console.error(`${name}: WARNING could not write ${path} (${e.message}) — back-off progress and the liveness ages the tick stamps will not persist`);
|
|
273
293
|
return false;
|
|
274
294
|
}
|
|
275
295
|
}
|
package/scripts/fleet-tick.mjs
CHANGED
|
@@ -56,6 +56,14 @@
|
|
|
56
56
|
// file reads DEFAULT-ONLY, the way the router (not in the tree
|
|
57
57
|
// yet) is to read it. pr-cost.mjs writes the main
|
|
58
58
|
// workspace's file by default, the one read here.
|
|
59
|
+
// seen `seen` in the heartbeat's state file, this script's own key:
|
|
60
|
+
// when it first saw each live token. The ledger records that a
|
|
61
|
+
// member was dispatched, never when, so the age of a live token
|
|
62
|
+
// exists nowhere else. A token live past its family's threshold
|
|
63
|
+
// prints one `CHECK-LIVENESS` line — advice to the controller to
|
|
64
|
+
// take the registry census, never an action: it resets no
|
|
65
|
+
// `quiet`, changes no row and frees no slot, and the tick never
|
|
66
|
+
// settles or kills anything.
|
|
59
67
|
//
|
|
60
68
|
// The pure half below is `reconcile()` over the counts `deriveRun()` reads off
|
|
61
69
|
// the ledger; main() does the I/O. Split so the guard table and the reading are
|
|
@@ -912,6 +920,64 @@ export function refreshWhy({ draining, status, entries, unclaimed: k, implCap })
|
|
|
912
920
|
return null;
|
|
913
921
|
}
|
|
914
922
|
|
|
923
|
+
// How long, in minutes, a live token of each family may run before the tick
|
|
924
|
+
// asks the controller to look at it. Each default is that family's p99
|
|
925
|
+
// run time: a token past it is rare among members that finished, so a line
|
|
926
|
+
// is worth a look and not noise. Measured 2026-10-11. The four member
|
|
927
|
+
// families are the maintainer's reading of the omp rows of
|
|
928
|
+
// member-outcomes.tsv, one family each; `review` is
|
|
929
|
+
// the nearest-rank p99 of (last - first entry timestamp) over every
|
|
930
|
+
// review-pr-*.jsonl under ~/.omp/agent/sessions — 77.09 min over 4751
|
|
931
|
+
// transcripts, rounded up. Re-derive by re-running that query; no script
|
|
932
|
+
// carries it.
|
|
933
|
+
export const LIVENESS_DEFAULTS = { impl: 78, "fix-pr": 56, "finisher-pr": 57, "merge-bot": 101, review: 78 };
|
|
934
|
+
// One flag per family, so a controller can tune what the defaults get wrong.
|
|
935
|
+
const LIVENESS_FLAGS = {
|
|
936
|
+
impl: "liveness-impl", "fix-pr": "liveness-fix-pr", "finisher-pr": "liveness-finisher",
|
|
937
|
+
"merge-bot": "liveness-merge-bot", review: "liveness-review",
|
|
938
|
+
};
|
|
939
|
+
|
|
940
|
+
// A live name's family: a member token's own, or `review` for the in-flight
|
|
941
|
+
// `review:PR#<n>` a `review=` launch is named by (it carries no member token).
|
|
942
|
+
const livenessFamily = (name) => (name.startsWith("review:") ? "review" : parseMember(name)?.family ?? null);
|
|
943
|
+
|
|
944
|
+
// The `seen` map for this tick: one stamp per live name, carried when it can
|
|
945
|
+
// be trusted and `now` when not. A stamp is the age of the token, so the rule
|
|
946
|
+
// is which ones still describe THIS run's token:
|
|
947
|
+
// - an `ancestor` rotate carried this run's tokens across — they are old, and
|
|
948
|
+
// their age must survive it;
|
|
949
|
+
// - `dead` and `none` carry nothing, so a stamp older than the record's `at`
|
|
950
|
+
// belongs to a token of the previous run, even one that shares its name
|
|
951
|
+
// (`merge-bot-1` is reused), and restarts at zero;
|
|
952
|
+
// - no record, or one with no `at`, leaves nothing to compare, so the stamp
|
|
953
|
+
// stands.
|
|
954
|
+
// A name that is no longer live is dropped: a settled member's stamp must not
|
|
955
|
+
// age a later token of the same name.
|
|
956
|
+
export function stampSeen({ live, prev, controller, now }) {
|
|
957
|
+
const thisRun = (at) => controller === null || controller.prior === "ancestor" || controller.at === null || at >= controller.at;
|
|
958
|
+
const next = {};
|
|
959
|
+
for (const name of live) next[name] = prev[name] !== undefined && thisRun(prev[name]) ? prev[name] : now;
|
|
960
|
+
return next;
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
// The CHECK-LIVENESS lines: one per live name past its family's threshold, as
|
|
964
|
+
// `{member, line}`. The age shows only as a multiple of the threshold, so the
|
|
965
|
+
// line — and the digest `--fold-unchanged` compares — changes when a member
|
|
966
|
+
// crosses into the next bucket and never on the minute between.
|
|
967
|
+
export function livenessLines({ live, seen, thresholds, now }) {
|
|
968
|
+
const out = [];
|
|
969
|
+
for (const member of live) {
|
|
970
|
+
const family = livenessFamily(member);
|
|
971
|
+
if (family === null || seen[member] === undefined) continue;
|
|
972
|
+
const minutes = thresholds[family];
|
|
973
|
+
const k = Math.floor(Math.max(0, now - seen[member]) / (minutes * 60_000));
|
|
974
|
+
if (k < 1) continue;
|
|
975
|
+
const label = family === "finisher-pr" ? "finisher" : family;
|
|
976
|
+
out.push({ member, line: `CHECK-LIVENESS ${member} (${label} > ${k === 1 ? "" : `${k}×`}p99 ${k * minutes}m)` });
|
|
977
|
+
}
|
|
978
|
+
return out;
|
|
979
|
+
}
|
|
980
|
+
|
|
915
981
|
// --------------------------------------------------------------------------
|
|
916
982
|
// I/O. Everything below runs only as a CLI — importing this file must never
|
|
917
983
|
// parse argv or touch the network, or the pure half stops being unit-testable.
|
|
@@ -987,6 +1053,8 @@ const OPTIONS = {
|
|
|
987
1053
|
// under parseArgs, and no path is legitimately empty, so empty resolves to
|
|
988
1054
|
// the default.
|
|
989
1055
|
state: { type: "string", default: "" },
|
|
1056
|
+
// Minutes a live token of each family may run before CHECK-LIVENESS names it.
|
|
1057
|
+
...Object.fromEntries(Object.entries(LIVENESS_FLAGS).map(([family, flag]) => [flag, { type: "string", default: String(LIVENESS_DEFAULTS[family]) }])),
|
|
990
1058
|
};
|
|
991
1059
|
|
|
992
1060
|
function options() {
|
|
@@ -1017,6 +1085,7 @@ function options() {
|
|
|
1017
1085
|
implCap: positive("implementer-cap"), reviewerCap,
|
|
1018
1086
|
maxReviews: values["max-reviews"] === undefined ? reviewerCap : positive("max-reviews"),
|
|
1019
1087
|
fold: values["fold-unchanged"], state: values.state || statePath(NAME),
|
|
1088
|
+
liveness: Object.fromEntries(Object.entries(LIVENESS_FLAGS).map(([family, flag]) => [family, positive(flag)])),
|
|
1020
1089
|
};
|
|
1021
1090
|
}
|
|
1022
1091
|
|
|
@@ -1242,7 +1311,7 @@ function claimed() {
|
|
|
1242
1311
|
}
|
|
1243
1312
|
|
|
1244
1313
|
function main() {
|
|
1245
|
-
const { fold, state: path, ...caps } = options();
|
|
1314
|
+
const { fold, state: path, liveness, ...caps } = options();
|
|
1246
1315
|
const listPath = shortlistPath();
|
|
1247
1316
|
const current = readShortlist(listPath);
|
|
1248
1317
|
// Beside the shortlist, off the same git probe rather than a second one.
|
|
@@ -1310,20 +1379,33 @@ function main() {
|
|
|
1310
1379
|
...caps, ...run, heads, supply, shortlistStatus: current.status, refresh: fresh, mainCheckout, router,
|
|
1311
1380
|
});
|
|
1312
1381
|
const said = describe(mainCheckout, run.live);
|
|
1313
|
-
|
|
1382
|
+
// Re-read rather than reusing priorState: this must be the freshest
|
|
1383
|
+
// snapshot, or this tick reverts whatever the heartbeat wrote while the
|
|
1384
|
+
// reads above were in flight. Read before the lines are built because the
|
|
1385
|
+
// liveness lines need the stamps it holds, and nothing between here and the
|
|
1386
|
+
// write below is I/O.
|
|
1387
|
+
const prev = readState(path, NAME);
|
|
1388
|
+
const now = Date.now();
|
|
1389
|
+
// Advisory only: a stale token prints a line asking the controller for the
|
|
1390
|
+
// registry census. It is not a row — it resets no `quiet`, moves no count
|
|
1391
|
+
// and frees no slot — and nothing here settles, kills or writes the ledger.
|
|
1392
|
+
const seen = stampSeen({ live: run.live, prev: prev.seen, controller: prev.controller, now });
|
|
1393
|
+
const stale = livenessLines({ live: run.live, seen, thresholds: liveness, now });
|
|
1394
|
+
const lines = [...(said === null ? [] : [said]), ...stale.map((s) => s.line), ...formatLines(rows)];
|
|
1314
1395
|
|
|
1315
1396
|
// The back-off streak and the fold digest, written on EVERY tick: an edge
|
|
1316
1397
|
// tick that dispatched is the clearest possible "there is work here". Digest,
|
|
1317
|
-
// not the lines themselves, so the file stays readable.
|
|
1318
|
-
// reusing priorState: this must be the freshest snapshot, or this tick
|
|
1319
|
-
// reverts whatever the heartbeat wrote while the reads above were in flight.
|
|
1320
|
-
const prev = readState(path, NAME);
|
|
1398
|
+
// not the lines themselves, so the file stays readable.
|
|
1321
1399
|
const digest = createHash("sha256").update(lines.join("\n")).digest("hex");
|
|
1322
1400
|
const acts = actionable(rows);
|
|
1323
1401
|
// `ticked`, fleet-tick's own liveness key: written on every tick that gets
|
|
1324
1402
|
// past its refusals — a refused tick (exit 2) writes none; this tick
|
|
1325
|
-
// reaching here IS the occurrence.
|
|
1326
|
-
|
|
1403
|
+
// reaching here IS the occurrence. `seen` is pruned to the live set on the
|
|
1404
|
+
// same write; an empty one removes the key.
|
|
1405
|
+
writeState(path, NAME, prev, {
|
|
1406
|
+
quiet: acts ? 0 : prev.quiet + 1, digest, ticked: { at: now },
|
|
1407
|
+
seen: Object.keys(seen).length > 0 ? seen : undefined,
|
|
1408
|
+
});
|
|
1327
1409
|
|
|
1328
1410
|
// Fold only when BOTH hold: nothing to act on, and nothing new to say. A
|
|
1329
1411
|
// main-checkout hold still holding dispatch stays named on the folded line
|
|
@@ -1333,7 +1415,11 @@ function main() {
|
|
|
1333
1415
|
if (fold && !acts && digest === prev.digest) {
|
|
1334
1416
|
const why = mainCheckoutHoldReason(mainCheckout);
|
|
1335
1417
|
const held = why === null ? "" : `; HOLD (${why}) persists`;
|
|
1336
|
-
|
|
1418
|
+
// A stale member persists the same way: advisory, so it never resets
|
|
1419
|
+
// `quiet`, and named on the folded line for as long as it stays live
|
|
1420
|
+
// past its threshold. The digest already moves when its bucket does.
|
|
1421
|
+
const aged = stale.map((s) => `; CHECK-LIVENESS ${s.member} persists`).join("");
|
|
1422
|
+
console.log(`fleet-tick: unchanged, nothing to act on (quiet=${prev.quiet + 1})${held}${aged} — full rows on the next change`);
|
|
1337
1423
|
return;
|
|
1338
1424
|
}
|
|
1339
1425
|
for (const line of lines) console.log(line);
|
package/skills/run-team/SKILL.md
CHANGED
|
@@ -1906,6 +1906,16 @@ depth** guard table applied in code. Act on each line as it reads:
|
|
|
1906
1906
|
baseline was recorded. Yours to fix: run Phase 0's `--record`. A baseline
|
|
1907
1907
|
that vanished mid-run is not a fresh run — report it to the maintainer
|
|
1908
1908
|
before recording over whatever the tree now holds.
|
|
1909
|
+
- `CHECK-LIVENESS <member> (<family> > p99 <t>m)`, printed above the rows — a
|
|
1910
|
+
live ledger token has stayed live past its family's p99 run time, counted
|
|
1911
|
+
from the first tick that saw it (`> 2×p99 …` from the second multiple of the
|
|
1912
|
+
threshold on; `review:PR#<n>` names an in-flight `review=` launch). Advice,
|
|
1913
|
+
not an action: it is not actionable, resets no `quiet`, changes no row and
|
|
1914
|
+
frees no slot, and the tick settles and kills nothing. Take the census the
|
|
1915
|
+
**Member unaccounted** row of **Failure handling** gives before you touch the
|
|
1916
|
+
token. It repeats every tick while the token stays live past the threshold.
|
|
1917
|
+
`--liveness-impl`, `--liveness-fix-pr`, `--liveness-finisher`,
|
|
1918
|
+
`--liveness-merge-bot` and `--liveness-review` set the thresholds, in minutes.
|
|
1909
1919
|
|
|
1910
1920
|
**Tier guards under Pull.** The per-dispatch `tier-check.mjs --batch` is
|
|
1911
1921
|
phase 2's own step: a mismatch it finds is the
|
|
@@ -1972,6 +1982,10 @@ ticket's own remedy. A dirty or unknown main checkout still folds — clearing i
|
|
|
1972
1982
|
is the maintainer's, so it is nothing for you to act on — but the folded line
|
|
1973
1983
|
names it (`…; HOLD (main checkout dirty) persists — …`) for as long as it holds
|
|
1974
1984
|
dispatch.
|
|
1985
|
+
A live member past its family's threshold folds the same way — it is advice, so
|
|
1986
|
+
nothing for the heartbeat to act on by itself — and the folded line names each
|
|
1987
|
+
such member (`…; CHECK-LIVENESS fix-pr-5 persists — …`) for as long as it stays
|
|
1988
|
+
live past the threshold.
|
|
1975
1989
|
|
|
1976
1990
|
**When you stop on purpose, say why — one command, and it is the last thing
|
|
1977
1991
|
you owe the next run.** The beat leaves a dated mark on every hold, so a run
|
|
@@ -4016,6 +4030,7 @@ Plan it as a PR the run's own merge queue lands, never as an edit-and-continue.
|
|
|
4016
4030
|
| Member silent or truncated | Send to ping or resume — same unit of work; see the state machine below |
|
|
4017
4031
|
| Member idle with work outstanding | Read the PR first, *then* ping. Idle ≠ done |
|
|
4018
4032
|
| Member **killed** (spend limit, API error, crash) | Confirm it is dead first — a frozen transcript does not establish that; see below — then new member, new name, prompt carries inherited state |
|
|
4033
|
+
| Member **unaccounted** (a live token the tick flags `CHECK-LIVENESS`, or any live token nothing has reported on) | Registry census first: `read history://` once. A row other than `running` → `read agent://<member>` and the artifacts (PR head, worktree, labels), settle the real outcome with the existing vocabulary, and resume a `parked` member with `write agent://<member>` when its work is worth continuing. A `running` row → the mtime discriminator below, two polls: moved means BLOCKED and alive; frozen at both polls **and** no artifact movement → `write proc://<member>/kill`, `ledger.mjs settle <member> killed`, fresh name. Three independent alive-signals — the first poll, the second poll, the artifacts — must all be silent before a kill; the tick settles and kills nothing |
|
|
4019
4034
|
|
|
4020
4035
|
A red PR never silently becomes `ready-to-merge`.
|
|
4021
4036
|
|
|
@@ -4035,8 +4050,8 @@ moved: **mtimes that moved between the two mean BLOCKED, a live member
|
|
|
4035
4050
|
still holding its claim and its worktree, so the killed row above does not
|
|
4036
4051
|
apply to it; mtimes unchanged at both are the dead signature.** Only the first
|
|
4037
4052
|
of those two verdicts is conclusive — a member wedged on a call that writes
|
|
4038
|
-
nothing freezes its scratch dir too — so settle the dead one on the
|
|
4039
|
-
|
|
4053
|
+
nothing freezes its scratch dir too — so settle the dead one on the Member
|
|
4054
|
+
unaccounted row above, never on the two listings alone. Measured once: a refuter
|
|
4040
4055
|
wedged forever on `until grep -q` for a marker `node --test` never writes, its
|
|
4041
4056
|
transcript's record counts unchanged across two polls while its own scratch
|
|
4042
4057
|
files carried fresh mtimes throughout — alive the entire time it read as dead.
|
|
@@ -4073,6 +4088,9 @@ One git-ignored `.fleet/ledger.md` per run, updated at **every** state change vi
|
|
|
4073
4088
|
durable thing in the run: it compacts, and a controller that loses the dispatch
|
|
4074
4089
|
map or the filed list redoes finished work. Two duplicate tickets shipped in one
|
|
4075
4090
|
run from exactly that.
|
|
4091
|
+
Compaction parks every in-flight member and nothing reports it, so on resume
|
|
4092
|
+
take the registry census first (**Member unaccounted**, in **Failure
|
|
4093
|
+
handling**) before you trust a live token.
|
|
4076
4094
|
|
|
4077
4095
|
**The file lives one run.** Phase 0 rotates it before anything reads it, so
|
|
4078
4096
|
no exclusion, filing, ruling or drain marker outlives the run that wrote it,
|