@feigi/fleet-ctl 3.47.0 → 3.47.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/fleet-heartbeat.mjs +17 -4
- package/scripts/fleet-state.mjs +30 -15
- package/scripts/fleet-tick.mjs +12 -8
- package/scripts/ledger.mjs +18 -8
- package/skills/run-team/SKILL.md +44 -19
package/package.json
CHANGED
|
@@ -193,7 +193,13 @@ function main() {
|
|
|
193
193
|
// No hold, and `elapsed` untouched: this invocation is not a beat, it is
|
|
194
194
|
// the run saying why there will not be another one. Leaving the partial
|
|
195
195
|
// interval where it is costs nothing — the next run reads a stopped mark
|
|
196
|
-
// and knows the streak behind it is a dead run's.
|
|
196
|
+
// and knows the streak behind it is a dead run's. Leave degraded files
|
|
197
|
+
// byte-identical, so a same-session rotate still refuses rather than
|
|
198
|
+
// judges none.
|
|
199
|
+
if (state.degraded) {
|
|
200
|
+
console.log(`heartbeat: stop NOT recorded (${stop}) — the heartbeat file is degraded and was left as it is, so a same-session rotate refuses on it until it is repaired`);
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
197
203
|
const persisted = writeState(path, NAME, state, { beat: mark(target, stop) });
|
|
198
204
|
// Announced on stdout as well as through writeState's own stderr warning,
|
|
199
205
|
// and still exit 0. An unrecorded stop is a real degradation — the next
|
|
@@ -231,13 +237,15 @@ function main() {
|
|
|
231
237
|
// taking one of its own. One write per invocation means a mark can never
|
|
232
238
|
// land while the progress it was taken beside did not, and the two can
|
|
233
239
|
// never disagree about which invocation they came from. It also keeps the
|
|
234
|
-
// failure story single: the warning below covers both keys at once.
|
|
240
|
+
// failure story single: the warning below covers both keys at once. Leave
|
|
241
|
+
// degraded files byte-identical, so a same-session rotate still refuses
|
|
242
|
+
// rather than judges none — a skipped write reports a fire like a failed one.
|
|
235
243
|
//
|
|
236
244
|
// `stopped: ""` on every ordinary beat, never carried forward: a run that
|
|
237
245
|
// recorded a stop and then kept beating is a run that did not stop, and a
|
|
238
246
|
// stale reason surviving into a live beat would have the next reader
|
|
239
247
|
// announcing a death that already un-happened.
|
|
240
|
-
const persisted = writeState(path, NAME, now, { elapsed: reached ? 0 : elapsed, beat: mark(target, "") });
|
|
248
|
+
const persisted = !now.degraded && writeState(path, NAME, now, { elapsed: reached ? 0 : elapsed, beat: mark(target, "") });
|
|
241
249
|
// A failed write is itself a fire. With nothing persisting the remainder,
|
|
242
250
|
// every invocation reads the same elapsed total, holds the same seconds and
|
|
243
251
|
// prints the same remainder: the interval can never complete, so fleet-tick
|
|
@@ -263,8 +271,13 @@ function main() {
|
|
|
263
271
|
// `quiet=` is the pre-hold streak, the one the interval was DERIVED from: a
|
|
264
272
|
// fresh streak printed beside a target computed from the old one would be two
|
|
265
273
|
// numbers that do not explain each other.
|
|
274
|
+
//
|
|
275
|
+
// A skipped write says so, on the fire it reports: the interval did not
|
|
276
|
+
// elapse, the file could not keep its progress, and this call fires again
|
|
277
|
+
// on every hold until the file is repaired.
|
|
278
|
+
const skippedNote = now.degraded ? " — the heartbeat file is degraded and was left as it is, so interval progress is not kept and every hold fires" : "";
|
|
266
279
|
console.log(done
|
|
267
|
-
? `heartbeat: held ${held}s, ${target}s interval elapsed (quiet=${state.quiet}) → run fleet-tick`
|
|
280
|
+
? `heartbeat: held ${held}s, ${target}s interval elapsed (quiet=${state.quiet}) → run fleet-tick${skippedNote}`
|
|
268
281
|
: `heartbeat: held ${held}s, ${target - elapsed}s of ${target}s remain (quiet=${state.quiet}) → re-issue this command now, do not end your turn`);
|
|
269
282
|
}
|
|
270
283
|
|
package/scripts/fleet-state.mjs
CHANGED
|
@@ -108,9 +108,11 @@ const PRIORS = ["dead", "ancestor", "none"];
|
|
|
108
108
|
// Absent is the ONLY silent case, because it is the only one that is not a
|
|
109
109
|
// fault: a fresh run legitimately has no file yet. An unreadable file and a
|
|
110
110
|
// corrupt one are both announced — each discards real state, and silence about
|
|
111
|
-
// a degraded read is the failure class this whole ticket exists to close.
|
|
112
|
-
//
|
|
113
|
-
//
|
|
111
|
+
// a degraded read is the failure class this whole ticket exists to close. So
|
|
112
|
+
// is a file that parses but whose `controller` key holds no record `ctl`
|
|
113
|
+
// below accepts: it reads as no record, and the next write would drop the
|
|
114
|
+
// key. All of them come back flagged `degraded`, for the one caller that
|
|
115
|
+
// must not read them as a file with nothing in it.
|
|
114
116
|
export function readState(path, name) {
|
|
115
117
|
// `beat: null` is the absent MARK, and it is not the same value as a mark
|
|
116
118
|
// whose fields are zero: nothing has been seen beating, so there is nothing
|
|
@@ -120,8 +122,10 @@ export function readState(path, name) {
|
|
|
120
122
|
const fresh = { quiet: 0, elapsed: 0, digest: "", beat: null, ticked: null, acted: null, controller: null, seen: {}, rest: {} };
|
|
121
123
|
// The same state, flagged: the file EXISTS and could not be used. A caller
|
|
122
124
|
// that acts on the absence of a record — ledger.mjs rotate — must tell this
|
|
123
|
-
// from `fresh`, which is a run that has simply not written one yet.
|
|
124
|
-
|
|
125
|
+
// from `fresh`, which is a run that has simply not written one yet. The
|
|
126
|
+
// `degraded` field is absent on a fresh read, else a phrase for the cause
|
|
127
|
+
// that completes "<file> …" in rotate's refusal.
|
|
128
|
+
const makeDegraded = (cause) => ({ ...fresh, degraded: cause });
|
|
125
129
|
let raw;
|
|
126
130
|
try {
|
|
127
131
|
raw = readFileSync(path, "utf8");
|
|
@@ -136,7 +140,7 @@ export function readState(path, name) {
|
|
|
136
140
|
// whole ticket exists to close.
|
|
137
141
|
if (e.code !== "ENOENT") {
|
|
138
142
|
console.error(`${name}: WARNING could not read ${path} (${e.message}) — restarting at the base interval`);
|
|
139
|
-
return
|
|
143
|
+
return makeDegraded("exists but cannot be read");
|
|
140
144
|
}
|
|
141
145
|
return fresh;
|
|
142
146
|
}
|
|
@@ -145,14 +149,16 @@ export function readState(path, name) {
|
|
|
145
149
|
parsed = JSON.parse(raw);
|
|
146
150
|
} catch (e) {
|
|
147
151
|
console.error(`${name}: WARNING ${path} is not JSON (${e.message}) — restarting at the base interval`);
|
|
148
|
-
return
|
|
152
|
+
return makeDegraded("is not JSON");
|
|
149
153
|
}
|
|
150
154
|
if (!isRecord(parsed)) {
|
|
151
155
|
console.error(`${name}: WARNING ${path} is not an object — restarting at the base interval`);
|
|
152
|
-
return
|
|
156
|
+
return makeDegraded("is not a JSON object");
|
|
153
157
|
}
|
|
154
158
|
// Per-field validation, not all-or-nothing: a file carrying a good `quiet`
|
|
155
159
|
// and a junk `elapsed` keeps the streak instead of losing both to one key.
|
|
160
|
+
// The exception is `controller`, below: a record that cannot be judged
|
|
161
|
+
// flags the whole file, because a write that dropped it is the loss.
|
|
156
162
|
const num = (v) => (typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : 0);
|
|
157
163
|
// `rest` is strictly what lies OUTSIDE the schema, which is what makes the
|
|
158
164
|
// patch write below a patch rather than a laundering step. A raw copy of
|
|
@@ -193,12 +199,13 @@ export function readState(path, name) {
|
|
|
193
199
|
};
|
|
194
200
|
// `controller` is validated as a unit for the mark's reason: a pid without
|
|
195
201
|
// the start time it was recorded with cannot be told from a reused pid. A
|
|
196
|
-
// pid outside the range a process id can take is
|
|
197
|
-
// no process could ever answer for it. `
|
|
198
|
-
//
|
|
199
|
-
//
|
|
200
|
-
//
|
|
201
|
-
//
|
|
202
|
+
// pid outside the range a process id can take is not a record either, since
|
|
203
|
+
// no process could ever answer for it. A `controller` key that fails this
|
|
204
|
+
// is not read as no record: it degrades the whole file, below. `prior`
|
|
205
|
+
// degrades like `stopped` does: a junk verdict proves nothing, so it reads
|
|
206
|
+
// as "none", and the record around it still names a controller to judge.
|
|
207
|
+
// `at` degrades the same way, to null: the record still names a
|
|
208
|
+
// controller, and only when it was written is unknown.
|
|
202
209
|
const ctl = (v) => {
|
|
203
210
|
if (!isRecord(v)) return null;
|
|
204
211
|
if (!Number.isInteger(v.pid) || v.pid < 1 || v.pid > 0x7fffffff) return null;
|
|
@@ -221,6 +228,14 @@ export function readState(path, name) {
|
|
|
221
228
|
return out;
|
|
222
229
|
};
|
|
223
230
|
const { quiet, elapsed, digest, beat, ticked, acted, controller, seen, ...rest } = parsed;
|
|
231
|
+
// A controller key that ctl() rejects is a record someone wrote and nobody
|
|
232
|
+
// can judge, not the absence of one — announced like the other degraded
|
|
233
|
+
// causes, and flagged.
|
|
234
|
+
const record = ctl(controller);
|
|
235
|
+
if ("controller" in parsed && record === null) {
|
|
236
|
+
console.error(`${name}: WARNING ${path} holds a controller record that cannot be judged — restarting at the base interval`);
|
|
237
|
+
return makeDegraded("holds a controller record that cannot be judged");
|
|
238
|
+
}
|
|
224
239
|
return {
|
|
225
240
|
quiet: num(quiet),
|
|
226
241
|
elapsed: num(elapsed),
|
|
@@ -228,7 +243,7 @@ export function readState(path, name) {
|
|
|
228
243
|
beat: mark(beat),
|
|
229
244
|
ticked: tick(ticked),
|
|
230
245
|
acted: tick(acted),
|
|
231
|
-
controller:
|
|
246
|
+
controller: record,
|
|
232
247
|
seen: stamps(seen),
|
|
233
248
|
rest,
|
|
234
249
|
};
|
package/scripts/fleet-tick.mjs
CHANGED
|
@@ -1526,14 +1526,18 @@ function main() {
|
|
|
1526
1526
|
const digest = createHash("sha256").update(lines.join("\n")).digest("hex");
|
|
1527
1527
|
const acts = actionable(rows) || over;
|
|
1528
1528
|
// `seen` is pruned to the live set on the same write; an empty one removes
|
|
1529
|
-
// the key.
|
|
1530
|
-
//
|
|
1531
|
-
//
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1529
|
+
// the key. A file that was degraded at this read is left byte-identical, so
|
|
1530
|
+
// a same-session rotate still refuses rather than judges none. `ticked` was
|
|
1531
|
+
// written at the top of the tick unless the file was degraded then; one
|
|
1532
|
+
// repaired since reaches this write without it, so this write carries the
|
|
1533
|
+
// stamp the top of the tick left out.
|
|
1534
|
+
if (!prev.degraded) {
|
|
1535
|
+
writeState(path, NAME, prev, {
|
|
1536
|
+
quiet: acts ? 0 : prev.quiet + 1, digest,
|
|
1537
|
+
seen: Object.keys(seen).length > 0 ? seen : undefined,
|
|
1538
|
+
...(priorState.degraded ? { ticked: { at: tickedAt } } : {}),
|
|
1539
|
+
});
|
|
1540
|
+
}
|
|
1537
1541
|
|
|
1538
1542
|
// Fold only when BOTH hold: nothing to act on, and nothing new to say. A
|
|
1539
1543
|
// main-checkout hold still holding dispatch stays named on the folded line
|
package/scripts/ledger.mjs
CHANGED
|
@@ -1484,12 +1484,14 @@ function runCheck() {
|
|
|
1484
1484
|
// allows the rotation. With no record the guard falls back to the mark, read
|
|
1485
1485
|
// through fleet-state.mjs's own reader and verdict: still beating — a beat
|
|
1486
1486
|
// inside its window, or past it with a tick or ledger write inside
|
|
1487
|
-
// theirs — refuses
|
|
1488
|
-
//
|
|
1487
|
+
// theirs — refuses, and so does a tick or ledger write inside its window
|
|
1488
|
+
// with no beat at all; a stalled mark, a recorded stop, or no mark at all
|
|
1489
|
+
// allows the rotation, and that fallback proves "not beating", never "dead".
|
|
1489
1490
|
//
|
|
1490
|
-
// A heartbeat file that exists but cannot be
|
|
1491
|
-
//
|
|
1492
|
-
//
|
|
1491
|
+
// A heartbeat file that exists but cannot be used — unreadable, not JSON, not
|
|
1492
|
+
// a JSON object, or holding a `controller` key that cannot be judged — gets
|
|
1493
|
+
// neither verdict: a record it may hold cannot be known, so rotate refuses
|
|
1494
|
+
// rather than read the fault as "no record".
|
|
1493
1495
|
//
|
|
1494
1496
|
// Its last act, whether or not there was a ledger to move, replaces the
|
|
1495
1497
|
// record with this run's own, carrying the verdict as `prior` — the next
|
|
@@ -1510,7 +1512,7 @@ function runRotate() {
|
|
|
1510
1512
|
// reading it as "no record" would let a live controller's ledger be moved
|
|
1511
1513
|
// by exactly the fault that hid its record.
|
|
1512
1514
|
if (state.degraded) {
|
|
1513
|
-
die(`refusing to rotate ${file}: ${beatFile}
|
|
1515
|
+
die(`refusing to rotate ${file}: ${beatFile} ${state.degraded} — repair it so its controller record is intact. Removing it makes this rotate judge none: if the previous run was this omp session's, its live members and retired names are not carried forward, so this run can dispatch onto their work and reuse their names.`);
|
|
1514
1516
|
}
|
|
1515
1517
|
let prior, ours;
|
|
1516
1518
|
try {
|
|
@@ -1527,9 +1529,17 @@ function runRotate() {
|
|
|
1527
1529
|
}
|
|
1528
1530
|
// The fallback guards a ledger, as it always has: with none to move there
|
|
1529
1531
|
// is nothing for a beating mark to protect. A ledger write is the
|
|
1530
|
-
// controller acting, and counts the way a beat does.
|
|
1532
|
+
// controller acting, and counts the way a beat does. Refuse when the beat
|
|
1533
|
+
// is beating, and also — the one state assessBeat reads as `none`, because
|
|
1534
|
+
// the cockpit shows no panel without a record — when there is no beat at
|
|
1535
|
+
// all and a tick or ledger write is still inside its window: a record lost
|
|
1536
|
+
// to a removed file leaves exactly that. A recorded stop reads `stopped`
|
|
1537
|
+
// through assessBeat, so it lets the rotation through.
|
|
1531
1538
|
const exists = existsSync(file);
|
|
1532
|
-
|
|
1539
|
+
const judgedAt = Date.now();
|
|
1540
|
+
const markWindowEnd = beat === null ? stallsAt({ ticked, acted }) : null;
|
|
1541
|
+
const actingWithoutBeat = markWindowEnd !== null && judgedAt <= markWindowEnd;
|
|
1542
|
+
if (prior === "none" && exists && (actingWithoutBeat || assessBeat({ beat, ticked, acted, now: judgedAt }).kind === "beating")) {
|
|
1533
1543
|
const [what, mark] = [["beat", beat], ["tick", ticked], ["ledger write", acted]]
|
|
1534
1544
|
.filter(([, m]) => m !== null)
|
|
1535
1545
|
.reduce((a, b) => (b[1].at > a[1].at ? b : a));
|
package/skills/run-team/SKILL.md
CHANGED
|
@@ -450,16 +450,25 @@ phase, or in any later one, asks the maintainer which tickets to take.
|
|
|
450
450
|
controller can stop beating and keep dispatching. A dead one, a pid
|
|
451
451
|
reused by another process, or this session's own earlier run lets it
|
|
452
452
|
rotate. With no record the heartbeat mark decides, as the only guard
|
|
453
|
-
there is: still beating or acting
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
453
|
+
there is: still beating or acting — a beat inside its window, or with no
|
|
454
|
+
beat at all a tick or ledger write inside its window — refuses, naming
|
|
455
|
+
the latest beat, tick or ledger write and when it would count as stalled;
|
|
456
|
+
a stalled mark, a recorded `--stop`, or no mark at all lets it rotate —
|
|
457
|
+
and that fallback proves "not beating", never "dead", and
|
|
457
458
|
guards a ledger only: with none to move the mark is not consulted. With no
|
|
458
459
|
ledger the record is still judged, so a live recorded controller refuses
|
|
459
460
|
even then. It also refuses, nothing moved, when the stamped archive name
|
|
460
461
|
already exists, when the process table cannot be read, or when
|
|
461
|
-
`.fleet/heartbeat.json` exists but cannot be
|
|
462
|
-
|
|
462
|
+
`.fleet/heartbeat.json` exists but cannot be used — unreadable, not JSON,
|
|
463
|
+
not a JSON object, or holding a `controller` that cannot be judged — a
|
|
464
|
+
record nobody could judge is never taken for no record. On that refusal
|
|
465
|
+
phase 0 stops and the refusal goes to the maintainer verbatim: the
|
|
466
|
+
controller never edits or removes `.fleet/heartbeat.json`, because
|
|
467
|
+
removing it inside a live omp session makes this rotate judge `none` and
|
|
468
|
+
forfeits the carry — the live members and the retired names — so this
|
|
469
|
+
run could dispatch onto their work and reuse their names. Every writer
|
|
470
|
+
(`fleet-tick.mjs`, `fleet-heartbeat.mjs`, the ledger's write subcommands)
|
|
471
|
+
leaves such a file as it is. Its last act, ledger or none, is
|
|
463
472
|
to replace the record with this run's own — the nearest `omp` process
|
|
464
473
|
above it, its start time, as `prior` the verdict it reached on the
|
|
465
474
|
record it found (`dead`, `ancestor` or `none`), and as `at` when it wrote
|
|
@@ -2185,9 +2194,13 @@ while :; do # one tick
|
|
|
2185
2194
|
cap_out=
|
|
2186
2195
|
fi
|
|
2187
2196
|
# gh pr list is GraphQL, so $rl still holds. The pass is budgeted at 4 core
|
|
2188
|
-
# REST units per listed PR on top of the 200 floor
|
|
2189
|
-
#
|
|
2190
|
-
#
|
|
2197
|
+
# REST units per listed PR on top of the 200 floor. The 4 is the measured
|
|
2198
|
+
# ceiling of one ci-state.mjs call on the counter `gh api rate_limit`
|
|
2199
|
+
# reports, for single-page reads: 2 on the common path, 4 for a PR whose
|
|
2200
|
+
# head dropped required jobs.
|
|
2201
|
+
# The workflow and run lookups bill a second core bucket that `rate_limit`
|
|
2202
|
+
# does not report and this gate does not guard. A shortfall pauses this
|
|
2203
|
+
# tick's pass.
|
|
2191
2204
|
if [ "$rl" -lt $((200 + 4 * count)) ]; then
|
|
2192
2205
|
if [ -z "$budget_out" ]; then
|
|
2193
2206
|
echo "WATCHER DEGRADED: core REST budget=$rl < $((200 + 4 * count)) needed for $count PRs — CI polling paused, silence is NOT green"
|
|
@@ -2283,12 +2296,21 @@ DEGRADED line names the rest as unwatched until a pass comes back under the cap.
|
|
|
2283
2296
|
|
|
2284
2297
|
**The budget is gated twice, and only the second gate announces recovery.** The
|
|
2285
2298
|
tick-top floor (`200`) covers the probes themselves. Once the open-PR list is
|
|
2286
|
-
read, the pass gate reserves `200 + 4 × count` core REST units —
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2299
|
+
read, the pass gate reserves `200 + 4 × count` core REST units — 4 per listed PR
|
|
2300
|
+
on top of the floor, so the reserve grows with the list instead of staying a
|
|
2301
|
+
flat 200 that a long list drains mid-pass. The 4 is the measured ceiling of one
|
|
2302
|
+
`ci-state.mjs` call's cost on the counter `gh api rate_limit` reports (measured
|
|
2303
|
+
2026-10-13): 2 on the common path, 4 when the head dropped required jobs, for up
|
|
2304
|
+
to 100 jobs and 100 rules — more pages cost more. That counter is not the only
|
|
2305
|
+
core bucket one call bills. The lookups `actions/workflows`,
|
|
2306
|
+
`actions/workflows/<id>`, `actions/workflows/<id>/runs` and `actions/runs/<id>`
|
|
2307
|
+
bill a second one that `rate_limit` does not report and the gate does not
|
|
2308
|
+
guard; the boundary is per route, so `actions/runs/<id>/jobs` is on the gated
|
|
2309
|
+
counter. Every response says `X-Ratelimit-Resource: core`, so only the reset
|
|
2310
|
+
epoch and the used sequence tell the two apart. An exhausted unreported bucket
|
|
2311
|
+
shows up only as per-PR `rate-limited` verdicts, which the blind-PR path
|
|
2312
|
+
already latches. A shortfall at either gate pauses the **whole** pass through
|
|
2313
|
+
the one budget latch and never polls a prefix of the list: a prefix runs
|
|
2292
2314
|
in list order, so the same tail PRs would go unread every tick with no line at
|
|
2293
2315
|
all (never polled, so nothing marks them blind), while the PRs that were polled
|
|
2294
2316
|
would run dry mid-pass and each emit a blind line. The DEGRADED line from the
|
|
@@ -4265,10 +4287,13 @@ live count from it, so nobody states one. Write neither by hand:
|
|
|
4265
4287
|
`## Retired` (phase 0 says which). Refused
|
|
4266
4288
|
while the recorded controller is alive and not this run's ancestor, ledger
|
|
4267
4289
|
or none — or, with no record and a ledger to move, while the heartbeat mark
|
|
4268
|
-
is still beating
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
|
|
4290
|
+
is still beating, or a tick or ledger write is inside its window with no
|
|
4291
|
+
beat — or when the archive name already exists, the process
|
|
4292
|
+
table cannot be read, or `.fleet/heartbeat.json` exists but cannot be used
|
|
4293
|
+
(unreadable, not JSON, not an object, or a `controller` it cannot judge);
|
|
4294
|
+
with no ledger and no such refusal it moves nothing and exits 0. Then it
|
|
4295
|
+
records this run's controller in `.fleet/heartbeat.json`. Phase 0 states
|
|
4296
|
+
the guard.
|
|
4272
4297
|
|
|
4273
4298
|
`row` refuses a malformed member token anywhere in its text — one whose outcome
|
|
4274
4299
|
is outside the vocabulary above, e.g. `merge-bot-1=dispatched` — at exit 2,
|