@cohortapp/agent-sdk 2.18.13 → 2.18.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +38 -1
- package/docs/runbooks/fleet-rollout.md +58 -7
- package/docs/runbooks/recovery-and-failover.md +18 -0
- package/lib/assurance/batch.mjs +353 -0
- package/lib/assurance/first-reply.mjs +423 -0
- package/lib/assurance/notice-voice.mjs +357 -0
- package/lib/assurance/plan-note.mjs +43 -0
- package/lib/assurance/room-budget.mjs +55 -6
- package/lib/cadence-failure-class.mjs +245 -0
- package/lib/claude-bin.mjs +26 -7
- package/lib/cli/doctor-checks.mjs +149 -1
- package/lib/comms/send-gate.mjs +59 -0
- package/lib/diagnostics/alerts.mjs +33 -0
- package/lib/engine/agents/usage.mjs +45 -0
- package/lib/engine/budget.mjs +293 -29
- package/lib/engine/cli.mjs +54 -5
- package/lib/engine/loop.mjs +30 -0
- package/lib/engine/output/json.mjs +26 -0
- package/lib/engine/wire/errors.mjs +179 -0
- package/lib/engine/wire/search.mjs +44 -8
- package/lib/identity/persona.mjs +31 -2
- package/lib/org/quota.mjs +27 -0
- package/lib/session/config.mjs +4 -0
- package/lib/session/identity.mjs +71 -7
- package/lib/session/launch-failure.mjs +251 -0
- package/lib/session/resume-target.mjs +86 -0
- package/lib/telemetry/alerts.mjs +94 -0
- package/lib/telemetry/collect.mjs +155 -2
- package/lib/upgrade/pinned-drift.mjs +467 -0
- package/package.json +1 -1
- package/scaffold/config/alerts.yaml +7 -0
- package/scripts/ci/check-cadence-prompts-exist.mjs +96 -0
- package/scripts/ci/check.mjs +3 -0
- package/scripts/daemon/agent-daemon.mjs +75 -5
- package/scripts/daemon/assurance.mjs +709 -44
- package/scripts/daemon/cadence-consumer.mjs +281 -34
- package/scripts/daemon/deliver.mjs +109 -0
- package/scripts/daemon/dispatcher.mjs +21 -3
- package/scripts/daemon/inbox-deferral.mjs +102 -9
- package/scripts/daemon/session-lock.mjs +41 -1
- package/scripts/emergency-stop.sh +114 -13
- package/scripts/fleet/rollout.mjs +256 -10
- package/scripts/healthcheck.sh +131 -33
- package/scripts/local-triggers/autoupdate.sh +144 -11
- package/scripts/resume-operations.sh +101 -6
- package/scripts/session/supervisor.mjs +198 -5
|
@@ -102,6 +102,7 @@ import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
|
102
102
|
import { spawnSync } from "node:child_process";
|
|
103
103
|
import { dirname, join, resolve as resolvePath } from "node:path";
|
|
104
104
|
import { fileURLToPath } from "node:url";
|
|
105
|
+
import { DEFAULT_THRESHOLDS } from "../../lib/telemetry/alerts.mjs";
|
|
105
106
|
|
|
106
107
|
const __filename = fileURLToPath(import.meta.url);
|
|
107
108
|
const REPO_ROOT = resolvePath(dirname(__filename), "..", "..");
|
|
@@ -262,13 +263,20 @@ export function gateLine(r) {
|
|
|
262
263
|
}
|
|
263
264
|
|
|
264
265
|
/**
|
|
265
|
-
* A note about paths the GATES themselves dirtied
|
|
266
|
-
*
|
|
267
|
-
* row would refuse on a file the first run wrote. The clean-tree gate is NOT
|
|
266
|
+
* A note about paths the GATES themselves dirtied: a second run in a row would
|
|
267
|
+
* otherwise refuse on a file the FIRST run wrote. The clean-tree gate is NOT
|
|
268
268
|
* weakened for this — publishing a tree you cannot describe stays a refusal —
|
|
269
269
|
* the run just says which paths are the gates' own leavings so the operator
|
|
270
270
|
* reverts them instead of hunting them. Pure.
|
|
271
271
|
*
|
|
272
|
+
* ~~"`npm test` appends to a tracked runtime ledger
|
|
273
|
+
* (.claude-flow/policy/state.json)"~~ — struck 2026-09-25: that file is the
|
|
274
|
+
* reason this function exists, and it is no longer tracked (it is a per-machine
|
|
275
|
+
* receipt chain, so it never should have been; `.gitignore` says why). This
|
|
276
|
+
* function stays because the SHAPE recurs — the next gate that writes into the
|
|
277
|
+
* tree will do the same thing — and because naming the paths beats hunting
|
|
278
|
+
* them. When it returns null for a whole rollout, that is the expected reading.
|
|
279
|
+
*
|
|
272
280
|
* @param {string[]} before dirty paths before the gates ran
|
|
273
281
|
* @param {string[]} after dirty paths after
|
|
274
282
|
* @returns {string|null}
|
|
@@ -387,6 +395,130 @@ export function seatVersions(entry) {
|
|
|
387
395
|
return { installed, running, version, basis: version === null ? null : running ? "running" : "installed" };
|
|
388
396
|
}
|
|
389
397
|
|
|
398
|
+
/**
|
|
399
|
+
* How long a failed upgrade attempt may stand, unrepeated, before a seat that
|
|
400
|
+
* is still on the version it failed FROM counts as stuck rather than merely
|
|
401
|
+
* recovering. Two full failed-target holds (the autoupdater's default is 24 h):
|
|
402
|
+
* one hold is the seat being deliberately left alone, two means the retry it
|
|
403
|
+
* was owed has been and gone.
|
|
404
|
+
*/
|
|
405
|
+
export const ATTEMPT_STALE_MS = 48 * 60 * 60 * 1000;
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Consecutive failed attempts that make a seat stuck.
|
|
409
|
+
*
|
|
410
|
+
* DERIVED, NOT RESTATED. This used to be `= 3` with a comment promising it
|
|
411
|
+
* "matches lib/telemetry/alerts.mjs upgradeStuck.warnStreak" — a prose promise
|
|
412
|
+
* about a literal, which is the exact shape the repo doctrine bans and which no
|
|
413
|
+
* test could have caught: this file's own tests use the constant symbolically
|
|
414
|
+
* (`at(STUCK_STREAK - 1)` / `at(STUCK_STREAK)`), so they pass at ANY value and
|
|
415
|
+
* the two copies could have drifted silently. The seat-side alert rule is the
|
|
416
|
+
* authority — it is what actually fires on a live beat — so this reads it.
|
|
417
|
+
*
|
|
418
|
+
* The third copy lives in the shell (`STUCK_STREAK` in
|
|
419
|
+
* scripts/local-triggers/autoupdate.sh), which cannot import this; it is pinned
|
|
420
|
+
* against this value by a source-reading test instead
|
|
421
|
+
* (scripts/local-triggers/autoupdate.test.mjs, "source: the shell's streak
|
|
422
|
+
* threshold is the same number the alert fires on").
|
|
423
|
+
*/
|
|
424
|
+
export const STUCK_STREAK = DEFAULT_THRESHOLDS.upgradeStuck.warnStreak;
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* The first SDK release whose `write_last` keeps a `failStreak` in
|
|
428
|
+
* state/autoupdate/last.json. A seat below this reports one upgrade outcome and
|
|
429
|
+
* no count — not because its beat is stale, but because the collector that
|
|
430
|
+
* would write the count is the code it has not got.
|
|
431
|
+
*
|
|
432
|
+
* This exists because the "attempt" basis used to explain itself with "this
|
|
433
|
+
* seat's beat is too old to carry a count", attached unconditionally to every
|
|
434
|
+
* attempt-basis row. At rollout time NO released SDK wrote `failStreak`, so
|
|
435
|
+
* every seat in the fleet — including ones beating every 30 seconds with
|
|
436
|
+
* `reason` and `healthy` on the record — was told its beat was too old. The
|
|
437
|
+
* beat age was never the reason; the seat's VERSION is.
|
|
438
|
+
*/
|
|
439
|
+
export const FAILSTREAK_SINCE = "2.18.15";
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* IS THIS SEAT STUCK — i.e. has it tried to reach @latest and failed, and will
|
|
443
|
+
* it keep failing?
|
|
444
|
+
*
|
|
445
|
+
* `classifySeat` says a seat is `stale`. That is a snapshot and it is not the
|
|
446
|
+
* finding: a seat is stale for twenty minutes after every publish, and three
|
|
447
|
+
* seats were stale for three weeks. What separates them is whether the seat has
|
|
448
|
+
* ATTEMPTED the hop and failed, and how many times — and that fact has always
|
|
449
|
+
* been in the beat (`machine.upgrade`) without anything reading it.
|
|
450
|
+
*
|
|
451
|
+
* TWO BASES, BECAUSE THE SEATS THIS MATTERS FOR REPORT THE LEAST. A seat that
|
|
452
|
+
* has been left behind for weeks is by definition running the oldest collector
|
|
453
|
+
* in the fleet, so it emits the smallest beat: on 2026-09-25 the three seats on
|
|
454
|
+
* 2.17.0 were the only three in the org with no `machine.daemon` block and no
|
|
455
|
+
* `machine.stranded` — staleness hides itself by removing the fields that would
|
|
456
|
+
* announce it. So this reads what a 2.17-era beat DOES carry, and names which
|
|
457
|
+
* basis it used rather than pretending to one answer:
|
|
458
|
+
*
|
|
459
|
+
* basis "streak" — the seat counted its own consecutive failures
|
|
460
|
+
* (`machine.upgrade.failStreak`, SDK {@link FAILSTREAK_SINCE}
|
|
461
|
+
* and later). Stuck at {@link STUCK_STREAK}. The real signal.
|
|
462
|
+
* basis "attempt" — an old beat, which carries one outcome and no count. A
|
|
463
|
+
* failed attempt (`ok:false`, `to` != `from`) with the seat
|
|
464
|
+
* STILL reporting `from` proves the hop did not take; older
|
|
465
|
+
* than {@link ATTEMPT_STALE_MS} it also proves the retry it
|
|
466
|
+
* was owed came and went. Below that age it is `failing`,
|
|
467
|
+
* not `stuck` — which is the same rule the streak obeys and
|
|
468
|
+
* the same rule (d) asks for: a seat that has not been
|
|
469
|
+
* ASKED again is not a stuck seat.
|
|
470
|
+
*
|
|
471
|
+
* Returns null when the beat carries no upgrade record at all: never tried is
|
|
472
|
+
* not the same as tried and failed, and must not print as one.
|
|
473
|
+
*
|
|
474
|
+
* Pure; `now` is a parameter because the "attempt" basis is about a duration.
|
|
475
|
+
*
|
|
476
|
+
* @param {object} entry a directory member, or anything {@link seatVersions} accepts
|
|
477
|
+
* @param {number} [now] epoch ms
|
|
478
|
+
* @returns {{verdict:"stuck"|"failing", basis:"streak"|"attempt", failStreak:number|null,
|
|
479
|
+
* target:string|null, since:string|null, ageMs:number|null}|null}
|
|
480
|
+
*/
|
|
481
|
+
export function seatStuck(entry, now = Date.now()) {
|
|
482
|
+
const e = entry && typeof entry === "object" ? entry : {};
|
|
483
|
+
const st = (e.status && typeof e.status === "object" && e.status) || {};
|
|
484
|
+
const mach = (st.machine && typeof st.machine === "object" && st.machine) || (e.machine && typeof e.machine === "object" && e.machine) || {};
|
|
485
|
+
const u = mach.upgrade && typeof mach.upgrade === "object" ? mach.upgrade : null;
|
|
486
|
+
if (!u) return null;
|
|
487
|
+
|
|
488
|
+
const streak = Number.isInteger(u.failStreak) && u.failStreak > 0 ? u.failStreak : null;
|
|
489
|
+
const target = firstString([u.streakTarget, u.to]);
|
|
490
|
+
if (streak !== null) {
|
|
491
|
+
const sinceS = firstString([u.stuckSince, u.at]);
|
|
492
|
+
const sinceMs = sinceS ? Date.parse(sinceS) : NaN;
|
|
493
|
+
return {
|
|
494
|
+
verdict: streak >= STUCK_STREAK ? "stuck" : "failing",
|
|
495
|
+
basis: "streak",
|
|
496
|
+
failStreak: streak,
|
|
497
|
+
target,
|
|
498
|
+
since: sinceS,
|
|
499
|
+
ageMs: Number.isFinite(sinceMs) ? now - sinceMs : null,
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// Degraded read: one outcome, no count.
|
|
504
|
+
if (u.ok !== false) return null;
|
|
505
|
+
const from = typeof u.from === "string" ? u.from : "";
|
|
506
|
+
const to = typeof u.to === "string" ? u.to : "";
|
|
507
|
+
if (!from || !to || from === to) return null; // not an upgrade attempt
|
|
508
|
+
const installed = seatVersions(e).installed;
|
|
509
|
+
if (installed && installed !== from) return null; // it has moved since; whatever failed is history
|
|
510
|
+
const atMs = typeof u.at === "string" ? Date.parse(u.at) : NaN;
|
|
511
|
+
const ageMs = Number.isFinite(atMs) ? now - atMs : null;
|
|
512
|
+
return {
|
|
513
|
+
verdict: ageMs !== null && ageMs > ATTEMPT_STALE_MS ? "stuck" : "failing",
|
|
514
|
+
basis: "attempt",
|
|
515
|
+
failStreak: null,
|
|
516
|
+
target: to,
|
|
517
|
+
since: typeof u.at === "string" ? u.at : null,
|
|
518
|
+
ageMs,
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
|
|
390
522
|
/**
|
|
391
523
|
* The single version a seat is judged on, or null when the read carries none.
|
|
392
524
|
* Kept as its own export because it is the whole of what `classifySeat` needs;
|
|
@@ -414,7 +546,7 @@ export function seatVersion(entry) {
|
|
|
414
546
|
* not written for.
|
|
415
547
|
*
|
|
416
548
|
* @param {object[]} members
|
|
417
|
-
* @param {{seatWindowMs?:number}} [o]
|
|
549
|
+
* @param {{seatWindowMs?:number, now?:number}} [o]
|
|
418
550
|
* @returns {{seats:object[], nonSeats:object[], readCarriesVersionField:boolean}}
|
|
419
551
|
*/
|
|
420
552
|
export function toSeats(members, o = {}) {
|
|
@@ -439,6 +571,7 @@ export function toSeats(members, o = {}) {
|
|
|
439
571
|
detail: (p && p.detail) || "",
|
|
440
572
|
lastBeatMs: beatMs,
|
|
441
573
|
...seatVersions(m),
|
|
574
|
+
stuck: seatStuck(m, o.now),
|
|
442
575
|
};
|
|
443
576
|
if (beatMs !== null && beatMs <= windowMs) seats.push(row);
|
|
444
577
|
else nonSeats.push(row);
|
|
@@ -473,15 +606,38 @@ export function summarise(fleet, target) {
|
|
|
473
606
|
const rows = (fleet.seats || []).map((s) => ({ ...s, verdict: classifySeat(s, target) }));
|
|
474
607
|
const counts = { current: 0, stale: 0, unverifiable: 0 };
|
|
475
608
|
for (const r of rows) counts[r.verdict] += 1;
|
|
609
|
+
// Behind AND proved it cannot arrive — a strictly stronger statement than
|
|
610
|
+
// `stale`, which clears itself on the next hourly run.
|
|
611
|
+
//
|
|
612
|
+
// `stuck` IS NOT A SUBSET OF `stale`, and the code used to assume it was. A
|
|
613
|
+
// seat's verdict is about THIS run's target; its streak is about whatever
|
|
614
|
+
// `@latest` was when it last tried, and the two are different versions
|
|
615
|
+
// whenever a rollout is verified against anything but the registry's head —
|
|
616
|
+
// `--dry-run`, `--verify-only`, or simply a publish that has since been
|
|
617
|
+
// followed by another. Demonstrated: one seat on target 2.18.14 whose record
|
|
618
|
+
// is {ok:false, from:"2.18.14", to:"2.18.15", failStreak:5} classifies
|
|
619
|
+
// `current`, so `stale` is empty, `done` was true, and renderVerdict's early
|
|
620
|
+
// return printed "All 1 seat(s) report 2.18.14." — five consecutive failed
|
|
621
|
+
// upgrades reported as a fully current fleet. So the two sets are kept and
|
|
622
|
+
// named apart, and `done` answers BOTH questions: every seat is on the target
|
|
623
|
+
// AND no seat is known to be unable to move.
|
|
624
|
+
const stuck = rows.filter((r) => r.stuck && r.stuck.verdict === "stuck");
|
|
625
|
+
const stale = rows.filter((r) => r.verdict === "stale");
|
|
626
|
+
const staleSlugs = new Set(stale.map((r) => r.slug));
|
|
476
627
|
return {
|
|
477
628
|
target,
|
|
478
629
|
rows,
|
|
479
630
|
counts,
|
|
480
631
|
nonSeats: fleet.nonSeats || [],
|
|
481
|
-
stale
|
|
632
|
+
stale,
|
|
482
633
|
unverifiable: rows.filter((r) => r.verdict === "unverifiable"),
|
|
634
|
+
stuck,
|
|
635
|
+
/** The stuck seats that are ALSO behind this run's target — "N of those". */
|
|
636
|
+
stuckBehind: stuck.filter((r) => staleSlugs.has(r.slug)),
|
|
637
|
+
/** Stuck against some other version while current on this one. Still a finding. */
|
|
638
|
+
stuckElsewhere: stuck.filter((r) => !staleSlugs.has(r.slug)),
|
|
483
639
|
readCarriesVersionField: fleet.readCarriesVersionField === true,
|
|
484
|
-
done: rows.length > 0 && counts.current === rows.length,
|
|
640
|
+
done: rows.length > 0 && counts.current === rows.length && stuck.length === 0,
|
|
485
641
|
};
|
|
486
642
|
}
|
|
487
643
|
|
|
@@ -511,7 +667,7 @@ export function renderSeatTable(summary, o = {}) {
|
|
|
511
667
|
r.slug,
|
|
512
668
|
ageLabel(r.lastBeatMs),
|
|
513
669
|
r.version ? (r.basis === "installed" ? `${r.version} (installed)` : r.version) : "unknown",
|
|
514
|
-
r.verdict,
|
|
670
|
+
r.stuck ? `${r.verdict} · ${r.stuck.verdict}${r.stuck.failStreak ? ` ×${r.stuck.failStreak}` : ""}` : r.verdict,
|
|
515
671
|
]);
|
|
516
672
|
const widths = head.map((h, i) => Math.max(h.length, ...body.map((row) => row[i].length), 0));
|
|
517
673
|
const line = (cells) => cells.map((c, i) => c.padEnd(widths[i])).join(" ").trimEnd();
|
|
@@ -531,9 +687,19 @@ export function renderSeatTable(summary, o = {}) {
|
|
|
531
687
|
return out.join("\n");
|
|
532
688
|
}
|
|
533
689
|
|
|
534
|
-
/**
|
|
690
|
+
/**
|
|
691
|
+
* The closing sentence naming exactly which seats are behind, and which of them
|
|
692
|
+
* will never arrive unaided. Pure.
|
|
693
|
+
*
|
|
694
|
+
* NO EARLY RETURN ON `done`. It used to open with
|
|
695
|
+
* `if (summary.done) return "All N seat(s) report X."`, which swallowed the
|
|
696
|
+
* STUCK line whole for any seat that was stuck against a version other than
|
|
697
|
+
* this run's target — see the note in {@link summarise}. `done` now accounts
|
|
698
|
+
* for stuck seats, and the "all clear" sentence is built at the END, from the
|
|
699
|
+
* absence of every finding, so a future finding cannot be skipped past by a
|
|
700
|
+
* `done` that did not learn about it.
|
|
701
|
+
*/
|
|
535
702
|
export function renderVerdict(summary) {
|
|
536
|
-
if (summary.done) return `All ${summary.rows.length} seat(s) report ${summary.target}.`;
|
|
537
703
|
const parts = [];
|
|
538
704
|
if (summary.stale.length) {
|
|
539
705
|
parts.push(
|
|
@@ -541,6 +707,27 @@ export function renderVerdict(summary) {
|
|
|
541
707
|
summary.stale.map((r) => `${r.name} (${r.slug}, reports ${r.version})`).join(", ")
|
|
542
708
|
);
|
|
543
709
|
}
|
|
710
|
+
const stuck = (summary.stuck && summary.stuck.length ? summary.stuck : []);
|
|
711
|
+
if (stuck.length) {
|
|
712
|
+
// Said SEPARATELY from "behind", and after it, because the response is
|
|
713
|
+
// different: a behind seat is waited on, a stuck one is visited. The line
|
|
714
|
+
// that was missing for three weeks is this one.
|
|
715
|
+
const behind = summary.stuckBehind || stuck.filter((r) => r.verdict === "stale");
|
|
716
|
+
const elsewhere = summary.stuckElsewhere || stuck.filter((r) => r.verdict !== "stale");
|
|
717
|
+
// "N of those" is only true of the ones that ARE among the seats just named
|
|
718
|
+
// as behind. A seat that is current on this target and stuck against a
|
|
719
|
+
// later one is not "one of those" and must not be counted as if it were.
|
|
720
|
+
const head =
|
|
721
|
+
elsewhere.length === 0
|
|
722
|
+
? `${behind.length} of those ${behind.length === 1 ? "is" : "are"} STUCK`
|
|
723
|
+
: behind.length === 0
|
|
724
|
+
? `${elsewhere.length} seat(s) ${elsewhere.length === 1 ? "is" : "are"} STUCK — on ${summary.target}, but unable to move off it`
|
|
725
|
+
: `${stuck.length} seat(s) are STUCK (${behind.length} of those just named as behind, ${elsewhere.length} current on ${summary.target} but unable to move off it)`;
|
|
726
|
+
parts.push(
|
|
727
|
+
`${head} — attempted and failed, and will not arrive unaided: ` +
|
|
728
|
+
stuck.map((r) => `${r.name} (${r.slug}, ${stuckClause(r, summary.target)})`).join(", ")
|
|
729
|
+
);
|
|
730
|
+
}
|
|
544
731
|
if (summary.unverifiable.length) {
|
|
545
732
|
parts.push(
|
|
546
733
|
`${summary.unverifiable.length} seat(s) could not be verified — the org read carries no sdkVersion: ` +
|
|
@@ -548,9 +735,46 @@ export function renderVerdict(summary) {
|
|
|
548
735
|
);
|
|
549
736
|
}
|
|
550
737
|
if (!summary.rows.length) parts.push("no beating seats found in the org directory — nothing to verify.");
|
|
738
|
+
if (!parts.length) return `All ${summary.rows.length} seat(s) report ${summary.target}.`;
|
|
551
739
|
return parts.join("\n");
|
|
552
740
|
}
|
|
553
741
|
|
|
742
|
+
/**
|
|
743
|
+
* One stuck seat's parenthetical: what it tried, how many times, and — only
|
|
744
|
+
* when it is true — why it cannot say how many times.
|
|
745
|
+
*
|
|
746
|
+
* THE "no count" SENTENCE IS GATED ON THE SEAT'S VERSION, NOT ON THE BRANCH
|
|
747
|
+
* THAT PRODUCED THE VERDICT. It used to hang off `basis === "attempt"` and
|
|
748
|
+
* read "this seat's beat is too old to carry a count". Every attempt-basis row
|
|
749
|
+
* got it, which at the time this landed was every row in the fleet — no
|
|
750
|
+
* released SDK wrote `failStreak` yet — including seats beating every thirty
|
|
751
|
+
* seconds whose records demonstrably carried `reason` and `healthy`. The beat
|
|
752
|
+
* was never old; the seat's SDK simply predates {@link FAILSTREAK_SINCE}. When
|
|
753
|
+
* the seat is new enough to count and still did not, that is a DIFFERENT and
|
|
754
|
+
* more interesting fact, so it gets its own words. Pure.
|
|
755
|
+
*
|
|
756
|
+
* @param {object} row a summarise() row with a non-null `stuck`
|
|
757
|
+
* @param {string} target
|
|
758
|
+
* @returns {string}
|
|
759
|
+
*/
|
|
760
|
+
export function stuckClause(row, target) {
|
|
761
|
+
const st = row.stuck;
|
|
762
|
+
const attempts = st.failStreak ? `${st.failStreak} consecutive failed attempts` : "a failed attempt";
|
|
763
|
+
const oldest = Number.isFinite(st.ageMs) ? `, oldest ${ageLabel(st.ageMs)} ago` : "";
|
|
764
|
+
let why = "";
|
|
765
|
+
if (st.basis === "attempt") {
|
|
766
|
+
const v = row.version;
|
|
767
|
+
if (v && compareVersions(v, FAILSTREAK_SINCE) < 0) {
|
|
768
|
+
why = `; it reports ${v}, which predates the failure count (SDK ${FAILSTREAK_SINCE}), so one attempt is all it can report`;
|
|
769
|
+
} else if (v) {
|
|
770
|
+
why = `; it reports ${v}, new enough to count its failures, but this record carries none — read state/autoupdate/last.json on the machine`;
|
|
771
|
+
} else {
|
|
772
|
+
why = "; this seat reports no version, so there is no telling whether it could have counted";
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
return `${attempts} to ${st.target || target}${oldest}${why}`;
|
|
776
|
+
}
|
|
777
|
+
|
|
554
778
|
/**
|
|
555
779
|
* What stage 3's exit code will be, and WHY — as data, so a caller can tell the
|
|
556
780
|
* three very different things a `3` means apart.
|
|
@@ -572,7 +796,7 @@ export function renderVerdict(summary) {
|
|
|
572
796
|
* Pure.
|
|
573
797
|
*
|
|
574
798
|
* @param {ReturnType<typeof summarise>|null|undefined} summary
|
|
575
|
-
* @returns {{code:number, reason:"verified"|"behind"|"unverifiable"|"empty-fleet"|"no-fleet-read", note:string|null}}
|
|
799
|
+
* @returns {{code:number, reason:"verified"|"stuck"|"behind"|"unverifiable"|"empty-fleet"|"no-fleet-read", note:string|null}}
|
|
576
800
|
*/
|
|
577
801
|
export function propagationOutcome(summary) {
|
|
578
802
|
if (!summary || !Array.isArray(summary.rows)) {
|
|
@@ -583,6 +807,28 @@ export function propagationOutcome(summary) {
|
|
|
583
807
|
};
|
|
584
808
|
}
|
|
585
809
|
if (summary.done) return { code: 0, reason: "verified", note: null };
|
|
810
|
+
// STUCK OUTRANKS BEHIND, and is checked before both `empty-fleet` and the
|
|
811
|
+
// unverifiable branch, because it is the only one of the three that a person
|
|
812
|
+
// must ACT on rather than wait out.
|
|
813
|
+
//
|
|
814
|
+
// This is also the automation the seat-side alert cannot provide. `upgrade_stuck`
|
|
815
|
+
// is derived by lib/telemetry/alerts.mjs running ON the seat, so a seat too
|
|
816
|
+
// stale to install the SDK never emits it — by construction the alert cannot
|
|
817
|
+
// reach the seats it is for. What CAN reach them is this: the fleet-side
|
|
818
|
+
// derivation runs on every publish, because stage 3 runs on every publish,
|
|
819
|
+
// and a stuck seat now makes the release command exit non-zero and name the
|
|
820
|
+
// machine. Nobody has to decide to go looking.
|
|
821
|
+
if (summary.stuck && summary.stuck.length) {
|
|
822
|
+
return {
|
|
823
|
+
code: 3,
|
|
824
|
+
reason: "stuck",
|
|
825
|
+
note:
|
|
826
|
+
`${summary.stuck.length} seat(s) have attempted @latest and failed repeatedly. They will not arrive on their own: ` +
|
|
827
|
+
`each needs a person at the machine (maestro doctor; state/autoupdate/last.json carries the reason). ` +
|
|
828
|
+
`Stage 3 still polls to its deadline in case someone clears one while it runs, but it will not clear itself. ` +
|
|
829
|
+
renderVerdict(summary),
|
|
830
|
+
};
|
|
831
|
+
}
|
|
586
832
|
if (!summary.rows.length) {
|
|
587
833
|
return {
|
|
588
834
|
code: 3,
|
package/scripts/healthcheck.sh
CHANGED
|
@@ -1,24 +1,109 @@
|
|
|
1
1
|
#!/bin/bash
|
|
2
2
|
# Health Check — Verifies all Maestro agent subsystems are operational
|
|
3
|
-
# Usage: ./scripts/healthcheck.sh
|
|
4
|
-
# Exit codes: 0 = healthy, 1 = degraded, 2 = critical
|
|
3
|
+
# Usage: ./scripts/healthcheck.sh [--ignore-emergency-stop] [--offline] [--help]
|
|
4
|
+
# Exit codes: 0 = healthy, 1 = degraded (warnings only), 2 = critical (errors)
|
|
5
|
+
#
|
|
6
|
+
# A refusal NAMES what blocked it. The summary always prints a `Blocking:`
|
|
7
|
+
# line listing the id of every failed check (and a `Warnings:` line for the
|
|
8
|
+
# soft ones), because the operator's next question after "1 errors" is always
|
|
9
|
+
# "which one?" — and on 2026-09-24 answering it cost a re-run with a filter
|
|
10
|
+
# while a seat stayed halted.
|
|
11
|
+
#
|
|
12
|
+
# --ignore-emergency-stop
|
|
13
|
+
# Demote check 1 (the .emergency-stop flag) from ERROR to NOTE. It is meant
|
|
14
|
+
# for one caller: resume-operations.sh, whose whole job is to lift that
|
|
15
|
+
# flag. Counting the flag as a reason not to lift the flag deadlocked the
|
|
16
|
+
# only tool that can lift it (a seat sat halted from 2026-09-24T08:16Z
|
|
17
|
+
# until it was cleared by hand).
|
|
18
|
+
#
|
|
19
|
+
# SCOPE, PLAINLY. This is a PUBLIC CLI flag. Any caller, script or operator
|
|
20
|
+
# can pass it; nothing refuses it, and nothing here pretends to. An earlier
|
|
21
|
+
# draft of this header said the exemption "is refused to everyone else" —
|
|
22
|
+
# it never was. The scoping is convention plus this comment, and a gate that
|
|
23
|
+
# exists only in a comment is exactly the thing this file was repaired for.
|
|
24
|
+
#
|
|
25
|
+
# What keeps it honest is scope, not exclusivity:
|
|
26
|
+
# - It demotes exactly ONE check. Every other check runs at full severity,
|
|
27
|
+
# so an exempted run still cannot green-light a broken seat.
|
|
28
|
+
# - It is off by default, so an operator running healthcheck on a halted
|
|
29
|
+
# seat is still told it is halted — the first thing they need to know.
|
|
30
|
+
# - It is explicit, greppable, and reviewable at its single call site.
|
|
31
|
+
# - The demotion is ANNOUNCED in the output ([NOTE] ...). That line is the
|
|
32
|
+
# receipt resume-operations.sh checks before it trusts the verdict, so a
|
|
33
|
+
# run where the flag was silently swallowed is caught rather than obeyed.
|
|
34
|
+
# Requiring a second token (an env var alongside the flag) would add another
|
|
35
|
+
# thing to forge, not a boundary: anyone who can run this script can set an
|
|
36
|
+
# env var. The real boundary is permission to execute it at all.
|
|
37
|
+
#
|
|
38
|
+
# The alternatives are worse on their merits, not on access control:
|
|
39
|
+
# - Dropping the check outright, or having it always ignore the flag, is
|
|
40
|
+
# wrong for the same reason as above: a halted seat must say so.
|
|
41
|
+
# - Re-ordering resume (remove flag, then check) is worse still: it lifts
|
|
42
|
+
# the stop on a genuinely broken seat and only then discovers it is
|
|
43
|
+
# broken, which is the failure the stop flag exists to prevent.
|
|
44
|
+
#
|
|
45
|
+
# --offline (or MAESTRO_HEALTHCHECK_SKIP_NETWORK=1)
|
|
46
|
+
# Record check 6 as SKIPPED instead of reaching the network. Not a weakening:
|
|
47
|
+
# the check is neither counted healthy nor counted failed, and the summary
|
|
48
|
+
# says it was skipped.
|
|
5
49
|
|
|
6
50
|
set -e
|
|
7
51
|
|
|
8
52
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
9
|
-
|
|
53
|
+
AGENT_DIR="$(dirname "$SCRIPT_DIR")"
|
|
10
54
|
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
11
|
-
HEALTHY=0
|
|
12
55
|
WARNINGS=0
|
|
13
56
|
ERRORS=0
|
|
57
|
+
BLOCKING="" # space-separated ids of failed checks
|
|
58
|
+
SOFT="" # space-separated ids of warned checks
|
|
59
|
+
|
|
60
|
+
IGNORE_STOP_FLAG=0
|
|
61
|
+
SKIP_NETWORK=0
|
|
62
|
+
case "${MAESTRO_HEALTHCHECK_SKIP_NETWORK:-0}" in 1 | true | yes) SKIP_NETWORK=1 ;; esac
|
|
63
|
+
|
|
64
|
+
while [ $# -gt 0 ]; do
|
|
65
|
+
case "$1" in
|
|
66
|
+
--ignore-emergency-stop) IGNORE_STOP_FLAG=1 ;;
|
|
67
|
+
--offline) SKIP_NETWORK=1 ;;
|
|
68
|
+
-h | --help)
|
|
69
|
+
sed -n '2,/^$/p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
|
70
|
+
exit 0
|
|
71
|
+
;;
|
|
72
|
+
*)
|
|
73
|
+
echo "healthcheck: unknown argument: $1" >&2
|
|
74
|
+
echo "Usage: healthcheck.sh [--ignore-emergency-stop] [--offline]" >&2
|
|
75
|
+
exit 2
|
|
76
|
+
;;
|
|
77
|
+
esac
|
|
78
|
+
shift
|
|
79
|
+
done
|
|
80
|
+
|
|
81
|
+
# fail <id> <message> — record a blocking condition under a stable id.
|
|
82
|
+
fail() {
|
|
83
|
+
echo "[FAIL] $2"
|
|
84
|
+
BLOCKING="$BLOCKING $1"
|
|
85
|
+
ERRORS=$((ERRORS + 1))
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
# warn <id> <message> — record a soft condition under a stable id.
|
|
89
|
+
warn() {
|
|
90
|
+
echo "[WARN] $2"
|
|
91
|
+
SOFT="$SOFT $1"
|
|
92
|
+
WARNINGS=$((WARNINGS + 1))
|
|
93
|
+
}
|
|
14
94
|
|
|
15
95
|
echo "Maestro Health Check — $TIMESTAMP"
|
|
16
96
|
echo "======================================="
|
|
17
97
|
|
|
18
98
|
# Check 1: Emergency stop flag
|
|
19
|
-
if [ -f "$
|
|
20
|
-
|
|
21
|
-
|
|
99
|
+
if [ -f "$AGENT_DIR/.emergency-stop" ]; then
|
|
100
|
+
if [ "$IGNORE_STOP_FLAG" -eq 1 ]; then
|
|
101
|
+
echo "[NOTE] Emergency stop flag is active — not counted (--ignore-emergency-stop)"
|
|
102
|
+
else
|
|
103
|
+
echo "[STOP] Emergency stop flag is active"
|
|
104
|
+
BLOCKING="$BLOCKING emergency-stop-flag"
|
|
105
|
+
ERRORS=$((ERRORS + 1))
|
|
106
|
+
fi
|
|
22
107
|
else
|
|
23
108
|
echo "[OK] No emergency stop flag"
|
|
24
109
|
fi
|
|
@@ -30,22 +115,20 @@ fi
|
|
|
30
115
|
# in .env strips the var and falls back to Keychain OAuth).
|
|
31
116
|
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
|
|
32
117
|
echo "[OK] Environment variable: ANTHROPIC_API_KEY"
|
|
33
|
-
elif [ -f "$
|
|
118
|
+
elif [ -f "$AGENT_DIR/.env" ] && grep -qE "^MAESTRO_PREFER_SUBSCRIPTION_AUTH=(1|true|yes)$" "$AGENT_DIR/.env"; then
|
|
34
119
|
echo "[OK] Anthropic auth: subscription mode (.env: MAESTRO_PREFER_SUBSCRIPTION_AUTH=1)"
|
|
35
|
-
elif [ -f "$
|
|
120
|
+
elif [ -f "$AGENT_DIR/.env" ] && grep -qE "^ANTHROPIC_API_KEY=." "$AGENT_DIR/.env"; then
|
|
36
121
|
echo "[OK] Anthropic auth: ANTHROPIC_API_KEY in .env"
|
|
37
122
|
else
|
|
38
|
-
|
|
39
|
-
ERRORS=$((ERRORS + 1))
|
|
123
|
+
fail "anthropic-auth" "Anthropic auth not configured (.env needs either ANTHROPIC_API_KEY or MAESTRO_PREFER_SUBSCRIPTION_AUTH=1)"
|
|
40
124
|
fi
|
|
41
125
|
|
|
42
126
|
# Check 3: Key directories exist and are writable
|
|
43
127
|
for dir in logs outputs knowledge/memory config; do
|
|
44
|
-
if [ -d "$
|
|
128
|
+
if [ -d "$AGENT_DIR/$dir" ] && [ -w "$AGENT_DIR/$dir" ]; then
|
|
45
129
|
echo "[OK] Directory writable: $dir"
|
|
46
130
|
else
|
|
47
|
-
|
|
48
|
-
ERRORS=$((ERRORS + 1))
|
|
131
|
+
fail "dir:$dir" "Directory missing or not writable: $dir"
|
|
49
132
|
fi
|
|
50
133
|
done
|
|
51
134
|
|
|
@@ -54,8 +137,7 @@ for repo in company-web innovation-lab partner-framework company-legal regulator
|
|
|
54
137
|
if [ -d "$HOME/$repo" ]; then
|
|
55
138
|
echo "[OK] Source repo accessible: ~/$repo"
|
|
56
139
|
else
|
|
57
|
-
|
|
58
|
-
WARNINGS=$((WARNINGS + 1))
|
|
140
|
+
warn "repo:$repo" "Source repo not found: ~/$repo"
|
|
59
141
|
fi
|
|
60
142
|
done
|
|
61
143
|
|
|
@@ -64,45 +146,61 @@ for app in Slack WhatsApp Safari; do
|
|
|
64
146
|
if [ -d "/Applications/$app.app" ] || [ -d "$HOME/Applications/$app.app" ]; then
|
|
65
147
|
echo "[OK] Application installed: $app"
|
|
66
148
|
else
|
|
67
|
-
|
|
68
|
-
WARNINGS=$((WARNINGS + 1))
|
|
149
|
+
warn "app:$app" "Application not found: $app"
|
|
69
150
|
fi
|
|
70
151
|
done
|
|
71
152
|
|
|
72
153
|
# Check 6: Network connectivity
|
|
73
|
-
if
|
|
154
|
+
if [ "$SKIP_NETWORK" -eq 1 ]; then
|
|
155
|
+
echo "[SKIP] Network: not probed (--offline)"
|
|
156
|
+
elif curl -s --max-time 5 https://api.anthropic.com > /dev/null 2>&1; then
|
|
74
157
|
echo "[OK] Network: Anthropic API reachable"
|
|
75
158
|
else
|
|
76
|
-
|
|
77
|
-
ERRORS=$((ERRORS + 1))
|
|
159
|
+
fail "network" "Network: Cannot reach Anthropic API"
|
|
78
160
|
fi
|
|
79
161
|
|
|
80
|
-
# Check 7: Disk space
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
162
|
+
# Check 7: Disk space. Ordered most-severe-first: the previous order tested
|
|
163
|
+
# >90 before >95, so the critical branch could never be reached and disk was
|
|
164
|
+
# warn-only in practice. Reaching it is a REAL severity change with a real
|
|
165
|
+
# consequence — resume-operations.sh refuses on any exit 2, so a seat above
|
|
166
|
+
# 95% can no longer be resumed until space is freed. That is deliberate, and
|
|
167
|
+
# it is not the deadlock this trio was repaired for: unlike the stop flag
|
|
168
|
+
# (which only resume could lift), a full disk is named, actionable, and
|
|
169
|
+
# fixable without running this script. Both branches — >95 blocking and >90
|
|
170
|
+
# warning — are pinned in resume-operations.test.mjs, at the healthcheck and
|
|
171
|
+
# at the resume gate, because they are the only severities that changed.
|
|
172
|
+
DISK_FREE=$(df -h "$AGENT_DIR" | awk 'NR==2{print $5}' | tr -d '%')
|
|
173
|
+
if [ "$DISK_FREE" -gt 95 ]; then
|
|
174
|
+
fail "disk" "Disk usage: ${DISK_FREE}% — critically low"
|
|
175
|
+
elif [ "$DISK_FREE" -gt 90 ]; then
|
|
176
|
+
warn "disk" "Disk usage: ${DISK_FREE}% — consider cleanup"
|
|
88
177
|
else
|
|
89
178
|
echo "[OK] Disk usage: ${DISK_FREE}%"
|
|
90
179
|
fi
|
|
91
180
|
|
|
92
181
|
# Check 8: Config files present
|
|
93
182
|
for config in priorities.yaml environment.yaml contacts.yaml; do
|
|
94
|
-
if [ -f "$
|
|
183
|
+
if [ -f "$AGENT_DIR/config/$config" ]; then
|
|
95
184
|
echo "[OK] Config present: $config"
|
|
96
185
|
else
|
|
97
|
-
|
|
98
|
-
ERRORS=$((ERRORS + 1))
|
|
186
|
+
fail "config:$config" "Config missing: $config"
|
|
99
187
|
fi
|
|
100
188
|
done
|
|
101
189
|
|
|
102
|
-
# Summary
|
|
190
|
+
# Summary. `Blocking:` / `Warnings:` are the machine-readable contract:
|
|
191
|
+
# resume-operations.sh parses `Blocking:` to name the condition it refused on.
|
|
103
192
|
echo ""
|
|
104
193
|
echo "======================================="
|
|
105
194
|
echo "Results: $ERRORS errors, $WARNINGS warnings"
|
|
195
|
+
# `if`, not `[ ... ] && echo` — but NOT for the reason first written here,
|
|
196
|
+
# which was false. `set -e` does NOT abort on a false short-circuit: bash
|
|
197
|
+
# exempts a failing command that is not the last in an && list. MEASURED:
|
|
198
|
+
# bash -c 'set -e; n=0; [ "$n" -gt 0 ] && echo hi; exit 0' → exit 0
|
|
199
|
+
# The genuine hazard is only that a false && list leaves status 1, which a
|
|
200
|
+
# script inherits if it is the last command run. The explicit exits below
|
|
201
|
+
# make that moot here; `if` is kept because it says what is meant.
|
|
202
|
+
if [ "$ERRORS" -gt 0 ]; then echo "Blocking:$BLOCKING"; fi
|
|
203
|
+
if [ "$WARNINGS" -gt 0 ]; then echo "Warnings:$SOFT"; fi
|
|
106
204
|
|
|
107
205
|
if [ "$ERRORS" -gt 0 ]; then
|
|
108
206
|
echo "Status: CRITICAL — fix errors before operating"
|