@cohortapp/agent-sdk 2.18.13 → 2.18.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/bin/maestro.mjs +38 -1
  2. package/docs/runbooks/fleet-rollout.md +58 -7
  3. package/docs/runbooks/recovery-and-failover.md +18 -0
  4. package/lib/assurance/batch.mjs +353 -0
  5. package/lib/assurance/first-reply.mjs +423 -0
  6. package/lib/assurance/notice-voice.mjs +357 -0
  7. package/lib/assurance/plan-note.mjs +43 -0
  8. package/lib/assurance/room-budget.mjs +55 -6
  9. package/lib/cadence-failure-class.mjs +245 -0
  10. package/lib/claude-bin.mjs +26 -7
  11. package/lib/cli/doctor-checks.mjs +149 -1
  12. package/lib/comms/send-gate.mjs +59 -0
  13. package/lib/diagnostics/alerts.mjs +33 -0
  14. package/lib/engine/agents/usage.mjs +45 -0
  15. package/lib/engine/budget.mjs +293 -29
  16. package/lib/engine/cli.mjs +54 -5
  17. package/lib/engine/loop.mjs +30 -0
  18. package/lib/engine/output/json.mjs +26 -0
  19. package/lib/engine/wire/errors.mjs +179 -0
  20. package/lib/engine/wire/search.mjs +44 -8
  21. package/lib/identity/persona.mjs +31 -2
  22. package/lib/org/quota.mjs +27 -0
  23. package/lib/session/config.mjs +4 -0
  24. package/lib/session/identity.mjs +71 -7
  25. package/lib/session/launch-failure.mjs +251 -0
  26. package/lib/session/resume-target.mjs +86 -0
  27. package/lib/telemetry/alerts.mjs +94 -0
  28. package/lib/telemetry/collect.mjs +155 -2
  29. package/lib/upgrade/pinned-drift.mjs +467 -0
  30. package/package.json +1 -1
  31. package/scaffold/config/alerts.yaml +7 -0
  32. package/scripts/ci/check-cadence-prompts-exist.mjs +96 -0
  33. package/scripts/ci/check.mjs +3 -0
  34. package/scripts/daemon/agent-daemon.mjs +75 -5
  35. package/scripts/daemon/assurance.mjs +709 -44
  36. package/scripts/daemon/cadence-consumer.mjs +281 -34
  37. package/scripts/daemon/deliver.mjs +109 -0
  38. package/scripts/daemon/dispatcher.mjs +21 -3
  39. package/scripts/daemon/inbox-deferral.mjs +102 -9
  40. package/scripts/daemon/session-lock.mjs +41 -1
  41. package/scripts/emergency-stop.sh +114 -13
  42. package/scripts/fleet/rollout.mjs +256 -10
  43. package/scripts/healthcheck.sh +131 -33
  44. package/scripts/local-triggers/autoupdate.sh +144 -11
  45. package/scripts/resume-operations.sh +101 -6
  46. package/scripts/session/supervisor.mjs +198 -5
@@ -102,6 +102,7 @@ import { existsSync, readFileSync, readdirSync } from "node:fs";
102
102
  import { spawnSync } from "node:child_process";
103
103
  import { dirname, join, resolve as resolvePath } from "node:path";
104
104
  import { fileURLToPath } from "node:url";
105
+ import { DEFAULT_THRESHOLDS } from "../../lib/telemetry/alerts.mjs";
105
106
 
106
107
  const __filename = fileURLToPath(import.meta.url);
107
108
  const REPO_ROOT = resolvePath(dirname(__filename), "..", "..");
@@ -262,13 +263,20 @@ export function gateLine(r) {
262
263
  }
263
264
 
264
265
  /**
265
- * A note about paths the GATES themselves dirtied. `npm test` appends to a
266
- * tracked runtime ledger (.claude-flow/policy/state.json), so a second run in a
267
- * row would refuse on a file the first run wrote. The clean-tree gate is NOT
266
+ * A note about paths the GATES themselves dirtied: a second run in a row would
267
+ * otherwise refuse on a file the FIRST run wrote. The clean-tree gate is NOT
268
268
  * weakened for this — publishing a tree you cannot describe stays a refusal —
269
269
  * the run just says which paths are the gates' own leavings so the operator
270
270
  * reverts them instead of hunting them. Pure.
271
271
  *
272
+ * ~~"`npm test` appends to a tracked runtime ledger
273
+ * (.claude-flow/policy/state.json)"~~ — struck 2026-09-25: that file is the
274
+ * reason this function exists, and it is no longer tracked (it is a per-machine
275
+ * receipt chain, so it never should have been; `.gitignore` says why). This
276
+ * function stays because the SHAPE recurs — the next gate that writes into the
277
+ * tree will do the same thing — and because naming the paths beats hunting
278
+ * them. When it returns null for a whole rollout, that is the expected reading.
279
+ *
272
280
  * @param {string[]} before dirty paths before the gates ran
273
281
  * @param {string[]} after dirty paths after
274
282
  * @returns {string|null}
@@ -387,6 +395,130 @@ export function seatVersions(entry) {
387
395
  return { installed, running, version, basis: version === null ? null : running ? "running" : "installed" };
388
396
  }
389
397
 
398
+ /**
399
+ * How long a failed upgrade attempt may stand, unrepeated, before a seat that
400
+ * is still on the version it failed FROM counts as stuck rather than merely
401
+ * recovering. Two full failed-target holds (the autoupdater's default is 24 h):
402
+ * one hold is the seat being deliberately left alone, two means the retry it
403
+ * was owed has been and gone.
404
+ */
405
+ export const ATTEMPT_STALE_MS = 48 * 60 * 60 * 1000;
406
+
407
+ /**
408
+ * Consecutive failed attempts that make a seat stuck.
409
+ *
410
+ * DERIVED, NOT RESTATED. This used to be `= 3` with a comment promising it
411
+ * "matches lib/telemetry/alerts.mjs upgradeStuck.warnStreak" — a prose promise
412
+ * about a literal, which is the exact shape the repo doctrine bans and which no
413
+ * test could have caught: this file's own tests use the constant symbolically
414
+ * (`at(STUCK_STREAK - 1)` / `at(STUCK_STREAK)`), so they pass at ANY value and
415
+ * the two copies could have drifted silently. The seat-side alert rule is the
416
+ * authority — it is what actually fires on a live beat — so this reads it.
417
+ *
418
+ * The third copy lives in the shell (`STUCK_STREAK` in
419
+ * scripts/local-triggers/autoupdate.sh), which cannot import this; it is pinned
420
+ * against this value by a source-reading test instead
421
+ * (scripts/local-triggers/autoupdate.test.mjs, "source: the shell's streak
422
+ * threshold is the same number the alert fires on").
423
+ */
424
+ export const STUCK_STREAK = DEFAULT_THRESHOLDS.upgradeStuck.warnStreak;
425
+
426
+ /**
427
+ * The first SDK release whose `write_last` keeps a `failStreak` in
428
+ * state/autoupdate/last.json. A seat below this reports one upgrade outcome and
429
+ * no count — not because its beat is stale, but because the collector that
430
+ * would write the count is the code it has not got.
431
+ *
432
+ * This exists because the "attempt" basis used to explain itself with "this
433
+ * seat's beat is too old to carry a count", attached unconditionally to every
434
+ * attempt-basis row. At rollout time NO released SDK wrote `failStreak`, so
435
+ * every seat in the fleet — including ones beating every 30 seconds with
436
+ * `reason` and `healthy` on the record — was told its beat was too old. The
437
+ * beat age was never the reason; the seat's VERSION is.
438
+ */
439
+ export const FAILSTREAK_SINCE = "2.18.15";
440
+
441
+ /**
442
+ * IS THIS SEAT STUCK — i.e. has it tried to reach @latest and failed, and will
443
+ * it keep failing?
444
+ *
445
+ * `classifySeat` says a seat is `stale`. That is a snapshot and it is not the
446
+ * finding: a seat is stale for twenty minutes after every publish, and three
447
+ * seats were stale for three weeks. What separates them is whether the seat has
448
+ * ATTEMPTED the hop and failed, and how many times — and that fact has always
449
+ * been in the beat (`machine.upgrade`) without anything reading it.
450
+ *
451
+ * TWO BASES, BECAUSE THE SEATS THIS MATTERS FOR REPORT THE LEAST. A seat that
452
+ * has been left behind for weeks is by definition running the oldest collector
453
+ * in the fleet, so it emits the smallest beat: on 2026-09-25 the three seats on
454
+ * 2.17.0 were the only three in the org with no `machine.daemon` block and no
455
+ * `machine.stranded` — staleness hides itself by removing the fields that would
456
+ * announce it. So this reads what a 2.17-era beat DOES carry, and names which
457
+ * basis it used rather than pretending to one answer:
458
+ *
459
+ * basis "streak" — the seat counted its own consecutive failures
460
+ * (`machine.upgrade.failStreak`, SDK {@link FAILSTREAK_SINCE}
461
+ * and later). Stuck at {@link STUCK_STREAK}. The real signal.
462
+ * basis "attempt" — an old beat, which carries one outcome and no count. A
463
+ * failed attempt (`ok:false`, `to` != `from`) with the seat
464
+ * STILL reporting `from` proves the hop did not take; older
465
+ * than {@link ATTEMPT_STALE_MS} it also proves the retry it
466
+ * was owed came and went. Below that age it is `failing`,
467
+ * not `stuck` — which is the same rule the streak obeys and
468
+ * the same rule (d) asks for: a seat that has not been
469
+ * ASKED again is not a stuck seat.
470
+ *
471
+ * Returns null when the beat carries no upgrade record at all: never tried is
472
+ * not the same as tried and failed, and must not print as one.
473
+ *
474
+ * Pure; `now` is a parameter because the "attempt" basis is about a duration.
475
+ *
476
+ * @param {object} entry a directory member, or anything {@link seatVersions} accepts
477
+ * @param {number} [now] epoch ms
478
+ * @returns {{verdict:"stuck"|"failing", basis:"streak"|"attempt", failStreak:number|null,
479
+ * target:string|null, since:string|null, ageMs:number|null}|null}
480
+ */
481
+ export function seatStuck(entry, now = Date.now()) {
482
+ const e = entry && typeof entry === "object" ? entry : {};
483
+ const st = (e.status && typeof e.status === "object" && e.status) || {};
484
+ const mach = (st.machine && typeof st.machine === "object" && st.machine) || (e.machine && typeof e.machine === "object" && e.machine) || {};
485
+ const u = mach.upgrade && typeof mach.upgrade === "object" ? mach.upgrade : null;
486
+ if (!u) return null;
487
+
488
+ const streak = Number.isInteger(u.failStreak) && u.failStreak > 0 ? u.failStreak : null;
489
+ const target = firstString([u.streakTarget, u.to]);
490
+ if (streak !== null) {
491
+ const sinceS = firstString([u.stuckSince, u.at]);
492
+ const sinceMs = sinceS ? Date.parse(sinceS) : NaN;
493
+ return {
494
+ verdict: streak >= STUCK_STREAK ? "stuck" : "failing",
495
+ basis: "streak",
496
+ failStreak: streak,
497
+ target,
498
+ since: sinceS,
499
+ ageMs: Number.isFinite(sinceMs) ? now - sinceMs : null,
500
+ };
501
+ }
502
+
503
+ // Degraded read: one outcome, no count.
504
+ if (u.ok !== false) return null;
505
+ const from = typeof u.from === "string" ? u.from : "";
506
+ const to = typeof u.to === "string" ? u.to : "";
507
+ if (!from || !to || from === to) return null; // not an upgrade attempt
508
+ const installed = seatVersions(e).installed;
509
+ if (installed && installed !== from) return null; // it has moved since; whatever failed is history
510
+ const atMs = typeof u.at === "string" ? Date.parse(u.at) : NaN;
511
+ const ageMs = Number.isFinite(atMs) ? now - atMs : null;
512
+ return {
513
+ verdict: ageMs !== null && ageMs > ATTEMPT_STALE_MS ? "stuck" : "failing",
514
+ basis: "attempt",
515
+ failStreak: null,
516
+ target: to,
517
+ since: typeof u.at === "string" ? u.at : null,
518
+ ageMs,
519
+ };
520
+ }
521
+
390
522
  /**
391
523
  * The single version a seat is judged on, or null when the read carries none.
392
524
  * Kept as its own export because it is the whole of what `classifySeat` needs;
@@ -414,7 +546,7 @@ export function seatVersion(entry) {
414
546
  * not written for.
415
547
  *
416
548
  * @param {object[]} members
417
- * @param {{seatWindowMs?:number}} [o]
549
+ * @param {{seatWindowMs?:number, now?:number}} [o]
418
550
  * @returns {{seats:object[], nonSeats:object[], readCarriesVersionField:boolean}}
419
551
  */
420
552
  export function toSeats(members, o = {}) {
@@ -439,6 +571,7 @@ export function toSeats(members, o = {}) {
439
571
  detail: (p && p.detail) || "",
440
572
  lastBeatMs: beatMs,
441
573
  ...seatVersions(m),
574
+ stuck: seatStuck(m, o.now),
442
575
  };
443
576
  if (beatMs !== null && beatMs <= windowMs) seats.push(row);
444
577
  else nonSeats.push(row);
@@ -473,15 +606,38 @@ export function summarise(fleet, target) {
473
606
  const rows = (fleet.seats || []).map((s) => ({ ...s, verdict: classifySeat(s, target) }));
474
607
  const counts = { current: 0, stale: 0, unverifiable: 0 };
475
608
  for (const r of rows) counts[r.verdict] += 1;
609
+ // Behind AND proved it cannot arrive — a strictly stronger statement than
610
+ // `stale`, which clears itself on the next hourly run.
611
+ //
612
+ // `stuck` IS NOT A SUBSET OF `stale`, and the code used to assume it was. A
613
+ // seat's verdict is about THIS run's target; its streak is about whatever
614
+ // `@latest` was when it last tried, and the two are different versions
615
+ // whenever a rollout is verified against anything but the registry's head —
616
+ // `--dry-run`, `--verify-only`, or simply a publish that has since been
617
+ // followed by another. Demonstrated: one seat on target 2.18.14 whose record
618
+ // is {ok:false, from:"2.18.14", to:"2.18.15", failStreak:5} classifies
619
+ // `current`, so `stale` is empty, `done` was true, and renderVerdict's early
620
+ // return printed "All 1 seat(s) report 2.18.14." — five consecutive failed
621
+ // upgrades reported as a fully current fleet. So the two sets are kept and
622
+ // named apart, and `done` answers BOTH questions: every seat is on the target
623
+ // AND no seat is known to be unable to move.
624
+ const stuck = rows.filter((r) => r.stuck && r.stuck.verdict === "stuck");
625
+ const stale = rows.filter((r) => r.verdict === "stale");
626
+ const staleSlugs = new Set(stale.map((r) => r.slug));
476
627
  return {
477
628
  target,
478
629
  rows,
479
630
  counts,
480
631
  nonSeats: fleet.nonSeats || [],
481
- stale: rows.filter((r) => r.verdict === "stale"),
632
+ stale,
482
633
  unverifiable: rows.filter((r) => r.verdict === "unverifiable"),
634
+ stuck,
635
+ /** The stuck seats that are ALSO behind this run's target — "N of those". */
636
+ stuckBehind: stuck.filter((r) => staleSlugs.has(r.slug)),
637
+ /** Stuck against some other version while current on this one. Still a finding. */
638
+ stuckElsewhere: stuck.filter((r) => !staleSlugs.has(r.slug)),
483
639
  readCarriesVersionField: fleet.readCarriesVersionField === true,
484
- done: rows.length > 0 && counts.current === rows.length,
640
+ done: rows.length > 0 && counts.current === rows.length && stuck.length === 0,
485
641
  };
486
642
  }
487
643
 
@@ -511,7 +667,7 @@ export function renderSeatTable(summary, o = {}) {
511
667
  r.slug,
512
668
  ageLabel(r.lastBeatMs),
513
669
  r.version ? (r.basis === "installed" ? `${r.version} (installed)` : r.version) : "unknown",
514
- r.verdict,
670
+ r.stuck ? `${r.verdict} · ${r.stuck.verdict}${r.stuck.failStreak ? ` ×${r.stuck.failStreak}` : ""}` : r.verdict,
515
671
  ]);
516
672
  const widths = head.map((h, i) => Math.max(h.length, ...body.map((row) => row[i].length), 0));
517
673
  const line = (cells) => cells.map((c, i) => c.padEnd(widths[i])).join(" ").trimEnd();
@@ -531,9 +687,19 @@ export function renderSeatTable(summary, o = {}) {
531
687
  return out.join("\n");
532
688
  }
533
689
 
534
- /** The closing sentence naming exactly which seats are behind. Pure. */
690
+ /**
691
+ * The closing sentence naming exactly which seats are behind, and which of them
692
+ * will never arrive unaided. Pure.
693
+ *
694
+ * NO EARLY RETURN ON `done`. It used to open with
695
+ * `if (summary.done) return "All N seat(s) report X."`, which swallowed the
696
+ * STUCK line whole for any seat that was stuck against a version other than
697
+ * this run's target — see the note in {@link summarise}. `done` now accounts
698
+ * for stuck seats, and the "all clear" sentence is built at the END, from the
699
+ * absence of every finding, so a future finding cannot be skipped past by a
700
+ * `done` that did not learn about it.
701
+ */
535
702
  export function renderVerdict(summary) {
536
- if (summary.done) return `All ${summary.rows.length} seat(s) report ${summary.target}.`;
537
703
  const parts = [];
538
704
  if (summary.stale.length) {
539
705
  parts.push(
@@ -541,6 +707,27 @@ export function renderVerdict(summary) {
541
707
  summary.stale.map((r) => `${r.name} (${r.slug}, reports ${r.version})`).join(", ")
542
708
  );
543
709
  }
710
+ const stuck = (summary.stuck && summary.stuck.length ? summary.stuck : []);
711
+ if (stuck.length) {
712
+ // Said SEPARATELY from "behind", and after it, because the response is
713
+ // different: a behind seat is waited on, a stuck one is visited. The line
714
+ // that was missing for three weeks is this one.
715
+ const behind = summary.stuckBehind || stuck.filter((r) => r.verdict === "stale");
716
+ const elsewhere = summary.stuckElsewhere || stuck.filter((r) => r.verdict !== "stale");
717
+ // "N of those" is only true of the ones that ARE among the seats just named
718
+ // as behind. A seat that is current on this target and stuck against a
719
+ // later one is not "one of those" and must not be counted as if it were.
720
+ const head =
721
+ elsewhere.length === 0
722
+ ? `${behind.length} of those ${behind.length === 1 ? "is" : "are"} STUCK`
723
+ : behind.length === 0
724
+ ? `${elsewhere.length} seat(s) ${elsewhere.length === 1 ? "is" : "are"} STUCK — on ${summary.target}, but unable to move off it`
725
+ : `${stuck.length} seat(s) are STUCK (${behind.length} of those just named as behind, ${elsewhere.length} current on ${summary.target} but unable to move off it)`;
726
+ parts.push(
727
+ `${head} — attempted and failed, and will not arrive unaided: ` +
728
+ stuck.map((r) => `${r.name} (${r.slug}, ${stuckClause(r, summary.target)})`).join(", ")
729
+ );
730
+ }
544
731
  if (summary.unverifiable.length) {
545
732
  parts.push(
546
733
  `${summary.unverifiable.length} seat(s) could not be verified — the org read carries no sdkVersion: ` +
@@ -548,9 +735,46 @@ export function renderVerdict(summary) {
548
735
  );
549
736
  }
550
737
  if (!summary.rows.length) parts.push("no beating seats found in the org directory — nothing to verify.");
738
+ if (!parts.length) return `All ${summary.rows.length} seat(s) report ${summary.target}.`;
551
739
  return parts.join("\n");
552
740
  }
553
741
 
742
+ /**
743
+ * One stuck seat's parenthetical: what it tried, how many times, and — only
744
+ * when it is true — why it cannot say how many times.
745
+ *
746
+ * THE "no count" SENTENCE IS GATED ON THE SEAT'S VERSION, NOT ON THE BRANCH
747
+ * THAT PRODUCED THE VERDICT. It used to hang off `basis === "attempt"` and
748
+ * read "this seat's beat is too old to carry a count". Every attempt-basis row
749
+ * got it, which at the time this landed was every row in the fleet — no
750
+ * released SDK wrote `failStreak` yet — including seats beating every thirty
751
+ * seconds whose records demonstrably carried `reason` and `healthy`. The beat
752
+ * was never old; the seat's SDK simply predates {@link FAILSTREAK_SINCE}. When
753
+ * the seat is new enough to count and still did not, that is a DIFFERENT and
754
+ * more interesting fact, so it gets its own words. Pure.
755
+ *
756
+ * @param {object} row a summarise() row with a non-null `stuck`
757
+ * @param {string} target
758
+ * @returns {string}
759
+ */
760
+ export function stuckClause(row, target) {
761
+ const st = row.stuck;
762
+ const attempts = st.failStreak ? `${st.failStreak} consecutive failed attempts` : "a failed attempt";
763
+ const oldest = Number.isFinite(st.ageMs) ? `, oldest ${ageLabel(st.ageMs)} ago` : "";
764
+ let why = "";
765
+ if (st.basis === "attempt") {
766
+ const v = row.version;
767
+ if (v && compareVersions(v, FAILSTREAK_SINCE) < 0) {
768
+ why = `; it reports ${v}, which predates the failure count (SDK ${FAILSTREAK_SINCE}), so one attempt is all it can report`;
769
+ } else if (v) {
770
+ why = `; it reports ${v}, new enough to count its failures, but this record carries none — read state/autoupdate/last.json on the machine`;
771
+ } else {
772
+ why = "; this seat reports no version, so there is no telling whether it could have counted";
773
+ }
774
+ }
775
+ return `${attempts} to ${st.target || target}${oldest}${why}`;
776
+ }
777
+
554
778
  /**
555
779
  * What stage 3's exit code will be, and WHY — as data, so a caller can tell the
556
780
  * three very different things a `3` means apart.
@@ -572,7 +796,7 @@ export function renderVerdict(summary) {
572
796
  * Pure.
573
797
  *
574
798
  * @param {ReturnType<typeof summarise>|null|undefined} summary
575
- * @returns {{code:number, reason:"verified"|"behind"|"unverifiable"|"empty-fleet"|"no-fleet-read", note:string|null}}
799
+ * @returns {{code:number, reason:"verified"|"stuck"|"behind"|"unverifiable"|"empty-fleet"|"no-fleet-read", note:string|null}}
576
800
  */
577
801
  export function propagationOutcome(summary) {
578
802
  if (!summary || !Array.isArray(summary.rows)) {
@@ -583,6 +807,28 @@ export function propagationOutcome(summary) {
583
807
  };
584
808
  }
585
809
  if (summary.done) return { code: 0, reason: "verified", note: null };
810
+ // STUCK OUTRANKS BEHIND, and is checked before both `empty-fleet` and the
811
+ // unverifiable branch, because it is the only one of the three that a person
812
+ // must ACT on rather than wait out.
813
+ //
814
+ // This is also the automation the seat-side alert cannot provide. `upgrade_stuck`
815
+ // is derived by lib/telemetry/alerts.mjs running ON the seat, so a seat too
816
+ // stale to install the SDK never emits it — by construction the alert cannot
817
+ // reach the seats it is for. What CAN reach them is this: the fleet-side
818
+ // derivation runs on every publish, because stage 3 runs on every publish,
819
+ // and a stuck seat now makes the release command exit non-zero and name the
820
+ // machine. Nobody has to decide to go looking.
821
+ if (summary.stuck && summary.stuck.length) {
822
+ return {
823
+ code: 3,
824
+ reason: "stuck",
825
+ note:
826
+ `${summary.stuck.length} seat(s) have attempted @latest and failed repeatedly. They will not arrive on their own: ` +
827
+ `each needs a person at the machine (maestro doctor; state/autoupdate/last.json carries the reason). ` +
828
+ `Stage 3 still polls to its deadline in case someone clears one while it runs, but it will not clear itself. ` +
829
+ renderVerdict(summary),
830
+ };
831
+ }
586
832
  if (!summary.rows.length) {
587
833
  return {
588
834
  code: 3,
@@ -1,24 +1,109 @@
1
1
  #!/bin/bash
2
2
  # Health Check — Verifies all Maestro agent subsystems are operational
3
- # Usage: ./scripts/healthcheck.sh
4
- # Exit codes: 0 = healthy, 1 = degraded, 2 = critical
3
+ # Usage: ./scripts/healthcheck.sh [--ignore-emergency-stop] [--offline] [--help]
4
+ # Exit codes: 0 = healthy, 1 = degraded (warnings only), 2 = critical (errors)
5
+ #
6
+ # A refusal NAMES what blocked it. The summary always prints a `Blocking:`
7
+ # line listing the id of every failed check (and a `Warnings:` line for the
8
+ # soft ones), because the operator's next question after "1 errors" is always
9
+ # "which one?" — and on 2026-09-24 answering it cost a re-run with a filter
10
+ # while a seat stayed halted.
11
+ #
12
+ # --ignore-emergency-stop
13
+ # Demote check 1 (the .emergency-stop flag) from ERROR to NOTE. It is meant
14
+ # for one caller: resume-operations.sh, whose whole job is to lift that
15
+ # flag. Counting the flag as a reason not to lift the flag deadlocked the
16
+ # only tool that can lift it (a seat sat halted from 2026-09-24T08:16Z
17
+ # until it was cleared by hand).
18
+ #
19
+ # SCOPE, PLAINLY. This is a PUBLIC CLI flag. Any caller, script or operator
20
+ # can pass it; nothing refuses it, and nothing here pretends to. An earlier
21
+ # draft of this header said the exemption "is refused to everyone else" —
22
+ # it never was. The scoping is convention plus this comment, and a gate that
23
+ # exists only in a comment is exactly the thing this file was repaired for.
24
+ #
25
+ # What keeps it honest is scope, not exclusivity:
26
+ # - It demotes exactly ONE check. Every other check runs at full severity,
27
+ # so an exempted run still cannot green-light a broken seat.
28
+ # - It is off by default, so an operator running healthcheck on a halted
29
+ # seat is still told it is halted — the first thing they need to know.
30
+ # - It is explicit, greppable, and reviewable at its single call site.
31
+ # - The demotion is ANNOUNCED in the output ([NOTE] ...). That line is the
32
+ # receipt resume-operations.sh checks before it trusts the verdict, so a
33
+ # run where the flag was silently swallowed is caught rather than obeyed.
34
+ # Requiring a second token (an env var alongside the flag) would add another
35
+ # thing to forge, not a boundary: anyone who can run this script can set an
36
+ # env var. The real boundary is permission to execute it at all.
37
+ #
38
+ # The alternatives are worse on their merits, not on access control:
39
+ # - Dropping the check outright, or having it always ignore the flag, is
40
+ # wrong for the same reason as above: a halted seat must say so.
41
+ # - Re-ordering resume (remove flag, then check) is worse still: it lifts
42
+ # the stop on a genuinely broken seat and only then discovers it is
43
+ # broken, which is the failure the stop flag exists to prevent.
44
+ #
45
+ # --offline (or MAESTRO_HEALTHCHECK_SKIP_NETWORK=1)
46
+ # Record check 6 as SKIPPED instead of reaching the network. Not a weakening:
47
+ # the check is neither counted healthy nor counted failed, and the summary
48
+ # says it was skipped.
5
49
 
6
50
  set -e
7
51
 
8
52
  SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
9
- SOPHIE_AI_DIR="$(dirname "$SCRIPT_DIR")"
53
+ AGENT_DIR="$(dirname "$SCRIPT_DIR")"
10
54
  TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
11
- HEALTHY=0
12
55
  WARNINGS=0
13
56
  ERRORS=0
57
+ BLOCKING="" # space-separated ids of failed checks
58
+ SOFT="" # space-separated ids of warned checks
59
+
60
+ IGNORE_STOP_FLAG=0
61
+ SKIP_NETWORK=0
62
+ case "${MAESTRO_HEALTHCHECK_SKIP_NETWORK:-0}" in 1 | true | yes) SKIP_NETWORK=1 ;; esac
63
+
64
+ while [ $# -gt 0 ]; do
65
+ case "$1" in
66
+ --ignore-emergency-stop) IGNORE_STOP_FLAG=1 ;;
67
+ --offline) SKIP_NETWORK=1 ;;
68
+ -h | --help)
69
+ sed -n '2,/^$/p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
70
+ exit 0
71
+ ;;
72
+ *)
73
+ echo "healthcheck: unknown argument: $1" >&2
74
+ echo "Usage: healthcheck.sh [--ignore-emergency-stop] [--offline]" >&2
75
+ exit 2
76
+ ;;
77
+ esac
78
+ shift
79
+ done
80
+
81
+ # fail <id> <message> — record a blocking condition under a stable id.
82
+ fail() {
83
+ echo "[FAIL] $2"
84
+ BLOCKING="$BLOCKING $1"
85
+ ERRORS=$((ERRORS + 1))
86
+ }
87
+
88
+ # warn <id> <message> — record a soft condition under a stable id.
89
+ warn() {
90
+ echo "[WARN] $2"
91
+ SOFT="$SOFT $1"
92
+ WARNINGS=$((WARNINGS + 1))
93
+ }
14
94
 
15
95
  echo "Maestro Health Check — $TIMESTAMP"
16
96
  echo "======================================="
17
97
 
18
98
  # Check 1: Emergency stop flag
19
- if [ -f "$SOPHIE_AI_DIR/.emergency-stop" ]; then
20
- echo "[STOP] Emergency stop flag is active"
21
- ERRORS=$((ERRORS + 1))
99
+ if [ -f "$AGENT_DIR/.emergency-stop" ]; then
100
+ if [ "$IGNORE_STOP_FLAG" -eq 1 ]; then
101
+ echo "[NOTE] Emergency stop flag is active — not counted (--ignore-emergency-stop)"
102
+ else
103
+ echo "[STOP] Emergency stop flag is active"
104
+ BLOCKING="$BLOCKING emergency-stop-flag"
105
+ ERRORS=$((ERRORS + 1))
106
+ fi
22
107
  else
23
108
  echo "[OK] No emergency stop flag"
24
109
  fi
@@ -30,22 +115,20 @@ fi
30
115
  # in .env strips the var and falls back to Keychain OAuth).
31
116
  if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
32
117
  echo "[OK] Environment variable: ANTHROPIC_API_KEY"
33
- elif [ -f "$SOPHIE_AI_DIR/.env" ] && grep -qE "^MAESTRO_PREFER_SUBSCRIPTION_AUTH=(1|true|yes)$" "$SOPHIE_AI_DIR/.env"; then
118
+ elif [ -f "$AGENT_DIR/.env" ] && grep -qE "^MAESTRO_PREFER_SUBSCRIPTION_AUTH=(1|true|yes)$" "$AGENT_DIR/.env"; then
34
119
  echo "[OK] Anthropic auth: subscription mode (.env: MAESTRO_PREFER_SUBSCRIPTION_AUTH=1)"
35
- elif [ -f "$SOPHIE_AI_DIR/.env" ] && grep -qE "^ANTHROPIC_API_KEY=." "$SOPHIE_AI_DIR/.env"; then
120
+ elif [ -f "$AGENT_DIR/.env" ] && grep -qE "^ANTHROPIC_API_KEY=." "$AGENT_DIR/.env"; then
36
121
  echo "[OK] Anthropic auth: ANTHROPIC_API_KEY in .env"
37
122
  else
38
- echo "[FAIL] Anthropic auth not configured (.env needs either ANTHROPIC_API_KEY or MAESTRO_PREFER_SUBSCRIPTION_AUTH=1)"
39
- ERRORS=$((ERRORS + 1))
123
+ fail "anthropic-auth" "Anthropic auth not configured (.env needs either ANTHROPIC_API_KEY or MAESTRO_PREFER_SUBSCRIPTION_AUTH=1)"
40
124
  fi
41
125
 
42
126
  # Check 3: Key directories exist and are writable
43
127
  for dir in logs outputs knowledge/memory config; do
44
- if [ -d "$SOPHIE_AI_DIR/$dir" ] && [ -w "$SOPHIE_AI_DIR/$dir" ]; then
128
+ if [ -d "$AGENT_DIR/$dir" ] && [ -w "$AGENT_DIR/$dir" ]; then
45
129
  echo "[OK] Directory writable: $dir"
46
130
  else
47
- echo "[FAIL] Directory missing or not writable: $dir"
48
- ERRORS=$((ERRORS + 1))
131
+ fail "dir:$dir" "Directory missing or not writable: $dir"
49
132
  fi
50
133
  done
51
134
 
@@ -54,8 +137,7 @@ for repo in company-web innovation-lab partner-framework company-legal regulator
54
137
  if [ -d "$HOME/$repo" ]; then
55
138
  echo "[OK] Source repo accessible: ~/$repo"
56
139
  else
57
- echo "[WARN] Source repo not found: ~/$repo"
58
- WARNINGS=$((WARNINGS + 1))
140
+ warn "repo:$repo" "Source repo not found: ~/$repo"
59
141
  fi
60
142
  done
61
143
 
@@ -64,45 +146,61 @@ for app in Slack WhatsApp Safari; do
64
146
  if [ -d "/Applications/$app.app" ] || [ -d "$HOME/Applications/$app.app" ]; then
65
147
  echo "[OK] Application installed: $app"
66
148
  else
67
- echo "[WARN] Application not found: $app"
68
- WARNINGS=$((WARNINGS + 1))
149
+ warn "app:$app" "Application not found: $app"
69
150
  fi
70
151
  done
71
152
 
72
153
  # Check 6: Network connectivity
73
- if curl -s --max-time 5 https://api.anthropic.com > /dev/null 2>&1; then
154
+ if [ "$SKIP_NETWORK" -eq 1 ]; then
155
+ echo "[SKIP] Network: not probed (--offline)"
156
+ elif curl -s --max-time 5 https://api.anthropic.com > /dev/null 2>&1; then
74
157
  echo "[OK] Network: Anthropic API reachable"
75
158
  else
76
- echo "[FAIL] Network: Cannot reach Anthropic API"
77
- ERRORS=$((ERRORS + 1))
159
+ fail "network" "Network: Cannot reach Anthropic API"
78
160
  fi
79
161
 
80
- # Check 7: Disk space
81
- DISK_FREE=$(df -h "$SOPHIE_AI_DIR" | awk 'NR==2{print $5}' | tr -d '%')
82
- if [ "$DISK_FREE" -gt 90 ]; then
83
- echo "[WARN] Disk usage: ${DISK_FREE}% — consider cleanup"
84
- WARNINGS=$((WARNINGS + 1))
85
- elif [ "$DISK_FREE" -gt 95 ]; then
86
- echo "[FAIL] Disk usage: ${DISK_FREE}% — critically low"
87
- ERRORS=$((ERRORS + 1))
162
+ # Check 7: Disk space. Ordered most-severe-first: the previous order tested
163
+ # >90 before >95, so the critical branch could never be reached and disk was
164
+ # warn-only in practice. Reaching it is a REAL severity change with a real
165
+ # consequence — resume-operations.sh refuses on any exit 2, so a seat above
166
+ # 95% can no longer be resumed until space is freed. That is deliberate, and
167
+ # it is not the deadlock this trio was repaired for: unlike the stop flag
168
+ # (which only resume could lift), a full disk is named, actionable, and
169
+ # fixable without running this script. Both branches — >95 blocking and >90
170
+ # warning — are pinned in resume-operations.test.mjs, at the healthcheck and
171
+ # at the resume gate, because they are the only severities that changed.
172
+ DISK_FREE=$(df -h "$AGENT_DIR" | awk 'NR==2{print $5}' | tr -d '%')
173
+ if [ "$DISK_FREE" -gt 95 ]; then
174
+ fail "disk" "Disk usage: ${DISK_FREE}% — critically low"
175
+ elif [ "$DISK_FREE" -gt 90 ]; then
176
+ warn "disk" "Disk usage: ${DISK_FREE}% — consider cleanup"
88
177
  else
89
178
  echo "[OK] Disk usage: ${DISK_FREE}%"
90
179
  fi
91
180
 
92
181
  # Check 8: Config files present
93
182
  for config in priorities.yaml environment.yaml contacts.yaml; do
94
- if [ -f "$SOPHIE_AI_DIR/config/$config" ]; then
183
+ if [ -f "$AGENT_DIR/config/$config" ]; then
95
184
  echo "[OK] Config present: $config"
96
185
  else
97
- echo "[FAIL] Config missing: $config"
98
- ERRORS=$((ERRORS + 1))
186
+ fail "config:$config" "Config missing: $config"
99
187
  fi
100
188
  done
101
189
 
102
- # Summary
190
+ # Summary. `Blocking:` / `Warnings:` are the machine-readable contract:
191
+ # resume-operations.sh parses `Blocking:` to name the condition it refused on.
103
192
  echo ""
104
193
  echo "======================================="
105
194
  echo "Results: $ERRORS errors, $WARNINGS warnings"
195
+ # `if`, not `[ ... ] && echo` — but NOT for the reason first written here,
196
+ # which was false. `set -e` does NOT abort on a false short-circuit: bash
197
+ # exempts a failing command that is not the last in an && list. MEASURED:
198
+ # bash -c 'set -e; n=0; [ "$n" -gt 0 ] && echo hi; exit 0' → exit 0
199
+ # The genuine hazard is only that a false && list leaves status 1, which a
200
+ # script inherits if it is the last command run. The explicit exits below
201
+ # make that moot here; `if` is kept because it says what is meant.
202
+ if [ "$ERRORS" -gt 0 ]; then echo "Blocking:$BLOCKING"; fi
203
+ if [ "$WARNINGS" -gt 0 ]; then echo "Warnings:$SOFT"; fi
106
204
 
107
205
  if [ "$ERRORS" -gt 0 ]; then
108
206
  echo "Status: CRITICAL — fix errors before operating"