@cohortapp/agent-sdk 2.18.14 → 2.18.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -102,6 +102,7 @@ import { existsSync, readFileSync, readdirSync } from "node:fs";
102
102
  import { spawnSync } from "node:child_process";
103
103
  import { dirname, join, resolve as resolvePath } from "node:path";
104
104
  import { fileURLToPath } from "node:url";
105
+ import { DEFAULT_THRESHOLDS } from "../../lib/telemetry/alerts.mjs";
105
106
 
106
107
  const __filename = fileURLToPath(import.meta.url);
107
108
  const REPO_ROOT = resolvePath(dirname(__filename), "..", "..");
@@ -394,6 +395,130 @@ export function seatVersions(entry) {
394
395
  return { installed, running, version, basis: version === null ? null : running ? "running" : "installed" };
395
396
  }
396
397
 
398
+ /**
399
+ * How long a failed upgrade attempt may stand, unrepeated, before a seat that
400
+ * is still on the version it failed FROM counts as stuck rather than merely
401
+ * recovering. Two full failed-target holds (the autoupdater's default is 24 h):
402
+ * one hold is the seat being deliberately left alone, two means the retry it
403
+ * was owed has been and gone.
404
+ */
405
+ export const ATTEMPT_STALE_MS = 48 * 60 * 60 * 1000;
406
+
407
+ /**
408
+ * Consecutive failed attempts that make a seat stuck.
409
+ *
410
+ * DERIVED, NOT RESTATED. This used to be `= 3` with a comment promising it
411
+ * "matches lib/telemetry/alerts.mjs upgradeStuck.warnStreak" — a prose promise
412
+ * about a literal, which is the exact shape the repo doctrine bans and which no
413
+ * test could have caught: this file's own tests use the constant symbolically
414
+ * (`at(STUCK_STREAK - 1)` / `at(STUCK_STREAK)`), so they pass at ANY value and
415
+ * the two copies could have drifted silently. The seat-side alert rule is the
416
+ * authority — it is what actually fires on a live beat — so this reads it.
417
+ *
418
+ * The third copy lives in the shell (`STUCK_STREAK` in
419
+ * scripts/local-triggers/autoupdate.sh), which cannot import this; it is pinned
420
+ * against this value by a source-reading test instead
421
+ * (scripts/local-triggers/autoupdate.test.mjs, "source: the shell's streak
422
+ * threshold is the same number the alert fires on").
423
+ */
424
+ export const STUCK_STREAK = DEFAULT_THRESHOLDS.upgradeStuck.warnStreak;
425
+
426
+ /**
427
+ * The first SDK release whose `write_last` keeps a `failStreak` in
428
+ * state/autoupdate/last.json. A seat below this reports one upgrade outcome and
429
+ * no count — not because its beat is stale, but because the collector that
430
+ * would write the count is the code it has not got.
431
+ *
432
+ * This exists because the "attempt" basis used to explain itself with "this
433
+ * seat's beat is too old to carry a count", attached unconditionally to every
434
+ * attempt-basis row. At rollout time NO released SDK wrote `failStreak`, so
435
+ * every seat in the fleet — including ones beating every 30 seconds with
436
+ * `reason` and `healthy` on the record — was told its beat was too old. The
437
+ * beat age was never the reason; the seat's VERSION is.
438
+ */
439
+ export const FAILSTREAK_SINCE = "2.18.15";
440
+
441
+ /**
442
+ * IS THIS SEAT STUCK — i.e. has it tried to reach @latest and failed, and will
443
+ * it keep failing?
444
+ *
445
+ * `classifySeat` says a seat is `stale`. That is a snapshot and it is not the
446
+ * finding: a seat is stale for twenty minutes after every publish, and three
447
+ * seats were stale for three weeks. What separates them is whether the seat has
448
+ * ATTEMPTED the hop and failed, and how many times — and that fact has always
449
+ * been in the beat (`machine.upgrade`) without anything reading it.
450
+ *
451
+ * TWO BASES, BECAUSE THE SEATS THIS MATTERS FOR REPORT THE LEAST. A seat that
452
+ * has been left behind for weeks is by definition running the oldest collector
453
+ * in the fleet, so it emits the smallest beat: on 2026-09-25 the three seats on
454
+ * 2.17.0 were the only three in the org with no `machine.daemon` block and no
455
+ * `machine.stranded` — staleness hides itself by removing the fields that would
456
+ * announce it. So this reads what a 2.17-era beat DOES carry, and names which
457
+ * basis it used rather than pretending to one answer:
458
+ *
459
+ * basis "streak" — the seat counted its own consecutive failures
460
+ * (`machine.upgrade.failStreak`, SDK {@link FAILSTREAK_SINCE}
461
+ * and later). Stuck at {@link STUCK_STREAK}. The real signal.
462
+ * basis "attempt" — an old beat, which carries one outcome and no count. A
463
+ * failed attempt (`ok:false`, `to` != `from`) with the seat
464
+ * STILL reporting `from` proves the hop did not take; older
465
+ * than {@link ATTEMPT_STALE_MS} it also proves the retry it
466
+ * was owed came and went. Below that age it is `failing`,
467
+ * not `stuck` — which is the same rule the streak obeys and
468
+ * the same rule (d) asks for: a seat that has not been
469
+ * ASKED again is not a stuck seat.
470
+ *
471
+ * Returns null when the beat carries no upgrade record at all: never tried is
472
+ * not the same as tried and failed, and must not print as one.
473
+ *
474
+ * Pure; `now` is a parameter because the "attempt" basis is about a duration.
475
+ *
476
+ * @param {object} entry a directory member, or anything {@link seatVersions} accepts
477
+ * @param {number} [now] epoch ms
478
+ * @returns {{verdict:"stuck"|"failing", basis:"streak"|"attempt", failStreak:number|null,
479
+ * target:string|null, since:string|null, ageMs:number|null}|null}
480
+ */
481
+ export function seatStuck(entry, now = Date.now()) {
482
+ const e = entry && typeof entry === "object" ? entry : {};
483
+ const st = (e.status && typeof e.status === "object" && e.status) || {};
484
+ const mach = (st.machine && typeof st.machine === "object" && st.machine) || (e.machine && typeof e.machine === "object" && e.machine) || {};
485
+ const u = mach.upgrade && typeof mach.upgrade === "object" ? mach.upgrade : null;
486
+ if (!u) return null;
487
+
488
+ const streak = Number.isInteger(u.failStreak) && u.failStreak > 0 ? u.failStreak : null;
489
+ const target = firstString([u.streakTarget, u.to]);
490
+ if (streak !== null) {
491
+ const sinceS = firstString([u.stuckSince, u.at]);
492
+ const sinceMs = sinceS ? Date.parse(sinceS) : NaN;
493
+ return {
494
+ verdict: streak >= STUCK_STREAK ? "stuck" : "failing",
495
+ basis: "streak",
496
+ failStreak: streak,
497
+ target,
498
+ since: sinceS,
499
+ ageMs: Number.isFinite(sinceMs) ? now - sinceMs : null,
500
+ };
501
+ }
502
+
503
+ // Degraded read: one outcome, no count.
504
+ if (u.ok !== false) return null;
505
+ const from = typeof u.from === "string" ? u.from : "";
506
+ const to = typeof u.to === "string" ? u.to : "";
507
+ if (!from || !to || from === to) return null; // not an upgrade attempt
508
+ const installed = seatVersions(e).installed;
509
+ if (installed && installed !== from) return null; // it has moved since; whatever failed is history
510
+ const atMs = typeof u.at === "string" ? Date.parse(u.at) : NaN;
511
+ const ageMs = Number.isFinite(atMs) ? now - atMs : null;
512
+ return {
513
+ verdict: ageMs !== null && ageMs > ATTEMPT_STALE_MS ? "stuck" : "failing",
514
+ basis: "attempt",
515
+ failStreak: null,
516
+ target: to,
517
+ since: typeof u.at === "string" ? u.at : null,
518
+ ageMs,
519
+ };
520
+ }
521
+
397
522
  /**
398
523
  * The single version a seat is judged on, or null when the read carries none.
399
524
  * Kept as its own export because it is the whole of what `classifySeat` needs;
@@ -421,7 +546,7 @@ export function seatVersion(entry) {
421
546
  * not written for.
422
547
  *
423
548
  * @param {object[]} members
424
- * @param {{seatWindowMs?:number}} [o]
549
+ * @param {{seatWindowMs?:number, now?:number}} [o]
425
550
  * @returns {{seats:object[], nonSeats:object[], readCarriesVersionField:boolean}}
426
551
  */
427
552
  export function toSeats(members, o = {}) {
@@ -446,6 +571,7 @@ export function toSeats(members, o = {}) {
446
571
  detail: (p && p.detail) || "",
447
572
  lastBeatMs: beatMs,
448
573
  ...seatVersions(m),
574
+ stuck: seatStuck(m, o.now),
449
575
  };
450
576
  if (beatMs !== null && beatMs <= windowMs) seats.push(row);
451
577
  else nonSeats.push(row);
@@ -480,15 +606,38 @@ export function summarise(fleet, target) {
480
606
  const rows = (fleet.seats || []).map((s) => ({ ...s, verdict: classifySeat(s, target) }));
481
607
  const counts = { current: 0, stale: 0, unverifiable: 0 };
482
608
  for (const r of rows) counts[r.verdict] += 1;
609
+ // Behind AND proved it cannot arrive — a strictly stronger statement than
610
+ // `stale`, which clears itself on the next hourly run.
611
+ //
612
+ // `stuck` IS NOT A SUBSET OF `stale`, and the code used to assume it was. A
613
+ // seat's verdict is about THIS run's target; its streak is about whatever
614
+ // `@latest` was when it last tried, and the two are different versions
615
+ // whenever a rollout is verified against anything but the registry's head —
616
+ // `--dry-run`, `--verify-only`, or simply a publish that has since been
617
+ // followed by another. Demonstrated: one seat on target 2.18.14 whose record
618
+ // is {ok:false, from:"2.18.14", to:"2.18.15", failStreak:5} classifies
619
+ // `current`, so `stale` is empty, `done` was true, and renderVerdict's early
620
+ // return printed "All 1 seat(s) report 2.18.14." — five consecutive failed
621
+ // upgrades reported as a fully current fleet. So the two sets are kept and
622
+ // named apart, and `done` answers BOTH questions: every seat is on the target
623
+ // AND no seat is known to be unable to move.
624
+ const stuck = rows.filter((r) => r.stuck && r.stuck.verdict === "stuck");
625
+ const stale = rows.filter((r) => r.verdict === "stale");
626
+ const staleSlugs = new Set(stale.map((r) => r.slug));
483
627
  return {
484
628
  target,
485
629
  rows,
486
630
  counts,
487
631
  nonSeats: fleet.nonSeats || [],
488
- stale: rows.filter((r) => r.verdict === "stale"),
632
+ stale,
489
633
  unverifiable: rows.filter((r) => r.verdict === "unverifiable"),
634
+ stuck,
635
+ /** The stuck seats that are ALSO behind this run's target — "N of those". */
636
+ stuckBehind: stuck.filter((r) => staleSlugs.has(r.slug)),
637
+ /** Stuck against some other version while current on this one. Still a finding. */
638
+ stuckElsewhere: stuck.filter((r) => !staleSlugs.has(r.slug)),
490
639
  readCarriesVersionField: fleet.readCarriesVersionField === true,
491
- done: rows.length > 0 && counts.current === rows.length,
640
+ done: rows.length > 0 && counts.current === rows.length && stuck.length === 0,
492
641
  };
493
642
  }
494
643
 
@@ -518,7 +667,7 @@ export function renderSeatTable(summary, o = {}) {
518
667
  r.slug,
519
668
  ageLabel(r.lastBeatMs),
520
669
  r.version ? (r.basis === "installed" ? `${r.version} (installed)` : r.version) : "unknown",
521
- r.verdict,
670
+ r.stuck ? `${r.verdict} · ${r.stuck.verdict}${r.stuck.failStreak ? ` ×${r.stuck.failStreak}` : ""}` : r.verdict,
522
671
  ]);
523
672
  const widths = head.map((h, i) => Math.max(h.length, ...body.map((row) => row[i].length), 0));
524
673
  const line = (cells) => cells.map((c, i) => c.padEnd(widths[i])).join(" ").trimEnd();
@@ -538,9 +687,19 @@ export function renderSeatTable(summary, o = {}) {
538
687
  return out.join("\n");
539
688
  }
540
689
 
541
- /** The closing sentence naming exactly which seats are behind. Pure. */
690
+ /**
691
+ * The closing sentence naming exactly which seats are behind, and which of them
692
+ * will never arrive unaided. Pure.
693
+ *
694
+ * NO EARLY RETURN ON `done`. It used to open with
695
+ * `if (summary.done) return "All N seat(s) report X."`, which swallowed the
696
+ * STUCK line whole for any seat that was stuck against a version other than
697
+ * this run's target — see the note in {@link summarise}. `done` now accounts
698
+ * for stuck seats, and the "all clear" sentence is built at the END, from the
699
+ * absence of every finding, so a future finding cannot be skipped past by a
700
+ * `done` that did not learn about it.
701
+ */
542
702
  export function renderVerdict(summary) {
543
- if (summary.done) return `All ${summary.rows.length} seat(s) report ${summary.target}.`;
544
703
  const parts = [];
545
704
  if (summary.stale.length) {
546
705
  parts.push(
@@ -548,6 +707,27 @@ export function renderVerdict(summary) {
548
707
  summary.stale.map((r) => `${r.name} (${r.slug}, reports ${r.version})`).join(", ")
549
708
  );
550
709
  }
710
+ const stuck = (summary.stuck && summary.stuck.length ? summary.stuck : []);
711
+ if (stuck.length) {
712
+ // Said SEPARATELY from "behind", and after it, because the response is
713
+ // different: a behind seat is waited on, a stuck one is visited. The line
714
+ // that was missing for three weeks is this one.
715
+ const behind = summary.stuckBehind || stuck.filter((r) => r.verdict === "stale");
716
+ const elsewhere = summary.stuckElsewhere || stuck.filter((r) => r.verdict !== "stale");
717
+ // "N of those" is only true of the ones that ARE among the seats just named
718
+ // as behind. A seat that is current on this target and stuck against a
719
+ // later one is not "one of those" and must not be counted as if it were.
720
+ const head =
721
+ elsewhere.length === 0
722
+ ? `${behind.length} of those ${behind.length === 1 ? "is" : "are"} STUCK`
723
+ : behind.length === 0
724
+ ? `${elsewhere.length} seat(s) ${elsewhere.length === 1 ? "is" : "are"} STUCK — on ${summary.target}, but unable to move off it`
725
+ : `${stuck.length} seat(s) are STUCK (${behind.length} of those just named as behind, ${elsewhere.length} current on ${summary.target} but unable to move off it)`;
726
+ parts.push(
727
+ `${head} — attempted and failed, and will not arrive unaided: ` +
728
+ stuck.map((r) => `${r.name} (${r.slug}, ${stuckClause(r, summary.target)})`).join(", ")
729
+ );
730
+ }
551
731
  if (summary.unverifiable.length) {
552
732
  parts.push(
553
733
  `${summary.unverifiable.length} seat(s) could not be verified — the org read carries no sdkVersion: ` +
@@ -555,9 +735,46 @@ export function renderVerdict(summary) {
555
735
  );
556
736
  }
557
737
  if (!summary.rows.length) parts.push("no beating seats found in the org directory — nothing to verify.");
738
+ if (!parts.length) return `All ${summary.rows.length} seat(s) report ${summary.target}.`;
558
739
  return parts.join("\n");
559
740
  }
560
741
 
742
+ /**
743
+ * One stuck seat's parenthetical: what it tried, how many times, and — only
744
+ * when it is true — why it cannot say how many times.
745
+ *
746
+ * THE "no count" SENTENCE IS GATED ON THE SEAT'S VERSION, NOT ON THE BRANCH
747
+ * THAT PRODUCED THE VERDICT. It used to hang off `basis === "attempt"` and
748
+ * read "this seat's beat is too old to carry a count". Every attempt-basis row
749
+ * got it, which at the time this landed was every row in the fleet — no
750
+ * released SDK wrote `failStreak` yet — including seats beating every thirty
751
+ * seconds whose records demonstrably carried `reason` and `healthy`. The beat
752
+ * was never old; the seat's SDK simply predates {@link FAILSTREAK_SINCE}. When
753
+ * the seat is new enough to count and still did not, that is a DIFFERENT and
754
+ * more interesting fact, so it gets its own words. Pure.
755
+ *
756
+ * @param {object} row a summarise() row with a non-null `stuck`
757
+ * @param {string} target
758
+ * @returns {string}
759
+ */
760
+ export function stuckClause(row, target) {
761
+ const st = row.stuck;
762
+ const attempts = st.failStreak ? `${st.failStreak} consecutive failed attempts` : "a failed attempt";
763
+ const oldest = Number.isFinite(st.ageMs) ? `, oldest ${ageLabel(st.ageMs)} ago` : "";
764
+ let why = "";
765
+ if (st.basis === "attempt") {
766
+ const v = row.version;
767
+ if (v && compareVersions(v, FAILSTREAK_SINCE) < 0) {
768
+ why = `; it reports ${v}, which predates the failure count (SDK ${FAILSTREAK_SINCE}), so one attempt is all it can report`;
769
+ } else if (v) {
770
+ why = `; it reports ${v}, new enough to count its failures, but this record carries none — read state/autoupdate/last.json on the machine`;
771
+ } else {
772
+ why = "; this seat reports no version, so there is no telling whether it could have counted";
773
+ }
774
+ }
775
+ return `${attempts} to ${st.target || target}${oldest}${why}`;
776
+ }
777
+
561
778
  /**
562
779
  * What stage 3's exit code will be, and WHY — as data, so a caller can tell the
563
780
  * three very different things a `3` means apart.
@@ -579,7 +796,7 @@ export function renderVerdict(summary) {
579
796
  * Pure.
580
797
  *
581
798
  * @param {ReturnType<typeof summarise>|null|undefined} summary
582
- * @returns {{code:number, reason:"verified"|"behind"|"unverifiable"|"empty-fleet"|"no-fleet-read", note:string|null}}
799
+ * @returns {{code:number, reason:"verified"|"stuck"|"behind"|"unverifiable"|"empty-fleet"|"no-fleet-read", note:string|null}}
583
800
  */
584
801
  export function propagationOutcome(summary) {
585
802
  if (!summary || !Array.isArray(summary.rows)) {
@@ -590,6 +807,28 @@ export function propagationOutcome(summary) {
590
807
  };
591
808
  }
592
809
  if (summary.done) return { code: 0, reason: "verified", note: null };
810
+ // STUCK OUTRANKS BEHIND, and is checked before both `empty-fleet` and the
811
+ // unverifiable branch, because it is the only one of the three that a person
812
+ // must ACT on rather than wait out.
813
+ //
814
+ // This is also the automation the seat-side alert cannot provide. `upgrade_stuck`
815
+ // is derived by lib/telemetry/alerts.mjs running ON the seat, so a seat too
816
+ // stale to install the SDK never emits it — by construction the alert cannot
817
+ // reach the seats it is for. What CAN reach them is this: the fleet-side
818
+ // derivation runs on every publish, because stage 3 runs on every publish,
819
+ // and a stuck seat now makes the release command exit non-zero and name the
820
+ // machine. Nobody has to decide to go looking.
821
+ if (summary.stuck && summary.stuck.length) {
822
+ return {
823
+ code: 3,
824
+ reason: "stuck",
825
+ note:
826
+ `${summary.stuck.length} seat(s) have attempted @latest and failed repeatedly. They will not arrive on their own: ` +
827
+ `each needs a person at the machine (maestro doctor; state/autoupdate/last.json carries the reason). ` +
828
+ `Stage 3 still polls to its deadline in case someone clears one while it runs, but it will not clear itself. ` +
829
+ renderVerdict(summary),
830
+ };
831
+ }
593
832
  if (!summary.rows.length) {
594
833
  return {
595
834
  code: 3,
@@ -481,18 +481,133 @@ write_notice(){ # $1 = from, $2 = to, [$3 = reason → healthy:false] — tell t
481
481
  }
482
482
  write_upgrade_notice(){ write_notice "$1" "$2"; }
483
483
  LAST_JSON="$AGENT_DIR/state/autoupdate/last.json"
484
- write_last(){ # $1 from, $2 to, $3 ok, $4 healthy, $5 reason — state/autoupdate/last.json (→ beat machine.upgrade)
485
- local dir="$AGENT_DIR/state/autoupdate" tmp
484
+ # How many CONSECUTIVE failed attempts to reach @latest make a seat STUCK, and
485
+ # at what point it stops being a warning. See the `failStreak` block below.
486
+ STUCK_STREAK="${MAESTRO_AUTOUPDATE_STUCK_STREAK:-3}"
487
+ # ── failStreak — "this seat cannot get to @latest" is a FACT, not a discovery ─
488
+ # The fleet learned on 2026-09-25 that three seats had sat on 2.17.0 for days.
489
+ # Nothing was broken in the sense anything reports: the launchd job fired every
490
+ # hour, npm answered, the install ran, the health gate failed, the rollback
491
+ # worked, last.json recorded it honestly, and the beat carried it. Each hour was
492
+ # a correct, self-contained failure — and a correct failure repeated forty times
493
+ # is a different fact from a correct failure once. Only the REPETITION says the
494
+ # seat will never arrive on its own, and no single record can hold it, so the
495
+ # count lives in last.json and rides the beat the rest of that record already
496
+ # rides. There is no second channel here and there must not be one: the seats
497
+ # this is about are, by construction, the ones running the oldest code.
498
+ #
499
+ # WHAT COUNTS AS AN ATTEMPT — the whole honesty of the number:
500
+ # · An ATTEMPT is a record whose `to` differs from its `from`: the upgrade
501
+ # path actually installed (or tried to install) a different version. Those
502
+ # are the only records that move the streak.
503
+ # · A FAILED attempt (`ok:false`) increments it and stamps `stuckSince` with
504
+ # the first failure of the run of failures, so a reader gets a DURATION and
505
+ # not merely a tally — "4 attempts since Monday" and "4 attempts in the last
506
+ # hour" are different seats.
507
+ # · A SUCCEEDED attempt (`ok:true`) clears all three fields. Arrival is the
508
+ # only thing that resets it; a fleet-wide publish does not, because a seat
509
+ # that fails 2.18.12, 2.18.13 and 2.18.14 in turn has failed three times to
510
+ # do the one thing asked of it, and restarting the count on every release
511
+ # would guarantee the number never reaches any threshold.
512
+ # · Everything else PRESERVES the fields untouched. A `to == from` record is
513
+ # a report on the installed version, not an attempt to leave it, so it must
514
+ # not move a count of ATTEMPTS in either direction. (Every such call site
515
+ # today sits on the up-to-date branch and therefore takes the arrival path
516
+ # below instead — this rule governs the writer, for the next caller that
517
+ # does not.) And a run that SKIPS writes no record at all, which is the
518
+ # same preservation by a shorter route. THIS IS THE POINT OF (d): a seat that has not been ASKED —
519
+ # nothing newer published, or held for the day after its last failure — is
520
+ # not a stuck seat, and must never accumulate a streak for sitting still.
521
+ # Only a real, completed, failed attempt does.
522
+ # · WHICH RUNS ACTUALLY SKIP — the exact list, because a wrong one was
523
+ # written here first. The paths that reach `exit 0` with NO write_last are:
524
+ # the failed-target hold, the overlap lock, the kill-switch, and a failed
525
+ # `npm view`. "SIMPLY BEING UP TO DATE" IS NOT AMONG THEM and never was —
526
+ # the up-to-date branch is the daemon health gate, and it calls write_last
527
+ # on every one of its outcomes (healthy, unhealthy-current,
528
+ # ahead-of-registry, recovered, and the stale-daemon kickstart). The
529
+ # original text claimed otherwise; nothing checked it, and the streak
530
+ # arithmetic on that branch is not what the sentence implied.
531
+ # · ARRIVAL IS BEING AT @latest, NOT MERELY INSTALLING IT. Every write on the
532
+ # up-to-date branch passes `at_latest=true`, which clears all three fields
533
+ # — because that branch is only entered when `CUR >= LATEST`, i.e. there is
534
+ # nothing left for this seat to reach. That covers the case the increment
535
+ # rule alone gets wrong: if a bad release is YANKED and @latest rolls back
536
+ # to the version a seat already runs, the seat stops attempting anything,
537
+ # so nothing would ever reset it and `upgrade_stuck` would fire forever
538
+ # (critical past 6) on a seat that is in fact perfectly current. It also
539
+ # makes the stale-daemon kickstart record honest rather than accidental:
540
+ # that hop really is an arrival.
541
+ # The write is done in node rather than printf because the record now depends on
542
+ # the record before it; a shell that reads a file it is about to overwrite gets
543
+ # that wrong at exactly the moments it matters.
544
+ write_last(){ # $1 from, $2 to, $3 ok, $4 healthy, $5 reason, [$6 at_latest] — state/autoupdate/last.json (→ beat machine.upgrade)
545
+ local dir="$AGENT_DIR/state/autoupdate" tmp streak at_latest="${6:-false}"
486
546
  mkdir -p "$dir" 2>/dev/null || return 0
487
547
  tmp="$(mktemp "$dir/.last.XXXXXX" 2>/dev/null)" || return 0
488
- printf '{\n "from": "%s",\n "to": "%s",\n "at": "%s",\n "ok": %s,\n "healthy": %s,\n "reason": "%s"\n}\n' \
489
- "$1" "$2" "$(date -u +%FT%TZ)" "$3" "$4" "$(json_str "$5")" > "$tmp" && mv -f "$tmp" "$LAST_JSON"
548
+ streak="$(node -e '
549
+ const fs = require("fs");
550
+ const [file, tmp, from, to, okS, healthyS, reason, stuckStreakS, atLatestS] = process.argv.slice(1);
551
+ const ok = okS === "true";
552
+ const at = new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
553
+ let prev = null;
554
+ try { prev = JSON.parse(fs.readFileSync(file, "utf8")); } catch {}
555
+ if (!prev || typeof prev !== "object") prev = {};
556
+ const prevStreak = Number.isInteger(prev.failStreak) && prev.failStreak > 0 ? prev.failStreak : 0;
557
+ const attempt = from !== to; // an upgrade was actually tried
558
+ const atLatest = atLatestS === "true"; // this seat is AT or past @latest — nothing left to reach
559
+ let failStreak, stuckSince, streakTarget;
560
+ if (atLatest || (attempt && ok)) { // arrival, by install OR because @latest came back to us (a yank)
561
+ failStreak = 0; stuckSince = ""; streakTarget = "";
562
+ } else if (!attempt) { // a report on the installed version — carry the count, do not touch it
563
+ failStreak = prevStreak;
564
+ stuckSince = typeof prev.stuckSince === "string" ? prev.stuckSince : "";
565
+ streakTarget = typeof prev.streakTarget === "string" ? prev.streakTarget : "";
566
+ } else {
567
+ failStreak = prevStreak + 1;
568
+ stuckSince = typeof prev.stuckSince === "string" && prev.stuckSince ? prev.stuckSince : at;
569
+ streakTarget = to;
570
+ }
571
+ const rec = { from, to, at, ok, healthy: healthyS === "true", reason };
572
+ if (failStreak > 0) {
573
+ rec.failStreak = failStreak;
574
+ if (stuckSince) rec.stuckSince = stuckSince;
575
+ if (streakTarget) rec.streakTarget = streakTarget;
576
+ }
577
+ fs.writeFileSync(tmp, JSON.stringify(rec, null, 2) + "\n");
578
+ // stdout is the shell'"'"'s only view of what was decided: "<streak> <target> <since>"
579
+ process.stdout.write(failStreak >= Number(stuckStreakS) && attempt && !ok ? `STUCK ${failStreak} ${streakTarget} ${stuckSince}` : "");
580
+ ' "$LAST_JSON" "$tmp" "$1" "$2" "$3" "$4" "$5" "$STUCK_STREAK" "$at_latest" 2>/dev/null)" || streak=""
581
+ if [ -s "$tmp" ]; then
582
+ mv -f "$tmp" "$LAST_JSON"
583
+ else
584
+ # The node writer produced nothing. Before this branch existed the tmp was
585
+ # simply removed and last.json was LEFT AT ITS PREVIOUS VALUE — so a write
586
+ # failure did not lose a record, it published a STALE one, and the 24 h
587
+ # failed-target hold (which reads `to` and `reason` out of this file) then
588
+ # made its decision from a previous run's outcome while believing it was
589
+ # this run's. A missing record is fail-open by design here; a wrong one is
590
+ # not. So write the event itself with printf, which needs no interpreter,
591
+ # and say loudly that the count did not survive: the streak is the one
592
+ # field that cannot be reconstructed without reading the old file, and a
593
+ # silently reset counter is the failure this whole block exists to prevent.
594
+ log "WARN: write_last could not run node — recording the bare outcome without the failStreak. The count restarts from this run; see state/autoupdate/last.json."
595
+ printf '{\n "from": "%s",\n "to": "%s",\n "at": "%s",\n "ok": %s,\n "healthy": %s,\n "reason": "%s"\n}\n' \
596
+ "$1" "$2" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$3" "$4" "$(printf '%s' "$5" | sed 's/[\\"]/\\&/g')" > "$LAST_JSON" 2>/dev/null || true
597
+ fi
490
598
  rm -f "$tmp" 2>/dev/null
599
+ if [ -n "$streak" ]; then
600
+ set -- $streak
601
+ log "STUCK: $2 consecutive failed attempts to reach $3 (since $4). This seat will not arrive on its own — the org now carries machine.upgrade.failStreak=$2 and an \`upgrade_stuck\` alert on every beat. maestro doctor; then rm state/autoupdate/last.json to retry immediately."
602
+ fi
491
603
  return 0
492
604
  }
493
605
  last_reason(){ # the reason field of last.json, or nothing
494
606
  node -e 'try { process.stdout.write(String(JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).reason || "")); } catch {}' "$LAST_JSON" 2>/dev/null
495
607
  }
608
+ last_streak(){ # the failStreak of last.json as a positive integer, or nothing
609
+ node -e 'try { const n = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).failStreak; if (Number.isInteger(n) && n > 0) process.stdout.write(String(n)); } catch {}' "$LAST_JSON" 2>/dev/null
610
+ }
496
611
  # The version the RUNNING daemon started on. scripts/daemon/health.mjs resolves
497
612
  # sdk_version once at load and writes it to state/dashboards/daemon-health.yaml
498
613
  # with its pid, precisely so a reader can tell the installed package from the
@@ -595,13 +710,13 @@ if [ "$CUR" = "$LATEST" ] || [ "$(printf '%s\n%s\n' "$CUR" "$LATEST" | sort -V |
595
710
  # the notice tells the session to restart itself onto the new code.
596
711
  log "OK: daemon now on $CUR (was $STALE_FROM) — the manual upgrade is complete"
597
712
  write_upgrade_notice "$STALE_FROM" "$CUR"
598
- write_last "$STALE_FROM" "$CUR" true true ""
713
+ write_last "$STALE_FROM" "$CUR" true true "" true
599
714
  elif [ -n "$STALE_FROM" ]; then
600
715
  # Same version, newer files: the daemon is reconciled; the session is
601
716
  # not told to restart (a touched or restored file must not cost it its
602
717
  # context — the version did not change).
603
718
  log "OK: daemon restarted onto the installed $CUR code; no session notice (same version)"
604
- write_last "$CUR" "$CUR" true true ""
719
+ write_last "$CUR" "$CUR" true true "" true
605
720
  elif [ -n "$AHEAD" ]; then
606
721
  # The DAEMON is well; the FLEET is not. Recorded as ok/healthy — because
607
722
  # it is — with the finding in `reason`, so the org learns that this seat
@@ -609,14 +724,32 @@ if [ "$CUR" = "$LATEST" ] || [ "$(printf '%s\n%s\n' "$CUR" "$LATEST" | sort -V |
609
724
  # per episode: unlike an unhealthy daemon this does not clear itself, and
610
725
  # a fact that ages out of one log file is how six days went by.
611
726
  log "OK: healthy on $CUR, but AHEAD of the registry ($LATEST) — recorded for the org as ahead-of-registry"
612
- write_last "$CUR" "$CUR" true true "ahead-of-registry: npm latest is $LATEST"
727
+ write_last "$CUR" "$CUR" true true "ahead-of-registry: npm latest is $LATEST" true
613
728
  else
614
- case "$PREV_REASON" in unhealthy-current*)
729
+ case "$PREV_REASON" in
730
+ unhealthy-current*)
615
731
  log "recovered: last.json no longer reports unhealthy-current"
616
- write_last "$CUR" "$CUR" true true "" ;;
732
+ write_last "$CUR" "$CUR" true true "" true ;;
617
733
  ahead-of-registry*)
618
734
  log "recovered: $CUR is on the registry now (latest $LATEST) — last.json no longer reports ahead-of-registry"
619
- write_last "$CUR" "$CUR" true true "" ;;
735
+ write_last "$CUR" "$CUR" true true "" true ;;
736
+ *)
737
+ # A healthy seat that is simply up to date writes NOTHING — there is
738
+ # nothing to report and an empty last.json is the honest state.
739
+ #
740
+ # The one exception is a standing failStreak, and it is the reason this
741
+ # branch exists at all. A streak only ever reset on ARRIVAL, i.e. on a
742
+ # successful hop. If a bad release is YANKED and @latest rolls back to
743
+ # the version this seat already runs, the seat stops attempting
744
+ # anything: every hour it takes this quiet path, writes nothing, and the
745
+ # count stands for ever — `upgrade_stuck` firing critical on a seat that
746
+ # is, in fact, exactly where the registry wants it. Being AT @latest is
747
+ # an arrival however you got there, so record one and say so.
748
+ STREAK="$(last_streak)"
749
+ if [ -n "$STREAK" ]; then
750
+ log "streak cleared: $CUR is @latest and this seat is healthy on it, but last.json still carried failStreak=$STREAK — @latest has come back to this version (a yank or a withdrawn release). Recording the arrival so the upgrade_stuck alert stops."
751
+ write_last "$CUR" "$CUR" true true "" true
752
+ fi ;;
620
753
  esac
621
754
  fi
622
755
  else
@@ -625,7 +758,7 @@ if [ "$CUR" = "$LATEST" ] || [ "$(printf '%s\n%s\n' "$CUR" "$LATEST" | sort -V |
625
758
  # and failed_hold_reason both match on it), so the ahead-of-registry fact
626
759
  # is appended, never prepended — an unpublished build that is also sick is
627
760
  # the worst case and must say both things.
628
- write_last "$CUR" "$CUR" false false "unhealthy-current: $HEALTH_REASON${AHEAD:+ (and ahead-of-registry: npm latest is $LATEST)}"
761
+ write_last "$CUR" "$CUR" false false "unhealthy-current: $HEALTH_REASON${AHEAD:+ (and ahead-of-registry: npm latest is $LATEST)}" true
629
762
  case "$PREV_REASON" in
630
763
  unhealthy-current*) log "session already notified this episode (last.json was unhealthy-current); notice not rewritten" ;;
631
764
  *) write_notice "$CUR" "$CUR" "unhealthy-current: $HEALTH_REASON" ;;