@cohortapp/agent-sdk 2.18.14 → 2.18.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/runbooks/fleet-rollout.md +45 -1
- package/lib/assurance/batch.mjs +353 -0
- package/lib/assurance/first-reply.mjs +423 -0
- package/lib/assurance/notice-voice.mjs +357 -0
- package/lib/assurance/plan-note.mjs +43 -0
- package/lib/assurance/room-budget.mjs +55 -6
- package/lib/comms/send-gate.mjs +59 -0
- package/lib/identity/persona.mjs +31 -2
- package/lib/org/inbound/hydrate.mjs +35 -1
- package/lib/org/inbound/project.mjs +3 -0
- package/lib/session/frontdoor.mjs +87 -8
- package/lib/session/handoffs.mjs +57 -0
- package/lib/session/inbox-claims.mjs +47 -2
- package/lib/session/revive.mjs +302 -5
- package/lib/telemetry/alerts.mjs +94 -0
- package/lib/telemetry/collect.mjs +250 -2
- package/package.json +1 -1
- package/scripts/daemon/agent-daemon.mjs +317 -14
- package/scripts/daemon/assurance.mjs +765 -45
- package/scripts/daemon/deliver.mjs +109 -0
- package/scripts/daemon/dispatcher.mjs +21 -3
- package/scripts/daemon/inbox-deferral.mjs +102 -9
- package/scripts/daemon/session-lock.mjs +41 -1
- package/scripts/fleet/rollout.mjs +292 -9
- package/scripts/hooks/pre-write-yaml-validate.mjs +63 -2
- package/scripts/local-triggers/autoupdate.sh +338 -20
|
@@ -102,6 +102,7 @@ import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
|
102
102
|
import { spawnSync } from "node:child_process";
|
|
103
103
|
import { dirname, join, resolve as resolvePath } from "node:path";
|
|
104
104
|
import { fileURLToPath } from "node:url";
|
|
105
|
+
import { DEFAULT_THRESHOLDS } from "../../lib/telemetry/alerts.mjs";
|
|
105
106
|
|
|
106
107
|
const __filename = fileURLToPath(import.meta.url);
|
|
107
108
|
const REPO_ROOT = resolvePath(dirname(__filename), "..", "..");
|
|
@@ -394,6 +395,130 @@ export function seatVersions(entry) {
|
|
|
394
395
|
return { installed, running, version, basis: version === null ? null : running ? "running" : "installed" };
|
|
395
396
|
}
|
|
396
397
|
|
|
398
|
+
/**
|
|
399
|
+
* How long a failed upgrade attempt may stand, unrepeated, before a seat that
|
|
400
|
+
* is still on the version it failed FROM counts as stuck rather than merely
|
|
401
|
+
* recovering. Two full failed-target holds (the autoupdater's default is 24 h):
|
|
402
|
+
* one hold is the seat being deliberately left alone, two means the retry it
|
|
403
|
+
* was owed has been and gone.
|
|
404
|
+
*/
|
|
405
|
+
export const ATTEMPT_STALE_MS = 48 * 60 * 60 * 1000;
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Consecutive failed attempts that make a seat stuck.
|
|
409
|
+
*
|
|
410
|
+
* DERIVED, NOT RESTATED. This used to be `= 3` with a comment promising it
|
|
411
|
+
* "matches lib/telemetry/alerts.mjs upgradeStuck.warnStreak" — a prose promise
|
|
412
|
+
* about a literal, which is the exact shape the repo doctrine bans and which no
|
|
413
|
+
* test could have caught: this file's own tests use the constant symbolically
|
|
414
|
+
* (`at(STUCK_STREAK - 1)` / `at(STUCK_STREAK)`), so they pass at ANY value and
|
|
415
|
+
* the two copies could have drifted silently. The seat-side alert rule is the
|
|
416
|
+
* authority — it is what actually fires on a live beat — so this reads it.
|
|
417
|
+
*
|
|
418
|
+
* The third copy lives in the shell (`STUCK_STREAK` in
|
|
419
|
+
* scripts/local-triggers/autoupdate.sh), which cannot import this; it is pinned
|
|
420
|
+
* against this value by a source-reading test instead
|
|
421
|
+
* (scripts/local-triggers/autoupdate.test.mjs, "source: the shell's streak
|
|
422
|
+
* threshold is the same number the alert fires on").
|
|
423
|
+
*/
|
|
424
|
+
export const STUCK_STREAK = DEFAULT_THRESHOLDS.upgradeStuck.warnStreak;
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* The first SDK release whose `write_last` keeps a `failStreak` in
|
|
428
|
+
* state/autoupdate/last.json. A seat below this reports one upgrade outcome and
|
|
429
|
+
* no count — not because its beat is stale, but because the collector that
|
|
430
|
+
* would write the count is the code it has not got.
|
|
431
|
+
*
|
|
432
|
+
* This exists because the "attempt" basis used to explain itself with "this
|
|
433
|
+
* seat's beat is too old to carry a count", attached unconditionally to every
|
|
434
|
+
* attempt-basis row. At rollout time NO released SDK wrote `failStreak`, so
|
|
435
|
+
* every seat in the fleet — including ones beating every 30 seconds with
|
|
436
|
+
* `reason` and `healthy` on the record — was told its beat was too old. The
|
|
437
|
+
* beat age was never the reason; the seat's VERSION is.
|
|
438
|
+
*/
|
|
439
|
+
export const FAILSTREAK_SINCE = "2.18.15";
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* IS THIS SEAT STUCK — i.e. has it tried to reach @latest and failed, and will
|
|
443
|
+
* it keep failing?
|
|
444
|
+
*
|
|
445
|
+
* `classifySeat` says a seat is `stale`. That is a snapshot and it is not the
|
|
446
|
+
* finding: a seat is stale for twenty minutes after every publish, and three
|
|
447
|
+
* seats were stale for three weeks. What separates them is whether the seat has
|
|
448
|
+
* ATTEMPTED the hop and failed, and how many times — and that fact has always
|
|
449
|
+
* been in the beat (`machine.upgrade`) without anything reading it.
|
|
450
|
+
*
|
|
451
|
+
* TWO BASES, BECAUSE THE SEATS THIS MATTERS FOR REPORT THE LEAST. A seat that
|
|
452
|
+
* has been left behind for weeks is by definition running the oldest collector
|
|
453
|
+
* in the fleet, so it emits the smallest beat: on 2026-09-25 the three seats on
|
|
454
|
+
* 2.17.0 were the only three in the org with no `machine.daemon` block and no
|
|
455
|
+
* `machine.stranded` — staleness hides itself by removing the fields that would
|
|
456
|
+
* announce it. So this reads what a 2.17-era beat DOES carry, and names which
|
|
457
|
+
* basis it used rather than pretending to one answer:
|
|
458
|
+
*
|
|
459
|
+
* basis "streak" — the seat counted its own consecutive failures
|
|
460
|
+
* (`machine.upgrade.failStreak`, SDK {@link FAILSTREAK_SINCE}
|
|
461
|
+
* and later). Stuck at {@link STUCK_STREAK}. The real signal.
|
|
462
|
+
* basis "attempt" — an old beat, which carries one outcome and no count. A
|
|
463
|
+
* failed attempt (`ok:false`, `to` != `from`) with the seat
|
|
464
|
+
* STILL reporting `from` proves the hop did not take; older
|
|
465
|
+
* than {@link ATTEMPT_STALE_MS} it also proves the retry it
|
|
466
|
+
* was owed came and went. Below that age it is `failing`,
|
|
467
|
+
* not `stuck` — which is the same rule the streak obeys and
|
|
468
|
+
* the same rule (d) asks for: a seat that has not been
|
|
469
|
+
* ASKED again is not a stuck seat.
|
|
470
|
+
*
|
|
471
|
+
* Returns null when the beat carries no upgrade record at all: never tried is
|
|
472
|
+
* not the same as tried and failed, and must not print as one.
|
|
473
|
+
*
|
|
474
|
+
* Pure; `now` is a parameter because the "attempt" basis is about a duration.
|
|
475
|
+
*
|
|
476
|
+
* @param {object} entry a directory member, or anything {@link seatVersions} accepts
|
|
477
|
+
* @param {number} [now] epoch ms
|
|
478
|
+
* @returns {{verdict:"stuck"|"failing", basis:"streak"|"attempt", failStreak:number|null,
|
|
479
|
+
* target:string|null, since:string|null, ageMs:number|null}|null}
|
|
480
|
+
*/
|
|
481
|
+
export function seatStuck(entry, now = Date.now()) {
|
|
482
|
+
const e = entry && typeof entry === "object" ? entry : {};
|
|
483
|
+
const st = (e.status && typeof e.status === "object" && e.status) || {};
|
|
484
|
+
const mach = (st.machine && typeof st.machine === "object" && st.machine) || (e.machine && typeof e.machine === "object" && e.machine) || {};
|
|
485
|
+
const u = mach.upgrade && typeof mach.upgrade === "object" ? mach.upgrade : null;
|
|
486
|
+
if (!u) return null;
|
|
487
|
+
|
|
488
|
+
const streak = Number.isInteger(u.failStreak) && u.failStreak > 0 ? u.failStreak : null;
|
|
489
|
+
const target = firstString([u.streakTarget, u.to]);
|
|
490
|
+
if (streak !== null) {
|
|
491
|
+
const sinceS = firstString([u.stuckSince, u.at]);
|
|
492
|
+
const sinceMs = sinceS ? Date.parse(sinceS) : NaN;
|
|
493
|
+
return {
|
|
494
|
+
verdict: streak >= STUCK_STREAK ? "stuck" : "failing",
|
|
495
|
+
basis: "streak",
|
|
496
|
+
failStreak: streak,
|
|
497
|
+
target,
|
|
498
|
+
since: sinceS,
|
|
499
|
+
ageMs: Number.isFinite(sinceMs) ? now - sinceMs : null,
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// Degraded read: one outcome, no count.
|
|
504
|
+
if (u.ok !== false) return null;
|
|
505
|
+
const from = typeof u.from === "string" ? u.from : "";
|
|
506
|
+
const to = typeof u.to === "string" ? u.to : "";
|
|
507
|
+
if (!from || !to || from === to) return null; // not an upgrade attempt
|
|
508
|
+
const installed = seatVersions(e).installed;
|
|
509
|
+
if (installed && installed !== from) return null; // it has moved since; whatever failed is history
|
|
510
|
+
const atMs = typeof u.at === "string" ? Date.parse(u.at) : NaN;
|
|
511
|
+
const ageMs = Number.isFinite(atMs) ? now - atMs : null;
|
|
512
|
+
return {
|
|
513
|
+
verdict: ageMs !== null && ageMs > ATTEMPT_STALE_MS ? "stuck" : "failing",
|
|
514
|
+
basis: "attempt",
|
|
515
|
+
failStreak: null,
|
|
516
|
+
target: to,
|
|
517
|
+
since: typeof u.at === "string" ? u.at : null,
|
|
518
|
+
ageMs,
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
|
|
397
522
|
/**
|
|
398
523
|
* The single version a seat is judged on, or null when the read carries none.
|
|
399
524
|
* Kept as its own export because it is the whole of what `classifySeat` needs;
|
|
@@ -420,8 +545,17 @@ export function seatVersion(entry) {
|
|
|
420
545
|
* cannot see, and it would be wrong on whichever side of the hq deploy it was
|
|
421
546
|
* not written for.
|
|
422
547
|
*
|
|
548
|
+
* Each row also carries `wedged` (bool) and `wedgeReason` (string|null),
|
|
549
|
+
* derived from the same `machine` block {@link seatStuck} reads: a seat is
|
|
550
|
+
* wedged when the beat itself reports `machine.wedge.reason` (published by
|
|
551
|
+
* collect.mjs's sessionWedge — beating, but not reading its inbox/handoffs),
|
|
552
|
+
* OR when `machine.frontDoor === "session"` and `machine.sessionLive === false`
|
|
553
|
+
* (the front door is shut outright). The version stays on the row either way —
|
|
554
|
+
* wedged is a verdict {@link classifySeat} applies, not a fact this function
|
|
555
|
+
* withholds.
|
|
556
|
+
*
|
|
423
557
|
* @param {object[]} members
|
|
424
|
-
* @param {{seatWindowMs?:number}} [o]
|
|
558
|
+
* @param {{seatWindowMs?:number, now?:number}} [o]
|
|
425
559
|
* @returns {{seats:object[], nonSeats:object[], readCarriesVersionField:boolean}}
|
|
426
560
|
*/
|
|
427
561
|
export function toSeats(members, o = {}) {
|
|
@@ -438,6 +572,10 @@ export function toSeats(members, o = {}) {
|
|
|
438
572
|
}
|
|
439
573
|
const p = m.presence && typeof m.presence === "object" ? m.presence : null;
|
|
440
574
|
const beatMs = p && Number.isFinite(p.lastBeatMs) ? p.lastBeatMs : null;
|
|
575
|
+
// Same accessor seatStuck uses: the beat's machine block, wherever it hangs.
|
|
576
|
+
const mach = (m.status && typeof m.status === "object" && m.status.machine) || m.machine || {};
|
|
577
|
+
const wedge = mach.wedge && typeof mach.wedge === "object" ? mach.wedge : null;
|
|
578
|
+
const doorDown = mach.frontDoor === "session" && mach.sessionLive === false;
|
|
441
579
|
const row = {
|
|
442
580
|
id: String(m.id || ""),
|
|
443
581
|
slug: String(m.slug || ""),
|
|
@@ -446,6 +584,9 @@ export function toSeats(members, o = {}) {
|
|
|
446
584
|
detail: (p && p.detail) || "",
|
|
447
585
|
lastBeatMs: beatMs,
|
|
448
586
|
...seatVersions(m),
|
|
587
|
+
stuck: seatStuck(m, o.now),
|
|
588
|
+
wedged: !!((wedge && wedge.reason) || doorDown),
|
|
589
|
+
wedgeReason: (wedge && typeof wedge.reason === "string" && wedge.reason) || (doorDown ? "front-door-not-live" : null),
|
|
449
590
|
};
|
|
450
591
|
if (beatMs !== null && beatMs <= windowMs) seats.push(row);
|
|
451
592
|
else nonSeats.push(row);
|
|
@@ -456,14 +597,18 @@ export function toSeats(members, o = {}) {
|
|
|
456
597
|
|
|
457
598
|
/**
|
|
458
599
|
* Verdict for one seat against the target version. Pure.
|
|
600
|
+
* wedged — beating, but its front door is not answering (a shut
|
|
601
|
+
* session or an overdue handoff); outranks current/stale —
|
|
602
|
+
* the version stays on the row for display, unread here
|
|
459
603
|
* current — reported version equals the target
|
|
460
604
|
* stale — reported a different (older or newer) version
|
|
461
605
|
* unverifiable — beating, but the read carries no version at all
|
|
462
606
|
* @param {object} seat
|
|
463
607
|
* @param {string} target
|
|
464
|
-
* @returns {"current"|"stale"|"unverifiable"}
|
|
608
|
+
* @returns {"wedged"|"current"|"stale"|"unverifiable"}
|
|
465
609
|
*/
|
|
466
610
|
export function classifySeat(seat, target) {
|
|
611
|
+
if (seat && seat.wedged) return "wedged";
|
|
467
612
|
const v = seat && seat.version;
|
|
468
613
|
if (!v) return "unverifiable";
|
|
469
614
|
return compareVersions(v, target) === 0 ? "current" : "stale";
|
|
@@ -478,17 +623,43 @@ export function classifySeat(seat, target) {
|
|
|
478
623
|
*/
|
|
479
624
|
export function summarise(fleet, target) {
|
|
480
625
|
const rows = (fleet.seats || []).map((s) => ({ ...s, verdict: classifySeat(s, target) }));
|
|
481
|
-
const counts = { current: 0, stale: 0, unverifiable: 0 };
|
|
626
|
+
const counts = { current: 0, stale: 0, unverifiable: 0, wedged: 0 };
|
|
482
627
|
for (const r of rows) counts[r.verdict] += 1;
|
|
628
|
+
// Behind AND proved it cannot arrive — a strictly stronger statement than
|
|
629
|
+
// `stale`, which clears itself on the next hourly run.
|
|
630
|
+
//
|
|
631
|
+
// `stuck` IS NOT A SUBSET OF `stale`, and the code used to assume it was. A
|
|
632
|
+
// seat's verdict is about THIS run's target; its streak is about whatever
|
|
633
|
+
// `@latest` was when it last tried, and the two are different versions
|
|
634
|
+
// whenever a rollout is verified against anything but the registry's head —
|
|
635
|
+
// `--dry-run`, `--verify-only`, or simply a publish that has since been
|
|
636
|
+
// followed by another. Demonstrated: one seat on target 2.18.14 whose record
|
|
637
|
+
// is {ok:false, from:"2.18.14", to:"2.18.15", failStreak:5} classifies
|
|
638
|
+
// `current`, so `stale` is empty, `done` was true, and renderVerdict's early
|
|
639
|
+
// return printed "All 1 seat(s) report 2.18.14." — five consecutive failed
|
|
640
|
+
// upgrades reported as a fully current fleet. So the two sets are kept and
|
|
641
|
+
// named apart, and `done` answers BOTH questions: every seat is on the target
|
|
642
|
+
// AND no seat is known to be unable to move.
|
|
643
|
+
const stuck = rows.filter((r) => r.stuck && r.stuck.verdict === "stuck");
|
|
644
|
+
const stale = rows.filter((r) => r.verdict === "stale");
|
|
645
|
+
const wedged = rows.filter((r) => r.verdict === "wedged");
|
|
646
|
+
const staleSlugs = new Set(stale.map((r) => r.slug));
|
|
483
647
|
return {
|
|
484
648
|
target,
|
|
485
649
|
rows,
|
|
486
650
|
counts,
|
|
487
651
|
nonSeats: fleet.nonSeats || [],
|
|
488
|
-
stale
|
|
652
|
+
stale,
|
|
489
653
|
unverifiable: rows.filter((r) => r.verdict === "unverifiable"),
|
|
654
|
+
stuck,
|
|
655
|
+
/** The stuck seats that are ALSO behind this run's target — "N of those". */
|
|
656
|
+
stuckBehind: stuck.filter((r) => staleSlugs.has(r.slug)),
|
|
657
|
+
/** Stuck against some other version while current on this one. Still a finding. */
|
|
658
|
+
stuckElsewhere: stuck.filter((r) => !staleSlugs.has(r.slug)),
|
|
659
|
+
/** Beating, but its front door is not answering — a 4th verdict beside stale. */
|
|
660
|
+
wedged,
|
|
490
661
|
readCarriesVersionField: fleet.readCarriesVersionField === true,
|
|
491
|
-
done: rows.length > 0 && counts.current === rows.length,
|
|
662
|
+
done: rows.length > 0 && counts.current === rows.length && stuck.length === 0 && wedged.length === 0,
|
|
492
663
|
};
|
|
493
664
|
}
|
|
494
665
|
|
|
@@ -518,7 +689,7 @@ export function renderSeatTable(summary, o = {}) {
|
|
|
518
689
|
r.slug,
|
|
519
690
|
ageLabel(r.lastBeatMs),
|
|
520
691
|
r.version ? (r.basis === "installed" ? `${r.version} (installed)` : r.version) : "unknown",
|
|
521
|
-
r.verdict,
|
|
692
|
+
r.stuck ? `${r.verdict} · ${r.stuck.verdict}${r.stuck.failStreak ? ` ×${r.stuck.failStreak}` : ""}` : r.verdict,
|
|
522
693
|
]);
|
|
523
694
|
const widths = head.map((h, i) => Math.max(h.length, ...body.map((row) => row[i].length), 0));
|
|
524
695
|
const line = (cells) => cells.map((c, i) => c.padEnd(widths[i])).join(" ").trimEnd();
|
|
@@ -538,9 +709,19 @@ export function renderSeatTable(summary, o = {}) {
|
|
|
538
709
|
return out.join("\n");
|
|
539
710
|
}
|
|
540
711
|
|
|
541
|
-
/**
|
|
712
|
+
/**
|
|
713
|
+
* The closing sentence naming exactly which seats are behind, and which of them
|
|
714
|
+
* will never arrive unaided. Pure.
|
|
715
|
+
*
|
|
716
|
+
* NO EARLY RETURN ON `done`. It used to open with
|
|
717
|
+
* `if (summary.done) return "All N seat(s) report X."`, which swallowed the
|
|
718
|
+
* STUCK line whole for any seat that was stuck against a version other than
|
|
719
|
+
* this run's target — see the note in {@link summarise}. `done` now accounts
|
|
720
|
+
* for stuck seats, and the "all clear" sentence is built at the END, from the
|
|
721
|
+
* absence of every finding, so a future finding cannot be skipped past by a
|
|
722
|
+
* `done` that did not learn about it.
|
|
723
|
+
*/
|
|
542
724
|
export function renderVerdict(summary) {
|
|
543
|
-
if (summary.done) return `All ${summary.rows.length} seat(s) report ${summary.target}.`;
|
|
544
725
|
const parts = [];
|
|
545
726
|
if (summary.stale.length) {
|
|
546
727
|
parts.push(
|
|
@@ -548,6 +729,27 @@ export function renderVerdict(summary) {
|
|
|
548
729
|
summary.stale.map((r) => `${r.name} (${r.slug}, reports ${r.version})`).join(", ")
|
|
549
730
|
);
|
|
550
731
|
}
|
|
732
|
+
const stuck = (summary.stuck && summary.stuck.length ? summary.stuck : []);
|
|
733
|
+
if (stuck.length) {
|
|
734
|
+
// Said SEPARATELY from "behind", and after it, because the response is
|
|
735
|
+
// different: a behind seat is waited on, a stuck one is visited. The line
|
|
736
|
+
// that was missing for three weeks is this one.
|
|
737
|
+
const behind = summary.stuckBehind || stuck.filter((r) => r.verdict === "stale");
|
|
738
|
+
const elsewhere = summary.stuckElsewhere || stuck.filter((r) => r.verdict !== "stale");
|
|
739
|
+
// "N of those" is only true of the ones that ARE among the seats just named
|
|
740
|
+
// as behind. A seat that is current on this target and stuck against a
|
|
741
|
+
// later one is not "one of those" and must not be counted as if it were.
|
|
742
|
+
const head =
|
|
743
|
+
elsewhere.length === 0
|
|
744
|
+
? `${behind.length} of those ${behind.length === 1 ? "is" : "are"} STUCK`
|
|
745
|
+
: behind.length === 0
|
|
746
|
+
? `${elsewhere.length} seat(s) ${elsewhere.length === 1 ? "is" : "are"} STUCK — on ${summary.target}, but unable to move off it`
|
|
747
|
+
: `${stuck.length} seat(s) are STUCK (${behind.length} of those just named as behind, ${elsewhere.length} current on ${summary.target} but unable to move off it)`;
|
|
748
|
+
parts.push(
|
|
749
|
+
`${head} — attempted and failed, and will not arrive unaided: ` +
|
|
750
|
+
stuck.map((r) => `${r.name} (${r.slug}, ${stuckClause(r, summary.target)})`).join(", ")
|
|
751
|
+
);
|
|
752
|
+
}
|
|
551
753
|
if (summary.unverifiable.length) {
|
|
552
754
|
parts.push(
|
|
553
755
|
`${summary.unverifiable.length} seat(s) could not be verified — the org read carries no sdkVersion: ` +
|
|
@@ -555,9 +757,46 @@ export function renderVerdict(summary) {
|
|
|
555
757
|
);
|
|
556
758
|
}
|
|
557
759
|
if (!summary.rows.length) parts.push("no beating seats found in the org directory — nothing to verify.");
|
|
760
|
+
if (!parts.length) return `All ${summary.rows.length} seat(s) report ${summary.target}.`;
|
|
558
761
|
return parts.join("\n");
|
|
559
762
|
}
|
|
560
763
|
|
|
764
|
+
/**
|
|
765
|
+
* One stuck seat's parenthetical: what it tried, how many times, and — only
|
|
766
|
+
* when it is true — why it cannot say how many times.
|
|
767
|
+
*
|
|
768
|
+
* THE "no count" SENTENCE IS GATED ON THE SEAT'S VERSION, NOT ON THE BRANCH
|
|
769
|
+
* THAT PRODUCED THE VERDICT. It used to hang off `basis === "attempt"` and
|
|
770
|
+
* read "this seat's beat is too old to carry a count". Every attempt-basis row
|
|
771
|
+
* got it, which at the time this landed was every row in the fleet — no
|
|
772
|
+
* released SDK wrote `failStreak` yet — including seats beating every thirty
|
|
773
|
+
* seconds whose records demonstrably carried `reason` and `healthy`. The beat
|
|
774
|
+
* was never old; the seat's SDK simply predates {@link FAILSTREAK_SINCE}. When
|
|
775
|
+
* the seat is new enough to count and still did not, that is a DIFFERENT and
|
|
776
|
+
* more interesting fact, so it gets its own words. Pure.
|
|
777
|
+
*
|
|
778
|
+
* @param {object} row a summarise() row with a non-null `stuck`
|
|
779
|
+
* @param {string} target
|
|
780
|
+
* @returns {string}
|
|
781
|
+
*/
|
|
782
|
+
export function stuckClause(row, target) {
|
|
783
|
+
const st = row.stuck;
|
|
784
|
+
const attempts = st.failStreak ? `${st.failStreak} consecutive failed attempts` : "a failed attempt";
|
|
785
|
+
const oldest = Number.isFinite(st.ageMs) ? `, oldest ${ageLabel(st.ageMs)} ago` : "";
|
|
786
|
+
let why = "";
|
|
787
|
+
if (st.basis === "attempt") {
|
|
788
|
+
const v = row.version;
|
|
789
|
+
if (v && compareVersions(v, FAILSTREAK_SINCE) < 0) {
|
|
790
|
+
why = `; it reports ${v}, which predates the failure count (SDK ${FAILSTREAK_SINCE}), so one attempt is all it can report`;
|
|
791
|
+
} else if (v) {
|
|
792
|
+
why = `; it reports ${v}, new enough to count its failures, but this record carries none — read state/autoupdate/last.json on the machine`;
|
|
793
|
+
} else {
|
|
794
|
+
why = "; this seat reports no version, so there is no telling whether it could have counted";
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
return `${attempts} to ${st.target || target}${oldest}${why}`;
|
|
798
|
+
}
|
|
799
|
+
|
|
561
800
|
/**
|
|
562
801
|
* What stage 3's exit code will be, and WHY — as data, so a caller can tell the
|
|
563
802
|
* three very different things a `3` means apart.
|
|
@@ -576,10 +815,18 @@ export function renderVerdict(summary) {
|
|
|
576
815
|
* exit code: `behind` and `unverifiable` are different problems with different
|
|
577
816
|
* fixes, and only the first is answered by waiting.
|
|
578
817
|
*
|
|
818
|
+
* `wedged` is a fourth problem with a fourth fix: the seat is beating (so it is
|
|
819
|
+
* not `unverifiable`, and waiting for its next beat proves nothing) but its
|
|
820
|
+
* front door is not answering — a shut session or an overdue handoff. It is
|
|
821
|
+
* checked after `stuck` (stuck is the longer-standing, already-acted-on
|
|
822
|
+
* signal) and before `empty-fleet`/`unverifiable`/`behind`: reporting a wedged
|
|
823
|
+
* seat as merely "behind" would invite waiting it out, and waiting does
|
|
824
|
+
* nothing for a seat that is not reading.
|
|
825
|
+
*
|
|
579
826
|
* Pure.
|
|
580
827
|
*
|
|
581
828
|
* @param {ReturnType<typeof summarise>|null|undefined} summary
|
|
582
|
-
* @returns {{code:number, reason:"verified"|"behind"|"unverifiable"|"empty-fleet"|"no-fleet-read", note:string|null}}
|
|
829
|
+
* @returns {{code:number, reason:"verified"|"stuck"|"wedged"|"behind"|"unverifiable"|"empty-fleet"|"no-fleet-read", note:string|null}}
|
|
583
830
|
*/
|
|
584
831
|
export function propagationOutcome(summary) {
|
|
585
832
|
if (!summary || !Array.isArray(summary.rows)) {
|
|
@@ -590,6 +837,42 @@ export function propagationOutcome(summary) {
|
|
|
590
837
|
};
|
|
591
838
|
}
|
|
592
839
|
if (summary.done) return { code: 0, reason: "verified", note: null };
|
|
840
|
+
// STUCK OUTRANKS BEHIND, and is checked before both `empty-fleet` and the
|
|
841
|
+
// unverifiable branch, because it is the only one of the three that a person
|
|
842
|
+
// must ACT on rather than wait out.
|
|
843
|
+
//
|
|
844
|
+
// This is also the automation the seat-side alert cannot provide. `upgrade_stuck`
|
|
845
|
+
// is derived by lib/telemetry/alerts.mjs running ON the seat, so a seat too
|
|
846
|
+
// stale to install the SDK never emits it — by construction the alert cannot
|
|
847
|
+
// reach the seats it is for. What CAN reach them is this: the fleet-side
|
|
848
|
+
// derivation runs on every publish, because stage 3 runs on every publish,
|
|
849
|
+
// and a stuck seat now makes the release command exit non-zero and name the
|
|
850
|
+
// machine. Nobody has to decide to go looking.
|
|
851
|
+
if (summary.stuck && summary.stuck.length) {
|
|
852
|
+
return {
|
|
853
|
+
code: 3,
|
|
854
|
+
reason: "stuck",
|
|
855
|
+
note:
|
|
856
|
+
`${summary.stuck.length} seat(s) have attempted @latest and failed repeatedly. They will not arrive on their own: ` +
|
|
857
|
+
`each needs a person at the machine (maestro doctor; state/autoupdate/last.json carries the reason). ` +
|
|
858
|
+
`Stage 3 still polls to its deadline in case someone clears one while it runs, but it will not clear itself. ` +
|
|
859
|
+
renderVerdict(summary),
|
|
860
|
+
};
|
|
861
|
+
}
|
|
862
|
+
// WEDGED comes after stuck (stuck is longer-standing and still the one a
|
|
863
|
+
// person must act on first) but before empty-fleet/unverifiable/behind: a
|
|
864
|
+
// seat that is beating but not reading is not a seat anyone can wait out
|
|
865
|
+
// either, and it must not be reported as merely "behind" — waiting changes
|
|
866
|
+
// nothing when the front door itself is what is not answering.
|
|
867
|
+
if (summary.wedged && summary.wedged.length) {
|
|
868
|
+
return {
|
|
869
|
+
code: 3,
|
|
870
|
+
reason: "wedged",
|
|
871
|
+
note:
|
|
872
|
+
`${summary.wedged.length} seat(s) are beating but their front door is not answering (wedged) — the ` +
|
|
873
|
+
`daemon revive or a person must clear them; a version cannot be verified on a seat that is not reading.`,
|
|
874
|
+
};
|
|
875
|
+
}
|
|
593
876
|
if (!summary.rows.length) {
|
|
594
877
|
return {
|
|
595
878
|
code: 3,
|
|
@@ -17,7 +17,14 @@
|
|
|
17
17
|
* old_string replaced by new_string) and REFUSES — exit 2, a permissionDecision:"deny" JSON on
|
|
18
18
|
* stdout and the reason on stderr — when:
|
|
19
19
|
* - an Edit's old_string is absent from the file, or matches more than once without
|
|
20
|
-
* replace_all (the edit would land somewhere the writer did not look at);
|
|
20
|
+
* replace_all (the edit would land somewhere the writer did not look at); when that repeated
|
|
21
|
+
* match is an `id:` line, the refusal names the real cause — the file already carries a
|
|
22
|
+
* duplicate id and is unwritable via id-anchored edits until the collision is resolved;
|
|
23
|
+
* - the would-be top-level `items:` list (or a top-level sequence) would carry two rows sharing
|
|
24
|
+
* the same `id` value that was not already colliding on disk. A duplicate id slips past js-yaml
|
|
25
|
+
* (the rows are distinct list entries, not duplicate map keys) but makes every id-anchored edit
|
|
26
|
+
* ambiguous — the "collision then silently unwritable all day" failure — so it is refused loudly
|
|
27
|
+
* as a WHOLE FILE BLOCK at the write boundary;
|
|
21
28
|
* - the result does not parse as YAML (js-yaml, which also rejects duplicate keys);
|
|
22
29
|
* - a provenance-keyed value NEW in this write ends on a dangling connector. The delta is
|
|
23
30
|
* judged, not the whole file, so a pre-existing truncated row never blocks an unrelated
|
|
@@ -108,6 +115,39 @@ export function guardedRowCount(doc) {
|
|
|
108
115
|
return null;
|
|
109
116
|
}
|
|
110
117
|
|
|
118
|
+
/** The top-level list the id-uniqueness guard protects: `items:` or a top-level sequence, else null. */
|
|
119
|
+
function guardedList(doc) {
|
|
120
|
+
if (Array.isArray(doc)) return doc;
|
|
121
|
+
if (doc && typeof doc === "object" && Array.isArray(doc.items)) return doc.items;
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* First `id` value that appears on two or more rows of the guarded top-level list, else null.
|
|
127
|
+
* A duplicate id makes every id-anchored edit ambiguous, so js-yaml lets it through (the rows are
|
|
128
|
+
* distinct list entries, not duplicate map keys) but the file becomes unwritable via id-anchored edits.
|
|
129
|
+
*/
|
|
130
|
+
export function duplicateItemId(doc) {
|
|
131
|
+
const list = guardedList(doc);
|
|
132
|
+
if (!list) return null;
|
|
133
|
+
const seen = new Set();
|
|
134
|
+
for (const row of list) {
|
|
135
|
+
if (!row || typeof row !== "object") continue;
|
|
136
|
+
const id = row.id;
|
|
137
|
+
if (typeof id !== "string" && typeof id !== "number") continue;
|
|
138
|
+
const key = String(id);
|
|
139
|
+
if (seen.has(key)) return key;
|
|
140
|
+
seen.add(key);
|
|
141
|
+
}
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** If `oldStr` is anchored on an `id:` line, return the id value; else null. */
|
|
146
|
+
function repeatedIdAnchor(oldStr) {
|
|
147
|
+
const m = /(?:^|\n)\s*(?:-\s+)?id:\s*(.+?)\s*(?:\n|$)/.exec(oldStr);
|
|
148
|
+
return m ? m[1].replace(/^["']|["']$/g, "") : null;
|
|
149
|
+
}
|
|
150
|
+
|
|
111
151
|
/** Count non-overlapping occurrences of `needle` in `hay`. */
|
|
112
152
|
function countOccurrences(hay, needle) {
|
|
113
153
|
if (!needle) return 0;
|
|
@@ -138,7 +178,15 @@ export function wouldBeContent(toolName, ti, current) {
|
|
|
138
178
|
if (typeof oldStr !== "string" || oldStr === "") return { refuse: "Edit with an empty old_string" };
|
|
139
179
|
const n = countOccurrences(current, oldStr);
|
|
140
180
|
if (n === 0) return { refuse: "old_string is not present in the file on disk — the edit would not land where you looked" };
|
|
141
|
-
if (n > 1 && !ti.replace_all)
|
|
181
|
+
if (n > 1 && !ti.replace_all) {
|
|
182
|
+
// A repeated `id:` line means the file already carries a duplicate id — the collision, not the
|
|
183
|
+
// anchor, is why every id-anchored edit matches twice. Name that instead of the generic advice.
|
|
184
|
+
const dupId = repeatedIdAnchor(oldStr);
|
|
185
|
+
if (dupId !== null) {
|
|
186
|
+
return { refuse: `WHOLE FILE BLOCKED — the file already contains a duplicate id '${dupId}', so this id-anchored edit matches ${n} places and cannot land. The file is unwritable via id-anchored edits until the duplicate id is resolved (give one of the colliding rows a unique id).` };
|
|
187
|
+
}
|
|
188
|
+
return { refuse: `old_string matches ${n} places — anchor it to one row or pass replace_all` };
|
|
189
|
+
}
|
|
142
190
|
return { content: ti.replace_all ? current.split(oldStr).join(newStr) : current.replace(oldStr, () => newStr) };
|
|
143
191
|
}
|
|
144
192
|
if (toolName === "MultiEdit" && Array.isArray(ti.edits)) {
|
|
@@ -200,6 +248,19 @@ export function evaluate(payload, o = {}) {
|
|
|
200
248
|
try { prev = yaml.load(current); prevParsed = true; } catch { prevParsed = false; }
|
|
201
249
|
}
|
|
202
250
|
|
|
251
|
+
// Duplicate-id guard (id-uniqueness reserved at the write boundary). A second row sharing an id
|
|
252
|
+
// slips past js-yaml — the rows are distinct list entries, not duplicate map keys — but it makes
|
|
253
|
+
// every id-anchored edit ambiguous and the file effectively unwritable. Judged on the DELTA: a
|
|
254
|
+
// brand-new collision this write introduces always blocks; a pre-existing one the write leaves in
|
|
255
|
+
// place is surfaced by the id-anchored-Edit match-count path, not penalised twice here.
|
|
256
|
+
const nextDup = duplicateItemId(next);
|
|
257
|
+
if (nextDup !== null && (!prevParsed || duplicateItemId(prev) !== nextDup)) {
|
|
258
|
+
return {
|
|
259
|
+
decision: "deny",
|
|
260
|
+
reason: `${toolName} ${fp}: WHOLE FILE BLOCKED — two rows share id '${nextDup}'. A duplicate id makes every id-anchored edit ambiguous and the file unwritable. Give the new row a unique id before writing.`,
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
|
|
203
264
|
// Provenance truncation — judged on the DELTA so pre-existing rows never block a new write.
|
|
204
265
|
const before = new Set(prevParsed ? provenanceHits(prev).map((h) => `${h.id}\u0000${h.key}\u0000${h.value}`) : []);
|
|
205
266
|
const fresh = provenanceHits(next).filter((h) => !before.has(`${h.id}\u0000${h.key}\u0000${h.value}`));
|