@cohortapp/agent-sdk 2.18.13 → 2.18.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/bin/maestro.mjs +38 -1
  2. package/docs/runbooks/fleet-rollout.md +58 -7
  3. package/docs/runbooks/recovery-and-failover.md +18 -0
  4. package/lib/assurance/batch.mjs +353 -0
  5. package/lib/assurance/first-reply.mjs +423 -0
  6. package/lib/assurance/notice-voice.mjs +357 -0
  7. package/lib/assurance/plan-note.mjs +43 -0
  8. package/lib/assurance/room-budget.mjs +55 -6
  9. package/lib/cadence-failure-class.mjs +245 -0
  10. package/lib/claude-bin.mjs +26 -7
  11. package/lib/cli/doctor-checks.mjs +149 -1
  12. package/lib/comms/send-gate.mjs +59 -0
  13. package/lib/diagnostics/alerts.mjs +33 -0
  14. package/lib/engine/agents/usage.mjs +45 -0
  15. package/lib/engine/budget.mjs +293 -29
  16. package/lib/engine/cli.mjs +54 -5
  17. package/lib/engine/loop.mjs +30 -0
  18. package/lib/engine/output/json.mjs +26 -0
  19. package/lib/engine/wire/errors.mjs +179 -0
  20. package/lib/engine/wire/search.mjs +44 -8
  21. package/lib/identity/persona.mjs +31 -2
  22. package/lib/org/quota.mjs +27 -0
  23. package/lib/session/config.mjs +4 -0
  24. package/lib/session/identity.mjs +71 -7
  25. package/lib/session/launch-failure.mjs +251 -0
  26. package/lib/session/resume-target.mjs +86 -0
  27. package/lib/telemetry/alerts.mjs +94 -0
  28. package/lib/telemetry/collect.mjs +155 -2
  29. package/lib/upgrade/pinned-drift.mjs +467 -0
  30. package/package.json +1 -1
  31. package/scaffold/config/alerts.yaml +7 -0
  32. package/scripts/ci/check-cadence-prompts-exist.mjs +96 -0
  33. package/scripts/ci/check.mjs +3 -0
  34. package/scripts/daemon/agent-daemon.mjs +75 -5
  35. package/scripts/daemon/assurance.mjs +709 -44
  36. package/scripts/daemon/cadence-consumer.mjs +281 -34
  37. package/scripts/daemon/deliver.mjs +109 -0
  38. package/scripts/daemon/dispatcher.mjs +21 -3
  39. package/scripts/daemon/inbox-deferral.mjs +102 -9
  40. package/scripts/daemon/session-lock.mjs +41 -1
  41. package/scripts/emergency-stop.sh +114 -13
  42. package/scripts/fleet/rollout.mjs +256 -10
  43. package/scripts/healthcheck.sh +131 -33
  44. package/scripts/local-triggers/autoupdate.sh +144 -11
  45. package/scripts/resume-operations.sh +101 -6
  46. package/scripts/session/supervisor.mjs +198 -5
@@ -96,6 +96,13 @@ import { writeHandoff, listHandoffs, expireHandoffs, pruneHandoffs, handoffPaths
96
96
  import { writeFileAtomic } from "../../lib/fs-atomic.mjs";
97
97
  import { resolveClaudeBin as sharedResolveClaude } from "../../lib/claude-bin.mjs";
98
98
  import { getCadenceDef } from "./cadence-handlers.mjs";
99
+ // PERMANENT vs TRANSIENT (lane/permanent-errors). A retry loop that never
100
+ // inspects its error class is how two seats burned down: a cadence whose prompt
101
+ // file does not exist retried at poll speed forever (measured 16 requeues per
102
+ // 30 s on Eli Rosenberg's seat, 2026-09-24). The taxonomy is pure and lives on
103
+ // its own so the policy can be argued with without a daemon.
104
+ import { classifyCadenceFailure, isPermanent, PERMANENT_MAX_ATTEMPTS } from "../../lib/cadence-failure-class.mjs";
105
+ import { bump as bumpCounter } from "../../lib/diagnostics/counters.mjs";
99
106
  import { obligationAllowedUnderPosture } from "../../lib/plan/compile.mjs";
100
107
  import { isHumanLaneCadence } from "../../lib/cadences.mjs";
101
108
  import { sessionPermissionArgs } from "../../lib/session-permissions.mjs";
@@ -636,6 +643,12 @@ export function startConsumer(opts = {}) {
636
643
  const budgetEscalateMs = opts.budgetEscalateMs ?? DEFAULT_BUDGET_ESCALATE_MS;
637
644
  const maxSpawnMs = opts.maxSpawnMs ?? DEFAULT_SPAWN_TIMEOUT_MS;
638
645
  const spawnSession = opts.spawnSession || realSpawnSession;
646
+ // Cadence registry lookup. Injectable because the config-driven half
647
+ // (`config/.cadence-registry.json`) is read once and memoised at module
648
+ // scope, which a hermetic test cannot re-point — and the permanent-error
649
+ // path is reached precisely through a CONFIGURED cadence whose prompt is
650
+ // absent, so it has to be reachable in a test.
651
+ const cadenceDef = typeof opts.getCadenceDef === "function" ? opts.getCadenceDef : getCadenceDef;
639
652
  const userLogger = opts.logger;
640
653
  // Test / tuning hooks for the reliability layer.
641
654
  const backoffSchedule = opts.backoffSchedule || BACKOFF_SCHEDULE_MS;
@@ -725,13 +738,64 @@ export function startConsumer(opts = {}) {
725
738
  * `renderCadencePromptBody`, so the handed-off prompt is byte-identical to
726
739
  * what a sub-session would have been given, parallelism directive included —
727
740
  * and land it under state/session/handoffs/prompts/<tickId>.md.
741
+ *
742
+ * WHY IT TAGS ITS OWN FAILURES. This function does two unrelated things: it
743
+ * READS the cadence's prompt (whose failure says the spawn cannot work
744
+ * either — the spawn opens the same path) and it WRITES a rendered copy into
745
+ * the handoff dir (whose failure says nothing about the spawn at all). The
746
+ * caller has to tell them apart, and the first attempt at that inferred it
747
+ * from the message — `msg.includes(promptPath)`.
748
+ *
749
+ * That inference is WRONG, measured 2026-09-25: `readFileSync` on a
750
+ * DIRECTORY throws exactly `EISDIR: illegal operation on a directory, read`,
751
+ * with no path in the text. A directory at the configured prompt path
752
+ * therefore passed the `existsSync` pre-check, threw here, was read as "not
753
+ * about the prompt", classified transient, and fell through to a spawn — the
754
+ * commonest unreadable case, and precisely the one the PROMPT_UNREADABLE code
755
+ * exists for. Node puts the path in the message for ENOENT and EACCES and
756
+ * not for EISDIR; no caller should have to know which.
757
+ *
758
+ * So the phase is not inferred, it is STAMPED, at the only two places that
759
+ * know it: `err.cadencePhase` is `"prompt"` on the read and `"handoff"` on
760
+ * everything after it. `err.cadenceErrno` carries `err.code` alongside, since
761
+ * the errno is the contract and the message is not.
728
762
  */
729
763
  function renderHandoffPrompt(tickId, promptPath) {
730
764
  const fullPrompt = join(agentRoot, promptPath);
731
- const body = renderCadencePromptBody(agentRoot, readFileSync(fullPrompt, "utf-8"));
732
- const out = join(handoffPaths(agentRoot).prompts, `${tickId}.md`);
733
- writeFileAtomic(out, body);
734
- return out;
765
+ let source;
766
+ try {
767
+ source = readFileSync(fullPrompt, "utf-8");
768
+ } catch (err) {
769
+ throw stampPhase(err, "prompt");
770
+ }
771
+ // Everything past the read is about OUR output, not the cadence's input:
772
+ // a render bug or an unwritable handoff dir leaves the spawn perfectly able
773
+ // to run, so neither may be read as a permanent prompt fault.
774
+ try {
775
+ const body = renderCadencePromptBody(agentRoot, source);
776
+ const out = join(handoffPaths(agentRoot).prompts, `${tickId}.md`);
777
+ writeFileAtomic(out, body);
778
+ return out;
779
+ } catch (err) {
780
+ throw stampPhase(err, "handoff");
781
+ }
782
+ }
783
+
784
+ /**
785
+ * Stamp an error with the phase it came from, so a catch does not have to
786
+ * guess. Non-destructive: an already-stamped error keeps its innermost phase
787
+ * (the read is nested inside nothing, but this keeps re-throws honest), and a
788
+ * non-object throw is wrapped rather than dropped.
789
+ */
790
+ function stampPhase(err, phase) {
791
+ if (!err || typeof err !== "object") {
792
+ const wrapped = new Error(String(err ?? "unknown error"));
793
+ wrapped.cadencePhase = phase;
794
+ return wrapped;
795
+ }
796
+ if (!err.cadencePhase) err.cadencePhase = phase;
797
+ if (!err.cadenceErrno && typeof err.code === "string") err.cadenceErrno = err.code;
798
+ return err;
735
799
  }
736
800
 
737
801
  /**
@@ -759,7 +823,7 @@ export function startConsumer(opts = {}) {
759
823
  // predicate both places — the daemon and the plan cannot disagree about
760
824
  // what is suspended.
761
825
  if (posture && cadence) {
762
- const def = getCadenceDef(cadence) || {};
826
+ const def = cadenceDef(cadence) || {};
763
827
  const verdict = obligationAllowedUnderPosture(
764
828
  {
765
829
  kind: "SCHEDULE",
@@ -824,6 +888,13 @@ export function startConsumer(opts = {}) {
824
888
  handed_off: 0,
825
889
  handoff_timeouts: 0,
826
890
  deferred: 0,
891
+ // A failure the classifier called PERMANENT — the cause cannot change by
892
+ // waiting, so the tick was dead-lettered instead of retried. Rides the
893
+ // heartbeat (writeHealth) so `maestro cadence status` and the beat carry it
894
+ // without a new channel; `last_permanent_failure` names the cadence and the
895
+ // code so the reader does not have to go back to the log to find out which.
896
+ permanent_failures: 0,
897
+ last_permanent_failure: null,
827
898
  last_event_id: null,
828
899
  last_decision: null,
829
900
  };
@@ -887,6 +958,99 @@ export function startConsumer(opts = {}) {
887
958
  return { ok: false, decision: "deferred", reason };
888
959
  }
889
960
 
961
+ /**
962
+ * A failure the classifier calls PERMANENT: stop retrying, record it
963
+ * durably, and make it visible to a person.
964
+ *
965
+ * THE FAULT THIS CLOSES (Eli Rosenberg's seat, 2026-09-24). A cadence whose
966
+ * configured prompt file was not on disk requeued forever — measured at 16
967
+ * per 30 seconds — because nothing in the retry path ever asked whether the
968
+ * cause could change by waiting. It cannot: a file that does not exist will
969
+ * not exist 30 seconds later.
970
+ *
971
+ * Three things happen, and all three matter:
972
+ *
973
+ * 1. STOP. `failTick` with `min(this consumer's budget,
974
+ * PERMANENT_MAX_ATTEMPTS)` — the permanent cap lowers a budget, never
975
+ * raises one. Two attempts, then dlq/: not because the cause can never be
976
+ * repaired (someone can drop the prompt onto the seat), but because it
977
+ * cannot be repaired BY WAITING.
978
+ *
979
+ * STATED HONESTLY (corrected 2026-09-25). `failTick`'s first outcome for
980
+ * attempt 1 is a REQUEUE, so what this buys is "one extra attempt, then
981
+ * stop" — not "the cadence's own schedule is the retry window", which is
982
+ * what an earlier draft of this comment claimed. The schedule only
983
+ * becomes the window once the last attempt reaches dlq/. Spacing the one
984
+ * extra attempt is a separate act, and it is done here:
985
+ * `holdCadenceForBackoff` moves the per-cadence gate forward so the retry
986
+ * cannot land in the same drain that produced the first failure. The
987
+ * measured before/after is 16 requeues per 30 s, unbounded → 2 attempts
988
+ * one backoff interval apart, then a durable stop.
989
+ *
990
+ * The budget is the smaller half of this; the bigger half is (2) and (3),
991
+ * and that a permanent fault no longer falls through to a second code
992
+ * path that opens the same missing file.
993
+ * 2. RECORD. The dlq/ file is the durable record and now carries
994
+ * `permanent:<code>` in `last_error`, so the reason survives a restart
995
+ * and a log rotation. A durable counter (`cadence.permanent_failure`)
996
+ * goes to logs/diagnostics/counters/<day>.jsonl alongside it.
997
+ * 3. BE VISIBLE. The counter is what the seat's EXISTING alert lane reads
998
+ * (lib/diagnostics/alerts.mjs `cadence_permanently_failing`, delivered by
999
+ * the alerts cadence through config/alerts.yaml's webhook, or logged
1000
+ * when there is none) — no new channel. And `stats.permanent_failures`
1001
+ * rides the heartbeat, so the seat's health file and everything
1002
+ * downstream of it say so without anyone tailing a log.
1003
+ *
1004
+ * Never throws: the counter bump is best-effort by contract and failTick is
1005
+ * already guarded.
1006
+ *
1007
+ * @param {object} event the claimed tick
1008
+ * @param {object} verdict a CadenceFailureVerdict (class "permanent")
1009
+ * @param {string} detail the raw error text, for the dlq record + log
1010
+ * @param {string} stage log stage naming WHERE it was caught
1011
+ */
1012
+ function failPermanently(event, verdict, detail, stage) {
1013
+ // The permanent cap LOWERS a budget, never raises one: if this consumer is
1014
+ // already stricter than the cap, its own number wins.
1015
+ const budget = Math.min(maxAttempts, verdict.maxAttempts ?? PERMANENT_MAX_ATTEMPTS);
1016
+ const reason = `permanent:${verdict.code} — ${verdict.reason}${detail ? `: ${detail}` : ""}`;
1017
+ log({
1018
+ level: "error",
1019
+ stage,
1020
+ permanent: true,
1021
+ code: verdict.code,
1022
+ id: event.id,
1023
+ cadence: event.cadence,
1024
+ error: detail || verdict.reason,
1025
+ max_attempts: budget,
1026
+ note: "cause cannot change by waiting — one more attempt after the cadence backoff, then dead-lettered",
1027
+ });
1028
+ try {
1029
+ bumpCounter("cadence.permanent_failure", { cadence: event.cadence, code: verdict.code }, { agentRoot });
1030
+ } catch { /* a counter must never crash the path it observes */ }
1031
+ stats.permanent_failures += 1;
1032
+ stats.last_permanent_failure = {
1033
+ cadence: event.cadence,
1034
+ code: verdict.code,
1035
+ at: new Date().toISOString(),
1036
+ };
1037
+ stats.last_decision = "permanent-failure";
1038
+ // Space the one remaining attempt. Without this the requeue below is
1039
+ // re-claimed by the very next drain pass — "bounded" but still at poll
1040
+ // speed, which is the behaviour this lane exists to remove.
1041
+ const heldUntil = holdCadenceForBackoff(event.cadence);
1042
+ const outcome = failTick(agentRoot, event.id, reason, { maxAttempts: budget });
1043
+ if (outcome?.destination === "dlq") stats.dlq += 1;
1044
+ else {
1045
+ stats.retries += 1;
1046
+ log({ level: "warn", stage: "permanent_retry_held", id: event.id, cadence: event.cadence, code: verdict.code, retry_at: new Date(heldUntil).toISOString() });
1047
+ }
1048
+ // The heartbeat is the visible surface; write it now rather than waiting up
1049
+ // to `heartbeatMs` for a reader to learn a cadence just died.
1050
+ heartbeat();
1051
+ return { ok: false, decision: outcome?.destination === "dlq" ? "dlq-permanent" : "failed-permanent", code: verdict.code };
1052
+ }
1053
+
890
1054
  // Ledger hygiene runs from the sweep at most this often.
891
1055
  const HANDOFF_PRUNE_EVERY_MS = 60 * 60_000;
892
1056
  let lastHandoffPruneAt = 0;
@@ -989,20 +1153,47 @@ export function startConsumer(opts = {}) {
989
1153
  s.failures += 1;
990
1154
  // Exponential back-off honouring the (test-overridable) schedule.
991
1155
  const idx = Math.min(s.failures, backoffSchedule.length - 1);
992
- s.nextAllowedAt = Date.now() + backoffSchedule[idx];
1156
+ s.nextAllowedAt = nowMs() + backoffSchedule[idx];
993
1157
  if (s.failures >= circuitThreshold) {
994
- s.openUntil = Date.now() + circuitDurationMs;
1158
+ s.openUntil = nowMs() + circuitDurationMs;
995
1159
  log({ level: "error", stage: "circuit_opened", cadence, failures: s.failures, open_until: new Date(s.openUntil).toISOString() });
996
1160
  writeCircuitFile();
997
1161
  }
998
1162
  }
999
1163
 
1164
+ /**
1165
+ * Hold a cadence off the spawn path for one backoff interval WITHOUT
1166
+ * advancing it toward an open circuit.
1167
+ *
1168
+ * WHY IT IS SEPARATE FROM `recordSubsessionFailure`. A permanent failure is
1169
+ * not a flaky sub-session: counting it toward `circuitThreshold` would trip a
1170
+ * breaker whose whole job is to ride out a bad patch, for a cause that has no
1171
+ * patch to ride out. What it DOES need is spacing — `PERMANENT_MAX_ATTEMPTS`
1172
+ * is a count, not a delay, and `failPermanently`'s first outcome is a requeue,
1173
+ * so without this the "one extra attempt" lands in the very same drain that
1174
+ * produced the first. `escalate` consults `isCadenceAllowed` before it reaches
1175
+ * the prompt at all, so moving `nextAllowedAt` forward is all it takes for the
1176
+ * extra attempt to cost a real interval rather than a millisecond.
1177
+ *
1178
+ * Idempotent-ish and monotonic: never moves the hold EARLIER, so a circuit or
1179
+ * a longer transient backoff already in force keeps its own deadline.
1180
+ */
1181
+ function holdCadenceForBackoff(cadence) {
1182
+ const s = getCadenceState(cadence);
1183
+ // The first rung is deliberately 0 ("retry immediately once"); a permanent
1184
+ // fault has already proved it needs a gap, so take the first NON-zero rung.
1185
+ const step = backoffSchedule.find((ms) => ms > 0) ?? 0;
1186
+ const until = nowMs() + step;
1187
+ if (until > s.nextAllowedAt) s.nextAllowedAt = until;
1188
+ return s.nextAllowedAt;
1189
+ }
1190
+
1000
1191
  function writeCircuitFile() {
1001
1192
  // Persist the open-circuit snapshot so doctor + the operator can see
1002
1193
  // which cadences are currently held back without scraping logs.
1003
1194
  const open = {};
1004
1195
  for (const [cad, s] of cadenceState.entries()) {
1005
- if (s.openUntil > Date.now()) {
1196
+ if (s.openUntil > nowMs()) {
1006
1197
  open[cad] = { failures: s.failures, open_until: new Date(s.openUntil).toISOString() };
1007
1198
  }
1008
1199
  }
@@ -1020,7 +1211,7 @@ export function startConsumer(opts = {}) {
1020
1211
 
1021
1212
  function isCadenceAllowed(cadence) {
1022
1213
  const s = getCadenceState(cadence);
1023
- const now = Date.now();
1214
+ const now = nowMs();
1024
1215
  if (s.openUntil > now) return { allowed: false, reason: "circuit-open", retry_at: s.openUntil };
1025
1216
  if (s.nextAllowedAt > now) return { allowed: false, reason: "backoff", retry_at: s.nextAllowedAt };
1026
1217
  // Circuit closes automatically when openUntil passes.
@@ -1107,7 +1298,7 @@ export function startConsumer(opts = {}) {
1107
1298
  }
1108
1299
  }
1109
1300
 
1110
- const def = getCadenceDef(event.cadence);
1301
+ const def = cadenceDef(event.cadence);
1111
1302
  let promptPath = def?.prompt;
1112
1303
  if (!promptPath) {
1113
1304
  // Unknown cadence — try the conventional location.
@@ -1121,6 +1312,15 @@ export function startConsumer(opts = {}) {
1121
1312
  stats.dlq += 1;
1122
1313
  return { ok: false, decision: "dlq-no-prompt" };
1123
1314
  }
1315
+ } else if (!existsSync(join(agentRoot, promptPath))) {
1316
+ // THE ELI FAULT, caught before it can loop. The `!promptPath` branch above
1317
+ // has checked the conventional path since it was written; a prompt named
1318
+ // by CONFIG was never checked at all — it was handed straight to the
1319
+ // handoff renderer and then, when that threw, to the spawn, which opens
1320
+ // the same path. Both fail, neither is terminal, and the tick comes back
1321
+ // on the bus 30 seconds later. Check it once, here, and classify.
1322
+ const verdict = classifyCadenceFailure({ phase: "prompt", error: `prompt not found: ${promptPath}` });
1323
+ return failPermanently(event, verdict, `configured prompt missing: ${promptPath}`, "escalate_prompt_missing");
1124
1324
  }
1125
1325
 
1126
1326
  // Per-cadence in-flight guard (F9): never the same cadence twice at once —
@@ -1139,31 +1339,69 @@ export function startConsumer(opts = {}) {
1139
1339
  const fd = frontDoorState();
1140
1340
  const verdict = shouldHandOffTick({ ...fd, mode: def?.mode, metadata: event.metadata });
1141
1341
  if (verdict.handOff) {
1342
+ // The PROMPT read is its own step, with its own catch. It used to sit
1343
+ // inside the handoff try/catch, so a prompt that could not be read was
1344
+ // indistinguishable from a handoff file that could not be written — and
1345
+ // the shared catch fell through to the spawn, which opens the same
1346
+ // prompt path. That is the busy loop: `handoff_failed_spawning_instead`
1347
+ // 16 times per 30 seconds on Eli Rosenberg's seat, 2026-09-24.
1348
+ // Split apart, each failure gets the answer it deserves: a permanent
1349
+ // prompt fault stops here; an unwritable handoff still falls through,
1350
+ // because the spawn genuinely might work.
1351
+ let rendered = null;
1142
1352
  try {
1143
- const rendered = renderHandoffPrompt(event.id, promptPath);
1144
- const h = writeHandoff(agentRoot, {
1145
- tickId: event.id,
1146
- cadence: event.cadence,
1147
- mode: def?.mode || "escalate",
1148
- promptPath: rendered,
1149
- metadata: { ...(event.metadata || {}), sourcePrompt: promptPath },
1150
- }, { now: nowMs(), deadlineMs: handoffDeadlineMs });
1151
- if (!h.ok) throw new Error(h.error || "handoff write failed");
1152
- completeTick(agentRoot, event.id, {
1153
- decision: "handed-to-session",
1154
- cadence: event.cadence,
1155
- prompt: promptPath,
1156
- handoff: h.path,
1157
- deadline_at: h.handoff && h.handoff.deadlineAt,
1158
- });
1159
- stats.handed_off += 1;
1160
- stats.last_decision = "handed-to-session";
1161
- log({ level: "info", stage: "handed_to_session", id: event.id, cadence: event.cadence, prompt: rendered, deadline_at: h.handoff && h.handoff.deadlineAt });
1162
- return { ok: true, decision: "handed-to-session" };
1353
+ rendered = renderHandoffPrompt(event.id, promptPath);
1163
1354
  } catch (err) {
1164
- // A handoff we could not write is not a reason to lose the tick —
1165
- // fall through to the legacy spawn, loudly.
1166
- log({ level: "warn", stage: "handoff_failed_spawning_instead", id: event.id, cadence: event.cadence, error: err && err.message });
1355
+ // renderHandoffPrompt stamps which half failed (see its doc): the
1356
+ // READ of the cadence prompt, or everything after it. Only the read
1357
+ // says anything about whether a spawn would work — the spawn opens
1358
+ // the same path — so only `cadencePhase === "prompt"` may reach a
1359
+ // permanent verdict. An unwritable handoff dir falls through to the
1360
+ // spawn like any other handoff fault.
1361
+ //
1362
+ // This used to be inferred from `msg.includes(promptPath)`, which is
1363
+ // false for EISDIR (Node omits the path), so the commonest unreadable
1364
+ // case — a DIRECTORY at the prompt path — fell through and spawned.
1365
+ // Never classify a path fault by whether the message quotes the path.
1366
+ const msg = (err && err.message) || "";
1367
+ const promptVerdict = classifyCadenceFailure({
1368
+ phase: err?.cadencePhase === "prompt" ? "prompt" : "spawn",
1369
+ error: msg,
1370
+ errno: err?.cadenceErrno,
1371
+ });
1372
+ if (isPermanent(promptVerdict)) {
1373
+ return failPermanently(event, promptVerdict, msg, "handoff_prompt_failed_permanently");
1374
+ }
1375
+ log({ level: "warn", stage: "handoff_render_failed_spawning_instead", id: event.id, cadence: event.cadence, phase: err?.cadencePhase || null, error: msg });
1376
+ }
1377
+ if (rendered) {
1378
+ try {
1379
+ const h = writeHandoff(agentRoot, {
1380
+ tickId: event.id,
1381
+ cadence: event.cadence,
1382
+ mode: def?.mode || "escalate",
1383
+ promptPath: rendered,
1384
+ metadata: { ...(event.metadata || {}), sourcePrompt: promptPath },
1385
+ }, { now: nowMs(), deadlineMs: handoffDeadlineMs });
1386
+ if (!h.ok) throw new Error(h.error || "handoff write failed");
1387
+ completeTick(agentRoot, event.id, {
1388
+ decision: "handed-to-session",
1389
+ cadence: event.cadence,
1390
+ prompt: promptPath,
1391
+ handoff: h.path,
1392
+ deadline_at: h.handoff && h.handoff.deadlineAt,
1393
+ });
1394
+ stats.handed_off += 1;
1395
+ stats.last_decision = "handed-to-session";
1396
+ log({ level: "info", stage: "handed_to_session", id: event.id, cadence: event.cadence, prompt: rendered, deadline_at: h.handoff && h.handoff.deadlineAt });
1397
+ return { ok: true, decision: "handed-to-session" };
1398
+ } catch (err) {
1399
+ // A handoff FILE we could not write is not a reason to lose the tick,
1400
+ // and it says nothing about whether the spawn would work — fall
1401
+ // through to the legacy spawn, loudly, exactly as before. The prompt
1402
+ // half of this, which DOES say so, is handled above.
1403
+ log({ level: "warn", stage: "handoff_failed_spawning_instead", id: event.id, cadence: event.cadence, error: err && err.message });
1404
+ }
1167
1405
  }
1168
1406
  }
1169
1407
  }
@@ -1292,6 +1530,15 @@ export function startConsumer(opts = {}) {
1292
1530
  stats.spawn_failures += 1;
1293
1531
  recordSubsessionFailure(event.cadence);
1294
1532
  const reason = result.error || (stderrTail ? `exit ${result.exit_code}: ${stderrTail}` : `exit ${result.exit_code}`);
1533
+ // Can this possibly succeed if we try again? `realSpawnSession` answers
1534
+ // -2 (prompt not on disk) / -3 (prompt unreadable) for causes that do not
1535
+ // move; those get the classifier's bounded budget and a durable,
1536
+ // visible record. Everything else — a timeout, a non-zero exit, a crash —
1537
+ // is transient and unchanged.
1538
+ const spawnVerdict = classifyCadenceFailure({ phase: "spawn", exitCode: result.exit_code, error: reason });
1539
+ if (isPermanent(spawnVerdict)) {
1540
+ return failPermanently(event, spawnVerdict, reason, "subsession_failed_permanently");
1541
+ }
1295
1542
  const outcome = failTick(agentRoot, event.id, reason, { maxAttempts });
1296
1543
  if (outcome?.destination === "dlq") stats.dlq += 1;
1297
1544
  else stats.retries += 1;
@@ -1304,7 +1551,7 @@ export function startConsumer(opts = {}) {
1304
1551
  stats.received += 1;
1305
1552
  stats.last_event_id = event.id;
1306
1553
 
1307
- const def = getCadenceDef(event.cadence);
1554
+ const def = cadenceDef(event.cadence);
1308
1555
  if (def?.mode === "inline" && typeof def.handler === "function") {
1309
1556
  try {
1310
1557
  const out = await def.handler({ event, agentRoot, log });
@@ -35,6 +35,7 @@ import { recordOutbound } from "../../lib/comms/receipts.mjs";
35
35
  // inbound projection stamps into `raw_ref`. Imported rather than re-listed so a
36
36
  // new surface cannot be added upstream without this file's switch noticing.
37
37
  import { ROOM_SURFACES, SURFACE_NAMES } from "../../lib/org/inbound/surfaces.mjs";
38
+ import { isRetiredFirstReply } from "../../lib/assurance/first-reply.mjs";
38
39
 
39
40
  const AGENT_REPO_DIR = process.env.AGENT_DIR || join(new URL(".", import.meta.url).pathname, "../..");
40
41
 
@@ -741,11 +742,117 @@ export function scaffoldMarkerLeak(text) {
741
742
  * escalate rather than lose the message to an exception. `permanent`
742
743
  * means retrying cannot help (no route, or a NOT_FOUND/BAD_REQUEST).
743
744
  */
745
+ /**
746
+ * Can this inbound be answered with a REACTION rather than a sentence?
747
+ *
748
+ * Only a Cohort message in a real room can: a reaction is attached to a message
749
+ * id in a channel, and that is the one surface where both exist. A board
750
+ * comment, a doc comment, an email and a Slack item all answer `false` here
751
+ * even where the underlying product has reactions, because this daemon has no
752
+ * route to them — and a shape decision that promises a reaction the transport
753
+ * cannot make would turn "acknowledge cheaply" into "say nothing at all".
754
+ *
755
+ * @param {object} item
756
+ * @returns {boolean}
757
+ */
758
+ export function canReactTo(item) {
759
+ if (!item || item.service !== "cohort") return false;
760
+ const route = cohortReplyRoute(item);
761
+ if (route.transport !== "channel") return false;
762
+ return Boolean(str(item.message_id));
763
+ }
764
+
765
+ /**
766
+ * Put a reaction on the message that arrived.
767
+ *
768
+ * The cheapest honest acknowledgement there is: it tells the sender their
769
+ * message reached a person, and it adds no row to the channel. That second
770
+ * property is the whole argument for it — the measured flood
771
+ * (`lib/assurance/tier.mjs`) was 3,069 acknowledgement ROWS, and a reaction is
772
+ * an acknowledgement that is not a row.
773
+ *
774
+ * Fail-open and quiet: a reaction that cannot be made is not an error worth
775
+ * escalating, it is a reason to write the line instead, and the caller treats
776
+ * `{sent:false}` exactly that way.
777
+ *
778
+ * @param {object} item
779
+ * @param {string} emoji
780
+ * @param {object} [o] {cfg, agentRoot, reactImpl, fetchImpl} test seams
781
+ * @returns {Promise<{sent:boolean, via:string|null, error?:string}>}
782
+ */
783
+ export async function deliverReaction(item, emoji, o = {}) {
784
+ if (!canReactTo(item)) return { sent: false, via: null, error: "no reaction surface" };
785
+ const e = str(emoji);
786
+ if (!e) return { sent: false, via: null, error: "no emoji" };
787
+ try {
788
+ const route = cohortReplyRoute(item);
789
+ const agentRoot = o.agentRoot || process.env.AGENT_ROOT || process.env.AGENT_DIR || process.cwd();
790
+ const cfg = o.cfg || (await import("../../lib/org/client.mjs")).loadOrgConfig(agentRoot);
791
+ const reactImpl = o.reactImpl || (await import("../../lib/org/messaging.mjs")).reactMessage;
792
+ const frame = await reactImpl(
793
+ { channel: route.channelId, messageId: str(item.message_id), emoji: e },
794
+ { cfg, agentRoot, idempotencyKey: `react-${str(item.message_id)}-${e}`, ...(o.fetchImpl ? { fetchImpl: o.fetchImpl } : {}) },
795
+ );
796
+ if (frame && frame.ok) {
797
+ recordOutbound({ service: "cohort", channel: route.channelId, kind: "reaction", via: "cohort", chars: 0, agentRoot });
798
+ return { sent: true, via: "cohort" };
799
+ }
800
+ return { sent: false, via: null, error: str(frame && frame.error && frame.error.message) || "react failed" };
801
+ } catch (err) {
802
+ return { sent: false, via: null, error: (err && err.message) || String(err) };
803
+ }
804
+ }
805
+
744
806
  export async function deliver(item, text, o = {}) {
745
807
  const kind = o.kind || "reply";
746
808
  if (!item || !text || !String(text).trim()) {
747
809
  return { sent: false, via: null, error: "nothing to deliver" };
748
810
  }
811
+ // ── THE REGISTER, AT THE CHOKEPOINT ──────────────────────────────────────
812
+ // Every outbound this daemon makes crosses this function — the answer, the
813
+ // acknowledgement and the assurance sweep alike — which is why the scaffold
814
+ // check below lives here rather than at three call sites. The retired
815
+ // placeholder replies get the same treatment, and for a reason the composer
816
+ // guards cannot cover:
817
+ //
818
+ // On 2026-09-25 two seats (Ravi Patel, Daniel Connors) were still emitting
819
+ // "On it." / "Looking now." / "On it — digging in now." — 251 of them in six
820
+ // days, the last at 11:28Z that morning — with `clientMsgId` ending `-ack`,
821
+ // i.e. through this very path, while `AgentStatus.machine.daemon` reported
822
+ // sdkVersion 2.18.13 and `versionsAgree: true`. The published package at that
823
+ // version contains none of those strings and `sanitiseAckText` returns null
824
+ // for all three. The explanation is that the daemon runs
825
+ // `$AGENT_ROOT/scripts/daemon/maestro-daemon.mjs` — the seat's PRESERVED
826
+ // LOCAL COPY — and `versionsAgree` compares two npm version numbers, which
827
+ // says nothing whatever about that tree.
828
+ //
829
+ // WHAT THIS GUARD DOES AND DOES NOT REACH — stated exactly, because the first
830
+ // version of this comment overclaimed it and a reader would have taken the
831
+ // stale-seat vector for closed.
832
+ //
833
+ // IT DOES protect every outbound of an UPDATED daemon, at the last point
834
+ // before bytes leave the machine, whatever composed the text:
835
+ // the answer, the acknowledgement, the assurance sweep, and any
836
+ // composer added later that nobody remembers to guard. That is
837
+ // the case for putting it here rather than at three call sites.
838
+ // IT DOES NOT reach the two seats that were actually emitting these
839
+ // sentences. This file is `scripts/daemon/deliver.mjs`, and it
840
+ // lives in the very tree those seats have preserved a stale copy
841
+ // of — a stale tree ships a stale deliver.mjs alongside its
842
+ // stale composer. THE REMEDY FOR THOSE SEATS IS THE ROLLOUT
843
+ // (publish, then `scripts/local-triggers/autoupdate.sh` on each
844
+ // box, verified by a server-recorded presence.beat rather than
845
+ // by `versionsAgree`). Nothing in this file can shorten that.
846
+ //
847
+ // The other plane is covered separately: `lib/comms/send-gate.screenOutbound`
848
+ // and `screenHookPayload` apply the same register to `messaging_send`,
849
+ // `email_send`, `email_draft_send` and `org_call_share_step`, which is how a
850
+ // SPAWNED SESSION typing "On it." itself gets refused — that path never
851
+ // crosses this function.
852
+ if (isRetiredFirstReply(text)) {
853
+ try { console.error(`[deliver] refused: retired placeholder reply in outbound body ${JSON.stringify(String(text).slice(0, 120))} — this is a stale composer, not a message (permanent, not retrying)`); } catch { /* */ }
854
+ return { sent: false, via: null, permanent: true, channel: item && item.channel_id, code: "FORBIDDEN_SCOPE", error: "FORBIDDEN_SCOPE: retired placeholder reply — the first line must be generated from the actual thread, or nothing" };
855
+ }
749
856
  const leak = scaffoldMarkerLeak(text);
750
857
  if (leak) {
751
858
  const surface = (item && item.channel_id) ? "room" : "unknown";
@@ -827,6 +934,8 @@ export async function deliverWithRetry(item, text, o = {}) {
827
934
  export default {
828
935
  deliver,
829
936
  deliverWithRetry,
937
+ canReactTo,
938
+ deliverReaction,
830
939
  resolveSlackChannel,
831
940
  replyTargetOf,
832
941
  canDeliverTo,
@@ -6,7 +6,7 @@ import { spawn } from "child_process";
6
6
  import { appendFileSync, mkdirSync, writeFileSync, readFileSync, renameSync, existsSync, readdirSync, unlinkSync } from "fs";
7
7
  import { randomUUID } from "crypto";
8
8
  import { join, dirname } from "path";
9
- import { releaseLock, releaseThreadLock, releaseRequestClaim, claimItem, releaseItemClaim } from "./session-lock.mjs";
9
+ import { releaseLock, releaseThreadLock, threadLockKey, releaseRequestClaim, claimItem, releaseItemClaim } from "./session-lock.mjs";
10
10
  import { promoteDeferred } from "./inbox-deferral.mjs";
11
11
  import { recordSession } from "./health.mjs";
12
12
  import { startTyping, stopTyping } from "./typing-registry.mjs";
@@ -2020,7 +2020,16 @@ function spawnSession(entry) {
2020
2020
  {
2021
2021
  const channel = item.channel_id || (item.raw_ref ? (item.raw_ref.match(/slack:([^:]+):/) || [])[1] : null) || item.channel;
2022
2022
  if (channel) {
2023
- releaseThreadLock(channel, item.thread_id);
2023
+ // THE SAME KEY THE DAEMON TOOK. `item.thread_id` alone is not that
2024
+ // key: an untreaded post locks `channel-main` and a Cohort DM locks
2025
+ // `dm-channel`, and neither is derivable from an empty thread id
2026
+ // without the item's own DM verdict. One derivation, both sides.
2027
+ releaseThreadLock(channel, threadLockKey({
2028
+ channel,
2029
+ threadId: item.thread_id,
2030
+ isDm: item.is_dm === true,
2031
+ channelMainLock: String(process.env.MAESTRO_CHANNEL_MAIN_LOCK ?? "1") !== "0",
2032
+ }));
2024
2033
  // Now that the lock is gone, promote any messages that were
2025
2034
  // deferred behind it. Latest-wins: a burst of N messages
2026
2035
  // collapses into ONE re-dispatch carrying the most recent
@@ -2157,7 +2166,16 @@ function spawnSession(entry) {
2157
2166
  {
2158
2167
  const channel = item.channel_id || (item.raw_ref ? (item.raw_ref.match(/slack:([^:]+):/) || [])[1] : null) || item.channel;
2159
2168
  if (channel) {
2160
- releaseThreadLock(channel, item.thread_id);
2169
+ // THE SAME KEY THE DAEMON TOOK. `item.thread_id` alone is not that
2170
+ // key: an untreaded post locks `channel-main` and a Cohort DM locks
2171
+ // `dm-channel`, and neither is derivable from an empty thread id
2172
+ // without the item's own DM verdict. One derivation, both sides.
2173
+ releaseThreadLock(channel, threadLockKey({
2174
+ channel,
2175
+ threadId: item.thread_id,
2176
+ isDm: item.is_dm === true,
2177
+ channelMainLock: String(process.env.MAESTRO_CHANNEL_MAIN_LOCK ?? "1") !== "0",
2178
+ }));
2161
2179
  const promo = promoteDeferred(channel, AGENT_REPO_DIR);
2162
2180
  if (promo.promoted > 0) {
2163
2181
  logSession({