@cohortapp/agent-sdk 2.18.13 → 2.18.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +38 -1
- package/docs/runbooks/fleet-rollout.md +58 -7
- package/docs/runbooks/recovery-and-failover.md +18 -0
- package/lib/assurance/batch.mjs +353 -0
- package/lib/assurance/first-reply.mjs +423 -0
- package/lib/assurance/notice-voice.mjs +357 -0
- package/lib/assurance/plan-note.mjs +43 -0
- package/lib/assurance/room-budget.mjs +55 -6
- package/lib/cadence-failure-class.mjs +245 -0
- package/lib/claude-bin.mjs +26 -7
- package/lib/cli/doctor-checks.mjs +149 -1
- package/lib/comms/send-gate.mjs +59 -0
- package/lib/diagnostics/alerts.mjs +33 -0
- package/lib/engine/agents/usage.mjs +45 -0
- package/lib/engine/budget.mjs +293 -29
- package/lib/engine/cli.mjs +54 -5
- package/lib/engine/loop.mjs +30 -0
- package/lib/engine/output/json.mjs +26 -0
- package/lib/engine/wire/errors.mjs +179 -0
- package/lib/engine/wire/search.mjs +44 -8
- package/lib/identity/persona.mjs +31 -2
- package/lib/org/quota.mjs +27 -0
- package/lib/session/config.mjs +4 -0
- package/lib/session/identity.mjs +71 -7
- package/lib/session/launch-failure.mjs +251 -0
- package/lib/session/resume-target.mjs +86 -0
- package/lib/telemetry/alerts.mjs +94 -0
- package/lib/telemetry/collect.mjs +155 -2
- package/lib/upgrade/pinned-drift.mjs +467 -0
- package/package.json +1 -1
- package/scaffold/config/alerts.yaml +7 -0
- package/scripts/ci/check-cadence-prompts-exist.mjs +96 -0
- package/scripts/ci/check.mjs +3 -0
- package/scripts/daemon/agent-daemon.mjs +75 -5
- package/scripts/daemon/assurance.mjs +709 -44
- package/scripts/daemon/cadence-consumer.mjs +281 -34
- package/scripts/daemon/deliver.mjs +109 -0
- package/scripts/daemon/dispatcher.mjs +21 -3
- package/scripts/daemon/inbox-deferral.mjs +102 -9
- package/scripts/daemon/session-lock.mjs +41 -1
- package/scripts/emergency-stop.sh +114 -13
- package/scripts/fleet/rollout.mjs +256 -10
- package/scripts/healthcheck.sh +131 -33
- package/scripts/local-triggers/autoupdate.sh +144 -11
- package/scripts/resume-operations.sh +101 -6
- package/scripts/session/supervisor.mjs +198 -5
|
@@ -96,6 +96,13 @@ import { writeHandoff, listHandoffs, expireHandoffs, pruneHandoffs, handoffPaths
|
|
|
96
96
|
import { writeFileAtomic } from "../../lib/fs-atomic.mjs";
|
|
97
97
|
import { resolveClaudeBin as sharedResolveClaude } from "../../lib/claude-bin.mjs";
|
|
98
98
|
import { getCadenceDef } from "./cadence-handlers.mjs";
|
|
99
|
+
// PERMANENT vs TRANSIENT (lane/permanent-errors). A retry loop that never
|
|
100
|
+
// inspects its error class is how two seats burned down: a cadence whose prompt
|
|
101
|
+
// file does not exist retried at poll speed forever (measured 16 requeues per
|
|
102
|
+
// 30 s on Eli Rosenberg's seat, 2026-09-24). The taxonomy is pure and lives on
|
|
103
|
+
// its own so the policy can be argued with without a daemon.
|
|
104
|
+
import { classifyCadenceFailure, isPermanent, PERMANENT_MAX_ATTEMPTS } from "../../lib/cadence-failure-class.mjs";
|
|
105
|
+
import { bump as bumpCounter } from "../../lib/diagnostics/counters.mjs";
|
|
99
106
|
import { obligationAllowedUnderPosture } from "../../lib/plan/compile.mjs";
|
|
100
107
|
import { isHumanLaneCadence } from "../../lib/cadences.mjs";
|
|
101
108
|
import { sessionPermissionArgs } from "../../lib/session-permissions.mjs";
|
|
@@ -636,6 +643,12 @@ export function startConsumer(opts = {}) {
|
|
|
636
643
|
const budgetEscalateMs = opts.budgetEscalateMs ?? DEFAULT_BUDGET_ESCALATE_MS;
|
|
637
644
|
const maxSpawnMs = opts.maxSpawnMs ?? DEFAULT_SPAWN_TIMEOUT_MS;
|
|
638
645
|
const spawnSession = opts.spawnSession || realSpawnSession;
|
|
646
|
+
// Cadence registry lookup. Injectable because the config-driven half
|
|
647
|
+
// (`config/.cadence-registry.json`) is read once and memoised at module
|
|
648
|
+
// scope, which a hermetic test cannot re-point — and the permanent-error
|
|
649
|
+
// path is reached precisely through a CONFIGURED cadence whose prompt is
|
|
650
|
+
// absent, so it has to be reachable in a test.
|
|
651
|
+
const cadenceDef = typeof opts.getCadenceDef === "function" ? opts.getCadenceDef : getCadenceDef;
|
|
639
652
|
const userLogger = opts.logger;
|
|
640
653
|
// Test / tuning hooks for the reliability layer.
|
|
641
654
|
const backoffSchedule = opts.backoffSchedule || BACKOFF_SCHEDULE_MS;
|
|
@@ -725,13 +738,64 @@ export function startConsumer(opts = {}) {
|
|
|
725
738
|
* `renderCadencePromptBody`, so the handed-off prompt is byte-identical to
|
|
726
739
|
* what a sub-session would have been given, parallelism directive included —
|
|
727
740
|
* and land it under state/session/handoffs/prompts/<tickId>.md.
|
|
741
|
+
*
|
|
742
|
+
* WHY IT TAGS ITS OWN FAILURES. This function does two unrelated things: it
|
|
743
|
+
* READS the cadence's prompt (whose failure says the spawn cannot work
|
|
744
|
+
* either — the spawn opens the same path) and it WRITES a rendered copy into
|
|
745
|
+
* the handoff dir (whose failure says nothing about the spawn at all). The
|
|
746
|
+
* caller has to tell them apart, and the first attempt at that inferred it
|
|
747
|
+
* from the message — `msg.includes(promptPath)`.
|
|
748
|
+
*
|
|
749
|
+
* That inference is WRONG, measured 2026-09-25: `readFileSync` on a
|
|
750
|
+
* DIRECTORY throws exactly `EISDIR: illegal operation on a directory, read`,
|
|
751
|
+
* with no path in the text. A directory at the configured prompt path
|
|
752
|
+
* therefore passed the `existsSync` pre-check, threw here, was read as "not
|
|
753
|
+
* about the prompt", classified transient, and fell through to a spawn — the
|
|
754
|
+
* commonest unreadable case, and precisely the one the PROMPT_UNREADABLE code
|
|
755
|
+
* exists for. Node puts the path in the message for ENOENT and EACCES and
|
|
756
|
+
* not for EISDIR; no caller should have to know which.
|
|
757
|
+
*
|
|
758
|
+
* So the phase is not inferred, it is STAMPED, at the only two places that
|
|
759
|
+
* know it: `err.cadencePhase` is `"prompt"` on the read and `"handoff"` on
|
|
760
|
+
* everything after it. `err.cadenceErrno` carries `err.code` alongside, since
|
|
761
|
+
* the errno is the contract and the message is not.
|
|
728
762
|
*/
|
|
729
763
|
function renderHandoffPrompt(tickId, promptPath) {
|
|
730
764
|
const fullPrompt = join(agentRoot, promptPath);
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
765
|
+
let source;
|
|
766
|
+
try {
|
|
767
|
+
source = readFileSync(fullPrompt, "utf-8");
|
|
768
|
+
} catch (err) {
|
|
769
|
+
throw stampPhase(err, "prompt");
|
|
770
|
+
}
|
|
771
|
+
// Everything past the read is about OUR output, not the cadence's input:
|
|
772
|
+
// a render bug or an unwritable handoff dir leaves the spawn perfectly able
|
|
773
|
+
// to run, so neither may be read as a permanent prompt fault.
|
|
774
|
+
try {
|
|
775
|
+
const body = renderCadencePromptBody(agentRoot, source);
|
|
776
|
+
const out = join(handoffPaths(agentRoot).prompts, `${tickId}.md`);
|
|
777
|
+
writeFileAtomic(out, body);
|
|
778
|
+
return out;
|
|
779
|
+
} catch (err) {
|
|
780
|
+
throw stampPhase(err, "handoff");
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* Stamp an error with the phase it came from, so a catch does not have to
|
|
786
|
+
* guess. Non-destructive: an already-stamped error keeps its innermost phase
|
|
787
|
+
* (the read is nested inside nothing, but this keeps re-throws honest), and a
|
|
788
|
+
* non-object throw is wrapped rather than dropped.
|
|
789
|
+
*/
|
|
790
|
+
function stampPhase(err, phase) {
|
|
791
|
+
if (!err || typeof err !== "object") {
|
|
792
|
+
const wrapped = new Error(String(err ?? "unknown error"));
|
|
793
|
+
wrapped.cadencePhase = phase;
|
|
794
|
+
return wrapped;
|
|
795
|
+
}
|
|
796
|
+
if (!err.cadencePhase) err.cadencePhase = phase;
|
|
797
|
+
if (!err.cadenceErrno && typeof err.code === "string") err.cadenceErrno = err.code;
|
|
798
|
+
return err;
|
|
735
799
|
}
|
|
736
800
|
|
|
737
801
|
/**
|
|
@@ -759,7 +823,7 @@ export function startConsumer(opts = {}) {
|
|
|
759
823
|
// predicate both places — the daemon and the plan cannot disagree about
|
|
760
824
|
// what is suspended.
|
|
761
825
|
if (posture && cadence) {
|
|
762
|
-
const def =
|
|
826
|
+
const def = cadenceDef(cadence) || {};
|
|
763
827
|
const verdict = obligationAllowedUnderPosture(
|
|
764
828
|
{
|
|
765
829
|
kind: "SCHEDULE",
|
|
@@ -824,6 +888,13 @@ export function startConsumer(opts = {}) {
|
|
|
824
888
|
handed_off: 0,
|
|
825
889
|
handoff_timeouts: 0,
|
|
826
890
|
deferred: 0,
|
|
891
|
+
// A failure the classifier called PERMANENT — the cause cannot change by
|
|
892
|
+
// waiting, so the tick was dead-lettered instead of retried. Rides the
|
|
893
|
+
// heartbeat (writeHealth) so `maestro cadence status` and the beat carry it
|
|
894
|
+
// without a new channel; `last_permanent_failure` names the cadence and the
|
|
895
|
+
// code so the reader does not have to go back to the log to find out which.
|
|
896
|
+
permanent_failures: 0,
|
|
897
|
+
last_permanent_failure: null,
|
|
827
898
|
last_event_id: null,
|
|
828
899
|
last_decision: null,
|
|
829
900
|
};
|
|
@@ -887,6 +958,99 @@ export function startConsumer(opts = {}) {
|
|
|
887
958
|
return { ok: false, decision: "deferred", reason };
|
|
888
959
|
}
|
|
889
960
|
|
|
961
|
+
/**
|
|
962
|
+
* A failure the classifier calls PERMANENT: stop retrying, record it
|
|
963
|
+
* durably, and make it visible to a person.
|
|
964
|
+
*
|
|
965
|
+
* THE FAULT THIS CLOSES (Eli Rosenberg's seat, 2026-09-24). A cadence whose
|
|
966
|
+
* configured prompt file was not on disk requeued forever — measured at 16
|
|
967
|
+
* per 30 seconds — because nothing in the retry path ever asked whether the
|
|
968
|
+
* cause could change by waiting. It cannot: a file that does not exist will
|
|
969
|
+
* not exist 30 seconds later.
|
|
970
|
+
*
|
|
971
|
+
* Three things happen, and all three matter:
|
|
972
|
+
*
|
|
973
|
+
* 1. STOP. `failTick` with `min(this consumer's budget,
|
|
974
|
+
* PERMANENT_MAX_ATTEMPTS)` — the permanent cap lowers a budget, never
|
|
975
|
+
* raises one. Two attempts, then dlq/: not because the cause can never be
|
|
976
|
+
* repaired (someone can drop the prompt onto the seat), but because it
|
|
977
|
+
* cannot be repaired BY WAITING.
|
|
978
|
+
*
|
|
979
|
+
* STATED HONESTLY (corrected 2026-09-25). `failTick`'s first outcome for
|
|
980
|
+
* attempt 1 is a REQUEUE, so what this buys is "one extra attempt, then
|
|
981
|
+
* stop" — not "the cadence's own schedule is the retry window", which is
|
|
982
|
+
* what an earlier draft of this comment claimed. The schedule only
|
|
983
|
+
* becomes the window once the last attempt reaches dlq/. Spacing the one
|
|
984
|
+
* extra attempt is a separate act, and it is done here:
|
|
985
|
+
* `holdCadenceForBackoff` moves the per-cadence gate forward so the retry
|
|
986
|
+
* cannot land in the same drain that produced the first failure. The
|
|
987
|
+
* measured before/after is 16 requeues per 30 s, unbounded → 2 attempts
|
|
988
|
+
* one backoff interval apart, then a durable stop.
|
|
989
|
+
*
|
|
990
|
+
* The budget is the smaller half of this; the bigger half is (2) and (3),
|
|
991
|
+
* and that a permanent fault no longer falls through to a second code
|
|
992
|
+
* path that opens the same missing file.
|
|
993
|
+
* 2. RECORD. The dlq/ file is the durable record and now carries
|
|
994
|
+
* `permanent:<code>` in `last_error`, so the reason survives a restart
|
|
995
|
+
* and a log rotation. A durable counter (`cadence.permanent_failure`)
|
|
996
|
+
* goes to logs/diagnostics/counters/<day>.jsonl alongside it.
|
|
997
|
+
* 3. BE VISIBLE. The counter is what the seat's EXISTING alert lane reads
|
|
998
|
+
* (lib/diagnostics/alerts.mjs `cadence_permanently_failing`, delivered by
|
|
999
|
+
* the alerts cadence through config/alerts.yaml's webhook, or logged
|
|
1000
|
+
* when there is none) — no new channel. And `stats.permanent_failures`
|
|
1001
|
+
* rides the heartbeat, so the seat's health file and everything
|
|
1002
|
+
* downstream of it say so without anyone tailing a log.
|
|
1003
|
+
*
|
|
1004
|
+
* Never throws: the counter bump is best-effort by contract and failTick is
|
|
1005
|
+
* already guarded.
|
|
1006
|
+
*
|
|
1007
|
+
* @param {object} event the claimed tick
|
|
1008
|
+
* @param {object} verdict a CadenceFailureVerdict (class "permanent")
|
|
1009
|
+
* @param {string} detail the raw error text, for the dlq record + log
|
|
1010
|
+
* @param {string} stage log stage naming WHERE it was caught
|
|
1011
|
+
*/
|
|
1012
|
+
function failPermanently(event, verdict, detail, stage) {
|
|
1013
|
+
// The permanent cap LOWERS a budget, never raises one: if this consumer is
|
|
1014
|
+
// already stricter than the cap, its own number wins.
|
|
1015
|
+
const budget = Math.min(maxAttempts, verdict.maxAttempts ?? PERMANENT_MAX_ATTEMPTS);
|
|
1016
|
+
const reason = `permanent:${verdict.code} — ${verdict.reason}${detail ? `: ${detail}` : ""}`;
|
|
1017
|
+
log({
|
|
1018
|
+
level: "error",
|
|
1019
|
+
stage,
|
|
1020
|
+
permanent: true,
|
|
1021
|
+
code: verdict.code,
|
|
1022
|
+
id: event.id,
|
|
1023
|
+
cadence: event.cadence,
|
|
1024
|
+
error: detail || verdict.reason,
|
|
1025
|
+
max_attempts: budget,
|
|
1026
|
+
note: "cause cannot change by waiting — one more attempt after the cadence backoff, then dead-lettered",
|
|
1027
|
+
});
|
|
1028
|
+
try {
|
|
1029
|
+
bumpCounter("cadence.permanent_failure", { cadence: event.cadence, code: verdict.code }, { agentRoot });
|
|
1030
|
+
} catch { /* a counter must never crash the path it observes */ }
|
|
1031
|
+
stats.permanent_failures += 1;
|
|
1032
|
+
stats.last_permanent_failure = {
|
|
1033
|
+
cadence: event.cadence,
|
|
1034
|
+
code: verdict.code,
|
|
1035
|
+
at: new Date().toISOString(),
|
|
1036
|
+
};
|
|
1037
|
+
stats.last_decision = "permanent-failure";
|
|
1038
|
+
// Space the one remaining attempt. Without this the requeue below is
|
|
1039
|
+
// re-claimed by the very next drain pass — "bounded" but still at poll
|
|
1040
|
+
// speed, which is the behaviour this lane exists to remove.
|
|
1041
|
+
const heldUntil = holdCadenceForBackoff(event.cadence);
|
|
1042
|
+
const outcome = failTick(agentRoot, event.id, reason, { maxAttempts: budget });
|
|
1043
|
+
if (outcome?.destination === "dlq") stats.dlq += 1;
|
|
1044
|
+
else {
|
|
1045
|
+
stats.retries += 1;
|
|
1046
|
+
log({ level: "warn", stage: "permanent_retry_held", id: event.id, cadence: event.cadence, code: verdict.code, retry_at: new Date(heldUntil).toISOString() });
|
|
1047
|
+
}
|
|
1048
|
+
// The heartbeat is the visible surface; write it now rather than waiting up
|
|
1049
|
+
// to `heartbeatMs` for a reader to learn a cadence just died.
|
|
1050
|
+
heartbeat();
|
|
1051
|
+
return { ok: false, decision: outcome?.destination === "dlq" ? "dlq-permanent" : "failed-permanent", code: verdict.code };
|
|
1052
|
+
}
|
|
1053
|
+
|
|
890
1054
|
// Ledger hygiene runs from the sweep at most this often.
|
|
891
1055
|
const HANDOFF_PRUNE_EVERY_MS = 60 * 60_000;
|
|
892
1056
|
let lastHandoffPruneAt = 0;
|
|
@@ -989,20 +1153,47 @@ export function startConsumer(opts = {}) {
|
|
|
989
1153
|
s.failures += 1;
|
|
990
1154
|
// Exponential back-off honouring the (test-overridable) schedule.
|
|
991
1155
|
const idx = Math.min(s.failures, backoffSchedule.length - 1);
|
|
992
|
-
s.nextAllowedAt =
|
|
1156
|
+
s.nextAllowedAt = nowMs() + backoffSchedule[idx];
|
|
993
1157
|
if (s.failures >= circuitThreshold) {
|
|
994
|
-
s.openUntil =
|
|
1158
|
+
s.openUntil = nowMs() + circuitDurationMs;
|
|
995
1159
|
log({ level: "error", stage: "circuit_opened", cadence, failures: s.failures, open_until: new Date(s.openUntil).toISOString() });
|
|
996
1160
|
writeCircuitFile();
|
|
997
1161
|
}
|
|
998
1162
|
}
|
|
999
1163
|
|
|
1164
|
+
/**
|
|
1165
|
+
* Hold a cadence off the spawn path for one backoff interval WITHOUT
|
|
1166
|
+
* advancing it toward an open circuit.
|
|
1167
|
+
*
|
|
1168
|
+
* WHY IT IS SEPARATE FROM `recordSubsessionFailure`. A permanent failure is
|
|
1169
|
+
* not a flaky sub-session: counting it toward `circuitThreshold` would trip a
|
|
1170
|
+
* breaker whose whole job is to ride out a bad patch, for a cause that has no
|
|
1171
|
+
* patch to ride out. What it DOES need is spacing — `PERMANENT_MAX_ATTEMPTS`
|
|
1172
|
+
* is a count, not a delay, and `failPermanently`'s first outcome is a requeue,
|
|
1173
|
+
* so without this the "one extra attempt" lands in the very same drain that
|
|
1174
|
+
* produced the first. `escalate` consults `isCadenceAllowed` before it reaches
|
|
1175
|
+
* the prompt at all, so moving `nextAllowedAt` forward is all it takes for the
|
|
1176
|
+
* extra attempt to cost a real interval rather than a millisecond.
|
|
1177
|
+
*
|
|
1178
|
+
* Idempotent-ish and monotonic: never moves the hold EARLIER, so a circuit or
|
|
1179
|
+
* a longer transient backoff already in force keeps its own deadline.
|
|
1180
|
+
*/
|
|
1181
|
+
function holdCadenceForBackoff(cadence) {
|
|
1182
|
+
const s = getCadenceState(cadence);
|
|
1183
|
+
// The first rung is deliberately 0 ("retry immediately once"); a permanent
|
|
1184
|
+
// fault has already proved it needs a gap, so take the first NON-zero rung.
|
|
1185
|
+
const step = backoffSchedule.find((ms) => ms > 0) ?? 0;
|
|
1186
|
+
const until = nowMs() + step;
|
|
1187
|
+
if (until > s.nextAllowedAt) s.nextAllowedAt = until;
|
|
1188
|
+
return s.nextAllowedAt;
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1000
1191
|
function writeCircuitFile() {
|
|
1001
1192
|
// Persist the open-circuit snapshot so doctor + the operator can see
|
|
1002
1193
|
// which cadences are currently held back without scraping logs.
|
|
1003
1194
|
const open = {};
|
|
1004
1195
|
for (const [cad, s] of cadenceState.entries()) {
|
|
1005
|
-
if (s.openUntil >
|
|
1196
|
+
if (s.openUntil > nowMs()) {
|
|
1006
1197
|
open[cad] = { failures: s.failures, open_until: new Date(s.openUntil).toISOString() };
|
|
1007
1198
|
}
|
|
1008
1199
|
}
|
|
@@ -1020,7 +1211,7 @@ export function startConsumer(opts = {}) {
|
|
|
1020
1211
|
|
|
1021
1212
|
function isCadenceAllowed(cadence) {
|
|
1022
1213
|
const s = getCadenceState(cadence);
|
|
1023
|
-
const now =
|
|
1214
|
+
const now = nowMs();
|
|
1024
1215
|
if (s.openUntil > now) return { allowed: false, reason: "circuit-open", retry_at: s.openUntil };
|
|
1025
1216
|
if (s.nextAllowedAt > now) return { allowed: false, reason: "backoff", retry_at: s.nextAllowedAt };
|
|
1026
1217
|
// Circuit closes automatically when openUntil passes.
|
|
@@ -1107,7 +1298,7 @@ export function startConsumer(opts = {}) {
|
|
|
1107
1298
|
}
|
|
1108
1299
|
}
|
|
1109
1300
|
|
|
1110
|
-
const def =
|
|
1301
|
+
const def = cadenceDef(event.cadence);
|
|
1111
1302
|
let promptPath = def?.prompt;
|
|
1112
1303
|
if (!promptPath) {
|
|
1113
1304
|
// Unknown cadence — try the conventional location.
|
|
@@ -1121,6 +1312,15 @@ export function startConsumer(opts = {}) {
|
|
|
1121
1312
|
stats.dlq += 1;
|
|
1122
1313
|
return { ok: false, decision: "dlq-no-prompt" };
|
|
1123
1314
|
}
|
|
1315
|
+
} else if (!existsSync(join(agentRoot, promptPath))) {
|
|
1316
|
+
// THE ELI FAULT, caught before it can loop. The `!promptPath` branch above
|
|
1317
|
+
// has checked the conventional path since it was written; a prompt named
|
|
1318
|
+
// by CONFIG was never checked at all — it was handed straight to the
|
|
1319
|
+
// handoff renderer and then, when that threw, to the spawn, which opens
|
|
1320
|
+
// the same path. Both fail, neither is terminal, and the tick comes back
|
|
1321
|
+
// on the bus 30 seconds later. Check it once, here, and classify.
|
|
1322
|
+
const verdict = classifyCadenceFailure({ phase: "prompt", error: `prompt not found: ${promptPath}` });
|
|
1323
|
+
return failPermanently(event, verdict, `configured prompt missing: ${promptPath}`, "escalate_prompt_missing");
|
|
1124
1324
|
}
|
|
1125
1325
|
|
|
1126
1326
|
// Per-cadence in-flight guard (F9): never the same cadence twice at once —
|
|
@@ -1139,31 +1339,69 @@ export function startConsumer(opts = {}) {
|
|
|
1139
1339
|
const fd = frontDoorState();
|
|
1140
1340
|
const verdict = shouldHandOffTick({ ...fd, mode: def?.mode, metadata: event.metadata });
|
|
1141
1341
|
if (verdict.handOff) {
|
|
1342
|
+
// The PROMPT read is its own step, with its own catch. It used to sit
|
|
1343
|
+
// inside the handoff try/catch, so a prompt that could not be read was
|
|
1344
|
+
// indistinguishable from a handoff file that could not be written — and
|
|
1345
|
+
// the shared catch fell through to the spawn, which opens the same
|
|
1346
|
+
// prompt path. That is the busy loop: `handoff_failed_spawning_instead`
|
|
1347
|
+
// 16 times per 30 seconds on Eli Rosenberg's seat, 2026-09-24.
|
|
1348
|
+
// Split apart, each failure gets the answer it deserves: a permanent
|
|
1349
|
+
// prompt fault stops here; an unwritable handoff still falls through,
|
|
1350
|
+
// because the spawn genuinely might work.
|
|
1351
|
+
let rendered = null;
|
|
1142
1352
|
try {
|
|
1143
|
-
|
|
1144
|
-
const h = writeHandoff(agentRoot, {
|
|
1145
|
-
tickId: event.id,
|
|
1146
|
-
cadence: event.cadence,
|
|
1147
|
-
mode: def?.mode || "escalate",
|
|
1148
|
-
promptPath: rendered,
|
|
1149
|
-
metadata: { ...(event.metadata || {}), sourcePrompt: promptPath },
|
|
1150
|
-
}, { now: nowMs(), deadlineMs: handoffDeadlineMs });
|
|
1151
|
-
if (!h.ok) throw new Error(h.error || "handoff write failed");
|
|
1152
|
-
completeTick(agentRoot, event.id, {
|
|
1153
|
-
decision: "handed-to-session",
|
|
1154
|
-
cadence: event.cadence,
|
|
1155
|
-
prompt: promptPath,
|
|
1156
|
-
handoff: h.path,
|
|
1157
|
-
deadline_at: h.handoff && h.handoff.deadlineAt,
|
|
1158
|
-
});
|
|
1159
|
-
stats.handed_off += 1;
|
|
1160
|
-
stats.last_decision = "handed-to-session";
|
|
1161
|
-
log({ level: "info", stage: "handed_to_session", id: event.id, cadence: event.cadence, prompt: rendered, deadline_at: h.handoff && h.handoff.deadlineAt });
|
|
1162
|
-
return { ok: true, decision: "handed-to-session" };
|
|
1353
|
+
rendered = renderHandoffPrompt(event.id, promptPath);
|
|
1163
1354
|
} catch (err) {
|
|
1164
|
-
//
|
|
1165
|
-
//
|
|
1166
|
-
|
|
1355
|
+
// renderHandoffPrompt stamps which half failed (see its doc): the
|
|
1356
|
+
// READ of the cadence prompt, or everything after it. Only the read
|
|
1357
|
+
// says anything about whether a spawn would work — the spawn opens
|
|
1358
|
+
// the same path — so only `cadencePhase === "prompt"` may reach a
|
|
1359
|
+
// permanent verdict. An unwritable handoff dir falls through to the
|
|
1360
|
+
// spawn like any other handoff fault.
|
|
1361
|
+
//
|
|
1362
|
+
// This used to be inferred from `msg.includes(promptPath)`, which is
|
|
1363
|
+
// false for EISDIR (Node omits the path), so the commonest unreadable
|
|
1364
|
+
// case — a DIRECTORY at the prompt path — fell through and spawned.
|
|
1365
|
+
// Never classify a path fault by whether the message quotes the path.
|
|
1366
|
+
const msg = (err && err.message) || "";
|
|
1367
|
+
const promptVerdict = classifyCadenceFailure({
|
|
1368
|
+
phase: err?.cadencePhase === "prompt" ? "prompt" : "spawn",
|
|
1369
|
+
error: msg,
|
|
1370
|
+
errno: err?.cadenceErrno,
|
|
1371
|
+
});
|
|
1372
|
+
if (isPermanent(promptVerdict)) {
|
|
1373
|
+
return failPermanently(event, promptVerdict, msg, "handoff_prompt_failed_permanently");
|
|
1374
|
+
}
|
|
1375
|
+
log({ level: "warn", stage: "handoff_render_failed_spawning_instead", id: event.id, cadence: event.cadence, phase: err?.cadencePhase || null, error: msg });
|
|
1376
|
+
}
|
|
1377
|
+
if (rendered) {
|
|
1378
|
+
try {
|
|
1379
|
+
const h = writeHandoff(agentRoot, {
|
|
1380
|
+
tickId: event.id,
|
|
1381
|
+
cadence: event.cadence,
|
|
1382
|
+
mode: def?.mode || "escalate",
|
|
1383
|
+
promptPath: rendered,
|
|
1384
|
+
metadata: { ...(event.metadata || {}), sourcePrompt: promptPath },
|
|
1385
|
+
}, { now: nowMs(), deadlineMs: handoffDeadlineMs });
|
|
1386
|
+
if (!h.ok) throw new Error(h.error || "handoff write failed");
|
|
1387
|
+
completeTick(agentRoot, event.id, {
|
|
1388
|
+
decision: "handed-to-session",
|
|
1389
|
+
cadence: event.cadence,
|
|
1390
|
+
prompt: promptPath,
|
|
1391
|
+
handoff: h.path,
|
|
1392
|
+
deadline_at: h.handoff && h.handoff.deadlineAt,
|
|
1393
|
+
});
|
|
1394
|
+
stats.handed_off += 1;
|
|
1395
|
+
stats.last_decision = "handed-to-session";
|
|
1396
|
+
log({ level: "info", stage: "handed_to_session", id: event.id, cadence: event.cadence, prompt: rendered, deadline_at: h.handoff && h.handoff.deadlineAt });
|
|
1397
|
+
return { ok: true, decision: "handed-to-session" };
|
|
1398
|
+
} catch (err) {
|
|
1399
|
+
// A handoff FILE we could not write is not a reason to lose the tick,
|
|
1400
|
+
// and it says nothing about whether the spawn would work — fall
|
|
1401
|
+
// through to the legacy spawn, loudly, exactly as before. The prompt
|
|
1402
|
+
// half of this, which DOES say so, is handled above.
|
|
1403
|
+
log({ level: "warn", stage: "handoff_failed_spawning_instead", id: event.id, cadence: event.cadence, error: err && err.message });
|
|
1404
|
+
}
|
|
1167
1405
|
}
|
|
1168
1406
|
}
|
|
1169
1407
|
}
|
|
@@ -1292,6 +1530,15 @@ export function startConsumer(opts = {}) {
|
|
|
1292
1530
|
stats.spawn_failures += 1;
|
|
1293
1531
|
recordSubsessionFailure(event.cadence);
|
|
1294
1532
|
const reason = result.error || (stderrTail ? `exit ${result.exit_code}: ${stderrTail}` : `exit ${result.exit_code}`);
|
|
1533
|
+
// Can this possibly succeed if we try again? `realSpawnSession` answers
|
|
1534
|
+
// -2 (prompt not on disk) / -3 (prompt unreadable) for causes that do not
|
|
1535
|
+
// move; those get the classifier's bounded budget and a durable,
|
|
1536
|
+
// visible record. Everything else — a timeout, a non-zero exit, a crash —
|
|
1537
|
+
// is transient and unchanged.
|
|
1538
|
+
const spawnVerdict = classifyCadenceFailure({ phase: "spawn", exitCode: result.exit_code, error: reason });
|
|
1539
|
+
if (isPermanent(spawnVerdict)) {
|
|
1540
|
+
return failPermanently(event, spawnVerdict, reason, "subsession_failed_permanently");
|
|
1541
|
+
}
|
|
1295
1542
|
const outcome = failTick(agentRoot, event.id, reason, { maxAttempts });
|
|
1296
1543
|
if (outcome?.destination === "dlq") stats.dlq += 1;
|
|
1297
1544
|
else stats.retries += 1;
|
|
@@ -1304,7 +1551,7 @@ export function startConsumer(opts = {}) {
|
|
|
1304
1551
|
stats.received += 1;
|
|
1305
1552
|
stats.last_event_id = event.id;
|
|
1306
1553
|
|
|
1307
|
-
const def =
|
|
1554
|
+
const def = cadenceDef(event.cadence);
|
|
1308
1555
|
if (def?.mode === "inline" && typeof def.handler === "function") {
|
|
1309
1556
|
try {
|
|
1310
1557
|
const out = await def.handler({ event, agentRoot, log });
|
|
@@ -35,6 +35,7 @@ import { recordOutbound } from "../../lib/comms/receipts.mjs";
|
|
|
35
35
|
// inbound projection stamps into `raw_ref`. Imported rather than re-listed so a
|
|
36
36
|
// new surface cannot be added upstream without this file's switch noticing.
|
|
37
37
|
import { ROOM_SURFACES, SURFACE_NAMES } from "../../lib/org/inbound/surfaces.mjs";
|
|
38
|
+
import { isRetiredFirstReply } from "../../lib/assurance/first-reply.mjs";
|
|
38
39
|
|
|
39
40
|
const AGENT_REPO_DIR = process.env.AGENT_DIR || join(new URL(".", import.meta.url).pathname, "../..");
|
|
40
41
|
|
|
@@ -741,11 +742,117 @@ export function scaffoldMarkerLeak(text) {
|
|
|
741
742
|
* escalate rather than lose the message to an exception. `permanent`
|
|
742
743
|
* means retrying cannot help (no route, or a NOT_FOUND/BAD_REQUEST).
|
|
743
744
|
*/
|
|
745
|
+
/**
|
|
746
|
+
* Can this inbound be answered with a REACTION rather than a sentence?
|
|
747
|
+
*
|
|
748
|
+
* Only a Cohort message in a real room can: a reaction is attached to a message
|
|
749
|
+
* id in a channel, and that is the one surface where both exist. A board
|
|
750
|
+
* comment, a doc comment, an email and a Slack item all answer `false` here
|
|
751
|
+
* even where the underlying product has reactions, because this daemon has no
|
|
752
|
+
* route to them — and a shape decision that promises a reaction the transport
|
|
753
|
+
* cannot make would turn "acknowledge cheaply" into "say nothing at all".
|
|
754
|
+
*
|
|
755
|
+
* @param {object} item
|
|
756
|
+
* @returns {boolean}
|
|
757
|
+
*/
|
|
758
|
+
export function canReactTo(item) {
|
|
759
|
+
if (!item || item.service !== "cohort") return false;
|
|
760
|
+
const route = cohortReplyRoute(item);
|
|
761
|
+
if (route.transport !== "channel") return false;
|
|
762
|
+
return Boolean(str(item.message_id));
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
/**
|
|
766
|
+
* Put a reaction on the message that arrived.
|
|
767
|
+
*
|
|
768
|
+
* The cheapest honest acknowledgement there is: it tells the sender their
|
|
769
|
+
* message reached a person, and it adds no row to the channel. That second
|
|
770
|
+
* property is the whole argument for it — the measured flood
|
|
771
|
+
* (`lib/assurance/tier.mjs`) was 3,069 acknowledgement ROWS, and a reaction is
|
|
772
|
+
* an acknowledgement that is not a row.
|
|
773
|
+
*
|
|
774
|
+
* Fail-open and quiet: a reaction that cannot be made is not an error worth
|
|
775
|
+
* escalating, it is a reason to write the line instead, and the caller treats
|
|
776
|
+
* `{sent:false}` exactly that way.
|
|
777
|
+
*
|
|
778
|
+
* @param {object} item
|
|
779
|
+
* @param {string} emoji
|
|
780
|
+
* @param {object} [o] {cfg, agentRoot, reactImpl, fetchImpl} test seams
|
|
781
|
+
* @returns {Promise<{sent:boolean, via:string|null, error?:string}>}
|
|
782
|
+
*/
|
|
783
|
+
export async function deliverReaction(item, emoji, o = {}) {
|
|
784
|
+
if (!canReactTo(item)) return { sent: false, via: null, error: "no reaction surface" };
|
|
785
|
+
const e = str(emoji);
|
|
786
|
+
if (!e) return { sent: false, via: null, error: "no emoji" };
|
|
787
|
+
try {
|
|
788
|
+
const route = cohortReplyRoute(item);
|
|
789
|
+
const agentRoot = o.agentRoot || process.env.AGENT_ROOT || process.env.AGENT_DIR || process.cwd();
|
|
790
|
+
const cfg = o.cfg || (await import("../../lib/org/client.mjs")).loadOrgConfig(agentRoot);
|
|
791
|
+
const reactImpl = o.reactImpl || (await import("../../lib/org/messaging.mjs")).reactMessage;
|
|
792
|
+
const frame = await reactImpl(
|
|
793
|
+
{ channel: route.channelId, messageId: str(item.message_id), emoji: e },
|
|
794
|
+
{ cfg, agentRoot, idempotencyKey: `react-${str(item.message_id)}-${e}`, ...(o.fetchImpl ? { fetchImpl: o.fetchImpl } : {}) },
|
|
795
|
+
);
|
|
796
|
+
if (frame && frame.ok) {
|
|
797
|
+
recordOutbound({ service: "cohort", channel: route.channelId, kind: "reaction", via: "cohort", chars: 0, agentRoot });
|
|
798
|
+
return { sent: true, via: "cohort" };
|
|
799
|
+
}
|
|
800
|
+
return { sent: false, via: null, error: str(frame && frame.error && frame.error.message) || "react failed" };
|
|
801
|
+
} catch (err) {
|
|
802
|
+
return { sent: false, via: null, error: (err && err.message) || String(err) };
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
|
|
744
806
|
export async function deliver(item, text, o = {}) {
|
|
745
807
|
const kind = o.kind || "reply";
|
|
746
808
|
if (!item || !text || !String(text).trim()) {
|
|
747
809
|
return { sent: false, via: null, error: "nothing to deliver" };
|
|
748
810
|
}
|
|
811
|
+
// ── THE REGISTER, AT THE CHOKEPOINT ──────────────────────────────────────
|
|
812
|
+
// Every outbound this daemon makes crosses this function — the answer, the
|
|
813
|
+
// acknowledgement and the assurance sweep alike — which is why the scaffold
|
|
814
|
+
// check below lives here rather than at three call sites. The retired
|
|
815
|
+
// placeholder replies get the same treatment, and for a reason the composer
|
|
816
|
+
// guards cannot cover:
|
|
817
|
+
//
|
|
818
|
+
// On 2026-09-25 two seats (Ravi Patel, Daniel Connors) were still emitting
|
|
819
|
+
// "On it." / "Looking now." / "On it — digging in now." — 251 of them in six
|
|
820
|
+
// days, the last at 11:28Z that morning — with `clientMsgId` ending `-ack`,
|
|
821
|
+
// i.e. through this very path, while `AgentStatus.machine.daemon` reported
|
|
822
|
+
// sdkVersion 2.18.13 and `versionsAgree: true`. The published package at that
|
|
823
|
+
// version contains none of those strings and `sanitiseAckText` returns null
|
|
824
|
+
// for all three. The explanation is that the daemon runs
|
|
825
|
+
// `$AGENT_ROOT/scripts/daemon/maestro-daemon.mjs` — the seat's PRESERVED
|
|
826
|
+
// LOCAL COPY — and `versionsAgree` compares two npm version numbers, which
|
|
827
|
+
// says nothing whatever about that tree.
|
|
828
|
+
//
|
|
829
|
+
// WHAT THIS GUARD DOES AND DOES NOT REACH — stated exactly, because the first
|
|
830
|
+
// version of this comment overclaimed it and a reader would have taken the
|
|
831
|
+
// stale-seat vector for closed.
|
|
832
|
+
//
|
|
833
|
+
// IT DOES protect every outbound of an UPDATED daemon, at the last point
|
|
834
|
+
// before bytes leave the machine, whatever composed the text:
|
|
835
|
+
// the answer, the acknowledgement, the assurance sweep, and any
|
|
836
|
+
// composer added later that nobody remembers to guard. That is
|
|
837
|
+
// the case for putting it here rather than at three call sites.
|
|
838
|
+
// IT DOES NOT reach the two seats that were actually emitting these
|
|
839
|
+
// sentences. This file is `scripts/daemon/deliver.mjs`, and it
|
|
840
|
+
// lives in the very tree those seats have preserved a stale copy
|
|
841
|
+
// of — a stale tree ships a stale deliver.mjs alongside its
|
|
842
|
+
// stale composer. THE REMEDY FOR THOSE SEATS IS THE ROLLOUT
|
|
843
|
+
// (publish, then `scripts/local-triggers/autoupdate.sh` on each
|
|
844
|
+
// box, verified by a server-recorded presence.beat rather than
|
|
845
|
+
// by `versionsAgree`). Nothing in this file can shorten that.
|
|
846
|
+
//
|
|
847
|
+
// The other plane is covered separately: `lib/comms/send-gate.screenOutbound`
|
|
848
|
+
// and `screenHookPayload` apply the same register to `messaging_send`,
|
|
849
|
+
// `email_send`, `email_draft_send` and `org_call_share_step`, which is how a
|
|
850
|
+
// SPAWNED SESSION typing "On it." itself gets refused — that path never
|
|
851
|
+
// crosses this function.
|
|
852
|
+
if (isRetiredFirstReply(text)) {
|
|
853
|
+
try { console.error(`[deliver] refused: retired placeholder reply in outbound body ${JSON.stringify(String(text).slice(0, 120))} — this is a stale composer, not a message (permanent, not retrying)`); } catch { /* */ }
|
|
854
|
+
return { sent: false, via: null, permanent: true, channel: item && item.channel_id, code: "FORBIDDEN_SCOPE", error: "FORBIDDEN_SCOPE: retired placeholder reply — the first line must be generated from the actual thread, or nothing" };
|
|
855
|
+
}
|
|
749
856
|
const leak = scaffoldMarkerLeak(text);
|
|
750
857
|
if (leak) {
|
|
751
858
|
const surface = (item && item.channel_id) ? "room" : "unknown";
|
|
@@ -827,6 +934,8 @@ export async function deliverWithRetry(item, text, o = {}) {
|
|
|
827
934
|
export default {
|
|
828
935
|
deliver,
|
|
829
936
|
deliverWithRetry,
|
|
937
|
+
canReactTo,
|
|
938
|
+
deliverReaction,
|
|
830
939
|
resolveSlackChannel,
|
|
831
940
|
replyTargetOf,
|
|
832
941
|
canDeliverTo,
|
|
@@ -6,7 +6,7 @@ import { spawn } from "child_process";
|
|
|
6
6
|
import { appendFileSync, mkdirSync, writeFileSync, readFileSync, renameSync, existsSync, readdirSync, unlinkSync } from "fs";
|
|
7
7
|
import { randomUUID } from "crypto";
|
|
8
8
|
import { join, dirname } from "path";
|
|
9
|
-
import { releaseLock, releaseThreadLock, releaseRequestClaim, claimItem, releaseItemClaim } from "./session-lock.mjs";
|
|
9
|
+
import { releaseLock, releaseThreadLock, threadLockKey, releaseRequestClaim, claimItem, releaseItemClaim } from "./session-lock.mjs";
|
|
10
10
|
import { promoteDeferred } from "./inbox-deferral.mjs";
|
|
11
11
|
import { recordSession } from "./health.mjs";
|
|
12
12
|
import { startTyping, stopTyping } from "./typing-registry.mjs";
|
|
@@ -2020,7 +2020,16 @@ function spawnSession(entry) {
|
|
|
2020
2020
|
{
|
|
2021
2021
|
const channel = item.channel_id || (item.raw_ref ? (item.raw_ref.match(/slack:([^:]+):/) || [])[1] : null) || item.channel;
|
|
2022
2022
|
if (channel) {
|
|
2023
|
-
|
|
2023
|
+
// THE SAME KEY THE DAEMON TOOK. `item.thread_id` alone is not that
|
|
2024
|
+
// key: an untreaded post locks `channel-main` and a Cohort DM locks
|
|
2025
|
+
// `dm-channel`, and neither is derivable from an empty thread id
|
|
2026
|
+
// without the item's own DM verdict. One derivation, both sides.
|
|
2027
|
+
releaseThreadLock(channel, threadLockKey({
|
|
2028
|
+
channel,
|
|
2029
|
+
threadId: item.thread_id,
|
|
2030
|
+
isDm: item.is_dm === true,
|
|
2031
|
+
channelMainLock: String(process.env.MAESTRO_CHANNEL_MAIN_LOCK ?? "1") !== "0",
|
|
2032
|
+
}));
|
|
2024
2033
|
// Now that the lock is gone, promote any messages that were
|
|
2025
2034
|
// deferred behind it. Latest-wins: a burst of N messages
|
|
2026
2035
|
// collapses into ONE re-dispatch carrying the most recent
|
|
@@ -2157,7 +2166,16 @@ function spawnSession(entry) {
|
|
|
2157
2166
|
{
|
|
2158
2167
|
const channel = item.channel_id || (item.raw_ref ? (item.raw_ref.match(/slack:([^:]+):/) || [])[1] : null) || item.channel;
|
|
2159
2168
|
if (channel) {
|
|
2160
|
-
|
|
2169
|
+
// THE SAME KEY THE DAEMON TOOK. `item.thread_id` alone is not that
|
|
2170
|
+
// key: an untreaded post locks `channel-main` and a Cohort DM locks
|
|
2171
|
+
// `dm-channel`, and neither is derivable from an empty thread id
|
|
2172
|
+
// without the item's own DM verdict. One derivation, both sides.
|
|
2173
|
+
releaseThreadLock(channel, threadLockKey({
|
|
2174
|
+
channel,
|
|
2175
|
+
threadId: item.thread_id,
|
|
2176
|
+
isDm: item.is_dm === true,
|
|
2177
|
+
channelMainLock: String(process.env.MAESTRO_CHANNEL_MAIN_LOCK ?? "1") !== "0",
|
|
2178
|
+
}));
|
|
2161
2179
|
const promo = promoteDeferred(channel, AGENT_REPO_DIR);
|
|
2162
2180
|
if (promo.promoted > 0) {
|
|
2163
2181
|
logSession({
|