@cohortapp/agent-sdk 2.18.13 → 2.18.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +38 -1
- package/docs/runbooks/fleet-rollout.md +58 -7
- package/docs/runbooks/recovery-and-failover.md +18 -0
- package/lib/assurance/batch.mjs +353 -0
- package/lib/assurance/first-reply.mjs +423 -0
- package/lib/assurance/notice-voice.mjs +357 -0
- package/lib/assurance/plan-note.mjs +43 -0
- package/lib/assurance/room-budget.mjs +55 -6
- package/lib/cadence-failure-class.mjs +245 -0
- package/lib/claude-bin.mjs +26 -7
- package/lib/cli/doctor-checks.mjs +149 -1
- package/lib/comms/send-gate.mjs +59 -0
- package/lib/diagnostics/alerts.mjs +33 -0
- package/lib/engine/agents/usage.mjs +45 -0
- package/lib/engine/budget.mjs +293 -29
- package/lib/engine/cli.mjs +54 -5
- package/lib/engine/loop.mjs +30 -0
- package/lib/engine/output/json.mjs +26 -0
- package/lib/engine/wire/errors.mjs +179 -0
- package/lib/engine/wire/search.mjs +44 -8
- package/lib/identity/persona.mjs +31 -2
- package/lib/org/quota.mjs +27 -0
- package/lib/session/config.mjs +4 -0
- package/lib/session/identity.mjs +71 -7
- package/lib/session/launch-failure.mjs +251 -0
- package/lib/session/resume-target.mjs +86 -0
- package/lib/telemetry/alerts.mjs +94 -0
- package/lib/telemetry/collect.mjs +155 -2
- package/lib/upgrade/pinned-drift.mjs +467 -0
- package/package.json +1 -1
- package/scaffold/config/alerts.yaml +7 -0
- package/scripts/ci/check-cadence-prompts-exist.mjs +96 -0
- package/scripts/ci/check.mjs +3 -0
- package/scripts/daemon/agent-daemon.mjs +75 -5
- package/scripts/daemon/assurance.mjs +709 -44
- package/scripts/daemon/cadence-consumer.mjs +281 -34
- package/scripts/daemon/deliver.mjs +109 -0
- package/scripts/daemon/dispatcher.mjs +21 -3
- package/scripts/daemon/inbox-deferral.mjs +102 -9
- package/scripts/daemon/session-lock.mjs +41 -1
- package/scripts/emergency-stop.sh +114 -13
- package/scripts/fleet/rollout.mjs +256 -10
- package/scripts/healthcheck.sh +131 -33
- package/scripts/local-triggers/autoupdate.sh +144 -11
- package/scripts/resume-operations.sh +101 -6
- package/scripts/session/supervisor.mjs +198 -5
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/cadence-failure-class.mjs — is this cadence failure worth retrying?
|
|
3
|
+
*
|
|
4
|
+
* THE FAULT THIS EXISTS FOR (measured, Eli Rosenberg's seat, 2026-09-24).
|
|
5
|
+
* His config scheduled the cadence `commitment-sweep` with
|
|
6
|
+
* `prompt: schedules/triggers/commitment-sweep.md`. That file was not on the
|
|
7
|
+
* seat. The consumer never checked: it rendered the handoff, `readFileSync`
|
|
8
|
+
* threw `ENOENT: no such file or directory`, the catch logged
|
|
9
|
+
* `handoff_failed_spawning_instead` and fell through to the spawn — which
|
|
10
|
+
* opens THE SAME PATH — and the tick went back on the bus. Measured at
|
|
11
|
+
* 16 requeues per 30 seconds, indefinitely. It is also the failure the seat's
|
|
12
|
+
* own emergency stop had been set to contain once before
|
|
13
|
+
* (`reason=meeting-action-capture requeue busy-loop containment`), so this
|
|
14
|
+
* shape has burned a seat down at least twice.
|
|
15
|
+
*
|
|
16
|
+
* The defect is not the missing file. It is that **every failure was treated
|
|
17
|
+
* as transient**. A file that does not exist will not exist on the next
|
|
18
|
+
* attempt 30 seconds later; retrying is not resilience, it is a busy loop with
|
|
19
|
+
* a log line. The bus already had a retry budget and a circuit breaker; what it
|
|
20
|
+
* did not have was the question *"can this possibly succeed if I try again?"*
|
|
21
|
+
*
|
|
22
|
+
* THE RULE, and its honest boundary. A permanent failure is one whose CAUSE
|
|
23
|
+
* cannot change as a result of waiting. That is not the same as "can never
|
|
24
|
+
* succeed": a human (or an `agent-sync`) could drop the missing prompt onto the
|
|
25
|
+
* seat between two attempts. So the policy here is NOT zero retries — it is a
|
|
26
|
+
* SMALL BOUNDED number of attempts (`PERMANENT_MAX_ATTEMPTS`) after which the
|
|
27
|
+
* tick is dead-lettered.
|
|
28
|
+
*
|
|
29
|
+
* ~~"and the long window is the cadence's OWN SCHEDULE: the next scheduled tick
|
|
30
|
+
* is a fresh event with a fresh budget, so a daily cadence re-tries twice a day
|
|
31
|
+
* ... What the classification removes is only the tight loop"~~ — struck
|
|
32
|
+
* 2026-09-25, on review of the lane that introduced it. The second half is
|
|
33
|
+
* true; the first half described a retry WINDOW this module does not enforce
|
|
34
|
+
* and cannot. `PERMANENT_MAX_ATTEMPTS` is a COUNT, not a delay. The next
|
|
35
|
+
* scheduled tick really is a fresh event with a fresh budget, but that only
|
|
36
|
+
* becomes "the window" AFTER the last attempt reaches dlq/ — and nothing here
|
|
37
|
+
* spaces the attempts before then. A caller that simply requeues gets its
|
|
38
|
+
* second attempt on the very next poll.
|
|
39
|
+
*
|
|
40
|
+
* What the classification actually buys, stated exactly: a fault whose cause
|
|
41
|
+
* cannot move goes from UNBOUNDED retries at drain speed (16 per 30 s, measured)
|
|
42
|
+
* to ONE extra attempt, then a stop with a durable record. Spacing that one
|
|
43
|
+
* extra attempt is the CALLER's job, because only the caller owns a clock —
|
|
44
|
+
* `cadence-consumer.mjs` `failPermanently` arms the per-cadence backoff gate for
|
|
45
|
+
* exactly that reason, so the retry cannot land in the same drain. Read the
|
|
46
|
+
* budget here as "how many", and look at the caller for "how far apart".
|
|
47
|
+
*
|
|
48
|
+
* THE IDIOM ALREADY EXISTED ONE LANE OVER. `scripts/daemon/assurance.mjs`
|
|
49
|
+
* `classifyFailure` asks exactly this question for the INBOX lane, and says why
|
|
50
|
+
* in the same words: "retrying a permanent fault burns another 45 minutes of
|
|
51
|
+
* the human's patience for the same outcome". The cadence lane never asked it.
|
|
52
|
+
* Two deliberate differences from that one: this classifier keys on the PHASE
|
|
53
|
+
* (we know whether we were touching the cadence's own prompt file, so an errno
|
|
54
|
+
* is unambiguous rather than a guess at a message), and it returns the attempt
|
|
55
|
+
* budget with the verdict so the caller does not re-derive the policy.
|
|
56
|
+
*
|
|
57
|
+
* ELSEWHERE IN THE DAEMON — the same shape, swept 2026-09-25. Four other retry
|
|
58
|
+
* loops were read for "does it ever inspect the error class?". None do. Three
|
|
59
|
+
* are BOUNDED, which is what keeps them out of this module for now; the fourth
|
|
60
|
+
* is not, and is deliberate:
|
|
61
|
+
*
|
|
62
|
+
* - `dispatcher.mjs` backlog retries (MAX_BACKLOG_RETRIES = 6, plus a
|
|
63
|
+
* post-failure cooldown). Bounded, but a permanent fault spends all six
|
|
64
|
+
* sessions and six cooldowns to learn what the first one knew.
|
|
65
|
+
* - `dispatcher.mjs` resume reconcile (RESUME_MAX_ATTEMPTS = 3 + a freshness
|
|
66
|
+
* window + retire-on-no-id/no-prompt). Bounded. A dead `--resume` id burns
|
|
67
|
+
* all three: "No conversation found with session ID" is as permanent as an
|
|
68
|
+
* ENOENT and is never read as such.
|
|
69
|
+
* - `lib/session/identity.mjs` `rotationDecision`. This one uses TIMING as a
|
|
70
|
+
* proxy for the class question — a resume that dies inside 20 s rotates to
|
|
71
|
+
* a fresh id, which is why James Kirkland's 2-second `--resume` failure is
|
|
72
|
+
* handled at HEAD. But a failure that takes LONGER than the window returns
|
|
73
|
+
* `{action:"relaunch"}`, and the relaunch re-uses the same dead id. A
|
|
74
|
+
* timing proxy answers the wrong question; the message says "permanent"
|
|
75
|
+
* outright.
|
|
76
|
+
* - `cadence-consumer.mjs` `sweepHandoffTimeouts` → `handoff_requeue_failed`:
|
|
77
|
+
* genuinely unbounded ("handoff kept open; retried next sweep"), and
|
|
78
|
+
* deliberately so — the alternative is losing the tick. It is at least
|
|
79
|
+
* loud (error level, every sweep). Nothing escalates it.
|
|
80
|
+
*
|
|
81
|
+
* PURE. Strings and numbers in, a verdict out. No fs, no clock, no env, no
|
|
82
|
+
* network — so the policy can be tested on its own (and reasoned about) without
|
|
83
|
+
* standing up a consumer. Never throws: a garbage input classifies as transient,
|
|
84
|
+
* because the safe default when we cannot tell is today's behaviour.
|
|
85
|
+
*
|
|
86
|
+
* @module lib/cadence-failure-class
|
|
87
|
+
*/
|
|
88
|
+
|
|
89
|
+
/** A failure whose cause cannot change by waiting. Bounded retries, then DLQ. */
|
|
90
|
+
export const PERMANENT = "permanent";
|
|
91
|
+
/** A failure that could plausibly succeed on the next attempt. Today's budget. */
|
|
92
|
+
export const TRANSIENT = "transient";
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* A CAP on how many attempts a PERMANENT failure gets before the tick is
|
|
96
|
+
* dead-lettered. A caller applies `min(its own budget, this)` — it lowers a
|
|
97
|
+
* budget, it never raises one.
|
|
98
|
+
*
|
|
99
|
+
* Why 2 and not 1: the cause can be repaired between attempts (someone writes
|
|
100
|
+
* the prompt file), and one extra attempt is cheap — a single spawn, not a
|
|
101
|
+
* loop. Why a cap at all, when the cadence consumer's own transient budget
|
|
102
|
+
* already happens to be 2: `failTick`'s default is 5 and other call sites use
|
|
103
|
+
* it, so the number has to be stated here rather than inherited from whichever
|
|
104
|
+
* lane happens to be calling.
|
|
105
|
+
*
|
|
106
|
+
* This is a COUNT and nothing else. It does not delay anything; two attempts
|
|
107
|
+
* with no spacing are two attempts in the same drain. The caller is what makes
|
|
108
|
+
* the extra attempt cost a backoff interval rather than a millisecond. The
|
|
109
|
+
* budget is the SMALLEST part of this fix; the larger parts are that a
|
|
110
|
+
* permanent failure stops where it is instead of falling through to a second
|
|
111
|
+
* code path that opens the same missing file, and that it is recorded and
|
|
112
|
+
* alerted rather than logged and forgotten.
|
|
113
|
+
*/
|
|
114
|
+
export const PERMANENT_MAX_ATTEMPTS = 2;
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* The permanent codes, as a frozen set, so a caller can branch on the code
|
|
118
|
+
* without re-deriving the taxonomy.
|
|
119
|
+
* @type {Readonly<Record<string,string>>}
|
|
120
|
+
*/
|
|
121
|
+
export const PERMANENT_CODES = Object.freeze({
|
|
122
|
+
/** The cadence's prompt file is not on disk at the configured path. */
|
|
123
|
+
PROMPT_MISSING: "prompt_missing",
|
|
124
|
+
/** The prompt path exists but cannot be read as a file (perms / it's a dir). */
|
|
125
|
+
PROMPT_UNREADABLE: "prompt_unreadable",
|
|
126
|
+
/** The cadence has no handler and no prompt anywhere to fall back to. */
|
|
127
|
+
NO_PROMPT_CONFIGURED: "no_prompt_configured",
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
/** errno tokens that mean "this path is not a readable file, and waiting won't help". */
|
|
131
|
+
const PERMANENT_PATH_ERRNOS = /\b(ENOENT|EACCES|EPERM|EISDIR|ENOTDIR|ELOOP|ENAMETOOLONG)\b/;
|
|
132
|
+
/** errno tokens on a path that CAN clear on their own (busy fs, full disk, IO blip). */
|
|
133
|
+
const TRANSIENT_PATH_ERRNOS = /\b(EBUSY|EAGAIN|EMFILE|ENFILE|ENOSPC|EIO|ETIMEDOUT)\b/;
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* `realSpawnSession` exit codes that are not process exits at all but the
|
|
137
|
+
* consumer's own pre-flight verdicts. Kept here rather than re-matched from
|
|
138
|
+
* prose, because the message text is a log string and the number is a contract.
|
|
139
|
+
*/
|
|
140
|
+
export const SPAWN_EXIT_PROMPT_MISSING = -2;
|
|
141
|
+
export const SPAWN_EXIT_PROMPT_READ_FAILED = -3;
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* @typedef {object} CadenceFailureVerdict
|
|
145
|
+
* @property {"permanent"|"transient"} class the verdict
|
|
146
|
+
* @property {string|null} code a stable machine code for a permanent failure
|
|
147
|
+
* (`PERMANENT_CODES.*`), else null
|
|
148
|
+
* @property {number|null} maxAttempts attempt budget the caller should apply;
|
|
149
|
+
* null means "the caller's own default"
|
|
150
|
+
* @property {string} reason one short human line saying why
|
|
151
|
+
*/
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Classify a cadence failure.
|
|
155
|
+
*
|
|
156
|
+
* @param {object} [failure]
|
|
157
|
+
* @param {string} [failure.error] the error message / stderr tail
|
|
158
|
+
* @param {string} [failure.errno] the Node error `code` (`err.code`), when
|
|
159
|
+
* the caller has the Error object. THE MESSAGE IS NOT A CONTRACT AND THE
|
|
160
|
+
* ERRNO IS — measured 2026-09-25: `readFileSync` on a DIRECTORY produces
|
|
161
|
+
* exactly `EISDIR: illegal operation on a directory, read`, with no path
|
|
162
|
+
* anywhere in it, so anything that recognises a path fault by finding the
|
|
163
|
+
* path in the text misses the commonest unreadable case outright. When
|
|
164
|
+
* `errno` is given it is authoritative and the text is not consulted for
|
|
165
|
+
* the class; the text stays the fallback for callers that only have a
|
|
166
|
+
* stderr tail (a sub-session's output is text, never an Error).
|
|
167
|
+
* @param {"prompt"|"spawn"} [failure.phase]
|
|
168
|
+
* WHERE the failure happened. `"prompt"` means we were resolving or
|
|
169
|
+
* reading the cadence's own prompt file — so a path errno is
|
|
170
|
+
* unambiguously ABOUT that file, and we do not have to guess from the
|
|
171
|
+
* message text. `"spawn"` (the default) means a sub-session ran.
|
|
172
|
+
* @returns {CadenceFailureVerdict}
|
|
173
|
+
*/
|
|
174
|
+
export function classifyCadenceFailure(failure = {}) {
|
|
175
|
+
try {
|
|
176
|
+
const text = typeof failure.error === "string" ? failure.error : String(failure.error ?? "");
|
|
177
|
+
const exitCode = Number.isFinite(failure.exitCode) ? failure.exitCode : null;
|
|
178
|
+
const phase = failure.phase === "prompt" ? "prompt" : "spawn";
|
|
179
|
+
// An explicit errno beats the message every time; falling back to the text
|
|
180
|
+
// is for callers holding a stderr tail rather than an Error.
|
|
181
|
+
const errno = typeof failure.errno === "string" && failure.errno ? failure.errno : null;
|
|
182
|
+
const hasErrno = (re) => (errno ? re.test(errno) : re.test(text));
|
|
183
|
+
|
|
184
|
+
// 1. The consumer's own pre-flight verdicts, by contract number.
|
|
185
|
+
if (exitCode === SPAWN_EXIT_PROMPT_MISSING) {
|
|
186
|
+
return permanent(PERMANENT_CODES.PROMPT_MISSING, "the cadence prompt file is not on disk");
|
|
187
|
+
}
|
|
188
|
+
if (exitCode === SPAWN_EXIT_PROMPT_READ_FAILED) {
|
|
189
|
+
// A read that failed for a reason that CAN clear (disk full, too many
|
|
190
|
+
// open files) is transient even though the exit code is the same.
|
|
191
|
+
if (hasErrno(TRANSIENT_PATH_ERRNOS)) {
|
|
192
|
+
return transient("prompt read failed for a cause that can clear on its own");
|
|
193
|
+
}
|
|
194
|
+
return permanent(PERMANENT_CODES.PROMPT_UNREADABLE, "the cadence prompt path is not a readable file");
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// 2. The prompt phase: any path errno here is about the prompt file.
|
|
198
|
+
if (phase === "prompt") {
|
|
199
|
+
if (hasErrno(TRANSIENT_PATH_ERRNOS)) {
|
|
200
|
+
return transient("prompt I/O failed for a cause that can clear on its own");
|
|
201
|
+
}
|
|
202
|
+
if (hasErrno(/\bENOENT\b/) || /prompt not found/i.test(text)) {
|
|
203
|
+
return permanent(PERMANENT_CODES.PROMPT_MISSING, "the cadence prompt file is not on disk");
|
|
204
|
+
}
|
|
205
|
+
if (hasErrno(PERMANENT_PATH_ERRNOS)) {
|
|
206
|
+
return permanent(PERMANENT_CODES.PROMPT_UNREADABLE, "the cadence prompt path is not a readable file");
|
|
207
|
+
}
|
|
208
|
+
// A prompt-phase failure we do not recognise (a render bug, say) keeps
|
|
209
|
+
// the ordinary budget: we do not dead-letter on a guess.
|
|
210
|
+
return transient("prompt step failed for an unrecognised reason");
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// 3. Spawn phase. `prompt not found:` is realSpawnSession's own wording and
|
|
214
|
+
// is definitive wherever it surfaces; a bare errno in a sub-session's
|
|
215
|
+
// stderr is NOT — it is far more likely to be about some file the session
|
|
216
|
+
// itself touched than about the cadence. So we do not read it.
|
|
217
|
+
if (/prompt not found/i.test(text)) {
|
|
218
|
+
return permanent(PERMANENT_CODES.PROMPT_MISSING, "the cadence prompt file is not on disk");
|
|
219
|
+
}
|
|
220
|
+
if (/no handler and no prompt/i.test(text)) {
|
|
221
|
+
return permanent(PERMANENT_CODES.NO_PROMPT_CONFIGURED, "the cadence has neither a handler nor a prompt");
|
|
222
|
+
}
|
|
223
|
+
return transient("no rule recognises this failure as permanent");
|
|
224
|
+
} catch {
|
|
225
|
+
// A classifier that can throw is a classifier that can wedge the lane it
|
|
226
|
+
// advises. Unknown → transient → today's behaviour.
|
|
227
|
+
return transient("classifier error; defaulting to the retryable path");
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Is a verdict (or a bare class string) permanent? */
|
|
232
|
+
export function isPermanent(v) {
|
|
233
|
+
if (!v) return false;
|
|
234
|
+
return (typeof v === "string" ? v : v.class) === PERMANENT;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function permanent(code, reason) {
|
|
238
|
+
return { class: PERMANENT, code, maxAttempts: PERMANENT_MAX_ATTEMPTS, reason };
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function transient(reason) {
|
|
242
|
+
return { class: TRANSIENT, code: null, maxAttempts: null, reason };
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
export default { classifyCadenceFailure, isPermanent, PERMANENT, TRANSIENT, PERMANENT_CODES, PERMANENT_MAX_ATTEMPTS };
|
package/lib/claude-bin.mjs
CHANGED
|
@@ -46,6 +46,31 @@ export function isSafeExecutable(p) {
|
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
/**
|
|
50
|
+
* The fixed paths `resolveClaudeBin` searches, in order, AFTER $CLAUDE_BIN.
|
|
51
|
+
* `~/` is the seat's home directory.
|
|
52
|
+
*
|
|
53
|
+
* THIS IS THE ONE COPY. `lib/cli/doctor-checks.mjs` reports on exactly these
|
|
54
|
+
* paths and imports this list rather than restating it: a hand-copied twin is
|
|
55
|
+
* how the doctor comes to say a path resolves that the resolver never looks at
|
|
56
|
+
* (and vice versa), and nothing fails when it does.
|
|
57
|
+
*/
|
|
58
|
+
export const CLAUDE_BIN_CANDIDATES = Object.freeze([
|
|
59
|
+
"~/.local/bin/claude",
|
|
60
|
+
"/opt/homebrew/bin/claude",
|
|
61
|
+
"/usr/local/bin/claude",
|
|
62
|
+
"/usr/bin/claude",
|
|
63
|
+
]);
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* `CLAUDE_BIN_CANDIDATES` with `~/` expanded against `home`.
|
|
67
|
+
* @param {string} [home] defaults to this process's home directory
|
|
68
|
+
* @returns {string[]}
|
|
69
|
+
*/
|
|
70
|
+
export function claudeBinCandidatePaths(home = homedir()) {
|
|
71
|
+
return CLAUDE_BIN_CANDIDATES.map((c) => (c.startsWith("~/") ? join(home, c.slice(2)) : c));
|
|
72
|
+
}
|
|
73
|
+
|
|
49
74
|
/**
|
|
50
75
|
* Return the absolute path to the Claude CLI. Searches, in order:
|
|
51
76
|
* 1. $CLAUDE_BIN env var (if set + executable on disk)
|
|
@@ -70,13 +95,7 @@ export function resolveClaudeBin() {
|
|
|
70
95
|
`non-writable executable. Falling back to default search paths.`
|
|
71
96
|
);
|
|
72
97
|
}
|
|
73
|
-
const
|
|
74
|
-
join(homedir(), ".local/bin/claude"),
|
|
75
|
-
"/opt/homebrew/bin/claude",
|
|
76
|
-
"/usr/local/bin/claude",
|
|
77
|
-
"/usr/bin/claude",
|
|
78
|
-
];
|
|
79
|
-
for (const c of candidates) {
|
|
98
|
+
for (const c of claudeBinCandidatePaths()) {
|
|
80
99
|
if (existsSync(c)) { _resolved = c; return c; }
|
|
81
100
|
}
|
|
82
101
|
// Last resort: bare name (lets the OS PATH + spawn error stay informative).
|
|
@@ -35,6 +35,7 @@ import { probeClaude as defaultProbeClaude } from "../setup/claude-probe.mjs";
|
|
|
35
35
|
import { readSeatEngine } from "../runtime/adapter.mjs";
|
|
36
36
|
import { sessionLiveness, SESSION_STALE_MS } from "../telemetry/collect.mjs";
|
|
37
37
|
import { CONSOLE_KEY_REMEDY, SEAT_AUTH_VARS, REQUIRE_CONSOLE_KEY_VAR, seatAuthVerdict } from "./seat-auth.mjs";
|
|
38
|
+
import { CLAUDE_BIN_CANDIDATES, isSafeExecutable } from "../claude-bin.mjs";
|
|
38
39
|
|
|
39
40
|
export const SDK_PACKAGE = "@cohortapp/agent-sdk";
|
|
40
41
|
/** The SDK checkout this module ships in — the source of the shipped hook copies. */
|
|
@@ -151,6 +152,132 @@ export function hookNodeResolvable({ onPath = "", isExecutable = () => false, ho
|
|
|
151
152
|
};
|
|
152
153
|
}
|
|
153
154
|
|
|
155
|
+
/**
|
|
156
|
+
* Where `lib/claude-bin.mjs#resolveClaudeBin` looks, in order. `~/` is the
|
|
157
|
+
* seat's home. IMPORTED, never restated: this row's whole job is to describe
|
|
158
|
+
* what the resolver does, so the list it reports on has to be the resolver's
|
|
159
|
+
* own. A hand-copied twin drifted silently the moment a path was added to one
|
|
160
|
+
* of them.
|
|
161
|
+
*/
|
|
162
|
+
export { CLAUDE_BIN_CANDIDATES };
|
|
163
|
+
/** Where the Claude Code installer keeps its versioned installs. */
|
|
164
|
+
export const CLAUDE_VERSIONS_DIR = "~/.local/share/claude/versions";
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Can anything on this seat actually RUN `claude`?
|
|
168
|
+
*
|
|
169
|
+
* ── THE SEAT THIS EXISTS FOR ────────────────────────────────────────────────
|
|
170
|
+
* James Kirkland's machine, measured 2026-09-25: no `~/.local/bin/claude` at
|
|
171
|
+
* all, while `~/.local/share/claude/versions/` held 2.1.273 and 2.1.278. Every
|
|
172
|
+
* launch exited 127 — for days. `resolveClaudeBin` falls back to the bare name
|
|
173
|
+
* `"claude"` when none of its fixed paths exist, deliberately, so the spawn's
|
|
174
|
+
* own error stays informative; but nobody was reading the spawn's error, and
|
|
175
|
+
* nothing anywhere said the plainest possible thing: the binary is not there.
|
|
176
|
+
*
|
|
177
|
+
* So this says it plainly, and — because the repair genuinely is one line —
|
|
178
|
+
* it prints the line. A `versions/` directory with installs in it and no
|
|
179
|
+
* symlink pointing at one is an interrupted install, not a missing product.
|
|
180
|
+
*
|
|
181
|
+
* ── THE ROW MUST ASK WHAT THE RESOLVER ASKS ────────────────────────────────
|
|
182
|
+
* Two probes, because `resolveClaudeBin` uses two different gates and an `ok`
|
|
183
|
+
* that answers the wrong one is the exact blind spot this row exists to close:
|
|
184
|
+
*
|
|
185
|
+
* $CLAUDE_BIN — accepted only if `isSafeExecutable`: a regular file,
|
|
186
|
+
* not group/world-writable, owned by this uid or root
|
|
187
|
+
* (M2). A 0o777 binary is executable AND ignored, so
|
|
188
|
+
* asking X_OK alone reports `ok` for an override that
|
|
189
|
+
* never gets used and a seat that exits 127 every spawn.
|
|
190
|
+
* the fixed paths — accepted on mere `existsSync`. A present-but-chmod-644
|
|
191
|
+
* `~/.local/bin/claude` is therefore what the resolver
|
|
192
|
+
* returns, and the spawn fails with EACCES — which is a
|
|
193
|
+
* different fault, and a different repair, from "no
|
|
194
|
+
* binary is installed".
|
|
195
|
+
*
|
|
196
|
+
* Pure over its inputs; the caller supplies the probes.
|
|
197
|
+
*
|
|
198
|
+
* @param {object} o
|
|
199
|
+
* @param {string} [o.onPath] `which claude` output ("" when it does not resolve)
|
|
200
|
+
* @param {(p:string)=>boolean} [o.isExecutable] can it be executed (X_OK)
|
|
201
|
+
* @param {(p:string)=>boolean} [o.isSafe] would claude-bin.mjs TRUST it as $CLAUDE_BIN (defaults to its own `isSafeExecutable`)
|
|
202
|
+
* @param {(p:string)=>boolean} [o.exists] is it on disk at all (defaults to `isExecutable`, i.e. the pre-existing behaviour)
|
|
203
|
+
* @param {string} [o.home] the seat's home directory
|
|
204
|
+
* @param {string[]} [o.versions] entries of ~/.local/share/claude/versions (newest need not be sorted)
|
|
205
|
+
* @param {string} [o.envOverride] $CLAUDE_BIN, when set
|
|
206
|
+
* @returns {{level:"ok"|"warn"|"fail", msg:string}}
|
|
207
|
+
*/
|
|
208
|
+
export function claudeBinResolvable({ onPath = "", isExecutable = () => false, isSafe = isSafeExecutable, exists = null, home = "", versions = [], envOverride = "" } = {}) {
|
|
209
|
+
const abs = (c) => (c.startsWith("~/") ? `${home}/${c.slice(2)}` : c);
|
|
210
|
+
const onDisk = typeof exists === "function" ? exists : isExecutable;
|
|
211
|
+
const override = String(envOverride || "").trim();
|
|
212
|
+
if (override) {
|
|
213
|
+
const trusted = isSafe(override);
|
|
214
|
+
if (trusted && isExecutable(override)) return { level: "ok", msg: `Claude CLI: $CLAUDE_BIN resolves (${override})` };
|
|
215
|
+
if (trusted) {
|
|
216
|
+
return {
|
|
217
|
+
level: "warn",
|
|
218
|
+
msg: `Claude CLI: $CLAUDE_BIN is set to ${override}, which lib/claude-bin.mjs accepts but the OS will not execute — every session launch and every daemon spawn fails on it; chmod +x ${override} or unset CLAUDE_BIN`,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
return {
|
|
222
|
+
level: "warn",
|
|
223
|
+
msg: `Claude CLI: $CLAUDE_BIN is set to ${override}, which lib/claude-bin.mjs IGNORES — it accepts an override only when it is a regular file, not group- or world-writable, and owned by you or root (M2); it falls through to the fixed paths instead. Unset it, or fix the file (chmod go-w ${override}) and point it at a real binary`,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
let present = "";
|
|
227
|
+
for (const c of CLAUDE_BIN_CANDIDATES) {
|
|
228
|
+
if (c.startsWith("~/") && !home) continue;
|
|
229
|
+
const p = abs(c);
|
|
230
|
+
if (isExecutable(p)) return { level: "ok", msg: `Claude CLI: resolves at ${p}` };
|
|
231
|
+
if (!present && onDisk(p)) present = p;
|
|
232
|
+
}
|
|
233
|
+
// The resolver searches with existsSync, so a file that is THERE and not
|
|
234
|
+
// executable is the one it returns — and the spawn, not the search, is what
|
|
235
|
+
// fails. Saying "install Claude Code" here would send an operator to fix a
|
|
236
|
+
// thing that is not broken.
|
|
237
|
+
if (present) {
|
|
238
|
+
return {
|
|
239
|
+
level: "fail",
|
|
240
|
+
msg: `Claude CLI: ${present} exists but is not executable — lib/claude-bin.mjs resolves it on presence alone, so every session launch and every daemon spawn fails on it. Fix: chmod +x ${present}`,
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
const found = String(onPath || "").trim();
|
|
244
|
+
if (found) {
|
|
245
|
+
return {
|
|
246
|
+
level: "warn",
|
|
247
|
+
msg: `Claude CLI: \`claude\` is on your PATH (${found}) but at none of the fixed paths lib/claude-bin.mjs searches (${CLAUDE_BIN_CANDIDATES.join(", ")}) — launchd jobs get a bare environment and will not find it; symlink it: ln -sfn ${found} ${home ? `${home}/.local/bin/claude` : "~/.local/bin/claude"}`,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
// An interrupted install: the product is here, the entry point is not.
|
|
251
|
+
const installs = (Array.isArray(versions) ? versions : []).map((v) => String(v && v.name ? v.name : v)).filter(Boolean);
|
|
252
|
+
if (installs.length > 0) {
|
|
253
|
+
const newest = [...installs].sort(compareVersionNames).pop();
|
|
254
|
+
const target = `${home || "~"}/.local/share/claude/versions/${newest}`;
|
|
255
|
+
const link = `${home || "~"}/.local/bin/claude`;
|
|
256
|
+
return {
|
|
257
|
+
level: "fail",
|
|
258
|
+
msg: `Claude CLI: no \`claude\` binary resolves, but ${installs.length} install(s) sit in ${CLAUDE_VERSIONS_DIR} (${installs.join(", ")}) — every session and every daemon spawn exits 127 until the symlink is back. Fix: mkdir -p ${home || "~"}/.local/bin && ln -sfn ${target} ${link}`,
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
return {
|
|
262
|
+
level: "fail",
|
|
263
|
+
msg: `Claude CLI: no \`claude\` binary resolves — not on PATH and at none of ${CLAUDE_BIN_CANDIDATES.join(", ")}, and ${CLAUDE_VERSIONS_DIR} holds no installs. Every session launch and every daemon spawn exits 127. Install Claude Code (curl -fsSL https://claude.ai/install.sh | bash), then re-run \`maestro doctor\``,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Sort helper: version-ish directory names, numerically where they look numeric. */
|
|
268
|
+
function compareVersionNames(a, b) {
|
|
269
|
+
const seg = (v) => String(v).split(/[.\-]/).map((x) => (/^\d+$/.test(x) ? Number(x) : x));
|
|
270
|
+
const A = seg(a); const B = seg(b);
|
|
271
|
+
for (let i = 0; i < Math.max(A.length, B.length); i += 1) {
|
|
272
|
+
const x = A[i]; const y = B[i];
|
|
273
|
+
if (x === undefined) return -1;
|
|
274
|
+
if (y === undefined) return 1;
|
|
275
|
+
if (typeof x === "number" && typeof y === "number") { if (x !== y) return x - y; continue; }
|
|
276
|
+
if (String(x) !== String(y)) return String(x) < String(y) ? -1 : 1;
|
|
277
|
+
}
|
|
278
|
+
return 0;
|
|
279
|
+
}
|
|
280
|
+
|
|
154
281
|
/**
|
|
155
282
|
* Do the seat's copies of the pre-send hook and the send gate match the
|
|
156
283
|
* versions this SDK ships? `maestro upgrade` keeps a locally modified copy (and
|
|
@@ -549,6 +676,27 @@ export async function runDoctorChecks(o = {}) {
|
|
|
549
676
|
}
|
|
550
677
|
}
|
|
551
678
|
|
|
679
|
+
// ── The Claude CLI itself (§3.1) ──────────────────────────────────────────
|
|
680
|
+
//
|
|
681
|
+
// Before "is the session job installed" comes "is there anything for it to
|
|
682
|
+
// run". A seat with a loaded job, a live supervisor and no binary exits 127
|
|
683
|
+
// on every launch and looks, from every other row here, entirely fine.
|
|
684
|
+
{
|
|
685
|
+
const home = o.home !== undefined ? o.home : homedir();
|
|
686
|
+
const isExecutable = o.isExecutable || ((p) => { try { accessSync(p, fsConstants.X_OK); return true; } catch { return false; } });
|
|
687
|
+
// The two gates `resolveClaudeBin` actually uses, wired to the real thing:
|
|
688
|
+
// `isSafeExecutable` for the $CLAUDE_BIN override (it is the resolver's own
|
|
689
|
+
// function, imported, not a restatement), `existsSync` for the fixed paths.
|
|
690
|
+
const isSafe = o.isSafe || isSafeExecutable;
|
|
691
|
+
const exists = o.existsSync || existsSync;
|
|
692
|
+
const listVersions = o.listClaudeVersions || (() => { try { return readdirSync(join(home, ".local", "share", "claude", "versions")); } catch { return []; } });
|
|
693
|
+
const binRow = claudeBinResolvable({
|
|
694
|
+
onPath: tryExec("which", ["claude"], { timeout: 2000 }).trim(),
|
|
695
|
+
isExecutable, isSafe, exists, home, versions: listVersions(), envOverride: env.CLAUDE_BIN || "",
|
|
696
|
+
});
|
|
697
|
+
rows.push(binRow);
|
|
698
|
+
}
|
|
699
|
+
|
|
552
700
|
// ── Main session job (§3.1) ───────────────────────────────────────────────
|
|
553
701
|
const listPlists = o.listPlists || (() => { try { return readdirSync(join(homedir(), "Library", "LaunchAgents")); } catch { return []; } });
|
|
554
702
|
const loaded = parseLaunchctlList(tryExec("launchctl", ["list"], { timeout: 4000 }));
|
|
@@ -569,4 +717,4 @@ function pick(obj, keys) {
|
|
|
569
717
|
return out;
|
|
570
718
|
}
|
|
571
719
|
|
|
572
|
-
export default { parseDotEnv, mask, authMode, looksLikeOrgId, orgIdShape, effectiveSource, sdkGlobalInstall, tailscaleSsh, sessionInstalled, hookNodeResolvable, hookCopyDrift, runDoctorChecks };
|
|
720
|
+
export default { parseDotEnv, mask, authMode, looksLikeOrgId, orgIdShape, effectiveSource, sdkGlobalInstall, tailscaleSsh, sessionInstalled, hookNodeResolvable, hookCopyDrift, claudeBinResolvable, runDoctorChecks };
|
package/lib/comms/send-gate.mjs
CHANGED
|
@@ -67,6 +67,7 @@ import {
|
|
|
67
67
|
} from "node:fs";
|
|
68
68
|
import { join } from "node:path";
|
|
69
69
|
import { writeFileAtomic, appendJsonl } from "../fs-atomic.mjs";
|
|
70
|
+
import { isRetiredFirstReply } from "../assurance/first-reply.mjs";
|
|
70
71
|
|
|
71
72
|
/* ─────────────────────────── policy file locations ───────────────────────── */
|
|
72
73
|
|
|
@@ -476,6 +477,47 @@ export function screenScaffold(text) {
|
|
|
476
477
|
return null;
|
|
477
478
|
}
|
|
478
479
|
|
|
480
|
+
/**
|
|
481
|
+
* (a1) THE RETIRED PLACEHOLDER REGISTER, on the session/MCP plane.
|
|
482
|
+
*
|
|
483
|
+
* `lib/assurance/first-reply.RETIRED_FIRST_REPLIES` holds the five sentences the
|
|
484
|
+
* owner screenshotted on 2026-09-25 — "On it.", "Looking now.", "On it —
|
|
485
|
+
* digging in now." and the two canned failure notices. `scripts/daemon/deliver`
|
|
486
|
+
* refuses them at the DAEMON's send. This is the same refusal on the other
|
|
487
|
+
* plane, and the two are genuinely different populations:
|
|
488
|
+
*
|
|
489
|
+
* deliver.mjs the daemon's own outbound — acks, notices, answers it posts
|
|
490
|
+
* on behalf of a session it spawned.
|
|
491
|
+
* here `messaging_send`, `email_send`, `email_draft_send` and
|
|
492
|
+
* `org_call_share_step`, native and MCP — i.e. a SPAWNED
|
|
493
|
+
* SESSION typing a message itself. A session that opens with
|
|
494
|
+
* "On it." never crosses `deliver` at all, so before this the
|
|
495
|
+
* headline claim ("none of the five can leave this code") was
|
|
496
|
+
* true of the daemon and false of the session.
|
|
497
|
+
*
|
|
498
|
+
* Why it sits here rather than in a composer: a composer guard protects only
|
|
499
|
+
* the callers that have the fixed composer, and the whole reason this register
|
|
500
|
+
* exists is that two seats were emitting these sentences from a tree nobody
|
|
501
|
+
* could see. Every send on this plane crosses this function.
|
|
502
|
+
*
|
|
503
|
+
* NOT a "generic opener" judgement — `GENERIC_OPENER` in
|
|
504
|
+
* `lib/assurance/plan-note.mjs` owns that, and it is arguable. This is the
|
|
505
|
+
* exact, un-arguable half: five literal sentences, matched per paragraph so one
|
|
506
|
+
* buried in a longer body is caught too.
|
|
507
|
+
*
|
|
508
|
+
* @param {string} text
|
|
509
|
+
* @returns {string|null}
|
|
510
|
+
*/
|
|
511
|
+
export function screenRetiredPlaceholder(text) {
|
|
512
|
+
if (!isRetiredFirstReply(text)) return null;
|
|
513
|
+
return (
|
|
514
|
+
"retired placeholder reply in the outbound body — these exact lines are " +
|
|
515
|
+
"retired (lib/assurance/first-reply.RETIRED_FIRST_REPLIES). The first thing " +
|
|
516
|
+
"said has to come out of reading the actual message: say what you have " +
|
|
517
|
+
"understood and what happens next, or send nothing."
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
|
|
479
521
|
/**
|
|
480
522
|
* (b) AI-disclosure screen. Resolves the active posture for the recipient's
|
|
481
523
|
* jurisdiction from ai-disclosure.yaml and enforces two things:
|
|
@@ -729,6 +771,14 @@ export async function screenOutbound({
|
|
|
729
771
|
const scaffoldReason = screenScaffold(text);
|
|
730
772
|
if (scaffoldReason) return blocked(scaffoldReason);
|
|
731
773
|
|
|
774
|
+
// (a1) The retired placeholder register — see screenRetiredPlaceholder.
|
|
775
|
+
// Ahead of the banned-phrase list because "On it." trips nothing in
|
|
776
|
+
// that list, and a reason naming the register tells the caller what to
|
|
777
|
+
// do about it ("write the first line from the thread") where a generic
|
|
778
|
+
// tone complaint would not.
|
|
779
|
+
const retiredReason = screenRetiredPlaceholder(text);
|
|
780
|
+
if (retiredReason) return blocked(retiredReason);
|
|
781
|
+
|
|
732
782
|
const bannedReason = screenBannedPhrases(text);
|
|
733
783
|
if (bannedReason) return blocked(bannedReason);
|
|
734
784
|
|
|
@@ -1030,6 +1080,15 @@ export function screenHookPayload({ channel, stdin, sessionNames = [], allowlist
|
|
|
1030
1080
|
const scaffoldReason = screenScaffold(text);
|
|
1031
1081
|
if (scaffoldReason) return deny(`Session-scaffold marker in outbound message: ${scaffoldReason}`);
|
|
1032
1082
|
|
|
1083
|
+
// (a1) The retired placeholder register, on the hook plane. Outside the
|
|
1084
|
+
// parity contract by construction, like the scaffold screen above it: the
|
|
1085
|
+
// frozen shell oracle (fixtures/pre-send-audit.legacy.sh) has no counterpart
|
|
1086
|
+
// rule, so this is an ADDITION to the hook rather than a divergence from the
|
|
1087
|
+
// port. scripts/hooks/pre-send-audit.sh is a thin wrapper over this function,
|
|
1088
|
+
// so it inherits the check with no edit of its own.
|
|
1089
|
+
const retiredReason = screenRetiredPlaceholder(text);
|
|
1090
|
+
if (retiredReason) return deny(`Retired placeholder reply: ${retiredReason}`);
|
|
1091
|
+
|
|
1033
1092
|
const lc = text.toLowerCase();
|
|
1034
1093
|
for (const phrase of HOOK_BANNED_SUBSTRINGS) {
|
|
1035
1094
|
if (lc.includes(phrase.toLowerCase())) {
|
|
@@ -91,6 +91,20 @@ export const DEFAULT_THRESHOLDS = Object.freeze({
|
|
|
91
91
|
warnSec: 120, // 2 min (heartbeat refreshes ~15s)
|
|
92
92
|
critSec: 300, // 5 min
|
|
93
93
|
}),
|
|
94
|
+
cadencePermanentFailure: Object.freeze({
|
|
95
|
+
// A cadence failed for a cause that cannot change by waiting — its prompt
|
|
96
|
+
// file is not on disk, or the path is not readable. Unlike every other rule
|
|
97
|
+
// here, this one fires at ONE. The pattern-not-single-failure principle is
|
|
98
|
+
// about failures that are individually meaningless; a permanent failure is
|
|
99
|
+
// not one of those. It means a scheduled obligation has stopped running and
|
|
100
|
+
// will not start again until a person puts the file back — and the
|
|
101
|
+
// alternative to alerting is what actually happened on Eli Rosenberg's seat
|
|
102
|
+
// (2026-09-24): 16 requeues per 30 seconds for a day and a half, contained
|
|
103
|
+
// eventually by an emergency stop rather than noticed.
|
|
104
|
+
counters: Object.freeze(["cadence.permanent_failure"]),
|
|
105
|
+
warnCount: 1,
|
|
106
|
+
critCount: 3,
|
|
107
|
+
}),
|
|
94
108
|
cadenceOutputMissing: Object.freeze({
|
|
95
109
|
// A scheduled workflow finished without leaving its output artefacts behind.
|
|
96
110
|
// This is the ONLY rule here not derived from the cadence bus, and that is
|
|
@@ -275,6 +289,7 @@ export function evaluateAlerts(args = {}) {
|
|
|
275
289
|
push(alerts, costPerSessionRule(ledger, t.costPerSessionP99));
|
|
276
290
|
push(alerts, absentAgentRule(presence, t.absentAgent));
|
|
277
291
|
push(alerts, daemonDownRule(presence, t.daemonDown));
|
|
292
|
+
push(alerts, cadencePermanentFailureRule(snapshot, t.cadencePermanentFailure));
|
|
278
293
|
push(alerts, cadenceOutputMissingRule(outputs, t.cadenceOutputMissing));
|
|
279
294
|
|
|
280
295
|
// Deterministic order: critical first, then by id, so identical inputs always
|
|
@@ -380,6 +395,24 @@ function daemonDownRule(presence, cfg = {}) {
|
|
|
380
395
|
};
|
|
381
396
|
}
|
|
382
397
|
|
|
398
|
+
/**
|
|
399
|
+
* A cadence failed permanently: the cause cannot change by waiting, so the tick
|
|
400
|
+
* was dead-lettered rather than retried (see lib/cadence-failure-class.mjs and
|
|
401
|
+
* the consumer's `failPermanently`). Fires at one, by design — see the
|
|
402
|
+
* threshold comment. The counter is durable (counters JSONL), so this stays
|
|
403
|
+
* true across a daemon restart within the day.
|
|
404
|
+
*/
|
|
405
|
+
function cadencePermanentFailureRule(snapshot, cfg = {}) {
|
|
406
|
+
const count = sumCounters(snapshot, cfg.counters);
|
|
407
|
+
const sev = severityFor(count, num(cfg.warnCount, Infinity), num(cfg.critCount, Infinity));
|
|
408
|
+
if (!sev) return null;
|
|
409
|
+
return {
|
|
410
|
+
id: "cadence_permanently_failing",
|
|
411
|
+
severity: sev,
|
|
412
|
+
detail: `${count} cadence tick(s) failed permanently today (warn ${num(cfg.warnCount)}, crit ${num(cfg.critCount)}) — a scheduled obligation is not running and retrying will not fix it. Read state/cadence-bus/dlq/ for the cadence and the code (prompt_missing = put the prompt file back).`,
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
|
|
383
416
|
/**
|
|
384
417
|
* A scheduled workflow completed without producing its output artefacts.
|
|
385
418
|
*
|