@cohortapp/agent-sdk 2.18.13 → 2.18.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +38 -1
- package/docs/runbooks/fleet-rollout.md +14 -7
- package/docs/runbooks/recovery-and-failover.md +18 -0
- package/lib/cadence-failure-class.mjs +245 -0
- package/lib/claude-bin.mjs +26 -7
- package/lib/cli/doctor-checks.mjs +149 -1
- package/lib/diagnostics/alerts.mjs +33 -0
- package/lib/engine/agents/usage.mjs +45 -0
- package/lib/engine/budget.mjs +293 -29
- package/lib/engine/cli.mjs +54 -5
- package/lib/engine/loop.mjs +30 -0
- package/lib/engine/output/json.mjs +26 -0
- package/lib/engine/wire/errors.mjs +179 -0
- package/lib/engine/wire/search.mjs +44 -8
- package/lib/org/quota.mjs +27 -0
- package/lib/session/config.mjs +4 -0
- package/lib/session/identity.mjs +71 -7
- package/lib/session/launch-failure.mjs +251 -0
- package/lib/session/resume-target.mjs +86 -0
- package/lib/telemetry/collect.mjs +129 -0
- package/lib/upgrade/pinned-drift.mjs +467 -0
- package/package.json +1 -1
- package/scaffold/config/alerts.yaml +7 -0
- package/scripts/ci/check-cadence-prompts-exist.mjs +96 -0
- package/scripts/ci/check.mjs +3 -0
- package/scripts/daemon/cadence-consumer.mjs +281 -34
- package/scripts/emergency-stop.sh +114 -13
- package/scripts/fleet/rollout.mjs +10 -3
- package/scripts/healthcheck.sh +131 -33
- package/scripts/resume-operations.sh +101 -6
- package/scripts/session/supervisor.mjs +198 -5
package/bin/maestro.mjs
CHANGED
|
@@ -38,6 +38,7 @@ import { selectProvider } from "../lib/secrets/providers.mjs";
|
|
|
38
38
|
import { syncSecrets, rotateSecret, makeBroker, auditLogPath } from "../lib/secrets/broker.mjs";
|
|
39
39
|
import { applyBrandEnvCompat } from "../lib/env-compat.mjs";
|
|
40
40
|
import { buildIgnoredDriftReport, formatIgnoredLine, IGNORED_DRIFT_REL } from "../lib/upgrade/ignored-drift.mjs";
|
|
41
|
+
import { summarisePinnedDrift, formatPinnedDriftWarning } from "../lib/upgrade/pinned-drift.mjs";
|
|
41
42
|
|
|
42
43
|
// Fleet back-compat FIRST: bridge NEOLITH_* ⇄ COHORT_* env names before any
|
|
43
44
|
// env-first resolution below — installed hosts still export the legacy names.
|
|
@@ -1613,7 +1614,6 @@ Per-file behaviour:
|
|
|
1613
1614
|
{ sdkVersion: readFrameworkVersion(), at: new Date().toISOString() },
|
|
1614
1615
|
);
|
|
1615
1616
|
for (const f of ignoredDrift.files) console.log(formatIgnoredLine(f));
|
|
1616
|
-
if (ignoredDrift.counts.drifts) warn(`${ignoredDrift.counts.drifts} protected file(s) drift from upstream — every upstream fix to those paths stops here until you port it (diff each against node_modules/@cohortapp/agent-sdk/<path>)`);
|
|
1617
1617
|
if (!flags.dryRun) {
|
|
1618
1618
|
try {
|
|
1619
1619
|
const out = join(cwd, IGNORED_DRIFT_REL);
|
|
@@ -1622,6 +1622,43 @@ Per-file behaviour:
|
|
|
1622
1622
|
} catch (e) { warn(`could not write ${IGNORED_DRIFT_REL}: ${e && e.message ? e.message : e}`); }
|
|
1623
1623
|
}
|
|
1624
1624
|
}
|
|
1625
|
+
// ── THE PIN THAT STRANDS A FIX ──────────────────────────────────────────
|
|
1626
|
+
// The per-file lines above say what every protected file looks like. They do
|
|
1627
|
+
// not say which of them MATTERS, and on 2026-09-25 that distinction was the
|
|
1628
|
+
// difference between a healthy fork and a seat dying at import time on a
|
|
1629
|
+
// pinned `deliver.mjs`. `summarisePinnedDrift` asks the one question the
|
|
1630
|
+
// report does not — is upstream carrying lines this pin refuses — and the
|
|
1631
|
+
// warning names the file, the distance, the runnable remedy, and (because
|
|
1632
|
+
// the obvious wrong response is to empty `.maestroignore`) which pins must
|
|
1633
|
+
// stay. See lib/upgrade/pinned-drift.mjs for why `onlyUpstream > 0` rather
|
|
1634
|
+
// than "drifts" is the predicate.
|
|
1635
|
+
{
|
|
1636
|
+
// `|| { files: [] }` and not `null`: here the upgrade has just RUN, so an
|
|
1637
|
+
// empty ignored list is a measured answer ("your patterns matched nothing")
|
|
1638
|
+
// and not the unreadable-report case. `null` would be read as unknown,
|
|
1639
|
+
// which is the right answer for the beat's disk read and the wrong one
|
|
1640
|
+
// here — the caller is the thing that took the measurement.
|
|
1641
|
+
const pinSummary = summarisePinnedDrift(ignoredDrift || { files: [] }, {
|
|
1642
|
+
pinCount: ignorePatterns ? ignorePatterns.length : 0,
|
|
1643
|
+
});
|
|
1644
|
+
const lines = formatPinnedDriftWarning(pinSummary);
|
|
1645
|
+
if (lines.length) {
|
|
1646
|
+
console.log();
|
|
1647
|
+
// Tone follows the FACT, not the topic. A seat whose pins strand nothing
|
|
1648
|
+
// gets a plain line — and it gets one rather than silence, because an
|
|
1649
|
+
// all-clear that is printed every run is what makes the loud version
|
|
1650
|
+
// believable the day it appears. Only a stranded fix, or a report that
|
|
1651
|
+
// could not be read, earns the warning chevron.
|
|
1652
|
+
const loud =
|
|
1653
|
+
pinSummary.unknown === true || pinSummary.stranded > 0;
|
|
1654
|
+
// The headline carries the chevron; the rest is the instruction under
|
|
1655
|
+
// it, and an instruction printed in warning-yellow reads as five more
|
|
1656
|
+
// problems rather than one problem and its fix.
|
|
1657
|
+
if (loud) warn(lines[0]);
|
|
1658
|
+
else console.log(` · ${lines[0]}`);
|
|
1659
|
+
for (const l of lines.slice(1)) console.log(` ${l}`);
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1625
1662
|
if (counts.mergeKept) console.log(` ~ ${counts.mergeKept} merge-mode kept (agents/ custom files preserved)`);
|
|
1626
1663
|
if (counts.preserved) warn(`${counts.preserved} preserved (local edits — kept your version)`);
|
|
1627
1664
|
if (counts.forced) warn(`${counts.forced} force-overwritten (backups in .maestro/backup/)`);
|
|
@@ -153,13 +153,20 @@ the version moved, the beat is fresh, `maestro session status` is not
|
|
|
153
153
|
fresh beat from a restarted DAEMON and still answer nothing — which is why the
|
|
154
154
|
beat alone is never the proof.
|
|
155
155
|
|
|
156
|
-
## One wrinkle: the gates dirty the tree
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
156
|
+
## One wrinkle: the gates can dirty the tree
|
|
157
|
+
|
|
158
|
+
If a gate writes into the working tree, a second run in a row refuses on a file
|
|
159
|
+
the *first* run wrote. The clean-tree gate is not relaxed for it — the run names
|
|
160
|
+
those paths and tells you to `git checkout --` them. The gate order also means a
|
|
161
|
+
first run is unaffected: cleanliness is read before the gates run.
|
|
162
|
+
|
|
163
|
+
~~"`npm test` appends to a tracked runtime ledger
|
|
164
|
+
(`.claude-flow/policy/state.json`)"~~ — struck 2026-09-25. That was the one
|
|
165
|
+
known instance, and it is fixed at the source rather than described: the file is
|
|
166
|
+
a per-machine receipt chain (one signed receipt per MCP tool call, ~160 lines
|
|
167
|
+
added per full `npm test`), it is now in `.gitignore`, and it is no longer
|
|
168
|
+
tracked. Nothing in a normal run should trip this section today; the machinery
|
|
169
|
+
stays because the next gate that writes into the tree will need it.
|
|
163
170
|
|
|
164
171
|
## Reading a round
|
|
165
172
|
|
|
@@ -193,6 +193,24 @@ scripts/resume-operations.sh # health check → rm .emergency-stop → reload
|
|
|
193
193
|
`maestro doctor` FAILS while `.emergency-stop` is present, pointing at
|
|
194
194
|
`resume-operations.sh`.
|
|
195
195
|
|
|
196
|
+
The health check it runs is passed `--ignore-emergency-stop`: the stop flag is
|
|
197
|
+
not a reason to refuse to lift the stop flag. (It was until 2026-09-25 —
|
|
198
|
+
`healthcheck.sh` counts the flag as an error and the resume refused on any
|
|
199
|
+
non-zero exit, so a seat sat halted until the flag was removed by hand.) Every
|
|
200
|
+
other check still gates the resume, and two things follow from healthcheck's
|
|
201
|
+
own exit contract:
|
|
202
|
+
|
|
203
|
+
- **CRITICAL (exit 2) refuses**, leaves the flag in place, and NAMES the
|
|
204
|
+
blocking condition — e.g. `Blocking condition(s): config:priorities.yaml`.
|
|
205
|
+
You should never have to re-run the health check with a filter to find out
|
|
206
|
+
what stopped you.
|
|
207
|
+
- **DEGRADED (exit 1 — warnings only) resumes**, printing the warnings.
|
|
208
|
+
"Operational with limitations" is what healthcheck means by degraded;
|
|
209
|
+
a missing optional repo is not grounds to keep a seat halted.
|
|
210
|
+
|
|
211
|
+
Run `scripts/healthcheck.sh` on its own and the flag is still reported as an
|
|
212
|
+
error, which is what an operator inspecting a halted seat needs to see.
|
|
213
|
+
|
|
196
214
|
---
|
|
197
215
|
|
|
198
216
|
## 7. Manual cadence-bus recovery (rarely needed)
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/cadence-failure-class.mjs — is this cadence failure worth retrying?
|
|
3
|
+
*
|
|
4
|
+
* THE FAULT THIS EXISTS FOR (measured, Eli Rosenberg's seat, 2026-09-24).
|
|
5
|
+
* His config scheduled the cadence `commitment-sweep` with
|
|
6
|
+
* `prompt: schedules/triggers/commitment-sweep.md`. That file was not on the
|
|
7
|
+
* seat. The consumer never checked: it rendered the handoff, `readFileSync`
|
|
8
|
+
* threw `ENOENT: no such file or directory`, the catch logged
|
|
9
|
+
* `handoff_failed_spawning_instead` and fell through to the spawn — which
|
|
10
|
+
* opens THE SAME PATH — and the tick went back on the bus. Measured at
|
|
11
|
+
* 16 requeues per 30 seconds, indefinitely. It is also the failure the seat's
|
|
12
|
+
* own emergency stop had been set to contain once before
|
|
13
|
+
* (`reason=meeting-action-capture requeue busy-loop containment`), so this
|
|
14
|
+
* shape has burned a seat down at least twice.
|
|
15
|
+
*
|
|
16
|
+
* The defect is not the missing file. It is that **every failure was treated
|
|
17
|
+
* as transient**. A file that does not exist will not exist on the next
|
|
18
|
+
* attempt 30 seconds later; retrying is not resilience, it is a busy loop with
|
|
19
|
+
* a log line. The bus already had a retry budget and a circuit breaker; what it
|
|
20
|
+
* did not have was the question *"can this possibly succeed if I try again?"*
|
|
21
|
+
*
|
|
22
|
+
* THE RULE, and its honest boundary. A permanent failure is one whose CAUSE
|
|
23
|
+
* cannot change as a result of waiting. That is not the same as "can never
|
|
24
|
+
* succeed": a human (or an `agent-sync`) could drop the missing prompt onto the
|
|
25
|
+
* seat between two attempts. So the policy here is NOT zero retries — it is a
|
|
26
|
+
* SMALL BOUNDED number of attempts (`PERMANENT_MAX_ATTEMPTS`) after which the
|
|
27
|
+
* tick is dead-lettered.
|
|
28
|
+
*
|
|
29
|
+
* ~~"and the long window is the cadence's OWN SCHEDULE: the next scheduled tick
|
|
30
|
+
* is a fresh event with a fresh budget, so a daily cadence re-tries twice a day
|
|
31
|
+
* ... What the classification removes is only the tight loop"~~ — struck
|
|
32
|
+
* 2026-09-25, on review of the lane that introduced it. The second half is
|
|
33
|
+
* true; the first half described a retry WINDOW this module does not enforce
|
|
34
|
+
* and cannot. `PERMANENT_MAX_ATTEMPTS` is a COUNT, not a delay. The next
|
|
35
|
+
* scheduled tick really is a fresh event with a fresh budget, but that only
|
|
36
|
+
* becomes "the window" AFTER the last attempt reaches dlq/ — and nothing here
|
|
37
|
+
* spaces the attempts before then. A caller that simply requeues gets its
|
|
38
|
+
* second attempt on the very next poll.
|
|
39
|
+
*
|
|
40
|
+
* What the classification actually buys, stated exactly: a fault whose cause
|
|
41
|
+
* cannot move goes from UNBOUNDED retries at drain speed (16 per 30 s, measured)
|
|
42
|
+
* to ONE extra attempt, then a stop with a durable record. Spacing that one
|
|
43
|
+
* extra attempt is the CALLER's job, because only the caller owns a clock —
|
|
44
|
+
* `cadence-consumer.mjs` `failPermanently` arms the per-cadence backoff gate for
|
|
45
|
+
* exactly that reason, so the retry cannot land in the same drain. Read the
|
|
46
|
+
* budget here as "how many", and look at the caller for "how far apart".
|
|
47
|
+
*
|
|
48
|
+
* THE IDIOM ALREADY EXISTED ONE LANE OVER. `scripts/daemon/assurance.mjs`
|
|
49
|
+
* `classifyFailure` asks exactly this question for the INBOX lane, and says why
|
|
50
|
+
* in the same words: "retrying a permanent fault burns another 45 minutes of
|
|
51
|
+
* the human's patience for the same outcome". The cadence lane never asked it.
|
|
52
|
+
* Two deliberate differences from that one: this classifier keys on the PHASE
|
|
53
|
+
* (we know whether we were touching the cadence's own prompt file, so an errno
|
|
54
|
+
* is unambiguous rather than a guess at a message), and it returns the attempt
|
|
55
|
+
* budget with the verdict so the caller does not re-derive the policy.
|
|
56
|
+
*
|
|
57
|
+
* ELSEWHERE IN THE DAEMON — the same shape, swept 2026-09-25. Four other retry
|
|
58
|
+
* loops were read for "does it ever inspect the error class?". None do. Three
|
|
59
|
+
* are BOUNDED, which is what keeps them out of this module for now; the fourth
|
|
60
|
+
* is not, and is deliberate:
|
|
61
|
+
*
|
|
62
|
+
* - `dispatcher.mjs` backlog retries (MAX_BACKLOG_RETRIES = 6, plus a
|
|
63
|
+
* post-failure cooldown). Bounded, but a permanent fault spends all six
|
|
64
|
+
* sessions and six cooldowns to learn what the first one knew.
|
|
65
|
+
* - `dispatcher.mjs` resume reconcile (RESUME_MAX_ATTEMPTS = 3 + a freshness
|
|
66
|
+
* window + retire-on-no-id/no-prompt). Bounded. A dead `--resume` id burns
|
|
67
|
+
* all three: "No conversation found with session ID" is as permanent as an
|
|
68
|
+
* ENOENT and is never read as such.
|
|
69
|
+
* - `lib/session/identity.mjs` `rotationDecision`. This one uses TIMING as a
|
|
70
|
+
* proxy for the class question — a resume that dies inside 20 s rotates to
|
|
71
|
+
* a fresh id, which is why James Kirkland's 2-second `--resume` failure is
|
|
72
|
+
* handled at HEAD. But a failure that takes LONGER than the window returns
|
|
73
|
+
* `{action:"relaunch"}`, and the relaunch re-uses the same dead id. A
|
|
74
|
+
* timing proxy answers the wrong question; the message says "permanent"
|
|
75
|
+
* outright.
|
|
76
|
+
* - `cadence-consumer.mjs` `sweepHandoffTimeouts` → `handoff_requeue_failed`:
|
|
77
|
+
* genuinely unbounded ("handoff kept open; retried next sweep"), and
|
|
78
|
+
* deliberately so — the alternative is losing the tick. It is at least
|
|
79
|
+
* loud (error level, every sweep). Nothing escalates it.
|
|
80
|
+
*
|
|
81
|
+
* PURE. Strings and numbers in, a verdict out. No fs, no clock, no env, no
|
|
82
|
+
* network — so the policy can be tested on its own (and reasoned about) without
|
|
83
|
+
* standing up a consumer. Never throws: a garbage input classifies as transient,
|
|
84
|
+
* because the safe default when we cannot tell is today's behaviour.
|
|
85
|
+
*
|
|
86
|
+
* @module lib/cadence-failure-class
|
|
87
|
+
*/
|
|
88
|
+
|
|
89
|
+
/** A failure whose cause cannot change by waiting. Bounded retries, then DLQ. */
|
|
90
|
+
export const PERMANENT = "permanent";
|
|
91
|
+
/** A failure that could plausibly succeed on the next attempt. Today's budget. */
|
|
92
|
+
export const TRANSIENT = "transient";
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* A CAP on how many attempts a PERMANENT failure gets before the tick is
|
|
96
|
+
* dead-lettered. A caller applies `min(its own budget, this)` — it lowers a
|
|
97
|
+
* budget, it never raises one.
|
|
98
|
+
*
|
|
99
|
+
* Why 2 and not 1: the cause can be repaired between attempts (someone writes
|
|
100
|
+
* the prompt file), and one extra attempt is cheap — a single spawn, not a
|
|
101
|
+
* loop. Why a cap at all, when the cadence consumer's own transient budget
|
|
102
|
+
* already happens to be 2: `failTick`'s default is 5 and other call sites use
|
|
103
|
+
* it, so the number has to be stated here rather than inherited from whichever
|
|
104
|
+
* lane happens to be calling.
|
|
105
|
+
*
|
|
106
|
+
* This is a COUNT and nothing else. It does not delay anything; two attempts
|
|
107
|
+
* with no spacing are two attempts in the same drain. The caller is what makes
|
|
108
|
+
* the extra attempt cost a backoff interval rather than a millisecond. The
|
|
109
|
+
* budget is the SMALLEST part of this fix; the larger parts are that a
|
|
110
|
+
* permanent failure stops where it is instead of falling through to a second
|
|
111
|
+
* code path that opens the same missing file, and that it is recorded and
|
|
112
|
+
* alerted rather than logged and forgotten.
|
|
113
|
+
*/
|
|
114
|
+
export const PERMANENT_MAX_ATTEMPTS = 2;
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* The permanent codes, as a frozen set, so a caller can branch on the code
|
|
118
|
+
* without re-deriving the taxonomy.
|
|
119
|
+
* @type {Readonly<Record<string,string>>}
|
|
120
|
+
*/
|
|
121
|
+
export const PERMANENT_CODES = Object.freeze({
|
|
122
|
+
/** The cadence's prompt file is not on disk at the configured path. */
|
|
123
|
+
PROMPT_MISSING: "prompt_missing",
|
|
124
|
+
/** The prompt path exists but cannot be read as a file (perms / it's a dir). */
|
|
125
|
+
PROMPT_UNREADABLE: "prompt_unreadable",
|
|
126
|
+
/** The cadence has no handler and no prompt anywhere to fall back to. */
|
|
127
|
+
NO_PROMPT_CONFIGURED: "no_prompt_configured",
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
/** errno tokens that mean "this path is not a readable file, and waiting won't help". */
|
|
131
|
+
const PERMANENT_PATH_ERRNOS = /\b(ENOENT|EACCES|EPERM|EISDIR|ENOTDIR|ELOOP|ENAMETOOLONG)\b/;
|
|
132
|
+
/** errno tokens on a path that CAN clear on their own (busy fs, full disk, IO blip). */
|
|
133
|
+
const TRANSIENT_PATH_ERRNOS = /\b(EBUSY|EAGAIN|EMFILE|ENFILE|ENOSPC|EIO|ETIMEDOUT)\b/;
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* `realSpawnSession` exit codes that are not process exits at all but the
|
|
137
|
+
* consumer's own pre-flight verdicts. Kept here rather than re-matched from
|
|
138
|
+
* prose, because the message text is a log string and the number is a contract.
|
|
139
|
+
*/
|
|
140
|
+
export const SPAWN_EXIT_PROMPT_MISSING = -2;
|
|
141
|
+
export const SPAWN_EXIT_PROMPT_READ_FAILED = -3;
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* @typedef {object} CadenceFailureVerdict
|
|
145
|
+
* @property {"permanent"|"transient"} class the verdict
|
|
146
|
+
* @property {string|null} code a stable machine code for a permanent failure
|
|
147
|
+
* (`PERMANENT_CODES.*`), else null
|
|
148
|
+
* @property {number|null} maxAttempts attempt budget the caller should apply;
|
|
149
|
+
* null means "the caller's own default"
|
|
150
|
+
* @property {string} reason one short human line saying why
|
|
151
|
+
*/
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Classify a cadence failure.
|
|
155
|
+
*
|
|
156
|
+
* @param {object} [failure]
|
|
157
|
+
* @param {string} [failure.error] the error message / stderr tail
|
|
158
|
+
* @param {string} [failure.errno] the Node error `code` (`err.code`), when
|
|
159
|
+
* the caller has the Error object. THE MESSAGE IS NOT A CONTRACT AND THE
|
|
160
|
+
* ERRNO IS — measured 2026-09-25: `readFileSync` on a DIRECTORY produces
|
|
161
|
+
* exactly `EISDIR: illegal operation on a directory, read`, with no path
|
|
162
|
+
* anywhere in it, so anything that recognises a path fault by finding the
|
|
163
|
+
* path in the text misses the commonest unreadable case outright. When
|
|
164
|
+
* `errno` is given it is authoritative and the text is not consulted for
|
|
165
|
+
* the class; the text stays the fallback for callers that only have a
|
|
166
|
+
* stderr tail (a sub-session's output is text, never an Error).
|
|
167
|
+
* @param {"prompt"|"spawn"} [failure.phase]
|
|
168
|
+
* WHERE the failure happened. `"prompt"` means we were resolving or
|
|
169
|
+
* reading the cadence's own prompt file — so a path errno is
|
|
170
|
+
* unambiguously ABOUT that file, and we do not have to guess from the
|
|
171
|
+
* message text. `"spawn"` (the default) means a sub-session ran.
|
|
172
|
+
* @returns {CadenceFailureVerdict}
|
|
173
|
+
*/
|
|
174
|
+
export function classifyCadenceFailure(failure = {}) {
|
|
175
|
+
try {
|
|
176
|
+
const text = typeof failure.error === "string" ? failure.error : String(failure.error ?? "");
|
|
177
|
+
const exitCode = Number.isFinite(failure.exitCode) ? failure.exitCode : null;
|
|
178
|
+
const phase = failure.phase === "prompt" ? "prompt" : "spawn";
|
|
179
|
+
// An explicit errno beats the message every time; falling back to the text
|
|
180
|
+
// is for callers holding a stderr tail rather than an Error.
|
|
181
|
+
const errno = typeof failure.errno === "string" && failure.errno ? failure.errno : null;
|
|
182
|
+
const hasErrno = (re) => (errno ? re.test(errno) : re.test(text));
|
|
183
|
+
|
|
184
|
+
// 1. The consumer's own pre-flight verdicts, by contract number.
|
|
185
|
+
if (exitCode === SPAWN_EXIT_PROMPT_MISSING) {
|
|
186
|
+
return permanent(PERMANENT_CODES.PROMPT_MISSING, "the cadence prompt file is not on disk");
|
|
187
|
+
}
|
|
188
|
+
if (exitCode === SPAWN_EXIT_PROMPT_READ_FAILED) {
|
|
189
|
+
// A read that failed for a reason that CAN clear (disk full, too many
|
|
190
|
+
// open files) is transient even though the exit code is the same.
|
|
191
|
+
if (hasErrno(TRANSIENT_PATH_ERRNOS)) {
|
|
192
|
+
return transient("prompt read failed for a cause that can clear on its own");
|
|
193
|
+
}
|
|
194
|
+
return permanent(PERMANENT_CODES.PROMPT_UNREADABLE, "the cadence prompt path is not a readable file");
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// 2. The prompt phase: any path errno here is about the prompt file.
|
|
198
|
+
if (phase === "prompt") {
|
|
199
|
+
if (hasErrno(TRANSIENT_PATH_ERRNOS)) {
|
|
200
|
+
return transient("prompt I/O failed for a cause that can clear on its own");
|
|
201
|
+
}
|
|
202
|
+
if (hasErrno(/\bENOENT\b/) || /prompt not found/i.test(text)) {
|
|
203
|
+
return permanent(PERMANENT_CODES.PROMPT_MISSING, "the cadence prompt file is not on disk");
|
|
204
|
+
}
|
|
205
|
+
if (hasErrno(PERMANENT_PATH_ERRNOS)) {
|
|
206
|
+
return permanent(PERMANENT_CODES.PROMPT_UNREADABLE, "the cadence prompt path is not a readable file");
|
|
207
|
+
}
|
|
208
|
+
// A prompt-phase failure we do not recognise (a render bug, say) keeps
|
|
209
|
+
// the ordinary budget: we do not dead-letter on a guess.
|
|
210
|
+
return transient("prompt step failed for an unrecognised reason");
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// 3. Spawn phase. `prompt not found:` is realSpawnSession's own wording and
|
|
214
|
+
// is definitive wherever it surfaces; a bare errno in a sub-session's
|
|
215
|
+
// stderr is NOT — it is far more likely to be about some file the session
|
|
216
|
+
// itself touched than about the cadence. So we do not read it.
|
|
217
|
+
if (/prompt not found/i.test(text)) {
|
|
218
|
+
return permanent(PERMANENT_CODES.PROMPT_MISSING, "the cadence prompt file is not on disk");
|
|
219
|
+
}
|
|
220
|
+
if (/no handler and no prompt/i.test(text)) {
|
|
221
|
+
return permanent(PERMANENT_CODES.NO_PROMPT_CONFIGURED, "the cadence has neither a handler nor a prompt");
|
|
222
|
+
}
|
|
223
|
+
return transient("no rule recognises this failure as permanent");
|
|
224
|
+
} catch {
|
|
225
|
+
// A classifier that can throw is a classifier that can wedge the lane it
|
|
226
|
+
// advises. Unknown → transient → today's behaviour.
|
|
227
|
+
return transient("classifier error; defaulting to the retryable path");
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Is a verdict (or a bare class string) permanent? */
|
|
232
|
+
export function isPermanent(v) {
|
|
233
|
+
if (!v) return false;
|
|
234
|
+
return (typeof v === "string" ? v : v.class) === PERMANENT;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function permanent(code, reason) {
|
|
238
|
+
return { class: PERMANENT, code, maxAttempts: PERMANENT_MAX_ATTEMPTS, reason };
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function transient(reason) {
|
|
242
|
+
return { class: TRANSIENT, code: null, maxAttempts: null, reason };
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
export default { classifyCadenceFailure, isPermanent, PERMANENT, TRANSIENT, PERMANENT_CODES, PERMANENT_MAX_ATTEMPTS };
|
package/lib/claude-bin.mjs
CHANGED
|
@@ -46,6 +46,31 @@ export function isSafeExecutable(p) {
|
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
/**
|
|
50
|
+
* The fixed paths `resolveClaudeBin` searches, in order, AFTER $CLAUDE_BIN.
|
|
51
|
+
* `~/` is the seat's home directory.
|
|
52
|
+
*
|
|
53
|
+
* THIS IS THE ONE COPY. `lib/cli/doctor-checks.mjs` reports on exactly these
|
|
54
|
+
* paths and imports this list rather than restating it: a hand-copied twin is
|
|
55
|
+
* how the doctor comes to say a path resolves that the resolver never looks at
|
|
56
|
+
* (and vice versa), and nothing fails when it does.
|
|
57
|
+
*/
|
|
58
|
+
export const CLAUDE_BIN_CANDIDATES = Object.freeze([
|
|
59
|
+
"~/.local/bin/claude",
|
|
60
|
+
"/opt/homebrew/bin/claude",
|
|
61
|
+
"/usr/local/bin/claude",
|
|
62
|
+
"/usr/bin/claude",
|
|
63
|
+
]);
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* `CLAUDE_BIN_CANDIDATES` with `~/` expanded against `home`.
|
|
67
|
+
* @param {string} [home] defaults to this process's home directory
|
|
68
|
+
* @returns {string[]}
|
|
69
|
+
*/
|
|
70
|
+
export function claudeBinCandidatePaths(home = homedir()) {
|
|
71
|
+
return CLAUDE_BIN_CANDIDATES.map((c) => (c.startsWith("~/") ? join(home, c.slice(2)) : c));
|
|
72
|
+
}
|
|
73
|
+
|
|
49
74
|
/**
|
|
50
75
|
* Return the absolute path to the Claude CLI. Searches, in order:
|
|
51
76
|
* 1. $CLAUDE_BIN env var (if set + executable on disk)
|
|
@@ -70,13 +95,7 @@ export function resolveClaudeBin() {
|
|
|
70
95
|
`non-writable executable. Falling back to default search paths.`
|
|
71
96
|
);
|
|
72
97
|
}
|
|
73
|
-
const
|
|
74
|
-
join(homedir(), ".local/bin/claude"),
|
|
75
|
-
"/opt/homebrew/bin/claude",
|
|
76
|
-
"/usr/local/bin/claude",
|
|
77
|
-
"/usr/bin/claude",
|
|
78
|
-
];
|
|
79
|
-
for (const c of candidates) {
|
|
98
|
+
for (const c of claudeBinCandidatePaths()) {
|
|
80
99
|
if (existsSync(c)) { _resolved = c; return c; }
|
|
81
100
|
}
|
|
82
101
|
// Last resort: bare name (lets the OS PATH + spawn error stay informative).
|
|
@@ -35,6 +35,7 @@ import { probeClaude as defaultProbeClaude } from "../setup/claude-probe.mjs";
|
|
|
35
35
|
import { readSeatEngine } from "../runtime/adapter.mjs";
|
|
36
36
|
import { sessionLiveness, SESSION_STALE_MS } from "../telemetry/collect.mjs";
|
|
37
37
|
import { CONSOLE_KEY_REMEDY, SEAT_AUTH_VARS, REQUIRE_CONSOLE_KEY_VAR, seatAuthVerdict } from "./seat-auth.mjs";
|
|
38
|
+
import { CLAUDE_BIN_CANDIDATES, isSafeExecutable } from "../claude-bin.mjs";
|
|
38
39
|
|
|
39
40
|
export const SDK_PACKAGE = "@cohortapp/agent-sdk";
|
|
40
41
|
/** The SDK checkout this module ships in — the source of the shipped hook copies. */
|
|
@@ -151,6 +152,132 @@ export function hookNodeResolvable({ onPath = "", isExecutable = () => false, ho
|
|
|
151
152
|
};
|
|
152
153
|
}
|
|
153
154
|
|
|
155
|
+
/**
|
|
156
|
+
* Where `lib/claude-bin.mjs#resolveClaudeBin` looks, in order. `~/` is the
|
|
157
|
+
* seat's home. IMPORTED, never restated: this row's whole job is to describe
|
|
158
|
+
* what the resolver does, so the list it reports on has to be the resolver's
|
|
159
|
+
* own. A hand-copied twin drifted silently the moment a path was added to one
|
|
160
|
+
* of them.
|
|
161
|
+
*/
|
|
162
|
+
export { CLAUDE_BIN_CANDIDATES };
|
|
163
|
+
/** Where the Claude Code installer keeps its versioned installs. */
|
|
164
|
+
export const CLAUDE_VERSIONS_DIR = "~/.local/share/claude/versions";
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Can anything on this seat actually RUN `claude`?
|
|
168
|
+
*
|
|
169
|
+
* ── THE SEAT THIS EXISTS FOR ────────────────────────────────────────────────
|
|
170
|
+
* James Kirkland's machine, measured 2026-09-25: no `~/.local/bin/claude` at
|
|
171
|
+
* all, while `~/.local/share/claude/versions/` held 2.1.273 and 2.1.278. Every
|
|
172
|
+
* launch exited 127 — for days. `resolveClaudeBin` falls back to the bare name
|
|
173
|
+
* `"claude"` when none of its fixed paths exist, deliberately, so the spawn's
|
|
174
|
+
* own error stays informative; but nobody was reading the spawn's error, and
|
|
175
|
+
* nothing anywhere said the plainest possible thing: the binary is not there.
|
|
176
|
+
*
|
|
177
|
+
* So this says it plainly, and — because the repair genuinely is one line —
|
|
178
|
+
* it prints the line. A `versions/` directory with installs in it and no
|
|
179
|
+
* symlink pointing at one is an interrupted install, not a missing product.
|
|
180
|
+
*
|
|
181
|
+
* ── THE ROW MUST ASK WHAT THE RESOLVER ASKS ────────────────────────────────
|
|
182
|
+
* Two probes, because `resolveClaudeBin` uses two different gates and an `ok`
|
|
183
|
+
* that answers the wrong one is the exact blind spot this row exists to close:
|
|
184
|
+
*
|
|
185
|
+
* $CLAUDE_BIN — accepted only if `isSafeExecutable`: a regular file,
|
|
186
|
+
* not group/world-writable, owned by this uid or root
|
|
187
|
+
* (M2). A 0o777 binary is executable AND ignored, so
|
|
188
|
+
* asking X_OK alone reports `ok` for an override that
|
|
189
|
+
* never gets used and a seat that exits 127 every spawn.
|
|
190
|
+
* the fixed paths — accepted on mere `existsSync`. A present-but-chmod-644
|
|
191
|
+
* `~/.local/bin/claude` is therefore what the resolver
|
|
192
|
+
* returns, and the spawn fails with EACCES — which is a
|
|
193
|
+
* different fault, and a different repair, from "no
|
|
194
|
+
* binary is installed".
|
|
195
|
+
*
|
|
196
|
+
* Pure over its inputs; the caller supplies the probes.
|
|
197
|
+
*
|
|
198
|
+
* @param {object} o
|
|
199
|
+
* @param {string} [o.onPath] `which claude` output ("" when it does not resolve)
|
|
200
|
+
* @param {(p:string)=>boolean} [o.isExecutable] can it be executed (X_OK)
|
|
201
|
+
* @param {(p:string)=>boolean} [o.isSafe] would claude-bin.mjs TRUST it as $CLAUDE_BIN (defaults to its own `isSafeExecutable`)
|
|
202
|
+
* @param {(p:string)=>boolean} [o.exists] is it on disk at all (defaults to `isExecutable`, i.e. the pre-existing behaviour)
|
|
203
|
+
* @param {string} [o.home] the seat's home directory
|
|
204
|
+
* @param {string[]} [o.versions] entries of ~/.local/share/claude/versions (newest need not be sorted)
|
|
205
|
+
* @param {string} [o.envOverride] $CLAUDE_BIN, when set
|
|
206
|
+
* @returns {{level:"ok"|"warn"|"fail", msg:string}}
|
|
207
|
+
*/
|
|
208
|
+
export function claudeBinResolvable({ onPath = "", isExecutable = () => false, isSafe = isSafeExecutable, exists = null, home = "", versions = [], envOverride = "" } = {}) {
|
|
209
|
+
const abs = (c) => (c.startsWith("~/") ? `${home}/${c.slice(2)}` : c);
|
|
210
|
+
const onDisk = typeof exists === "function" ? exists : isExecutable;
|
|
211
|
+
const override = String(envOverride || "").trim();
|
|
212
|
+
if (override) {
|
|
213
|
+
const trusted = isSafe(override);
|
|
214
|
+
if (trusted && isExecutable(override)) return { level: "ok", msg: `Claude CLI: $CLAUDE_BIN resolves (${override})` };
|
|
215
|
+
if (trusted) {
|
|
216
|
+
return {
|
|
217
|
+
level: "warn",
|
|
218
|
+
msg: `Claude CLI: $CLAUDE_BIN is set to ${override}, which lib/claude-bin.mjs accepts but the OS will not execute — every session launch and every daemon spawn fails on it; chmod +x ${override} or unset CLAUDE_BIN`,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
return {
|
|
222
|
+
level: "warn",
|
|
223
|
+
msg: `Claude CLI: $CLAUDE_BIN is set to ${override}, which lib/claude-bin.mjs IGNORES — it accepts an override only when it is a regular file, not group- or world-writable, and owned by you or root (M2); it falls through to the fixed paths instead. Unset it, or fix the file (chmod go-w ${override}) and point it at a real binary`,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
let present = "";
|
|
227
|
+
for (const c of CLAUDE_BIN_CANDIDATES) {
|
|
228
|
+
if (c.startsWith("~/") && !home) continue;
|
|
229
|
+
const p = abs(c);
|
|
230
|
+
if (isExecutable(p)) return { level: "ok", msg: `Claude CLI: resolves at ${p}` };
|
|
231
|
+
if (!present && onDisk(p)) present = p;
|
|
232
|
+
}
|
|
233
|
+
// The resolver searches with existsSync, so a file that is THERE and not
|
|
234
|
+
// executable is the one it returns — and the spawn, not the search, is what
|
|
235
|
+
// fails. Saying "install Claude Code" here would send an operator to fix a
|
|
236
|
+
// thing that is not broken.
|
|
237
|
+
if (present) {
|
|
238
|
+
return {
|
|
239
|
+
level: "fail",
|
|
240
|
+
msg: `Claude CLI: ${present} exists but is not executable — lib/claude-bin.mjs resolves it on presence alone, so every session launch and every daemon spawn fails on it. Fix: chmod +x ${present}`,
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
const found = String(onPath || "").trim();
|
|
244
|
+
if (found) {
|
|
245
|
+
return {
|
|
246
|
+
level: "warn",
|
|
247
|
+
msg: `Claude CLI: \`claude\` is on your PATH (${found}) but at none of the fixed paths lib/claude-bin.mjs searches (${CLAUDE_BIN_CANDIDATES.join(", ")}) — launchd jobs get a bare environment and will not find it; symlink it: ln -sfn ${found} ${home ? `${home}/.local/bin/claude` : "~/.local/bin/claude"}`,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
// An interrupted install: the product is here, the entry point is not.
|
|
251
|
+
const installs = (Array.isArray(versions) ? versions : []).map((v) => String(v && v.name ? v.name : v)).filter(Boolean);
|
|
252
|
+
if (installs.length > 0) {
|
|
253
|
+
const newest = [...installs].sort(compareVersionNames).pop();
|
|
254
|
+
const target = `${home || "~"}/.local/share/claude/versions/${newest}`;
|
|
255
|
+
const link = `${home || "~"}/.local/bin/claude`;
|
|
256
|
+
return {
|
|
257
|
+
level: "fail",
|
|
258
|
+
msg: `Claude CLI: no \`claude\` binary resolves, but ${installs.length} install(s) sit in ${CLAUDE_VERSIONS_DIR} (${installs.join(", ")}) — every session and every daemon spawn exits 127 until the symlink is back. Fix: mkdir -p ${home || "~"}/.local/bin && ln -sfn ${target} ${link}`,
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
return {
|
|
262
|
+
level: "fail",
|
|
263
|
+
msg: `Claude CLI: no \`claude\` binary resolves — not on PATH and at none of ${CLAUDE_BIN_CANDIDATES.join(", ")}, and ${CLAUDE_VERSIONS_DIR} holds no installs. Every session launch and every daemon spawn exits 127. Install Claude Code (curl -fsSL https://claude.ai/install.sh | bash), then re-run \`maestro doctor\``,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Sort helper: version-ish directory names, numerically where they look numeric. */
|
|
268
|
+
function compareVersionNames(a, b) {
|
|
269
|
+
const seg = (v) => String(v).split(/[.\-]/).map((x) => (/^\d+$/.test(x) ? Number(x) : x));
|
|
270
|
+
const A = seg(a); const B = seg(b);
|
|
271
|
+
for (let i = 0; i < Math.max(A.length, B.length); i += 1) {
|
|
272
|
+
const x = A[i]; const y = B[i];
|
|
273
|
+
if (x === undefined) return -1;
|
|
274
|
+
if (y === undefined) return 1;
|
|
275
|
+
if (typeof x === "number" && typeof y === "number") { if (x !== y) return x - y; continue; }
|
|
276
|
+
if (String(x) !== String(y)) return String(x) < String(y) ? -1 : 1;
|
|
277
|
+
}
|
|
278
|
+
return 0;
|
|
279
|
+
}
|
|
280
|
+
|
|
154
281
|
/**
|
|
155
282
|
* Do the seat's copies of the pre-send hook and the send gate match the
|
|
156
283
|
* versions this SDK ships? `maestro upgrade` keeps a locally modified copy (and
|
|
@@ -549,6 +676,27 @@ export async function runDoctorChecks(o = {}) {
|
|
|
549
676
|
}
|
|
550
677
|
}
|
|
551
678
|
|
|
679
|
+
// ── The Claude CLI itself (§3.1) ──────────────────────────────────────────
|
|
680
|
+
//
|
|
681
|
+
// Before "is the session job installed" comes "is there anything for it to
|
|
682
|
+
// run". A seat with a loaded job, a live supervisor and no binary exits 127
|
|
683
|
+
// on every launch and looks, from every other row here, entirely fine.
|
|
684
|
+
{
|
|
685
|
+
const home = o.home !== undefined ? o.home : homedir();
|
|
686
|
+
const isExecutable = o.isExecutable || ((p) => { try { accessSync(p, fsConstants.X_OK); return true; } catch { return false; } });
|
|
687
|
+
// The two gates `resolveClaudeBin` actually uses, wired to the real thing:
|
|
688
|
+
// `isSafeExecutable` for the $CLAUDE_BIN override (it is the resolver's own
|
|
689
|
+
// function, imported, not a restatement), `existsSync` for the fixed paths.
|
|
690
|
+
const isSafe = o.isSafe || isSafeExecutable;
|
|
691
|
+
const exists = o.existsSync || existsSync;
|
|
692
|
+
const listVersions = o.listClaudeVersions || (() => { try { return readdirSync(join(home, ".local", "share", "claude", "versions")); } catch { return []; } });
|
|
693
|
+
const binRow = claudeBinResolvable({
|
|
694
|
+
onPath: tryExec("which", ["claude"], { timeout: 2000 }).trim(),
|
|
695
|
+
isExecutable, isSafe, exists, home, versions: listVersions(), envOverride: env.CLAUDE_BIN || "",
|
|
696
|
+
});
|
|
697
|
+
rows.push(binRow);
|
|
698
|
+
}
|
|
699
|
+
|
|
552
700
|
// ── Main session job (§3.1) ───────────────────────────────────────────────
|
|
553
701
|
const listPlists = o.listPlists || (() => { try { return readdirSync(join(homedir(), "Library", "LaunchAgents")); } catch { return []; } });
|
|
554
702
|
const loaded = parseLaunchctlList(tryExec("launchctl", ["list"], { timeout: 4000 }));
|
|
@@ -569,4 +717,4 @@ function pick(obj, keys) {
|
|
|
569
717
|
return out;
|
|
570
718
|
}
|
|
571
719
|
|
|
572
|
-
export default { parseDotEnv, mask, authMode, looksLikeOrgId, orgIdShape, effectiveSource, sdkGlobalInstall, tailscaleSsh, sessionInstalled, hookNodeResolvable, hookCopyDrift, runDoctorChecks };
|
|
720
|
+
export default { parseDotEnv, mask, authMode, looksLikeOrgId, orgIdShape, effectiveSource, sdkGlobalInstall, tailscaleSsh, sessionInstalled, hookNodeResolvable, hookCopyDrift, claudeBinResolvable, runDoctorChecks };
|
|
@@ -91,6 +91,20 @@ export const DEFAULT_THRESHOLDS = Object.freeze({
|
|
|
91
91
|
warnSec: 120, // 2 min (heartbeat refreshes ~15s)
|
|
92
92
|
critSec: 300, // 5 min
|
|
93
93
|
}),
|
|
94
|
+
cadencePermanentFailure: Object.freeze({
|
|
95
|
+
// A cadence failed for a cause that cannot change by waiting — its prompt
|
|
96
|
+
// file is not on disk, or the path is not readable. Unlike every other rule
|
|
97
|
+
// here, this one fires at ONE. The pattern-not-single-failure principle is
|
|
98
|
+
// about failures that are individually meaningless; a permanent failure is
|
|
99
|
+
// not one of those. It means a scheduled obligation has stopped running and
|
|
100
|
+
// will not start again until a person puts the file back — and the
|
|
101
|
+
// alternative to alerting is what actually happened on Eli Rosenberg's seat
|
|
102
|
+
// (2026-09-24): 16 requeues per 30 seconds for a day and a half, contained
|
|
103
|
+
// eventually by an emergency stop rather than noticed.
|
|
104
|
+
counters: Object.freeze(["cadence.permanent_failure"]),
|
|
105
|
+
warnCount: 1,
|
|
106
|
+
critCount: 3,
|
|
107
|
+
}),
|
|
94
108
|
cadenceOutputMissing: Object.freeze({
|
|
95
109
|
// A scheduled workflow finished without leaving its output artefacts behind.
|
|
96
110
|
// This is the ONLY rule here not derived from the cadence bus, and that is
|
|
@@ -275,6 +289,7 @@ export function evaluateAlerts(args = {}) {
|
|
|
275
289
|
push(alerts, costPerSessionRule(ledger, t.costPerSessionP99));
|
|
276
290
|
push(alerts, absentAgentRule(presence, t.absentAgent));
|
|
277
291
|
push(alerts, daemonDownRule(presence, t.daemonDown));
|
|
292
|
+
push(alerts, cadencePermanentFailureRule(snapshot, t.cadencePermanentFailure));
|
|
278
293
|
push(alerts, cadenceOutputMissingRule(outputs, t.cadenceOutputMissing));
|
|
279
294
|
|
|
280
295
|
// Deterministic order: critical first, then by id, so identical inputs always
|
|
@@ -380,6 +395,24 @@ function daemonDownRule(presence, cfg = {}) {
|
|
|
380
395
|
};
|
|
381
396
|
}
|
|
382
397
|
|
|
398
|
+
/**
|
|
399
|
+
* A cadence failed permanently: the cause cannot change by waiting, so the tick
|
|
400
|
+
* was dead-lettered rather than retried (see lib/cadence-failure-class.mjs and
|
|
401
|
+
* the consumer's `failPermanently`). Fires at one, by design — see the
|
|
402
|
+
* threshold comment. The counter is durable (counters JSONL), so this stays
|
|
403
|
+
* true across a daemon restart within the day.
|
|
404
|
+
*/
|
|
405
|
+
function cadencePermanentFailureRule(snapshot, cfg = {}) {
|
|
406
|
+
const count = sumCounters(snapshot, cfg.counters);
|
|
407
|
+
const sev = severityFor(count, num(cfg.warnCount, Infinity), num(cfg.critCount, Infinity));
|
|
408
|
+
if (!sev) return null;
|
|
409
|
+
return {
|
|
410
|
+
id: "cadence_permanently_failing",
|
|
411
|
+
severity: sev,
|
|
412
|
+
detail: `${count} cadence tick(s) failed permanently today (warn ${num(cfg.warnCount)}, crit ${num(cfg.critCount)}) — a scheduled obligation is not running and retrying will not fix it. Read state/cadence-bus/dlq/ for the cadence and the code (prompt_missing = put the prompt file back).`,
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
|
|
383
416
|
/**
|
|
384
417
|
* A scheduled workflow completed without producing its output artefacts.
|
|
385
418
|
*
|