@cohortapp/agent-sdk 2.18.12 → 2.18.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +38 -1
- package/docs/runbooks/fleet-rollout.md +14 -7
- package/docs/runbooks/recovery-and-failover.md +18 -0
- package/lib/cadence-failure-class.mjs +245 -0
- package/lib/claude-bin.mjs +26 -7
- package/lib/cli/doctor-checks.mjs +149 -1
- package/lib/comms/send-gate.mjs +6 -4
- package/lib/diagnostics/alerts.mjs +33 -0
- package/lib/engine/agents/usage.mjs +45 -0
- package/lib/engine/budget.mjs +293 -29
- package/lib/engine/cli.mjs +54 -5
- package/lib/engine/loop.mjs +30 -0
- package/lib/engine/output/json.mjs +26 -0
- package/lib/engine/wire/errors.mjs +179 -0
- package/lib/engine/wire/search.mjs +44 -8
- package/lib/identity/claude-md.mjs +107 -0
- package/lib/identity/disclosure-instructions.mjs +148 -0
- package/lib/identity/disclosure-scrub.mjs +207 -0
- package/lib/identity/persona.mjs +141 -6
- package/lib/org/inbound/conversation-frame.mjs +289 -0
- package/lib/org/inbound/directedness.mjs +27 -7
- package/lib/org/quota.mjs +27 -0
- package/lib/session/config.mjs +4 -0
- package/lib/session/identity.mjs +71 -7
- package/lib/session/launch-failure.mjs +251 -0
- package/lib/session/resume-target.mjs +86 -0
- package/lib/telemetry/collect.mjs +129 -0
- package/lib/upgrade/pinned-drift.mjs +467 -0
- package/package.json +1 -1
- package/plugins/maestro-skills/skills/persona-discipline.md +24 -2
- package/scaffold/config/alerts.yaml +7 -0
- package/scripts/ci/check-cadence-prompts-exist.mjs +96 -0
- package/scripts/ci/check.mjs +3 -0
- package/scripts/ci/run-tests.mjs +16 -2
- package/scripts/daemon/cadence-consumer.mjs +281 -34
- package/scripts/daemon/context-compiler.mjs +9 -1
- package/scripts/daemon/prompt-builder.mjs +219 -137
- package/scripts/daemon/responder.mjs +226 -26
- package/scripts/emergency-stop.sh +114 -13
- package/scripts/fleet/rollout.mjs +10 -3
- package/scripts/healthcheck.sh +131 -33
- package/scripts/resume-operations.sh +101 -6
- package/scripts/session/supervisor.mjs +198 -5
|
@@ -15,6 +15,13 @@ alerts:
|
|
|
15
15
|
# Cost guardrails (USD).
|
|
16
16
|
costPerSessionP99USD: 2.50 # p99 session cost over the daily ledger
|
|
17
17
|
dailySpendWarnUSD: 20 # soft warn before the hard budget cap
|
|
18
|
+
# A cadence failed for a cause that retrying cannot fix — its prompt file is
|
|
19
|
+
# missing or unreadable. Fires at ONE: a permanent failure means a scheduled
|
|
20
|
+
# obligation has stopped and will not restart until a person puts the file
|
|
21
|
+
# back. (Nested form, matching the built-in defaults.)
|
|
22
|
+
cadencePermanentFailure:
|
|
23
|
+
warnCount: 1
|
|
24
|
+
critCount: 3
|
|
18
25
|
# Liveness (seconds).
|
|
19
26
|
absentAgentSec: 900 # a peer with no heartbeat in 15min
|
|
20
27
|
daemonHealthStaleSec: 180 # daemon health.json older than 3min
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* check-cadence-prompts-exist.mjs — every built-in cadence's prompt file is on disk.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS GUARD EXISTS. On 2026-09-24 Eli Rosenberg's seat scheduled the
|
|
5
|
+
* cadence `commitment-sweep` with `prompt: schedules/triggers/commitment-sweep.md`,
|
|
6
|
+
* and that file was not there. The consumer never checked: the handoff renderer
|
|
7
|
+
* threw ENOENT, the catch fell through to the spawn — which opens the same path
|
|
8
|
+
* — and the tick went back on the bus. Measured at 16 requeues per 30 seconds,
|
|
9
|
+
* indefinitely, until an emergency stop halted the whole seat to contain it.
|
|
10
|
+
*
|
|
11
|
+
* Two fixes, and this is the cheap half. The runtime half is
|
|
12
|
+
* `lib/cadence-failure-class.mjs` + the consumer's `failPermanently`: a failure
|
|
13
|
+
* whose cause cannot change by waiting stops, records itself and alerts,
|
|
14
|
+
* instead of retrying at poll speed. But a cadence the FRAMEWORK ships should
|
|
15
|
+
* never reach that path at all — its prompt is a file in this repo, so its
|
|
16
|
+
* absence is a build-time fact, and a build-time fact belongs in a check rather
|
|
17
|
+
* than in a seat's log at 3 a.m.
|
|
18
|
+
*
|
|
19
|
+
* Scope, stated honestly: this guard covers the hardcoded CADENCE_REGISTRY in
|
|
20
|
+
* `scripts/daemon/cadence-handlers.mjs` — the cadences every seat gets. It
|
|
21
|
+
* CANNOT cover a seat's `config/.cadence-registry.json` (written by the plan
|
|
22
|
+
* compiler on the seat, naming prompts the seat is meant to author), which is
|
|
23
|
+
* exactly where Eli's came from. That case is the runtime half's job.
|
|
24
|
+
*
|
|
25
|
+
* Usage (standalone): `node scripts/ci/check-cadence-prompts-exist.mjs`
|
|
26
|
+
* exit 0 → every registry prompt resolves; exit 1 → one is missing.
|
|
27
|
+
*
|
|
28
|
+
* @module scripts/ci/check-cadence-prompts-exist
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
"use strict";
|
|
32
|
+
|
|
33
|
+
import { existsSync } from "node:fs";
|
|
34
|
+
import path from "node:path";
|
|
35
|
+
import { fileURLToPath } from "node:url";
|
|
36
|
+
|
|
37
|
+
/** Repo root: two levels up from scripts/ci/. @type {string} */
|
|
38
|
+
const REPO_ROOT = path.resolve(fileURLToPath(new URL(".", import.meta.url)), "..", "..");
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Check every hardcoded cadence definition that names a prompt.
|
|
42
|
+
*
|
|
43
|
+
* @param {object} [opts]
|
|
44
|
+
* @param {string} [opts.cwd=REPO_ROOT]
|
|
45
|
+
* @param {object} [opts.registry] pre-supplied registry (skips the import)
|
|
46
|
+
* @returns {Promise<{ok:boolean, checked:number, missing:Array<{cadence:string, prompt:string, resolved:string}>}>}
|
|
47
|
+
*/
|
|
48
|
+
export async function checkCadencePromptsExist(opts = {}) {
|
|
49
|
+
const cwd = opts.cwd || REPO_ROOT;
|
|
50
|
+
let registry = opts.registry;
|
|
51
|
+
if (!registry) {
|
|
52
|
+
const mod = await import(path.join(cwd, "scripts/daemon/cadence-handlers.mjs"));
|
|
53
|
+
registry = mod.CADENCE_REGISTRY || {};
|
|
54
|
+
}
|
|
55
|
+
const missing = [];
|
|
56
|
+
let checked = 0;
|
|
57
|
+
for (const [cadence, def] of Object.entries(registry)) {
|
|
58
|
+
const prompt = def && typeof def.prompt === "string" ? def.prompt : null;
|
|
59
|
+
if (!prompt) continue; // inline/guarded-only cadences have no prompt to ship
|
|
60
|
+
checked++;
|
|
61
|
+
const resolved = path.resolve(cwd, prompt);
|
|
62
|
+
if (!existsSync(resolved)) missing.push({ cadence, prompt, resolved });
|
|
63
|
+
}
|
|
64
|
+
return { ok: missing.length === 0, checked, missing };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Run the check and print a human report.
|
|
69
|
+
* @param {string} [cwd=REPO_ROOT]
|
|
70
|
+
* @returns {Promise<number>} 0 = ok, 1 = a prompt is missing
|
|
71
|
+
*/
|
|
72
|
+
export async function run(cwd = REPO_ROOT) {
|
|
73
|
+
const { ok, checked, missing } = await checkCadencePromptsExist({ cwd });
|
|
74
|
+
if (ok) {
|
|
75
|
+
console.log(`check-cadence-prompts-exist: OK (${checked} built-in cadence prompt(s) on disk)`);
|
|
76
|
+
return 0;
|
|
77
|
+
}
|
|
78
|
+
console.error("check-cadence-prompts-exist: FAIL — a shipped cadence names a prompt that is not in the repo:");
|
|
79
|
+
for (const m of missing) {
|
|
80
|
+
console.error(` ${m.cadence} → "${m.prompt}" (not found: ${m.resolved})`);
|
|
81
|
+
}
|
|
82
|
+
console.error(
|
|
83
|
+
`check-cadence-prompts-exist: ${missing.length} missing of ${checked} checked. ` +
|
|
84
|
+
"On a seat this is not a log line — it is a cadence that stops, loudly, every time it is due."
|
|
85
|
+
);
|
|
86
|
+
return 1;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
90
|
+
run()
|
|
91
|
+
.then((code) => process.exit(code))
|
|
92
|
+
.catch((err) => {
|
|
93
|
+
console.error("check-cadence-prompts-exist: ERROR", err && err.message ? err.message : err);
|
|
94
|
+
process.exit(2);
|
|
95
|
+
});
|
|
96
|
+
}
|
package/scripts/ci/check.mjs
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
* - check-subagent-frontmatter: every agents/*.md passes the registry validator
|
|
23
23
|
* - check-durable-write-seam : durable JSON writes under lib/ go through fs-atomic
|
|
24
24
|
* - check-skill-packs : vendored design skill packs match their pinned manifests
|
|
25
|
+
* - check-cadence-prompts-exist : every built-in cadence's prompt file is on disk
|
|
25
26
|
*
|
|
26
27
|
* Usage: `node scripts/ci/check.mjs`
|
|
27
28
|
* exit 0 → all checks passed; exit 1 → one or more failed.
|
|
@@ -42,6 +43,7 @@ import { run as runDocsAccuracy } from "./check-docs-accuracy.mjs";
|
|
|
42
43
|
import { run as runSubagentFrontmatter } from "./check-subagent-frontmatter.mjs";
|
|
43
44
|
import { run as runDurableWriteSeam } from "./check-durable-write-seam.mjs";
|
|
44
45
|
import { run as runSkillPacks } from "./check-skill-packs.mjs";
|
|
46
|
+
import { run as runCadencePrompts } from "./check-cadence-prompts-exist.mjs";
|
|
45
47
|
|
|
46
48
|
/**
|
|
47
49
|
* The ordered list of guards this aggregator runs.
|
|
@@ -60,6 +62,7 @@ export const CHECKS = [
|
|
|
60
62
|
{ name: "check-subagent-frontmatter", run: runSubagentFrontmatter },
|
|
61
63
|
{ name: "check-durable-write-seam", run: runDurableWriteSeam },
|
|
62
64
|
{ name: "check-skill-packs", run: runSkillPacks },
|
|
65
|
+
{ name: "check-cadence-prompts-exist", run: runCadencePrompts },
|
|
63
66
|
];
|
|
64
67
|
|
|
65
68
|
/**
|
package/scripts/ci/run-tests.mjs
CHANGED
|
@@ -75,7 +75,17 @@ function main(argv) {
|
|
|
75
75
|
}
|
|
76
76
|
|
|
77
77
|
if (argv.includes("--list")) {
|
|
78
|
-
|
|
78
|
+
// ONE write, not one per file. `console.log` to a PIPE is asynchronous, and
|
|
79
|
+
// the caller below used to `process.exit()` the moment main() returned —
|
|
80
|
+
// which discards whatever has not drained yet. It only shows up under load:
|
|
81
|
+
// run-tests.test.mjs#"--list prints the discovered set" spawns this with a
|
|
82
|
+
// pipe from INSIDE the full suite, where a machine running `node --test`
|
|
83
|
+
// across every core does not drain it in time, and the child exited 0
|
|
84
|
+
// having printed a list truncated mid-alphabet. A green run and a silently
|
|
85
|
+
// short inventory is exactly the failure this module exists to prevent —
|
|
86
|
+
// see the header — so it is fixed on both halves: one write here, and
|
|
87
|
+
// `process.exitCode` rather than `process.exit()` at the bottom.
|
|
88
|
+
process.stdout.write(files.map((f) => relative(REPO_ROOT, f)).join("\n") + "\n");
|
|
79
89
|
console.error(`run-tests: ${files.length} test file(s) discovered (nothing was run).`);
|
|
80
90
|
return 0;
|
|
81
91
|
}
|
|
@@ -95,5 +105,9 @@ function main(argv) {
|
|
|
95
105
|
}
|
|
96
106
|
|
|
97
107
|
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
98
|
-
process.
|
|
108
|
+
// `process.exitCode`, NOT `process.exit()`: the latter tears the process down
|
|
109
|
+
// immediately and drops any stdout still queued on a pipe. Node exits with
|
|
110
|
+
// this code on its own once the event loop drains, which is the only way a
|
|
111
|
+
// piped `--list` is guaranteed to arrive whole.
|
|
112
|
+
process.exitCode = main(process.argv.slice(2));
|
|
99
113
|
}
|
|
@@ -96,6 +96,13 @@ import { writeHandoff, listHandoffs, expireHandoffs, pruneHandoffs, handoffPaths
|
|
|
96
96
|
import { writeFileAtomic } from "../../lib/fs-atomic.mjs";
|
|
97
97
|
import { resolveClaudeBin as sharedResolveClaude } from "../../lib/claude-bin.mjs";
|
|
98
98
|
import { getCadenceDef } from "./cadence-handlers.mjs";
|
|
99
|
+
// PERMANENT vs TRANSIENT (lane/permanent-errors). A retry loop that never
|
|
100
|
+
// inspects its error class is how two seats burned down: a cadence whose prompt
|
|
101
|
+
// file does not exist retried at poll speed forever (measured 16 requeues per
|
|
102
|
+
// 30 s on Eli Rosenberg's seat, 2026-09-24). The taxonomy is pure and lives on
|
|
103
|
+
// its own so the policy can be argued with without a daemon.
|
|
104
|
+
import { classifyCadenceFailure, isPermanent, PERMANENT_MAX_ATTEMPTS } from "../../lib/cadence-failure-class.mjs";
|
|
105
|
+
import { bump as bumpCounter } from "../../lib/diagnostics/counters.mjs";
|
|
99
106
|
import { obligationAllowedUnderPosture } from "../../lib/plan/compile.mjs";
|
|
100
107
|
import { isHumanLaneCadence } from "../../lib/cadences.mjs";
|
|
101
108
|
import { sessionPermissionArgs } from "../../lib/session-permissions.mjs";
|
|
@@ -636,6 +643,12 @@ export function startConsumer(opts = {}) {
|
|
|
636
643
|
const budgetEscalateMs = opts.budgetEscalateMs ?? DEFAULT_BUDGET_ESCALATE_MS;
|
|
637
644
|
const maxSpawnMs = opts.maxSpawnMs ?? DEFAULT_SPAWN_TIMEOUT_MS;
|
|
638
645
|
const spawnSession = opts.spawnSession || realSpawnSession;
|
|
646
|
+
// Cadence registry lookup. Injectable because the config-driven half
|
|
647
|
+
// (`config/.cadence-registry.json`) is read once and memoised at module
|
|
648
|
+
// scope, which a hermetic test cannot re-point — and the permanent-error
|
|
649
|
+
// path is reached precisely through a CONFIGURED cadence whose prompt is
|
|
650
|
+
// absent, so it has to be reachable in a test.
|
|
651
|
+
const cadenceDef = typeof opts.getCadenceDef === "function" ? opts.getCadenceDef : getCadenceDef;
|
|
639
652
|
const userLogger = opts.logger;
|
|
640
653
|
// Test / tuning hooks for the reliability layer.
|
|
641
654
|
const backoffSchedule = opts.backoffSchedule || BACKOFF_SCHEDULE_MS;
|
|
@@ -725,13 +738,64 @@ export function startConsumer(opts = {}) {
|
|
|
725
738
|
* `renderCadencePromptBody`, so the handed-off prompt is byte-identical to
|
|
726
739
|
* what a sub-session would have been given, parallelism directive included —
|
|
727
740
|
* and land it under state/session/handoffs/prompts/<tickId>.md.
|
|
741
|
+
*
|
|
742
|
+
* WHY IT TAGS ITS OWN FAILURES. This function does two unrelated things: it
|
|
743
|
+
* READS the cadence's prompt (whose failure says the spawn cannot work
|
|
744
|
+
* either — the spawn opens the same path) and it WRITES a rendered copy into
|
|
745
|
+
* the handoff dir (whose failure says nothing about the spawn at all). The
|
|
746
|
+
* caller has to tell them apart, and the first attempt at that inferred it
|
|
747
|
+
* from the message — `msg.includes(promptPath)`.
|
|
748
|
+
*
|
|
749
|
+
* That inference is WRONG, measured 2026-09-25: `readFileSync` on a
|
|
750
|
+
* DIRECTORY throws exactly `EISDIR: illegal operation on a directory, read`,
|
|
751
|
+
* with no path in the text. A directory at the configured prompt path
|
|
752
|
+
* therefore passed the `existsSync` pre-check, threw here, was read as "not
|
|
753
|
+
* about the prompt", classified transient, and fell through to a spawn — the
|
|
754
|
+
* commonest unreadable case, and precisely the one the PROMPT_UNREADABLE code
|
|
755
|
+
* exists for. Node puts the path in the message for ENOENT and EACCES and
|
|
756
|
+
* not for EISDIR; no caller should have to know which.
|
|
757
|
+
*
|
|
758
|
+
* So the phase is not inferred, it is STAMPED, at the only two places that
|
|
759
|
+
* know it: `err.cadencePhase` is `"prompt"` on the read and `"handoff"` on
|
|
760
|
+
* everything after it. `err.cadenceErrno` carries `err.code` alongside, since
|
|
761
|
+
* the errno is the contract and the message is not.
|
|
728
762
|
*/
|
|
729
763
|
function renderHandoffPrompt(tickId, promptPath) {
|
|
730
764
|
const fullPrompt = join(agentRoot, promptPath);
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
765
|
+
let source;
|
|
766
|
+
try {
|
|
767
|
+
source = readFileSync(fullPrompt, "utf-8");
|
|
768
|
+
} catch (err) {
|
|
769
|
+
throw stampPhase(err, "prompt");
|
|
770
|
+
}
|
|
771
|
+
// Everything past the read is about OUR output, not the cadence's input:
|
|
772
|
+
// a render bug or an unwritable handoff dir leaves the spawn perfectly able
|
|
773
|
+
// to run, so neither may be read as a permanent prompt fault.
|
|
774
|
+
try {
|
|
775
|
+
const body = renderCadencePromptBody(agentRoot, source);
|
|
776
|
+
const out = join(handoffPaths(agentRoot).prompts, `${tickId}.md`);
|
|
777
|
+
writeFileAtomic(out, body);
|
|
778
|
+
return out;
|
|
779
|
+
} catch (err) {
|
|
780
|
+
throw stampPhase(err, "handoff");
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* Stamp an error with the phase it came from, so a catch does not have to
|
|
786
|
+
* guess. Non-destructive: an already-stamped error keeps its innermost phase
|
|
787
|
+
* (the read is nested inside nothing, but this keeps re-throws honest), and a
|
|
788
|
+
* non-object throw is wrapped rather than dropped.
|
|
789
|
+
*/
|
|
790
|
+
function stampPhase(err, phase) {
|
|
791
|
+
if (!err || typeof err !== "object") {
|
|
792
|
+
const wrapped = new Error(String(err ?? "unknown error"));
|
|
793
|
+
wrapped.cadencePhase = phase;
|
|
794
|
+
return wrapped;
|
|
795
|
+
}
|
|
796
|
+
if (!err.cadencePhase) err.cadencePhase = phase;
|
|
797
|
+
if (!err.cadenceErrno && typeof err.code === "string") err.cadenceErrno = err.code;
|
|
798
|
+
return err;
|
|
735
799
|
}
|
|
736
800
|
|
|
737
801
|
/**
|
|
@@ -759,7 +823,7 @@ export function startConsumer(opts = {}) {
|
|
|
759
823
|
// predicate both places — the daemon and the plan cannot disagree about
|
|
760
824
|
// what is suspended.
|
|
761
825
|
if (posture && cadence) {
|
|
762
|
-
const def =
|
|
826
|
+
const def = cadenceDef(cadence) || {};
|
|
763
827
|
const verdict = obligationAllowedUnderPosture(
|
|
764
828
|
{
|
|
765
829
|
kind: "SCHEDULE",
|
|
@@ -824,6 +888,13 @@ export function startConsumer(opts = {}) {
|
|
|
824
888
|
handed_off: 0,
|
|
825
889
|
handoff_timeouts: 0,
|
|
826
890
|
deferred: 0,
|
|
891
|
+
// A failure the classifier called PERMANENT — the cause cannot change by
|
|
892
|
+
// waiting, so the tick was dead-lettered instead of retried. Rides the
|
|
893
|
+
// heartbeat (writeHealth) so `maestro cadence status` and the beat carry it
|
|
894
|
+
// without a new channel; `last_permanent_failure` names the cadence and the
|
|
895
|
+
// code so the reader does not have to go back to the log to find out which.
|
|
896
|
+
permanent_failures: 0,
|
|
897
|
+
last_permanent_failure: null,
|
|
827
898
|
last_event_id: null,
|
|
828
899
|
last_decision: null,
|
|
829
900
|
};
|
|
@@ -887,6 +958,99 @@ export function startConsumer(opts = {}) {
|
|
|
887
958
|
return { ok: false, decision: "deferred", reason };
|
|
888
959
|
}
|
|
889
960
|
|
|
961
|
+
/**
|
|
962
|
+
* A failure the classifier calls PERMANENT: stop retrying, record it
|
|
963
|
+
* durably, and make it visible to a person.
|
|
964
|
+
*
|
|
965
|
+
* THE FAULT THIS CLOSES (Eli Rosenberg's seat, 2026-09-24). A cadence whose
|
|
966
|
+
* configured prompt file was not on disk requeued forever — measured at 16
|
|
967
|
+
* per 30 seconds — because nothing in the retry path ever asked whether the
|
|
968
|
+
* cause could change by waiting. It cannot: a file that does not exist will
|
|
969
|
+
* not exist 30 seconds later.
|
|
970
|
+
*
|
|
971
|
+
* Three things happen, and all three matter:
|
|
972
|
+
*
|
|
973
|
+
* 1. STOP. `failTick` with `min(this consumer's budget,
|
|
974
|
+
* PERMANENT_MAX_ATTEMPTS)` — the permanent cap lowers a budget, never
|
|
975
|
+
* raises one. Two attempts, then dlq/: not because the cause can never be
|
|
976
|
+
* repaired (someone can drop the prompt onto the seat), but because it
|
|
977
|
+
* cannot be repaired BY WAITING.
|
|
978
|
+
*
|
|
979
|
+
* STATED HONESTLY (corrected 2026-09-25). `failTick`'s first outcome for
|
|
980
|
+
* attempt 1 is a REQUEUE, so what this buys is "one extra attempt, then
|
|
981
|
+
* stop" — not "the cadence's own schedule is the retry window", which is
|
|
982
|
+
* what an earlier draft of this comment claimed. The schedule only
|
|
983
|
+
* becomes the window once the last attempt reaches dlq/. Spacing the one
|
|
984
|
+
* extra attempt is a separate act, and it is done here:
|
|
985
|
+
* `holdCadenceForBackoff` moves the per-cadence gate forward so the retry
|
|
986
|
+
* cannot land in the same drain that produced the first failure. The
|
|
987
|
+
* measured before/after is 16 requeues per 30 s, unbounded → 2 attempts
|
|
988
|
+
* one backoff interval apart, then a durable stop.
|
|
989
|
+
*
|
|
990
|
+
* The budget is the smaller half of this; the bigger half is (2) and (3),
|
|
991
|
+
* and that a permanent fault no longer falls through to a second code
|
|
992
|
+
* path that opens the same missing file.
|
|
993
|
+
* 2. RECORD. The dlq/ file is the durable record and now carries
|
|
994
|
+
* `permanent:<code>` in `last_error`, so the reason survives a restart
|
|
995
|
+
* and a log rotation. A durable counter (`cadence.permanent_failure`)
|
|
996
|
+
* goes to logs/diagnostics/counters/<day>.jsonl alongside it.
|
|
997
|
+
* 3. BE VISIBLE. The counter is what the seat's EXISTING alert lane reads
|
|
998
|
+
* (lib/diagnostics/alerts.mjs `cadence_permanently_failing`, delivered by
|
|
999
|
+
* the alerts cadence through config/alerts.yaml's webhook, or logged
|
|
1000
|
+
* when there is none) — no new channel. And `stats.permanent_failures`
|
|
1001
|
+
* rides the heartbeat, so the seat's health file and everything
|
|
1002
|
+
* downstream of it say so without anyone tailing a log.
|
|
1003
|
+
*
|
|
1004
|
+
* Never throws: the counter bump is best-effort by contract and failTick is
|
|
1005
|
+
* already guarded.
|
|
1006
|
+
*
|
|
1007
|
+
* @param {object} event the claimed tick
|
|
1008
|
+
* @param {object} verdict a CadenceFailureVerdict (class "permanent")
|
|
1009
|
+
* @param {string} detail the raw error text, for the dlq record + log
|
|
1010
|
+
* @param {string} stage log stage naming WHERE it was caught
|
|
1011
|
+
*/
|
|
1012
|
+
function failPermanently(event, verdict, detail, stage) {
|
|
1013
|
+
// The permanent cap LOWERS a budget, never raises one: if this consumer is
|
|
1014
|
+
// already stricter than the cap, its own number wins.
|
|
1015
|
+
const budget = Math.min(maxAttempts, verdict.maxAttempts ?? PERMANENT_MAX_ATTEMPTS);
|
|
1016
|
+
const reason = `permanent:${verdict.code} — ${verdict.reason}${detail ? `: ${detail}` : ""}`;
|
|
1017
|
+
log({
|
|
1018
|
+
level: "error",
|
|
1019
|
+
stage,
|
|
1020
|
+
permanent: true,
|
|
1021
|
+
code: verdict.code,
|
|
1022
|
+
id: event.id,
|
|
1023
|
+
cadence: event.cadence,
|
|
1024
|
+
error: detail || verdict.reason,
|
|
1025
|
+
max_attempts: budget,
|
|
1026
|
+
note: "cause cannot change by waiting — one more attempt after the cadence backoff, then dead-lettered",
|
|
1027
|
+
});
|
|
1028
|
+
try {
|
|
1029
|
+
bumpCounter("cadence.permanent_failure", { cadence: event.cadence, code: verdict.code }, { agentRoot });
|
|
1030
|
+
} catch { /* a counter must never crash the path it observes */ }
|
|
1031
|
+
stats.permanent_failures += 1;
|
|
1032
|
+
stats.last_permanent_failure = {
|
|
1033
|
+
cadence: event.cadence,
|
|
1034
|
+
code: verdict.code,
|
|
1035
|
+
at: new Date().toISOString(),
|
|
1036
|
+
};
|
|
1037
|
+
stats.last_decision = "permanent-failure";
|
|
1038
|
+
// Space the one remaining attempt. Without this the requeue below is
|
|
1039
|
+
// re-claimed by the very next drain pass — "bounded" but still at poll
|
|
1040
|
+
// speed, which is the behaviour this lane exists to remove.
|
|
1041
|
+
const heldUntil = holdCadenceForBackoff(event.cadence);
|
|
1042
|
+
const outcome = failTick(agentRoot, event.id, reason, { maxAttempts: budget });
|
|
1043
|
+
if (outcome?.destination === "dlq") stats.dlq += 1;
|
|
1044
|
+
else {
|
|
1045
|
+
stats.retries += 1;
|
|
1046
|
+
log({ level: "warn", stage: "permanent_retry_held", id: event.id, cadence: event.cadence, code: verdict.code, retry_at: new Date(heldUntil).toISOString() });
|
|
1047
|
+
}
|
|
1048
|
+
// The heartbeat is the visible surface; write it now rather than waiting up
|
|
1049
|
+
// to `heartbeatMs` for a reader to learn a cadence just died.
|
|
1050
|
+
heartbeat();
|
|
1051
|
+
return { ok: false, decision: outcome?.destination === "dlq" ? "dlq-permanent" : "failed-permanent", code: verdict.code };
|
|
1052
|
+
}
|
|
1053
|
+
|
|
890
1054
|
// Ledger hygiene runs from the sweep at most this often.
|
|
891
1055
|
const HANDOFF_PRUNE_EVERY_MS = 60 * 60_000;
|
|
892
1056
|
let lastHandoffPruneAt = 0;
|
|
@@ -989,20 +1153,47 @@ export function startConsumer(opts = {}) {
|
|
|
989
1153
|
s.failures += 1;
|
|
990
1154
|
// Exponential back-off honouring the (test-overridable) schedule.
|
|
991
1155
|
const idx = Math.min(s.failures, backoffSchedule.length - 1);
|
|
992
|
-
s.nextAllowedAt =
|
|
1156
|
+
s.nextAllowedAt = nowMs() + backoffSchedule[idx];
|
|
993
1157
|
if (s.failures >= circuitThreshold) {
|
|
994
|
-
s.openUntil =
|
|
1158
|
+
s.openUntil = nowMs() + circuitDurationMs;
|
|
995
1159
|
log({ level: "error", stage: "circuit_opened", cadence, failures: s.failures, open_until: new Date(s.openUntil).toISOString() });
|
|
996
1160
|
writeCircuitFile();
|
|
997
1161
|
}
|
|
998
1162
|
}
|
|
999
1163
|
|
|
1164
|
+
/**
|
|
1165
|
+
* Hold a cadence off the spawn path for one backoff interval WITHOUT
|
|
1166
|
+
* advancing it toward an open circuit.
|
|
1167
|
+
*
|
|
1168
|
+
* WHY IT IS SEPARATE FROM `recordSubsessionFailure`. A permanent failure is
|
|
1169
|
+
* not a flaky sub-session: counting it toward `circuitThreshold` would trip a
|
|
1170
|
+
* breaker whose whole job is to ride out a bad patch, for a cause that has no
|
|
1171
|
+
* patch to ride out. What it DOES need is spacing — `PERMANENT_MAX_ATTEMPTS`
|
|
1172
|
+
* is a count, not a delay, and `failPermanently`'s first outcome is a requeue,
|
|
1173
|
+
* so without this the "one extra attempt" lands in the very same drain that
|
|
1174
|
+
* produced the first. `escalate` consults `isCadenceAllowed` before it reaches
|
|
1175
|
+
* the prompt at all, so moving `nextAllowedAt` forward is all it takes for the
|
|
1176
|
+
* extra attempt to cost a real interval rather than a millisecond.
|
|
1177
|
+
*
|
|
1178
|
+
* Idempotent-ish and monotonic: never moves the hold EARLIER, so a circuit or
|
|
1179
|
+
* a longer transient backoff already in force keeps its own deadline.
|
|
1180
|
+
*/
|
|
1181
|
+
function holdCadenceForBackoff(cadence) {
|
|
1182
|
+
const s = getCadenceState(cadence);
|
|
1183
|
+
// The first rung is deliberately 0 ("retry immediately once"); a permanent
|
|
1184
|
+
// fault has already proved it needs a gap, so take the first NON-zero rung.
|
|
1185
|
+
const step = backoffSchedule.find((ms) => ms > 0) ?? 0;
|
|
1186
|
+
const until = nowMs() + step;
|
|
1187
|
+
if (until > s.nextAllowedAt) s.nextAllowedAt = until;
|
|
1188
|
+
return s.nextAllowedAt;
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1000
1191
|
function writeCircuitFile() {
|
|
1001
1192
|
// Persist the open-circuit snapshot so doctor + the operator can see
|
|
1002
1193
|
// which cadences are currently held back without scraping logs.
|
|
1003
1194
|
const open = {};
|
|
1004
1195
|
for (const [cad, s] of cadenceState.entries()) {
|
|
1005
|
-
if (s.openUntil >
|
|
1196
|
+
if (s.openUntil > nowMs()) {
|
|
1006
1197
|
open[cad] = { failures: s.failures, open_until: new Date(s.openUntil).toISOString() };
|
|
1007
1198
|
}
|
|
1008
1199
|
}
|
|
@@ -1020,7 +1211,7 @@ export function startConsumer(opts = {}) {
|
|
|
1020
1211
|
|
|
1021
1212
|
function isCadenceAllowed(cadence) {
|
|
1022
1213
|
const s = getCadenceState(cadence);
|
|
1023
|
-
const now =
|
|
1214
|
+
const now = nowMs();
|
|
1024
1215
|
if (s.openUntil > now) return { allowed: false, reason: "circuit-open", retry_at: s.openUntil };
|
|
1025
1216
|
if (s.nextAllowedAt > now) return { allowed: false, reason: "backoff", retry_at: s.nextAllowedAt };
|
|
1026
1217
|
// Circuit closes automatically when openUntil passes.
|
|
@@ -1107,7 +1298,7 @@ export function startConsumer(opts = {}) {
|
|
|
1107
1298
|
}
|
|
1108
1299
|
}
|
|
1109
1300
|
|
|
1110
|
-
const def =
|
|
1301
|
+
const def = cadenceDef(event.cadence);
|
|
1111
1302
|
let promptPath = def?.prompt;
|
|
1112
1303
|
if (!promptPath) {
|
|
1113
1304
|
// Unknown cadence — try the conventional location.
|
|
@@ -1121,6 +1312,15 @@ export function startConsumer(opts = {}) {
|
|
|
1121
1312
|
stats.dlq += 1;
|
|
1122
1313
|
return { ok: false, decision: "dlq-no-prompt" };
|
|
1123
1314
|
}
|
|
1315
|
+
} else if (!existsSync(join(agentRoot, promptPath))) {
|
|
1316
|
+
// THE ELI FAULT, caught before it can loop. The `!promptPath` branch above
|
|
1317
|
+
// has checked the conventional path since it was written; a prompt named
|
|
1318
|
+
// by CONFIG was never checked at all — it was handed straight to the
|
|
1319
|
+
// handoff renderer and then, when that threw, to the spawn, which opens
|
|
1320
|
+
// the same path. Both fail, neither is terminal, and the tick comes back
|
|
1321
|
+
// on the bus 30 seconds later. Check it once, here, and classify.
|
|
1322
|
+
const verdict = classifyCadenceFailure({ phase: "prompt", error: `prompt not found: ${promptPath}` });
|
|
1323
|
+
return failPermanently(event, verdict, `configured prompt missing: ${promptPath}`, "escalate_prompt_missing");
|
|
1124
1324
|
}
|
|
1125
1325
|
|
|
1126
1326
|
// Per-cadence in-flight guard (F9): never the same cadence twice at once —
|
|
@@ -1139,31 +1339,69 @@ export function startConsumer(opts = {}) {
|
|
|
1139
1339
|
const fd = frontDoorState();
|
|
1140
1340
|
const verdict = shouldHandOffTick({ ...fd, mode: def?.mode, metadata: event.metadata });
|
|
1141
1341
|
if (verdict.handOff) {
|
|
1342
|
+
// The PROMPT read is its own step, with its own catch. It used to sit
|
|
1343
|
+
// inside the handoff try/catch, so a prompt that could not be read was
|
|
1344
|
+
// indistinguishable from a handoff file that could not be written — and
|
|
1345
|
+
// the shared catch fell through to the spawn, which opens the same
|
|
1346
|
+
// prompt path. That is the busy loop: `handoff_failed_spawning_instead`
|
|
1347
|
+
// 16 times per 30 seconds on Eli Rosenberg's seat, 2026-09-24.
|
|
1348
|
+
// Split apart, each failure gets the answer it deserves: a permanent
|
|
1349
|
+
// prompt fault stops here; an unwritable handoff still falls through,
|
|
1350
|
+
// because the spawn genuinely might work.
|
|
1351
|
+
let rendered = null;
|
|
1142
1352
|
try {
|
|
1143
|
-
|
|
1144
|
-
const h = writeHandoff(agentRoot, {
|
|
1145
|
-
tickId: event.id,
|
|
1146
|
-
cadence: event.cadence,
|
|
1147
|
-
mode: def?.mode || "escalate",
|
|
1148
|
-
promptPath: rendered,
|
|
1149
|
-
metadata: { ...(event.metadata || {}), sourcePrompt: promptPath },
|
|
1150
|
-
}, { now: nowMs(), deadlineMs: handoffDeadlineMs });
|
|
1151
|
-
if (!h.ok) throw new Error(h.error || "handoff write failed");
|
|
1152
|
-
completeTick(agentRoot, event.id, {
|
|
1153
|
-
decision: "handed-to-session",
|
|
1154
|
-
cadence: event.cadence,
|
|
1155
|
-
prompt: promptPath,
|
|
1156
|
-
handoff: h.path,
|
|
1157
|
-
deadline_at: h.handoff && h.handoff.deadlineAt,
|
|
1158
|
-
});
|
|
1159
|
-
stats.handed_off += 1;
|
|
1160
|
-
stats.last_decision = "handed-to-session";
|
|
1161
|
-
log({ level: "info", stage: "handed_to_session", id: event.id, cadence: event.cadence, prompt: rendered, deadline_at: h.handoff && h.handoff.deadlineAt });
|
|
1162
|
-
return { ok: true, decision: "handed-to-session" };
|
|
1353
|
+
rendered = renderHandoffPrompt(event.id, promptPath);
|
|
1163
1354
|
} catch (err) {
|
|
1164
|
-
//
|
|
1165
|
-
//
|
|
1166
|
-
|
|
1355
|
+
// renderHandoffPrompt stamps which half failed (see its doc): the
|
|
1356
|
+
// READ of the cadence prompt, or everything after it. Only the read
|
|
1357
|
+
// says anything about whether a spawn would work — the spawn opens
|
|
1358
|
+
// the same path — so only `cadencePhase === "prompt"` may reach a
|
|
1359
|
+
// permanent verdict. An unwritable handoff dir falls through to the
|
|
1360
|
+
// spawn like any other handoff fault.
|
|
1361
|
+
//
|
|
1362
|
+
// This used to be inferred from `msg.includes(promptPath)`, which is
|
|
1363
|
+
// false for EISDIR (Node omits the path), so the commonest unreadable
|
|
1364
|
+
// case — a DIRECTORY at the prompt path — fell through and spawned.
|
|
1365
|
+
// Never classify a path fault by whether the message quotes the path.
|
|
1366
|
+
const msg = (err && err.message) || "";
|
|
1367
|
+
const promptVerdict = classifyCadenceFailure({
|
|
1368
|
+
phase: err?.cadencePhase === "prompt" ? "prompt" : "spawn",
|
|
1369
|
+
error: msg,
|
|
1370
|
+
errno: err?.cadenceErrno,
|
|
1371
|
+
});
|
|
1372
|
+
if (isPermanent(promptVerdict)) {
|
|
1373
|
+
return failPermanently(event, promptVerdict, msg, "handoff_prompt_failed_permanently");
|
|
1374
|
+
}
|
|
1375
|
+
log({ level: "warn", stage: "handoff_render_failed_spawning_instead", id: event.id, cadence: event.cadence, phase: err?.cadencePhase || null, error: msg });
|
|
1376
|
+
}
|
|
1377
|
+
if (rendered) {
|
|
1378
|
+
try {
|
|
1379
|
+
const h = writeHandoff(agentRoot, {
|
|
1380
|
+
tickId: event.id,
|
|
1381
|
+
cadence: event.cadence,
|
|
1382
|
+
mode: def?.mode || "escalate",
|
|
1383
|
+
promptPath: rendered,
|
|
1384
|
+
metadata: { ...(event.metadata || {}), sourcePrompt: promptPath },
|
|
1385
|
+
}, { now: nowMs(), deadlineMs: handoffDeadlineMs });
|
|
1386
|
+
if (!h.ok) throw new Error(h.error || "handoff write failed");
|
|
1387
|
+
completeTick(agentRoot, event.id, {
|
|
1388
|
+
decision: "handed-to-session",
|
|
1389
|
+
cadence: event.cadence,
|
|
1390
|
+
prompt: promptPath,
|
|
1391
|
+
handoff: h.path,
|
|
1392
|
+
deadline_at: h.handoff && h.handoff.deadlineAt,
|
|
1393
|
+
});
|
|
1394
|
+
stats.handed_off += 1;
|
|
1395
|
+
stats.last_decision = "handed-to-session";
|
|
1396
|
+
log({ level: "info", stage: "handed_to_session", id: event.id, cadence: event.cadence, prompt: rendered, deadline_at: h.handoff && h.handoff.deadlineAt });
|
|
1397
|
+
return { ok: true, decision: "handed-to-session" };
|
|
1398
|
+
} catch (err) {
|
|
1399
|
+
// A handoff FILE we could not write is not a reason to lose the tick,
|
|
1400
|
+
// and it says nothing about whether the spawn would work — fall
|
|
1401
|
+
// through to the legacy spawn, loudly, exactly as before. The prompt
|
|
1402
|
+
// half of this, which DOES say so, is handled above.
|
|
1403
|
+
log({ level: "warn", stage: "handoff_failed_spawning_instead", id: event.id, cadence: event.cadence, error: err && err.message });
|
|
1404
|
+
}
|
|
1167
1405
|
}
|
|
1168
1406
|
}
|
|
1169
1407
|
}
|
|
@@ -1292,6 +1530,15 @@ export function startConsumer(opts = {}) {
|
|
|
1292
1530
|
stats.spawn_failures += 1;
|
|
1293
1531
|
recordSubsessionFailure(event.cadence);
|
|
1294
1532
|
const reason = result.error || (stderrTail ? `exit ${result.exit_code}: ${stderrTail}` : `exit ${result.exit_code}`);
|
|
1533
|
+
// Can this possibly succeed if we try again? `realSpawnSession` answers
|
|
1534
|
+
// -2 (prompt not on disk) / -3 (prompt unreadable) for causes that do not
|
|
1535
|
+
// move; those get the classifier's bounded budget and a durable,
|
|
1536
|
+
// visible record. Everything else — a timeout, a non-zero exit, a crash —
|
|
1537
|
+
// is transient and unchanged.
|
|
1538
|
+
const spawnVerdict = classifyCadenceFailure({ phase: "spawn", exitCode: result.exit_code, error: reason });
|
|
1539
|
+
if (isPermanent(spawnVerdict)) {
|
|
1540
|
+
return failPermanently(event, spawnVerdict, reason, "subsession_failed_permanently");
|
|
1541
|
+
}
|
|
1295
1542
|
const outcome = failTick(agentRoot, event.id, reason, { maxAttempts });
|
|
1296
1543
|
if (outcome?.destination === "dlq") stats.dlq += 1;
|
|
1297
1544
|
else stats.retries += 1;
|
|
@@ -1304,7 +1551,7 @@ export function startConsumer(opts = {}) {
|
|
|
1304
1551
|
stats.received += 1;
|
|
1305
1552
|
stats.last_event_id = event.id;
|
|
1306
1553
|
|
|
1307
|
-
const def =
|
|
1554
|
+
const def = cadenceDef(event.cadence);
|
|
1308
1555
|
if (def?.mode === "inline" && typeof def.handler === "function") {
|
|
1309
1556
|
try {
|
|
1310
1557
|
const out = await def.handler({ event, agentRoot, log });
|
|
@@ -17,6 +17,7 @@ import { join } from "path";
|
|
|
17
17
|
import { isEnabled as orgEnabled, loadOrgConfig } from "../../lib/org/client.mjs";
|
|
18
18
|
import { recall as orgRecall } from "../../lib/org/knowledge.mjs";
|
|
19
19
|
import { isPrivateConversation, historyDirNames } from "../../lib/context/history-scope.mjs";
|
|
20
|
+
import { stripDisclosureInstructionsAndWarn } from "../../lib/identity/disclosure-instructions.mjs";
|
|
20
21
|
import { fitSections } from "../../lib/context/budget.mjs";
|
|
21
22
|
import { currentWorkBlock, audienceForItem } from "../../lib/session/current-work.mjs";
|
|
22
23
|
|
|
@@ -590,9 +591,16 @@ export async function compileContext(item, classResult, options = {}) {
|
|
|
590
591
|
sections.push("--- COMPILED CONTEXT (do not repeat verbatim) ---");
|
|
591
592
|
sections.push("");
|
|
592
593
|
|
|
594
|
+
// The sender profile is SEAT-AUTHORED prose (memory/profiles/users/*.yaml),
|
|
595
|
+
// not inbound data, and it lands in a context block the model reads as
|
|
596
|
+
// guidance. A profile line ordering a self-introduction would defeat the
|
|
597
|
+
// rest of the fix for exactly one correspondent — the hardest version of
|
|
598
|
+
// the bug to reproduce — and the repo cannot see the file. Filtered like
|
|
599
|
+
// the CLAUDE.md scrape; the transcript below is deliberately NOT filtered,
|
|
600
|
+
// because it is a record of what people said, not an instruction.
|
|
593
601
|
if (profileText) {
|
|
594
602
|
sections.push("## Sender Profile");
|
|
595
|
-
sections.push(profileText);
|
|
603
|
+
sections.push(stripDisclosureInstructionsAndWarn(profileText, "memory/profiles/users/*.yaml"));
|
|
596
604
|
sections.push("");
|
|
597
605
|
}
|
|
598
606
|
|