@cohortapp/agent-sdk 2.5.1 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +185 -88
- package/bin/maestro.test.mjs +175 -48
- package/docs/runbooks/backup-restore.md +65 -33
- package/framework-features.json +4 -4
- package/lib/backup/policy.mjs +710 -0
- package/lib/backup/policy.test.mjs +305 -0
- package/lib/budget-escalate.mjs +133 -0
- package/lib/budget-escalate.test.mjs +232 -0
- package/lib/budget-guard.envelope.test.mjs +476 -0
- package/lib/budget-guard.mjs +853 -75
- package/lib/budget-guard.test.mjs +91 -42
- package/lib/cadences.mjs +33 -0
- package/lib/channels/orgmail/adapter.mjs +88 -3
- package/lib/channels/orgmail/adapter.test.mjs +137 -0
- package/lib/channels/repeat-suppressor.mjs +198 -0
- package/lib/channels/repeat-suppressor.test.mjs +134 -0
- package/lib/comms/receipts.mjs +297 -0
- package/lib/cost/ledger-row.mjs +333 -0
- package/lib/cost/ledger-row.test.mjs +183 -0
- package/lib/execution/drive.mjs +28 -1
- package/lib/execution/effects.mjs +191 -12
- package/lib/execution/effects.test.mjs +50 -11
- package/lib/goals/admission.mjs +13 -1
- package/lib/goals/admission.test.mjs +26 -1
- package/lib/goals/loop.mjs +13 -0
- package/lib/kpi-sensors.test.mjs +3 -0
- package/lib/mandate/cache.mjs +13 -5
- package/lib/mandate/derive.mjs +146 -21
- package/lib/mandate/derive.test.mjs +50 -6
- package/lib/mandate/model.mjs +32 -4
- package/lib/mandate/refresh.test.mjs +16 -2
- package/lib/mcp/server.test.mjs +12 -3
- package/lib/model-router/economics.mjs +107 -76
- package/lib/model-router/economics.test.mjs +64 -46
- package/lib/model-router/integration-coverage.test.mjs +39 -37
- package/lib/model-router/ledger.mjs +75 -22
- package/lib/model-router/ledger.test.mjs +35 -2
- package/lib/org/client.mjs +14 -0
- package/lib/org/cost-sync.mjs +16 -2
- package/lib/org/doctor.mjs +62 -1
- package/lib/org/doctor.test.mjs +36 -3
- package/lib/org/email-remedy.mjs +49 -0
- package/lib/org/engagement-ledger.mjs +376 -0
- package/lib/org/engagement-ledger.test.mjs +112 -0
- package/lib/org/engagement.mjs +1056 -0
- package/lib/org/engagement.test.mjs +739 -0
- package/lib/org/messaging.mjs +230 -3
- package/lib/org/messaging.test.mjs +110 -1
- package/lib/org/param-contract.mjs +56 -2
- package/lib/org/param-contract.test.mjs +26 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +5 -0
- package/lib/org/protocol.test.mjs +7 -1
- package/lib/org/tool-surface.mjs +506 -10
- package/lib/org/tool-surface.test.mjs +191 -7
- package/lib/org/ui-parity.mjs +333 -6
- package/lib/org/ui-parity.test.mjs +96 -3
- package/lib/org/work-ledger.mjs +241 -0
- package/lib/org/work-ledger.test.mjs +237 -0
- package/lib/plan/adoption-e2e.test.mjs +366 -0
- package/lib/plan/budget-enforcement.test.mjs +400 -0
- package/lib/plan/budget-runtime.mjs +215 -0
- package/lib/plan/compile.mjs +201 -5
- package/lib/plan/compile.test.mjs +19 -5
- package/lib/plan/emit.mjs +8 -0
- package/lib/plan/emit.test.mjs +18 -0
- package/lib/resource-governor.mjs +58 -12
- package/lib/resource-governor.test.mjs +41 -1
- package/lib/security/audit-engine.mjs +45 -8
- package/lib/security/audit-engine.test.mjs +35 -0
- package/lib/setup/enroll-from-cohort.mjs +14 -1
- package/lib/setup/sections/mandate.mjs +48 -7
- package/lib/setup/sections/mandate.test.mjs +17 -2
- package/lib/setup/sections/orgmail.mjs +10 -2
- package/lib/setup/state.mjs +83 -2
- package/lib/telemetry/collect.mjs +360 -20
- package/lib/telemetry/collect.test.mjs +266 -0
- package/package.json +1 -1
- package/scripts/cost/track-claude-usage.mjs +207 -48
- package/scripts/cost/track-claude-usage.test.mjs +148 -0
- package/scripts/daemon/agent-daemon.mjs +315 -17
- package/scripts/daemon/assurance-e2e.test.mjs +421 -0
- package/scripts/daemon/assurance.mjs +944 -0
- package/scripts/daemon/assurance.test.mjs +668 -0
- package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
- package/scripts/daemon/cadence-consumer.mjs +147 -9
- package/scripts/daemon/cadence-consumer.test.mjs +6 -0
- package/scripts/daemon/cadence-handlers.mjs +158 -0
- package/scripts/daemon/cadence-handlers.test.mjs +64 -0
- package/scripts/daemon/deliver.mjs +314 -0
- package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
- package/scripts/daemon/dispatcher.mjs +64 -6
- package/scripts/daemon/responder-cost.test.mjs +68 -0
- package/scripts/daemon/responder.mjs +351 -298
- package/scripts/local-triggers/generate-plists.test.mjs +7 -4
- package/scripts/maintenance/backup-run.mjs +415 -0
- package/scripts/maintenance/backup-to-cloud.sh +16 -116
- package/scripts/org/send-orgmail.mjs +16 -0
- package/scripts/record-receipt.sh +63 -0
- package/scripts/restore-from-backup.sh +14 -3
- package/scripts/restore-from-backup.test.mjs +8 -5
- package/scripts/send-email-threaded.py +47 -0
- package/scripts/send-sms.sh +4 -0
- package/scripts/send-whatsapp.sh +4 -0
- package/scripts/setup/init-backup.mjs +93 -38
- package/scripts/slack-send.sh +12 -0
|
@@ -18,6 +18,9 @@ import { join, dirname } from "node:path";
|
|
|
18
18
|
|
|
19
19
|
import { enqueueTick, busDepth, listInbox, getBusPaths } from "../../lib/cadence-bus.mjs";
|
|
20
20
|
import { startConsumer } from "./cadence-consumer.mjs";
|
|
21
|
+
import { obligationAllowedUnderPosture } from "../../lib/plan/compile.mjs";
|
|
22
|
+
import { postureForBand } from "../../lib/budget-guard.mjs";
|
|
23
|
+
import { isHumanLaneCadence } from "../../lib/cadences.mjs";
|
|
21
24
|
|
|
22
25
|
async function makeAgentRoot() {
|
|
23
26
|
const path = join(tmpdir(), `consumer-gov-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
|
|
@@ -218,3 +221,56 @@ test("L2: a 429 sub-session failure does NOT trip the per-cadence circuit", asyn
|
|
|
218
221
|
assert.ok(stats.retries >= 1, "the tick was requeued");
|
|
219
222
|
} finally { await consumer.stop(); await rmRoot(root); }
|
|
220
223
|
});
|
|
224
|
+
|
|
225
|
+
test("band 150 does not silence the inbox at EITHER gate", async () => {
|
|
226
|
+
// Two gates sit between a cadence tick and a spawn, and the 150% rung has to
|
|
227
|
+
// agree with itself across both:
|
|
228
|
+
// 1. `obligationAllowedUnderPosture` — was `kind !== "REACT"`, so every
|
|
229
|
+
// cadence (all SCHEDULEs) was refused, including `messaging-inbound`, the
|
|
230
|
+
// only puller of org DMs into the inbox pipeline.
|
|
231
|
+
// 2. `resource-governor.admit` — treats only `source:"inbox"` as a human
|
|
232
|
+
// reply, and every cadence tick arrives as `source:"cadence"`, so fixing
|
|
233
|
+
// gate 1 alone just moves the silence one hop down.
|
|
234
|
+
const posture = postureForBand(150);
|
|
235
|
+
const gov = await import("../../lib/resource-governor.mjs");
|
|
236
|
+
// An IDLE, roomy machine, injected. `defaultDeps` reads real freemem/loadavg/
|
|
237
|
+
// live-process counts, so asserting against it makes the test a measure of
|
|
238
|
+
// whatever else is running — this test is about the BUDGET rung, nothing else.
|
|
239
|
+
const deps = {
|
|
240
|
+
totalmem: 64 * 1024 ** 3,
|
|
241
|
+
freemem: 48 * 1024 ** 3,
|
|
242
|
+
cpus: 10,
|
|
243
|
+
loadavg: [0.1],
|
|
244
|
+
liveClaude: { count: 0, rssMB: 0 },
|
|
245
|
+
throttleCeiling: Infinity,
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
for (const id of ["inbox-processor", "messaging-inbound"]) {
|
|
249
|
+
assert.equal(
|
|
250
|
+
obligationAllowedUnderPosture(
|
|
251
|
+
{ kind: "SCHEDULE", mode: "guarded", status: "active", human_lane: isHumanLaneCadence(id) },
|
|
252
|
+
posture
|
|
253
|
+
).allowed,
|
|
254
|
+
true,
|
|
255
|
+
`${id} passes the obligation gate`
|
|
256
|
+
);
|
|
257
|
+
assert.notEqual(
|
|
258
|
+
gov.admit({ source: "cadence", mode: posture.mode, humanReply: isHumanLaneCadence(id) }, deps).decision,
|
|
259
|
+
"DEFER",
|
|
260
|
+
`${id} is not deferred by the spawn governor either`
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// And discretionary cadences ARE still deferred, at both gates.
|
|
265
|
+
assert.equal(
|
|
266
|
+
obligationAllowedUnderPosture(
|
|
267
|
+
{ kind: "SCHEDULE", mode: "guarded", status: "active", human_lane: isHumanLaneCadence("backlog-executor") },
|
|
268
|
+
posture
|
|
269
|
+
).allowed,
|
|
270
|
+
false
|
|
271
|
+
);
|
|
272
|
+
assert.equal(
|
|
273
|
+
gov.admit({ source: "cadence", mode: posture.mode, humanReply: isHumanLaneCadence("backlog-executor") }, deps).decision,
|
|
274
|
+
"DEFER"
|
|
275
|
+
);
|
|
276
|
+
});
|
|
@@ -36,6 +36,8 @@
|
|
|
36
36
|
* pollMs drain interval (default 2_000).
|
|
37
37
|
* heartbeatMs health.json refresh interval (default 15_000).
|
|
38
38
|
* recoveryMs stale-claim sweep interval (default 5 * 60_000).
|
|
39
|
+
* budgetEscalateMs how often to check whether a budget-band notice is due
|
|
40
|
+
* for the SUPERVISOR (default 5 * 60_000; 0 disables).
|
|
39
41
|
* spawnSession injected spawner for tests; defaults to a real
|
|
40
42
|
* `claude --print <permission-args> <prompt>` child_process
|
|
41
43
|
* .spawn. <permission-args> comes from
|
|
@@ -70,6 +72,8 @@ import {
|
|
|
70
72
|
} from "../../lib/cadence-bus.mjs";
|
|
71
73
|
import { resolveClaudeBin as sharedResolveClaude, augmentedPath, daemonClaudeArgs } from "../../lib/claude-bin.mjs";
|
|
72
74
|
import { getCadenceDef } from "./cadence-handlers.mjs";
|
|
75
|
+
import { obligationAllowedUnderPosture } from "../../lib/plan/compile.mjs";
|
|
76
|
+
import { isHumanLaneCadence } from "../../lib/cadences.mjs";
|
|
73
77
|
import { sessionPermissionArgs } from "../../lib/session-permissions.mjs";
|
|
74
78
|
import { renderTemplate, buildContext } from "../../lib/render.mjs";
|
|
75
79
|
// Model router (opt-in). A `schema_version: 2` config routes each cadence
|
|
@@ -104,6 +108,7 @@ const RATE_PROVIDER = process.env.MAESTRO_RATE_PROVIDER || "anthropic";
|
|
|
104
108
|
const DEFAULT_POLL_MS = 2_000;
|
|
105
109
|
const DEFAULT_HEARTBEAT_MS = 15_000;
|
|
106
110
|
const DEFAULT_RECOVERY_MS = 5 * 60_000;
|
|
111
|
+
const DEFAULT_BUDGET_ESCALATE_MS = 5 * 60_000;
|
|
107
112
|
const DEFAULT_SPAWN_TIMEOUT_MS = 30 * 60_000;
|
|
108
113
|
|
|
109
114
|
// Concurrency: at most one sub-session at a time per cadence consumer.
|
|
@@ -196,6 +201,10 @@ export function parseUsageFromStdout(stdoutPath) {
|
|
|
196
201
|
const out = { ok: true, inputTokens, outputTokens };
|
|
197
202
|
const cacheRead = Number(usage.cache_read_input_tokens);
|
|
198
203
|
if (Number.isFinite(cacheRead)) out.cacheReadTokens = cacheRead;
|
|
204
|
+
// Cache CREATION tokens bill at 1.25x input; dropping them was a large slice
|
|
205
|
+
// of the gap between the local estimate and the CLI's authoritative cost.
|
|
206
|
+
const cacheWrite = Number(usage.cache_creation_input_tokens);
|
|
207
|
+
if (Number.isFinite(cacheWrite)) out.cacheWriteTokens = cacheWrite;
|
|
199
208
|
const totalCost = Number(obj.total_cost_usd);
|
|
200
209
|
if (Number.isFinite(totalCost) && totalCost >= 0) out.totalCostUsd = totalCost;
|
|
201
210
|
// The CLI may report the resolved model (e.g. "claude-sonnet-4-6"); map it to
|
|
@@ -244,7 +253,8 @@ export function routeCadenceSpawn(agentRoot, cadence) {
|
|
|
244
253
|
let band = 0;
|
|
245
254
|
try {
|
|
246
255
|
const st = budgetGuardForBand.dailyStatus({ agentRoot });
|
|
247
|
-
|
|
256
|
+
// The governor's band, not a re-derivation — see dispatcher.currentBudgetBand.
|
|
257
|
+
band = budgetLadder(st.spentUSD, st.capUSD, { band: st.band }).band;
|
|
248
258
|
} catch { /* band 0 on read failure */ }
|
|
249
259
|
const req = {
|
|
250
260
|
agent_role: "cadence",
|
|
@@ -465,11 +475,15 @@ function realSpawnSession({ agentRoot, cadence, promptPath, timeoutMs, log }) {
|
|
|
465
475
|
trackerArgs.push("--input-tokens", String(usage.inputTokens));
|
|
466
476
|
trackerArgs.push("--output-tokens", String(usage.outputTokens));
|
|
467
477
|
if (usage.cacheReadTokens != null) trackerArgs.push("--cache-read-tokens", String(usage.cacheReadTokens));
|
|
478
|
+
if (usage.cacheWriteTokens != null) trackerArgs.push("--cache-creation-tokens", String(usage.cacheWriteTokens));
|
|
468
479
|
if (usage.totalCostUsd != null) trackerArgs.push("--total-cost-usd", String(usage.totalCostUsd));
|
|
469
480
|
} else {
|
|
470
|
-
// Parse failure:
|
|
471
|
-
//
|
|
472
|
-
//
|
|
481
|
+
// Parse failure: record the session as EXPLICITLY UNMEASURED. Omitting
|
|
482
|
+
// the token flags used to leave the tracker defaulting them to 0, so a
|
|
483
|
+
// systematic parse regression read as a run of free sessions and the
|
|
484
|
+
// budget governor went quiet exactly when telemetry broke. The row now
|
|
485
|
+
// says "unknown", and the bus log names the parse failure.
|
|
486
|
+
trackerArgs.push("--tokens-unknown", String(usage.reason || "usage-parse-failed"));
|
|
473
487
|
log({ level: "warn", stage: "cost_usage_parse_failed", cadence, reason: usage.reason, stdout: stdoutPath });
|
|
474
488
|
}
|
|
475
489
|
spawn(process.execPath, trackerArgs, { stdio: "ignore", env: { ...env, AGENT_ROOT: agentRoot } }).unref();
|
|
@@ -519,6 +533,10 @@ export function startConsumer(opts = {}) {
|
|
|
519
533
|
const pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
|
|
520
534
|
const heartbeatMs = opts.heartbeatMs ?? DEFAULT_HEARTBEAT_MS;
|
|
521
535
|
const recoveryMs = opts.recoveryMs ?? DEFAULT_RECOVERY_MS;
|
|
536
|
+
// How often the seat checks whether a budget band notice is due for its
|
|
537
|
+
// supervisor. 0 disables it (tests, and any deployment that delivers budget
|
|
538
|
+
// notices out of band).
|
|
539
|
+
const budgetEscalateMs = opts.budgetEscalateMs ?? DEFAULT_BUDGET_ESCALATE_MS;
|
|
522
540
|
const maxSpawnMs = opts.maxSpawnMs ?? DEFAULT_SPAWN_TIMEOUT_MS;
|
|
523
541
|
const spawnSession = opts.spawnSession || realSpawnSession;
|
|
524
542
|
const userLogger = opts.logger;
|
|
@@ -538,14 +556,55 @@ export function startConsumer(opts = {}) {
|
|
|
538
556
|
* any throw → admit (Invariant: a governance bug must not wedge cadences).
|
|
539
557
|
* Returns { admit:boolean, reason }.
|
|
540
558
|
*/
|
|
541
|
-
function governanceGate() {
|
|
559
|
+
function governanceGate(cadence) {
|
|
542
560
|
try {
|
|
543
561
|
const rb = rateGuard.checkRateLimit(RATE_PROVIDER, { agentRoot });
|
|
544
562
|
if (!rb.allowed) return { admit: false, reason: "rate-limited" };
|
|
545
563
|
let mode = null;
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
564
|
+
let posture = null;
|
|
565
|
+
try {
|
|
566
|
+
const st = budgetGuard.dailyStatus({ agentRoot });
|
|
567
|
+
posture = st.posture;
|
|
568
|
+
if (st.mode && st.mode !== "normal") mode = st.mode;
|
|
569
|
+
} catch { /* budget read best-effort */ }
|
|
570
|
+
|
|
571
|
+
// THE 125% RUNG, PER OBLIGATION. `applyBudgetPosture` makes this transition
|
|
572
|
+
// on a whole plan; this is its single-obligation form on the hot path, so a
|
|
573
|
+
// breach suspends the OUTCOME work it is supposed to suspend and leaves the
|
|
574
|
+
// inline/offline-safe cadences that keep the seat alive running. Same
|
|
575
|
+
// predicate both places — the daemon and the plan cannot disagree about
|
|
576
|
+
// what is suspended.
|
|
577
|
+
if (posture && cadence) {
|
|
578
|
+
const def = getCadenceDef(cadence) || {};
|
|
579
|
+
const verdict = obligationAllowedUnderPosture(
|
|
580
|
+
{
|
|
581
|
+
kind: "SCHEDULE",
|
|
582
|
+
scope: def.scope || null,
|
|
583
|
+
objective_id: def.objectiveId || null,
|
|
584
|
+
offline_safe: def.offlineSafe === true,
|
|
585
|
+
// THE HUMAN LANE. Without this every cadence reaching the gate is a
|
|
586
|
+
// bare SCHEDULE, so the 150% rung ("refuse anything that is not a
|
|
587
|
+
// direct human reply") deferred `inbox-processor` and
|
|
588
|
+
// `messaging-inbound` too — the seat stopped reading its inbox at a
|
|
589
|
+
// budget breach, which is the bricking Invariant #1 forbids.
|
|
590
|
+
human_lane: isHumanLaneCadence(cadence),
|
|
591
|
+
mode: def.mode,
|
|
592
|
+
},
|
|
593
|
+
posture
|
|
594
|
+
);
|
|
595
|
+
if (!verdict.allowed) return { admit: false, reason: verdict.reason };
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
// THE HUMAN LANE, ONE LAYER DOWN. The obligation gate above lets
|
|
599
|
+
// `inbox-processor` and `messaging-inbound` through at band 150, and the
|
|
600
|
+
// governor would then DEFER them anyway — it treats only `source:"inbox"`
|
|
601
|
+
// as a human reply, and every cadence tick arrives as `source:"cadence"`.
|
|
602
|
+
// Two gates, one predicate, or the rung silences the inbox on the second
|
|
603
|
+
// hop instead of the first.
|
|
604
|
+
const adm = governor.admit(
|
|
605
|
+
{ source: "cadence", mode, humanReply: cadence ? isHumanLaneCadence(cadence) : false },
|
|
606
|
+
governor.defaultDeps({ agentRoot })
|
|
607
|
+
);
|
|
549
608
|
if (adm.decision !== "ADMIT") return { admit: false, reason: adm.reason || adm.decision };
|
|
550
609
|
return { admit: true };
|
|
551
610
|
} catch {
|
|
@@ -553,6 +612,16 @@ export function startConsumer(opts = {}) {
|
|
|
553
612
|
}
|
|
554
613
|
}
|
|
555
614
|
|
|
615
|
+
/** The seat's memberId from the cached mandate body. Never throws. */
|
|
616
|
+
function readMandateMemberId(root) {
|
|
617
|
+
try {
|
|
618
|
+
const raw = JSON.parse(readFileSync(join(root, "state", "mandate", "cache.json"), "utf-8"));
|
|
619
|
+
return (raw && raw.body && (raw.body.memberId || raw.body.member_id)) || null;
|
|
620
|
+
} catch {
|
|
621
|
+
return null;
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
|
|
556
625
|
const stats = {
|
|
557
626
|
started_at: new Date().toISOString(),
|
|
558
627
|
received: 0,
|
|
@@ -738,7 +807,7 @@ export function startConsumer(opts = {}) {
|
|
|
738
807
|
// not a per-event failure — burning retry budget here would eventually DLQ
|
|
739
808
|
// a perfectly good cadence just because the box was busy).
|
|
740
809
|
{
|
|
741
|
-
const gov = governanceGate();
|
|
810
|
+
const gov = governanceGate(event.cadence);
|
|
742
811
|
if (!gov.admit) {
|
|
743
812
|
log({
|
|
744
813
|
level: "info",
|
|
@@ -1032,6 +1101,75 @@ export function startConsumer(opts = {}) {
|
|
|
1032
1101
|
recoveryTimer.unref?.();
|
|
1033
1102
|
timers.push(recoveryTimer);
|
|
1034
1103
|
|
|
1104
|
+
// THE 80% RUNG'S DELIVERY LEG. `maybeNotify` fires at most once per band per
|
|
1105
|
+
// period and addresses the notice to the SUPERVISOR (never to this seat), so
|
|
1106
|
+
// this timer is cheap: after the first fire every subsequent call reads two
|
|
1107
|
+
// small files and returns `notified:false`. It shares the recovery cadence
|
|
1108
|
+
// because a spend escalation that arrives tomorrow morning is not an
|
|
1109
|
+
// escalation — the goal-steward runs daily at 07:15, which is far too coarse
|
|
1110
|
+
// for money. Fail-open and LOUD: an undeliverable notice logs an error rather
|
|
1111
|
+
// than disappearing.
|
|
1112
|
+
const escalateTimer = budgetEscalateMs > 0 ? setInterval(() => {
|
|
1113
|
+
(async () => {
|
|
1114
|
+
const { escalateBudget } = await import("../../lib/budget-escalate.mjs");
|
|
1115
|
+
let conn = null;
|
|
1116
|
+
try {
|
|
1117
|
+
const { resolveOrgToolConfig } = await import("../../lib/org/tool-surface.mjs");
|
|
1118
|
+
const c = resolveOrgToolConfig({ agentRoot });
|
|
1119
|
+
conn = c ? { base: c.base || "", token: c.token || "", orgId: c.orgId || "" } : null;
|
|
1120
|
+
} catch { /* no credential → escalateBudget reports it rather than sending */ }
|
|
1121
|
+
const r = await escalateBudget(
|
|
1122
|
+
{ agentRoot, conn },
|
|
1123
|
+
{ log: (level, msg) => log({ level, stage: "budget_escalation", message: msg }) }
|
|
1124
|
+
);
|
|
1125
|
+
if (r.notified) {
|
|
1126
|
+
log({ level: r.delivered ? "warn" : "error", stage: "budget_escalation", band: r.band, delivered: r.delivered, reason: r.reason });
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
// ── THE RETURN LEG, ON THE SAME CLOCK AS THE MONEY ────────────────────
|
|
1130
|
+
// `reconcileBudgetPosture` used to be reachable ONLY from the goal-steward
|
|
1131
|
+
// guard, which runs daily at 07:15 — while the band it reads resets at UTC
|
|
1132
|
+
// midnight. A seat could hit 150% every afternoon for a month and hq would
|
|
1133
|
+
// never see one `budget_breach` drift row, because by 07:15 the next
|
|
1134
|
+
// morning the band it observed was 0 again. Local enforcement was fine
|
|
1135
|
+
// (`governanceGate` re-reads the posture per tick); it was specifically the
|
|
1136
|
+
// record — the thing the person who FUNDS the seat looks at — that never
|
|
1137
|
+
// arrived. Five minutes is the right clock for money.
|
|
1138
|
+
try {
|
|
1139
|
+
const bg = await import("../../lib/budget-guard.mjs");
|
|
1140
|
+
const st = bg.dailyStatus({ agentRoot });
|
|
1141
|
+
const { reconcileBudgetPosture } = await import("../../lib/plan/budget-runtime.mjs");
|
|
1142
|
+
let mirrorDrift;
|
|
1143
|
+
// The seat's own member id: the drift row is filed AGAINST this member.
|
|
1144
|
+
// Same resolution ladder the goal-steward uses (env first, then the
|
|
1145
|
+
// mandate cache the seat already holds), so the two agree.
|
|
1146
|
+
const memberId = process.env.COHORT_AGENT_ID || readMandateMemberId(agentRoot);
|
|
1147
|
+
if (conn && conn.base && conn.token && memberId) {
|
|
1148
|
+
const { call } = await import("../../lib/org/client.mjs");
|
|
1149
|
+
mirrorDrift = async ({ kind, key, detail, resolve }) =>
|
|
1150
|
+
call("mandate.reportDrift", { memberId, kind, key, detail: detail || null, ...(resolve ? { resolve: true } : {}) }, conn);
|
|
1151
|
+
} else if (st.band >= 125) {
|
|
1152
|
+
log({ level: "warn", stage: "budget_posture", message: "band >=125% but no org credential/memberId — the suspension is recorded LOCALLY ONLY and hq will not show it" });
|
|
1153
|
+
}
|
|
1154
|
+
const rec = await reconcileBudgetPosture(
|
|
1155
|
+
{ agentRoot, posture: st.posture, status: st },
|
|
1156
|
+
{ log: (level, msg) => log({ level, stage: "budget_posture", message: msg }), mirrorDrift }
|
|
1157
|
+
);
|
|
1158
|
+
if (rec.recorded > 0 || rec.previousBand !== rec.band) {
|
|
1159
|
+
log({ level: "warn", stage: "budget_posture", band: rec.band, previous_band: rec.previousBand, recorded: rec.recorded, mirrored: rec.mirrored, suspended: rec.suspended.length });
|
|
1160
|
+
}
|
|
1161
|
+
} catch (err) {
|
|
1162
|
+
log({ level: "error", stage: "budget_posture_failed", error: err?.message || String(err) });
|
|
1163
|
+
}
|
|
1164
|
+
})().catch((err) => {
|
|
1165
|
+
log({ level: "error", stage: "budget_escalation_failed", error: err?.message || String(err) });
|
|
1166
|
+
});
|
|
1167
|
+
}, budgetEscalateMs) : null;
|
|
1168
|
+
if (escalateTimer) {
|
|
1169
|
+
escalateTimer.unref?.();
|
|
1170
|
+
timers.push(escalateTimer);
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1035
1173
|
// Initial sweep + heartbeat
|
|
1036
1174
|
recoverStaleClaimsGuarded();
|
|
1037
1175
|
heartbeat();
|
|
@@ -646,6 +646,12 @@ test("0.3: realSpawnSession argv requests --output-format json and records real
|
|
|
646
646
|
new URL("../cost/track-claude-usage.mjs", import.meta.url),
|
|
647
647
|
join(root, "scripts/cost/track-claude-usage.mjs")
|
|
648
648
|
);
|
|
649
|
+
// ...and the shared billing contract it imports relative to itself.
|
|
650
|
+
mkdirSync(join(root, "lib/cost"), { recursive: true });
|
|
651
|
+
await fsp.copyFile(
|
|
652
|
+
new URL("../../lib/cost/ledger-row.mjs", import.meta.url),
|
|
653
|
+
join(root, "lib/cost/ledger-row.mjs")
|
|
654
|
+
);
|
|
649
655
|
|
|
650
656
|
const argvFile = join(root, "shim-argv.txt");
|
|
651
657
|
const shim = join(root, "claude-shim");
|
|
@@ -426,6 +426,72 @@ function loadAgentCfg(agentRoot) {
|
|
|
426
426
|
return agent;
|
|
427
427
|
}
|
|
428
428
|
|
|
429
|
+
/**
|
|
430
|
+
* nightly-backup (inline): the disaster-recovery job.
|
|
431
|
+
*
|
|
432
|
+
* Runs scripts/maintenance/backup-run.mjs as a child process rather than
|
|
433
|
+
* in-process, deliberately: it shells out to `tar` (and possibly gsutil/aws),
|
|
434
|
+
* can take tens of seconds on a large state tree, and must not be able to wedge
|
|
435
|
+
* the persistent daemon's event loop or leave a half-written archive if the
|
|
436
|
+
* daemon is restarted mid-run. A non-zero exit is REPORTED, never swallowed —
|
|
437
|
+
* the whole reason this gap existed is that the old shell driver exited 0 in
|
|
438
|
+
* silence when unconfigured.
|
|
439
|
+
*
|
|
440
|
+
* Injectable via `opts.runImpl` for tests (no tar, no filesystem).
|
|
441
|
+
*/
|
|
442
|
+
async function handleNightlyBackup({ event, agentRoot, log }, opts = {}) {
|
|
443
|
+
const started = Date.now();
|
|
444
|
+
try {
|
|
445
|
+
const runner =
|
|
446
|
+
opts.runImpl ||
|
|
447
|
+
(async () => {
|
|
448
|
+
const { execFile } = await import("node:child_process");
|
|
449
|
+
const script = join(agentRoot, "scripts", "maintenance", "backup-run.mjs");
|
|
450
|
+
const fallback = join(__dirname, "..", "maintenance", "backup-run.mjs");
|
|
451
|
+
const target = existsSync(script) ? script : fallback;
|
|
452
|
+
return new Promise((resolvePromise) => {
|
|
453
|
+
execFile(
|
|
454
|
+
process.execPath,
|
|
455
|
+
[target, "--agent-dir", agentRoot],
|
|
456
|
+
{ timeout: 20 * 60 * 1000, maxBuffer: 8 * 1024 * 1024 },
|
|
457
|
+
(err, stdout, stderr) => {
|
|
458
|
+
resolvePromise({
|
|
459
|
+
code: err ? (typeof err.code === "number" ? err.code : 1) : 0,
|
|
460
|
+
stdout: String(stdout || ""),
|
|
461
|
+
stderr: String(stderr || ""),
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
);
|
|
465
|
+
});
|
|
466
|
+
});
|
|
467
|
+
|
|
468
|
+
const res = await runner({ agentRoot });
|
|
469
|
+
const durationMs = Date.now() - started;
|
|
470
|
+
// Surface the runner's own warn/error lines into the daemon log so a
|
|
471
|
+
// degraded backup (offsite unconfigured, offsite push failed, include path
|
|
472
|
+
// dropped by the credential deny-list) is visible without opening a file.
|
|
473
|
+
const notable = `${res.stdout || ""}${res.stderr || ""}`
|
|
474
|
+
.split("\n")
|
|
475
|
+
.filter((l) => /\[(warn|error)\]/.test(l));
|
|
476
|
+
if (typeof log === "function") {
|
|
477
|
+
for (const line of notable.slice(0, 10)) log(res.code === 0 ? "warn" : "error", `[nightly-backup] ${line.trim()}`);
|
|
478
|
+
}
|
|
479
|
+
if (res.code !== 0) {
|
|
480
|
+
return {
|
|
481
|
+
ok: false,
|
|
482
|
+
decision: "inline",
|
|
483
|
+
cadence: event.cadence,
|
|
484
|
+
durationMs,
|
|
485
|
+
error: `backup-run.mjs exited ${res.code}`,
|
|
486
|
+
detail: notable.slice(0, 5),
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
return { ok: true, decision: "inline", cadence: event.cadence, durationMs, warnings: notable.length };
|
|
490
|
+
} catch (err) {
|
|
491
|
+
return { ok: false, decision: "inline", cadence: event.cadence, error: err && err.message ? err.message : String(err) };
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
429
495
|
/**
|
|
430
496
|
* nightly-cost-reconcile (inline): three-way spend reconcile of today's local
|
|
431
497
|
* router ledger against the Anthropic Admin Cost API. We pass NO fetchImpl, so
|
|
@@ -1198,6 +1264,84 @@ async function guardGoalSteward({ event, agentRoot, log }, opts = {}) {
|
|
|
1198
1264
|
}
|
|
1199
1265
|
}
|
|
1200
1266
|
|
|
1267
|
+
// 2d. THE BUDGET LADDER'S 100% RUNG: "self-directed (D3) work stops."
|
|
1268
|
+
// THIS loop is D3 — the only thing in the system that creates work for
|
|
1269
|
+
// the seat out of its own gaps. Above the degrade rung it does not run
|
|
1270
|
+
// at all; below it, the seat's remaining envelope and the fan-out
|
|
1271
|
+
// ceiling are handed to `routeRung`, which already knows how to enqueue
|
|
1272
|
+
// rather than downgrade an over-budget item (`remainingCents`/`maxRung`
|
|
1273
|
+
// were parameters with no caller — this is the caller).
|
|
1274
|
+
//
|
|
1275
|
+
// Measuring is NOT self-directed work and is deliberately not gated: a
|
|
1276
|
+
// seat that stops measuring because it is over budget also stops being
|
|
1277
|
+
// able to prove it, and the KPI series goes dark exactly when someone
|
|
1278
|
+
// needs it. Only the CREATE half is stopped, via enforcement=observe.
|
|
1279
|
+
let budgetDeps = opts.budgetDeps;
|
|
1280
|
+
if (budgetDeps === undefined) {
|
|
1281
|
+
budgetDeps = {};
|
|
1282
|
+
try {
|
|
1283
|
+
const bg = await import("../../lib/budget-guard.mjs");
|
|
1284
|
+
const st = bg.dailyStatus({ agentRoot });
|
|
1285
|
+
if (!st.posture.selfDirected && enforcement === "active") {
|
|
1286
|
+
say(
|
|
1287
|
+
"warn",
|
|
1288
|
+
`[goal-steward] budget ${st.mode} (${st.pct}% of $${st.capUSD}/day, ${st.capSource}) — self-directed work STOPS: ` +
|
|
1289
|
+
"measuring continues, creating does not"
|
|
1290
|
+
);
|
|
1291
|
+
enforcement = "observe";
|
|
1292
|
+
}
|
|
1293
|
+
budgetDeps = {
|
|
1294
|
+
budgetStatus: st,
|
|
1295
|
+
maxRung: st.posture.maxRung ?? undefined,
|
|
1296
|
+
// The seat's remaining envelope for the period, in cents — the same
|
|
1297
|
+
// envelope hq funded, not a local guess. Null envelope ⇒ undefined ⇒
|
|
1298
|
+
// routeRung's budget gate stays inert rather than inventing a ceiling.
|
|
1299
|
+
remainingCents: st.envelope && Number.isFinite(st.envelope.remainingUSD)
|
|
1300
|
+
? Math.max(0, Math.round(st.envelope.remainingUSD * 100))
|
|
1301
|
+
: undefined,
|
|
1302
|
+
};
|
|
1303
|
+
|
|
1304
|
+
// THE 125% RUNG, on the plan. Reconcile the compiled plan against the
|
|
1305
|
+
// live band: OUTCOME obligations (and the measure/review cadences that
|
|
1306
|
+
// serve them) are suspended, one `budget_breach` drift row each, on the
|
|
1307
|
+
// band EDGE only. The plan file itself is not rewritten — see
|
|
1308
|
+
// lib/plan/budget-runtime.mjs for why a schedule that flaps with spend
|
|
1309
|
+
// would be worse than the breach.
|
|
1310
|
+
try {
|
|
1311
|
+
const { reconcileBudgetPosture } = await import("../../lib/plan/budget-runtime.mjs");
|
|
1312
|
+
|
|
1313
|
+
// The return leg. A suspension only this laptop knows about is
|
|
1314
|
+
// invisible to the person who funds the seat — and funding is the only
|
|
1315
|
+
// thing that clears it. hq dedupes an open row per
|
|
1316
|
+
// (memberId, kind, key), so re-reporting is safe.
|
|
1317
|
+
let mirrorDrift;
|
|
1318
|
+
if (conn && conn.base && conn.token) {
|
|
1319
|
+
const { call } = await import("../../lib/org/client.mjs");
|
|
1320
|
+
mirrorDrift = async ({ kind, key, detail, resolve }) =>
|
|
1321
|
+
call(
|
|
1322
|
+
"mandate.reportDrift",
|
|
1323
|
+
{ memberId, kind, key, detail: detail || null, ...(resolve ? { resolve: true } : {}) },
|
|
1324
|
+
{ ...conn, fetchImpl: opts.fetchImpl }
|
|
1325
|
+
);
|
|
1326
|
+
} else {
|
|
1327
|
+
say("warn", "[goal-steward] no org credential — a budget suspension will be recorded LOCALLY ONLY and hq will not show it");
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1330
|
+
const rec = await reconcileBudgetPosture(
|
|
1331
|
+
{ agentRoot, posture: st.posture, status: st },
|
|
1332
|
+
{ now: opts.now, log: (level, msg) => say(level, `[goal-steward] ${msg}`), mirrorDrift }
|
|
1333
|
+
);
|
|
1334
|
+
if (rec.recorded > 0) {
|
|
1335
|
+
say("warn", `[goal-steward] recorded ${rec.recorded} budget_breach drift row(s) at band ${rec.band}% (${rec.mirrored} mirrored to hq)`);
|
|
1336
|
+
}
|
|
1337
|
+
} catch (err) {
|
|
1338
|
+
say("error", `[goal-steward] budget/plan reconcile failed (${err && err.message ? err.message : err}) — obligations are NOT suspended this tick`);
|
|
1339
|
+
}
|
|
1340
|
+
} catch (err) {
|
|
1341
|
+
say("warn", `[goal-steward] could not read the budget posture (${err && err.message ? err.message : err}) — running UNMETERED this tick`);
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1201
1345
|
const run = opts.runImpl || (await import("../../lib/goals/loop.mjs")).runGoalSteward;
|
|
1202
1346
|
const report = await run({
|
|
1203
1347
|
agentRoot,
|
|
@@ -1207,6 +1351,7 @@ async function guardGoalSteward({ event, agentRoot, log }, opts = {}) {
|
|
|
1207
1351
|
sensors,
|
|
1208
1352
|
reachable,
|
|
1209
1353
|
memberId,
|
|
1354
|
+
...budgetDeps,
|
|
1210
1355
|
...(conn || {}),
|
|
1211
1356
|
...(mirrorSample ? { mirrorSample } : {}),
|
|
1212
1357
|
...(opts.runDeps || {}),
|
|
@@ -1371,6 +1516,11 @@ export const CADENCE_REGISTRY = {
|
|
|
1371
1516
|
|
|
1372
1517
|
// Economics / scheduling cadences (WS-economics). Deterministic → inline,
|
|
1373
1518
|
// except dynamic-jobs (guarded escalate-when-due).
|
|
1519
|
+
"nightly-backup": {
|
|
1520
|
+
mode: "inline",
|
|
1521
|
+
handler: handleNightlyBackup,
|
|
1522
|
+
description: "Nightly DR backup — tar.gz of durable state to local restore points outside the repo, plus the offsite copy when configured. Inline (no LLM); a failed run is reported, never swallowed.",
|
|
1523
|
+
},
|
|
1374
1524
|
"nightly-cost-reconcile": {
|
|
1375
1525
|
mode: "inline",
|
|
1376
1526
|
handler: handleNightlyCostReconcile,
|
|
@@ -1525,6 +1675,13 @@ export function getCadenceDef(name) {
|
|
|
1525
1675
|
if (Number.isFinite(cfg.budgetCents)) def.budgetCents = cfg.budgetCents;
|
|
1526
1676
|
else _warnUnmetered(name, cfg.obligationKey);
|
|
1527
1677
|
if (cfg.obligationKey) def.obligationKey = cfg.obligationKey;
|
|
1678
|
+
// Provenance the RUNTIME budget gate needs (lib/plan/emit.toRegistry): which
|
|
1679
|
+
// obligation this cadence serves, and whether it is free to keep running at
|
|
1680
|
+
// a breached band. Without them the consumer cannot tell a funded
|
|
1681
|
+
// objective's measure-cadence from the seat's own heartbeat.
|
|
1682
|
+
if (cfg.scope) def.scope = cfg.scope;
|
|
1683
|
+
if (cfg.objectiveId) def.objectiveId = cfg.objectiveId;
|
|
1684
|
+
def.offlineSafe = cfg.offlineSafe === true;
|
|
1528
1685
|
return def;
|
|
1529
1686
|
}
|
|
1530
1687
|
return null;
|
|
@@ -1548,6 +1705,7 @@ export { guardSkillCurator, maybeSpawnConsolidation };
|
|
|
1548
1705
|
// them via CADENCE_REGISTRY). Each takes an optional second arg of injectable
|
|
1549
1706
|
// seams (lib fn / clock / cfg) so tests never touch the network or real config.
|
|
1550
1707
|
export {
|
|
1708
|
+
handleNightlyBackup,
|
|
1551
1709
|
handleNightlyCostReconcile,
|
|
1552
1710
|
handleFleetCostDigest,
|
|
1553
1711
|
handleOrgCostSync,
|
|
@@ -20,6 +20,7 @@ import { tmpdir } from "node:os";
|
|
|
20
20
|
import { join } from "node:path";
|
|
21
21
|
|
|
22
22
|
import { guardSkillCurator, maybeSpawnConsolidation, CADENCE_REGISTRY,
|
|
23
|
+
handleNightlyBackup,
|
|
23
24
|
handleNightlyCostReconcile, handleFleetCostDigest, handleOrgCostSync, guardDynamicJobs,
|
|
24
25
|
guardMessagingInbound, guardDirectoryHygiene, guardBrandSteward } from "./cadence-handlers.mjs";
|
|
25
26
|
import { writeSkill } from "../../lib/learning/skill-writer.mjs";
|
|
@@ -181,6 +182,69 @@ test("CADENCE_REGISTRY wires the economics/scheduling cadences with the right mo
|
|
|
181
182
|
assert.ok(CADENCE_REGISTRY["dynamic-jobs"].prompt);
|
|
182
183
|
});
|
|
183
184
|
|
|
185
|
+
// ---------------------------------------------------------------------------
|
|
186
|
+
// nightly-backup (DR) — the whole point is that a failed run is never silent
|
|
187
|
+
// ---------------------------------------------------------------------------
|
|
188
|
+
|
|
189
|
+
test("CADENCE_REGISTRY wires nightly-backup as an inline handler", () => {
|
|
190
|
+
assert.equal(CADENCE_REGISTRY["nightly-backup"].mode, "inline");
|
|
191
|
+
assert.equal(typeof CADENCE_REGISTRY["nightly-backup"].handler, "function");
|
|
192
|
+
// Inline cadences must NOT carry a prompt — a prompt would let the consumer
|
|
193
|
+
// spawn a Claude session for a tar job.
|
|
194
|
+
assert.ok(!CADENCE_REGISTRY["nightly-backup"].prompt);
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
test("nightly-backup: a clean run is ok and reports no warnings", async () => {
|
|
198
|
+
const res = await handleNightlyBackup(ev("nightly-backup"), {
|
|
199
|
+
runImpl: async () => ({ code: 0, stdout: "[info] backup complete (tier=offsite).\n", stderr: "" }),
|
|
200
|
+
});
|
|
201
|
+
assert.equal(res.ok, true);
|
|
202
|
+
assert.equal(res.decision, "inline");
|
|
203
|
+
assert.equal(res.cadence, "nightly-backup");
|
|
204
|
+
assert.equal(res.warnings, 0);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
test("nightly-backup: a non-zero exit is REPORTED, not swallowed", async () => {
|
|
208
|
+
const logs = [];
|
|
209
|
+
const res = await handleNightlyBackup(
|
|
210
|
+
{ event: { cadence: "nightly-backup" }, agentRoot: tmpRoot(), log: (lvl, m) => logs.push([lvl, m]) },
|
|
211
|
+
{ runImpl: async () => ({ code: 1, stdout: "", stderr: "[error] REFUSING: no .maestro/backup-config.yaml\n" }) }
|
|
212
|
+
);
|
|
213
|
+
assert.equal(res.ok, false);
|
|
214
|
+
assert.match(res.error, /exited 1/);
|
|
215
|
+
assert.ok(logs.some(([lvl, m]) => lvl === "error" && /REFUSING/.test(m)), "the refusal must reach the daemon log");
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
test("nightly-backup: a degraded-but-successful run surfaces its warnings in the daemon log", async () => {
|
|
219
|
+
const logs = [];
|
|
220
|
+
const res = await handleNightlyBackup(
|
|
221
|
+
{ event: { cadence: "nightly-backup" }, agentRoot: tmpRoot(), log: (lvl, m) => logs.push([lvl, m]) },
|
|
222
|
+
{
|
|
223
|
+
runImpl: async () => ({
|
|
224
|
+
code: 0,
|
|
225
|
+
stdout:
|
|
226
|
+
"[info] plan: tier=local\n" +
|
|
227
|
+
"[warn] offsite is NOT configured — these restore points do not survive losing this machine.\n" +
|
|
228
|
+
"[warn] include path DROPPED by the credential deny-list: .env (matched .env)\n",
|
|
229
|
+
stderr: "",
|
|
230
|
+
}),
|
|
231
|
+
}
|
|
232
|
+
);
|
|
233
|
+
assert.equal(res.ok, true);
|
|
234
|
+
assert.equal(res.warnings, 2);
|
|
235
|
+
assert.ok(logs.some(([, m]) => /offsite is NOT configured/.test(m)));
|
|
236
|
+
assert.ok(logs.some(([, m]) => /credential deny-list/.test(m)));
|
|
237
|
+
assert.ok(!logs.some(([, m]) => /\[info\]/.test(m)), "info noise stays in the backup log, not the daemon log");
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
test("nightly-backup: a throwing runner degrades to ok:false, never an unhandled rejection", async () => {
|
|
241
|
+
const res = await handleNightlyBackup(ev("nightly-backup"), {
|
|
242
|
+
runImpl: async () => { throw new Error("spawn ENOENT"); },
|
|
243
|
+
});
|
|
244
|
+
assert.equal(res.ok, false);
|
|
245
|
+
assert.match(res.error, /spawn ENOENT/);
|
|
246
|
+
});
|
|
247
|
+
|
|
184
248
|
test("nightly-cost-reconcile: inline, surfaces driftPct, passes NO fetchImpl (fails open)", async () => {
|
|
185
249
|
let sawFetch = "unset";
|
|
186
250
|
const fake = async (o) => {
|