@cohortapp/agent-sdk 2.5.1 → 2.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/bin/maestro.mjs +305 -89
  2. package/bin/maestro.test.mjs +357 -48
  3. package/docs/runbooks/backup-restore.md +65 -33
  4. package/framework-features.json +4 -4
  5. package/lib/backup/policy.mjs +710 -0
  6. package/lib/backup/policy.test.mjs +305 -0
  7. package/lib/budget-escalate.mjs +133 -0
  8. package/lib/budget-escalate.test.mjs +232 -0
  9. package/lib/budget-guard.envelope.test.mjs +476 -0
  10. package/lib/budget-guard.mjs +853 -75
  11. package/lib/budget-guard.test.mjs +91 -42
  12. package/lib/cadences.mjs +33 -0
  13. package/lib/channels/orgmail/adapter.mjs +88 -3
  14. package/lib/channels/orgmail/adapter.test.mjs +137 -0
  15. package/lib/channels/repeat-suppressor.mjs +198 -0
  16. package/lib/channels/repeat-suppressor.test.mjs +134 -0
  17. package/lib/comms/receipts.mjs +297 -0
  18. package/lib/cost/ledger-row.mjs +333 -0
  19. package/lib/cost/ledger-row.test.mjs +183 -0
  20. package/lib/execution/drive.mjs +28 -1
  21. package/lib/execution/effects.mjs +191 -12
  22. package/lib/execution/effects.test.mjs +50 -11
  23. package/lib/goals/admission.mjs +13 -1
  24. package/lib/goals/admission.test.mjs +26 -1
  25. package/lib/goals/loop.mjs +13 -0
  26. package/lib/kpi-sensors.test.mjs +3 -0
  27. package/lib/mandate/cache.mjs +13 -5
  28. package/lib/mandate/derive.mjs +146 -21
  29. package/lib/mandate/derive.test.mjs +50 -6
  30. package/lib/mandate/model.mjs +32 -4
  31. package/lib/mandate/refresh.test.mjs +16 -2
  32. package/lib/mcp/server.test.mjs +12 -3
  33. package/lib/model-router/economics.mjs +107 -76
  34. package/lib/model-router/economics.test.mjs +64 -46
  35. package/lib/model-router/integration-coverage.test.mjs +39 -37
  36. package/lib/model-router/ledger.mjs +75 -22
  37. package/lib/model-router/ledger.test.mjs +35 -2
  38. package/lib/org/client.mjs +14 -0
  39. package/lib/org/cost-sync.mjs +16 -2
  40. package/lib/org/doctor.mjs +62 -1
  41. package/lib/org/doctor.test.mjs +36 -3
  42. package/lib/org/email-remedy.mjs +49 -0
  43. package/lib/org/engagement-ledger.mjs +376 -0
  44. package/lib/org/engagement-ledger.test.mjs +112 -0
  45. package/lib/org/engagement.mjs +1056 -0
  46. package/lib/org/engagement.test.mjs +739 -0
  47. package/lib/org/messaging.mjs +230 -3
  48. package/lib/org/messaging.test.mjs +110 -1
  49. package/lib/org/param-contract.mjs +56 -2
  50. package/lib/org/param-contract.test.mjs +26 -0
  51. package/lib/org/protocol.checksum +1 -1
  52. package/lib/org/protocol.mjs +5 -0
  53. package/lib/org/protocol.test.mjs +7 -1
  54. package/lib/org/tool-surface.mjs +506 -10
  55. package/lib/org/tool-surface.test.mjs +191 -7
  56. package/lib/org/ui-parity.mjs +333 -6
  57. package/lib/org/ui-parity.test.mjs +96 -3
  58. package/lib/org/work-ledger.mjs +241 -0
  59. package/lib/org/work-ledger.test.mjs +237 -0
  60. package/lib/plan/adoption-e2e.test.mjs +366 -0
  61. package/lib/plan/budget-enforcement.test.mjs +400 -0
  62. package/lib/plan/budget-runtime.mjs +215 -0
  63. package/lib/plan/compile.mjs +201 -5
  64. package/lib/plan/compile.test.mjs +19 -5
  65. package/lib/plan/emit.mjs +8 -0
  66. package/lib/plan/emit.test.mjs +18 -0
  67. package/lib/resource-governor.mjs +58 -12
  68. package/lib/resource-governor.test.mjs +41 -1
  69. package/lib/security/audit-engine.mjs +45 -8
  70. package/lib/security/audit-engine.test.mjs +35 -0
  71. package/lib/setup/enroll-from-cohort.mjs +14 -1
  72. package/lib/setup/sections/mandate.mjs +48 -7
  73. package/lib/setup/sections/mandate.test.mjs +17 -2
  74. package/lib/setup/sections/orgmail.mjs +10 -2
  75. package/lib/setup/state.mjs +83 -2
  76. package/lib/telemetry/collect.mjs +360 -20
  77. package/lib/telemetry/collect.test.mjs +266 -0
  78. package/package.json +1 -1
  79. package/scripts/cost/track-claude-usage.mjs +207 -48
  80. package/scripts/cost/track-claude-usage.test.mjs +148 -0
  81. package/scripts/daemon/agent-daemon.mjs +315 -17
  82. package/scripts/daemon/assurance-e2e.test.mjs +421 -0
  83. package/scripts/daemon/assurance.mjs +944 -0
  84. package/scripts/daemon/assurance.test.mjs +668 -0
  85. package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
  86. package/scripts/daemon/cadence-consumer.mjs +147 -9
  87. package/scripts/daemon/cadence-consumer.test.mjs +6 -0
  88. package/scripts/daemon/cadence-handlers.mjs +158 -0
  89. package/scripts/daemon/cadence-handlers.test.mjs +64 -0
  90. package/scripts/daemon/classifier.test.mjs +18 -9
  91. package/scripts/daemon/deliver.mjs +314 -0
  92. package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
  93. package/scripts/daemon/dispatcher.mjs +64 -6
  94. package/scripts/daemon/responder-cost.test.mjs +68 -0
  95. package/scripts/daemon/responder.mjs +351 -298
  96. package/scripts/local-triggers/generate-plists.test.mjs +7 -4
  97. package/scripts/maintenance/backup-run.mjs +415 -0
  98. package/scripts/maintenance/backup-to-cloud.sh +16 -116
  99. package/scripts/org/send-orgmail.mjs +16 -0
  100. package/scripts/record-receipt.sh +63 -0
  101. package/scripts/restore-from-backup.sh +14 -3
  102. package/scripts/restore-from-backup.test.mjs +8 -5
  103. package/scripts/send-email-threaded.py +47 -0
  104. package/scripts/send-sms.sh +4 -0
  105. package/scripts/send-whatsapp.sh +4 -0
  106. package/scripts/setup/init-backup.mjs +93 -38
  107. package/scripts/slack-send.sh +12 -0
@@ -18,6 +18,9 @@ import { join, dirname } from "node:path";
18
18
 
19
19
  import { enqueueTick, busDepth, listInbox, getBusPaths } from "../../lib/cadence-bus.mjs";
20
20
  import { startConsumer } from "./cadence-consumer.mjs";
21
+ import { obligationAllowedUnderPosture } from "../../lib/plan/compile.mjs";
22
+ import { postureForBand } from "../../lib/budget-guard.mjs";
23
+ import { isHumanLaneCadence } from "../../lib/cadences.mjs";
21
24
 
22
25
  async function makeAgentRoot() {
23
26
  const path = join(tmpdir(), `consumer-gov-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
@@ -218,3 +221,56 @@ test("L2: a 429 sub-session failure does NOT trip the per-cadence circuit", asyn
218
221
  assert.ok(stats.retries >= 1, "the tick was requeued");
219
222
  } finally { await consumer.stop(); await rmRoot(root); }
220
223
  });
224
+
225
+ test("band 150 does not silence the inbox at EITHER gate", async () => {
226
+ // Two gates sit between a cadence tick and a spawn, and the 150% rung has to
227
+ // agree with itself across both:
228
+ // 1. `obligationAllowedUnderPosture` — was `kind !== "REACT"`, so every
229
+ // cadence (all SCHEDULEs) was refused, including `messaging-inbound`, the
230
+ // only puller of org DMs into the inbox pipeline.
231
+ // 2. `resource-governor.admit` — treats only `source:"inbox"` as a human
232
+ // reply, and every cadence tick arrives as `source:"cadence"`, so fixing
233
+ // gate 1 alone just moves the silence one hop down.
234
+ const posture = postureForBand(150);
235
+ const gov = await import("../../lib/resource-governor.mjs");
236
+ // An IDLE, roomy machine, injected. `defaultDeps` reads real freemem/loadavg/
237
+ // live-process counts, so asserting against it makes the test a measure of
238
+ // whatever else is running — this test is about the BUDGET rung, nothing else.
239
+ const deps = {
240
+ totalmem: 64 * 1024 ** 3,
241
+ freemem: 48 * 1024 ** 3,
242
+ cpus: 10,
243
+ loadavg: [0.1],
244
+ liveClaude: { count: 0, rssMB: 0 },
245
+ throttleCeiling: Infinity,
246
+ };
247
+
248
+ for (const id of ["inbox-processor", "messaging-inbound"]) {
249
+ assert.equal(
250
+ obligationAllowedUnderPosture(
251
+ { kind: "SCHEDULE", mode: "guarded", status: "active", human_lane: isHumanLaneCadence(id) },
252
+ posture
253
+ ).allowed,
254
+ true,
255
+ `${id} passes the obligation gate`
256
+ );
257
+ assert.notEqual(
258
+ gov.admit({ source: "cadence", mode: posture.mode, humanReply: isHumanLaneCadence(id) }, deps).decision,
259
+ "DEFER",
260
+ `${id} is not deferred by the spawn governor either`
261
+ );
262
+ }
263
+
264
+ // And discretionary cadences ARE still deferred, at both gates.
265
+ assert.equal(
266
+ obligationAllowedUnderPosture(
267
+ { kind: "SCHEDULE", mode: "guarded", status: "active", human_lane: isHumanLaneCadence("backlog-executor") },
268
+ posture
269
+ ).allowed,
270
+ false
271
+ );
272
+ assert.equal(
273
+ gov.admit({ source: "cadence", mode: posture.mode, humanReply: isHumanLaneCadence("backlog-executor") }, deps).decision,
274
+ "DEFER"
275
+ );
276
+ });
@@ -36,6 +36,8 @@
36
36
  * pollMs drain interval (default 2_000).
37
37
  * heartbeatMs health.json refresh interval (default 15_000).
38
38
  * recoveryMs stale-claim sweep interval (default 5 * 60_000).
39
+ * budgetEscalateMs how often to check whether a budget-band notice is due
40
+ * for the SUPERVISOR (default 5 * 60_000; 0 disables).
39
41
  * spawnSession injected spawner for tests; defaults to a real
40
42
  * `claude --print <permission-args> <prompt>` child_process
41
43
  * .spawn. <permission-args> comes from
@@ -70,6 +72,8 @@ import {
70
72
  } from "../../lib/cadence-bus.mjs";
71
73
  import { resolveClaudeBin as sharedResolveClaude, augmentedPath, daemonClaudeArgs } from "../../lib/claude-bin.mjs";
72
74
  import { getCadenceDef } from "./cadence-handlers.mjs";
75
+ import { obligationAllowedUnderPosture } from "../../lib/plan/compile.mjs";
76
+ import { isHumanLaneCadence } from "../../lib/cadences.mjs";
73
77
  import { sessionPermissionArgs } from "../../lib/session-permissions.mjs";
74
78
  import { renderTemplate, buildContext } from "../../lib/render.mjs";
75
79
  // Model router (opt-in). A `schema_version: 2` config routes each cadence
@@ -104,6 +108,7 @@ const RATE_PROVIDER = process.env.MAESTRO_RATE_PROVIDER || "anthropic";
104
108
  const DEFAULT_POLL_MS = 2_000;
105
109
  const DEFAULT_HEARTBEAT_MS = 15_000;
106
110
  const DEFAULT_RECOVERY_MS = 5 * 60_000;
111
+ const DEFAULT_BUDGET_ESCALATE_MS = 5 * 60_000;
107
112
  const DEFAULT_SPAWN_TIMEOUT_MS = 30 * 60_000;
108
113
 
109
114
  // Concurrency: at most one sub-session at a time per cadence consumer.
@@ -196,6 +201,10 @@ export function parseUsageFromStdout(stdoutPath) {
196
201
  const out = { ok: true, inputTokens, outputTokens };
197
202
  const cacheRead = Number(usage.cache_read_input_tokens);
198
203
  if (Number.isFinite(cacheRead)) out.cacheReadTokens = cacheRead;
204
+ // Cache CREATION tokens bill at 1.25x input; dropping them was a large slice
205
+ // of the gap between the local estimate and the CLI's authoritative cost.
206
+ const cacheWrite = Number(usage.cache_creation_input_tokens);
207
+ if (Number.isFinite(cacheWrite)) out.cacheWriteTokens = cacheWrite;
199
208
  const totalCost = Number(obj.total_cost_usd);
200
209
  if (Number.isFinite(totalCost) && totalCost >= 0) out.totalCostUsd = totalCost;
201
210
  // The CLI may report the resolved model (e.g. "claude-sonnet-4-6"); map it to
@@ -244,7 +253,8 @@ export function routeCadenceSpawn(agentRoot, cadence) {
244
253
  let band = 0;
245
254
  try {
246
255
  const st = budgetGuardForBand.dailyStatus({ agentRoot });
247
- band = budgetLadder(st.spentUSD, st.capUSD).band;
256
+ // The governor's band, not a re-derivation — see dispatcher.currentBudgetBand.
257
+ band = budgetLadder(st.spentUSD, st.capUSD, { band: st.band }).band;
248
258
  } catch { /* band 0 on read failure */ }
249
259
  const req = {
250
260
  agent_role: "cadence",
@@ -465,11 +475,15 @@ function realSpawnSession({ agentRoot, cadence, promptPath, timeoutMs, log }) {
465
475
  trackerArgs.push("--input-tokens", String(usage.inputTokens));
466
476
  trackerArgs.push("--output-tokens", String(usage.outputTokens));
467
477
  if (usage.cacheReadTokens != null) trackerArgs.push("--cache-read-tokens", String(usage.cacheReadTokens));
478
+ if (usage.cacheWriteTokens != null) trackerArgs.push("--cache-creation-tokens", String(usage.cacheWriteTokens));
468
479
  if (usage.totalCostUsd != null) trackerArgs.push("--total-cost-usd", String(usage.totalCostUsd));
469
480
  } else {
470
- // Parse failure: do NOT pass fabricated zeros — omit token flags
471
- // (tracker defaults them to 0) and surface the gap in the bus log so
472
- // a systematic parse regression is visible rather than silently $0.
481
+ // Parse failure: record the session as EXPLICITLY UNMEASURED. Omitting
482
+ // the token flags used to leave the tracker defaulting them to 0, so a
483
+ // systematic parse regression read as a run of free sessions and the
484
+ // budget governor went quiet exactly when telemetry broke. The row now
485
+ // says "unknown", and the bus log names the parse failure.
486
+ trackerArgs.push("--tokens-unknown", String(usage.reason || "usage-parse-failed"));
473
487
  log({ level: "warn", stage: "cost_usage_parse_failed", cadence, reason: usage.reason, stdout: stdoutPath });
474
488
  }
475
489
  spawn(process.execPath, trackerArgs, { stdio: "ignore", env: { ...env, AGENT_ROOT: agentRoot } }).unref();
@@ -519,6 +533,10 @@ export function startConsumer(opts = {}) {
519
533
  const pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
520
534
  const heartbeatMs = opts.heartbeatMs ?? DEFAULT_HEARTBEAT_MS;
521
535
  const recoveryMs = opts.recoveryMs ?? DEFAULT_RECOVERY_MS;
536
+ // How often the seat checks whether a budget band notice is due for its
537
+ // supervisor. 0 disables it (tests, and any deployment that delivers budget
538
+ // notices out of band).
539
+ const budgetEscalateMs = opts.budgetEscalateMs ?? DEFAULT_BUDGET_ESCALATE_MS;
522
540
  const maxSpawnMs = opts.maxSpawnMs ?? DEFAULT_SPAWN_TIMEOUT_MS;
523
541
  const spawnSession = opts.spawnSession || realSpawnSession;
524
542
  const userLogger = opts.logger;
@@ -538,14 +556,55 @@ export function startConsumer(opts = {}) {
538
556
  * any throw → admit (Invariant: a governance bug must not wedge cadences).
539
557
  * Returns { admit:boolean, reason }.
540
558
  */
541
- function governanceGate() {
559
+ function governanceGate(cadence) {
542
560
  try {
543
561
  const rb = rateGuard.checkRateLimit(RATE_PROVIDER, { agentRoot });
544
562
  if (!rb.allowed) return { admit: false, reason: "rate-limited" };
545
563
  let mode = null;
546
- try { if (budgetGuard.dailyStatus({ agentRoot }).essentialOnly) mode = "essential-only"; }
547
- catch { /* budget read best-effort */ }
548
- const adm = governor.admit({ source: "cadence", mode }, governor.defaultDeps({ agentRoot }));
564
+ let posture = null;
565
+ try {
566
+ const st = budgetGuard.dailyStatus({ agentRoot });
567
+ posture = st.posture;
568
+ if (st.mode && st.mode !== "normal") mode = st.mode;
569
+ } catch { /* budget read best-effort */ }
570
+
571
+ // THE 125% RUNG, PER OBLIGATION. `applyBudgetPosture` makes this transition
572
+ // on a whole plan; this is its single-obligation form on the hot path, so a
573
+ // breach suspends the OUTCOME work it is supposed to suspend and leaves the
574
+ // inline/offline-safe cadences that keep the seat alive running. Same
575
+ // predicate both places — the daemon and the plan cannot disagree about
576
+ // what is suspended.
577
+ if (posture && cadence) {
578
+ const def = getCadenceDef(cadence) || {};
579
+ const verdict = obligationAllowedUnderPosture(
580
+ {
581
+ kind: "SCHEDULE",
582
+ scope: def.scope || null,
583
+ objective_id: def.objectiveId || null,
584
+ offline_safe: def.offlineSafe === true,
585
+ // THE HUMAN LANE. Without this every cadence reaching the gate is a
586
+ // bare SCHEDULE, so the 150% rung ("refuse anything that is not a
587
+ // direct human reply") deferred `inbox-processor` and
588
+ // `messaging-inbound` too — the seat stopped reading its inbox at a
589
+ // budget breach, which is the bricking Invariant #1 forbids.
590
+ human_lane: isHumanLaneCadence(cadence),
591
+ mode: def.mode,
592
+ },
593
+ posture
594
+ );
595
+ if (!verdict.allowed) return { admit: false, reason: verdict.reason };
596
+ }
597
+
598
+ // THE HUMAN LANE, ONE LAYER DOWN. The obligation gate above lets
599
+ // `inbox-processor` and `messaging-inbound` through at band 150, and the
600
+ // governor would then DEFER them anyway — it treats only `source:"inbox"`
601
+ // as a human reply, and every cadence tick arrives as `source:"cadence"`.
602
+ // Two gates, one predicate, or the rung silences the inbox on the second
603
+ // hop instead of the first.
604
+ const adm = governor.admit(
605
+ { source: "cadence", mode, humanReply: cadence ? isHumanLaneCadence(cadence) : false },
606
+ governor.defaultDeps({ agentRoot })
607
+ );
549
608
  if (adm.decision !== "ADMIT") return { admit: false, reason: adm.reason || adm.decision };
550
609
  return { admit: true };
551
610
  } catch {
@@ -553,6 +612,16 @@ export function startConsumer(opts = {}) {
553
612
  }
554
613
  }
555
614
 
615
+ /** The seat's memberId from the cached mandate body. Never throws. */
616
+ function readMandateMemberId(root) {
617
+ try {
618
+ const raw = JSON.parse(readFileSync(join(root, "state", "mandate", "cache.json"), "utf-8"));
619
+ return (raw && raw.body && (raw.body.memberId || raw.body.member_id)) || null;
620
+ } catch {
621
+ return null;
622
+ }
623
+ }
624
+
556
625
  const stats = {
557
626
  started_at: new Date().toISOString(),
558
627
  received: 0,
@@ -738,7 +807,7 @@ export function startConsumer(opts = {}) {
738
807
  // not a per-event failure — burning retry budget here would eventually DLQ
739
808
  // a perfectly good cadence just because the box was busy).
740
809
  {
741
- const gov = governanceGate();
810
+ const gov = governanceGate(event.cadence);
742
811
  if (!gov.admit) {
743
812
  log({
744
813
  level: "info",
@@ -1032,6 +1101,75 @@ export function startConsumer(opts = {}) {
1032
1101
  recoveryTimer.unref?.();
1033
1102
  timers.push(recoveryTimer);
1034
1103
 
1104
+ // THE 80% RUNG'S DELIVERY LEG. `maybeNotify` fires at most once per band per
1105
+ // period and addresses the notice to the SUPERVISOR (never to this seat), so
1106
+ // this timer is cheap: after the first fire every subsequent call reads two
1107
+ // small files and returns `notified:false`. It shares the recovery cadence
1108
+ // because a spend escalation that arrives tomorrow morning is not an
1109
+ // escalation — the goal-steward runs daily at 07:15, which is far too coarse
1110
+ // for money. Fail-open and LOUD: an undeliverable notice logs an error rather
1111
+ // than disappearing.
1112
+ const escalateTimer = budgetEscalateMs > 0 ? setInterval(() => {
1113
+ (async () => {
1114
+ const { escalateBudget } = await import("../../lib/budget-escalate.mjs");
1115
+ let conn = null;
1116
+ try {
1117
+ const { resolveOrgToolConfig } = await import("../../lib/org/tool-surface.mjs");
1118
+ const c = resolveOrgToolConfig({ agentRoot });
1119
+ conn = c ? { base: c.base || "", token: c.token || "", orgId: c.orgId || "" } : null;
1120
+ } catch { /* no credential → escalateBudget reports it rather than sending */ }
1121
+ const r = await escalateBudget(
1122
+ { agentRoot, conn },
1123
+ { log: (level, msg) => log({ level, stage: "budget_escalation", message: msg }) }
1124
+ );
1125
+ if (r.notified) {
1126
+ log({ level: r.delivered ? "warn" : "error", stage: "budget_escalation", band: r.band, delivered: r.delivered, reason: r.reason });
1127
+ }
1128
+
1129
+ // ── THE RETURN LEG, ON THE SAME CLOCK AS THE MONEY ────────────────────
1130
+ // `reconcileBudgetPosture` used to be reachable ONLY from the goal-steward
1131
+ // guard, which runs daily at 07:15 — while the band it reads resets at UTC
1132
+ // midnight. A seat could hit 150% every afternoon for a month and hq would
1133
+ // never see one `budget_breach` drift row, because by 07:15 the next
1134
+ // morning the band it observed was 0 again. Local enforcement was fine
1135
+ // (`governanceGate` re-reads the posture per tick); it was specifically the
1136
+ // record — the thing the person who FUNDS the seat looks at — that never
1137
+ // arrived. Five minutes is the right clock for money.
1138
+ try {
1139
+ const bg = await import("../../lib/budget-guard.mjs");
1140
+ const st = bg.dailyStatus({ agentRoot });
1141
+ const { reconcileBudgetPosture } = await import("../../lib/plan/budget-runtime.mjs");
1142
+ let mirrorDrift;
1143
+ // The seat's own member id: the drift row is filed AGAINST this member.
1144
+ // Same resolution ladder the goal-steward uses (env first, then the
1145
+ // mandate cache the seat already holds), so the two agree.
1146
+ const memberId = process.env.COHORT_AGENT_ID || readMandateMemberId(agentRoot);
1147
+ if (conn && conn.base && conn.token && memberId) {
1148
+ const { call } = await import("../../lib/org/client.mjs");
1149
+ mirrorDrift = async ({ kind, key, detail, resolve }) =>
1150
+ call("mandate.reportDrift", { memberId, kind, key, detail: detail || null, ...(resolve ? { resolve: true } : {}) }, conn);
1151
+ } else if (st.band >= 125) {
1152
+ log({ level: "warn", stage: "budget_posture", message: "band >=125% but no org credential/memberId — the suspension is recorded LOCALLY ONLY and hq will not show it" });
1153
+ }
1154
+ const rec = await reconcileBudgetPosture(
1155
+ { agentRoot, posture: st.posture, status: st },
1156
+ { log: (level, msg) => log({ level, stage: "budget_posture", message: msg }), mirrorDrift }
1157
+ );
1158
+ if (rec.recorded > 0 || rec.previousBand !== rec.band) {
1159
+ log({ level: "warn", stage: "budget_posture", band: rec.band, previous_band: rec.previousBand, recorded: rec.recorded, mirrored: rec.mirrored, suspended: rec.suspended.length });
1160
+ }
1161
+ } catch (err) {
1162
+ log({ level: "error", stage: "budget_posture_failed", error: err?.message || String(err) });
1163
+ }
1164
+ })().catch((err) => {
1165
+ log({ level: "error", stage: "budget_escalation_failed", error: err?.message || String(err) });
1166
+ });
1167
+ }, budgetEscalateMs) : null;
1168
+ if (escalateTimer) {
1169
+ escalateTimer.unref?.();
1170
+ timers.push(escalateTimer);
1171
+ }
1172
+
1035
1173
  // Initial sweep + heartbeat
1036
1174
  recoverStaleClaimsGuarded();
1037
1175
  heartbeat();
@@ -646,6 +646,12 @@ test("0.3: realSpawnSession argv requests --output-format json and records real
646
646
  new URL("../cost/track-claude-usage.mjs", import.meta.url),
647
647
  join(root, "scripts/cost/track-claude-usage.mjs")
648
648
  );
649
+ // ...and the shared billing contract it imports relative to itself.
650
+ mkdirSync(join(root, "lib/cost"), { recursive: true });
651
+ await fsp.copyFile(
652
+ new URL("../../lib/cost/ledger-row.mjs", import.meta.url),
653
+ join(root, "lib/cost/ledger-row.mjs")
654
+ );
649
655
 
650
656
  const argvFile = join(root, "shim-argv.txt");
651
657
  const shim = join(root, "claude-shim");
@@ -426,6 +426,72 @@ function loadAgentCfg(agentRoot) {
426
426
  return agent;
427
427
  }
428
428
 
429
+ /**
430
+ * nightly-backup (inline): the disaster-recovery job.
431
+ *
432
+ * Runs scripts/maintenance/backup-run.mjs as a child process rather than
433
+ * in-process, deliberately: it shells out to `tar` (and possibly gsutil/aws),
434
+ * can take tens of seconds on a large state tree, and must not be able to wedge
435
+ * the persistent daemon's event loop or leave a half-written archive if the
436
+ * daemon is restarted mid-run. A non-zero exit is REPORTED, never swallowed —
437
+ * the whole reason this gap existed is that the old shell driver exited 0 in
438
+ * silence when unconfigured.
439
+ *
440
+ * Injectable via `opts.runImpl` for tests (no tar, no filesystem).
441
+ */
442
+ async function handleNightlyBackup({ event, agentRoot, log }, opts = {}) {
443
+ const started = Date.now();
444
+ try {
445
+ const runner =
446
+ opts.runImpl ||
447
+ (async () => {
448
+ const { execFile } = await import("node:child_process");
449
+ const script = join(agentRoot, "scripts", "maintenance", "backup-run.mjs");
450
+ const fallback = join(__dirname, "..", "maintenance", "backup-run.mjs");
451
+ const target = existsSync(script) ? script : fallback;
452
+ return new Promise((resolvePromise) => {
453
+ execFile(
454
+ process.execPath,
455
+ [target, "--agent-dir", agentRoot],
456
+ { timeout: 20 * 60 * 1000, maxBuffer: 8 * 1024 * 1024 },
457
+ (err, stdout, stderr) => {
458
+ resolvePromise({
459
+ code: err ? (typeof err.code === "number" ? err.code : 1) : 0,
460
+ stdout: String(stdout || ""),
461
+ stderr: String(stderr || ""),
462
+ });
463
+ }
464
+ );
465
+ });
466
+ });
467
+
468
+ const res = await runner({ agentRoot });
469
+ const durationMs = Date.now() - started;
470
+ // Surface the runner's own warn/error lines into the daemon log so a
471
+ // degraded backup (offsite unconfigured, offsite push failed, include path
472
+ // dropped by the credential deny-list) is visible without opening a file.
473
+ const notable = `${res.stdout || ""}${res.stderr || ""}`
474
+ .split("\n")
475
+ .filter((l) => /\[(warn|error)\]/.test(l));
476
+ if (typeof log === "function") {
477
+ for (const line of notable.slice(0, 10)) log(res.code === 0 ? "warn" : "error", `[nightly-backup] ${line.trim()}`);
478
+ }
479
+ if (res.code !== 0) {
480
+ return {
481
+ ok: false,
482
+ decision: "inline",
483
+ cadence: event.cadence,
484
+ durationMs,
485
+ error: `backup-run.mjs exited ${res.code}`,
486
+ detail: notable.slice(0, 5),
487
+ };
488
+ }
489
+ return { ok: true, decision: "inline", cadence: event.cadence, durationMs, warnings: notable.length };
490
+ } catch (err) {
491
+ return { ok: false, decision: "inline", cadence: event.cadence, error: err && err.message ? err.message : String(err) };
492
+ }
493
+ }
494
+
429
495
  /**
430
496
  * nightly-cost-reconcile (inline): three-way spend reconcile of today's local
431
497
  * router ledger against the Anthropic Admin Cost API. We pass NO fetchImpl, so
@@ -1198,6 +1264,84 @@ async function guardGoalSteward({ event, agentRoot, log }, opts = {}) {
1198
1264
  }
1199
1265
  }
1200
1266
 
1267
+ // 2d. THE BUDGET LADDER'S 100% RUNG: "self-directed (D3) work stops."
1268
+ // THIS loop is D3 — the only thing in the system that creates work for
1269
+ // the seat out of its own gaps. Above the degrade rung it does not run
1270
+ // at all; below it, the seat's remaining envelope and the fan-out
1271
+ // ceiling are handed to `routeRung`, which already knows how to enqueue
1272
+ // rather than downgrade an over-budget item (`remainingCents`/`maxRung`
1273
+ // were parameters with no caller — this is the caller).
1274
+ //
1275
+ // Measuring is NOT self-directed work and is deliberately not gated: a
1276
+ // seat that stops measuring because it is over budget also stops being
1277
+ // able to prove it, and the KPI series goes dark exactly when someone
1278
+ // needs it. Only the CREATE half is stopped, via enforcement=observe.
1279
+ let budgetDeps = opts.budgetDeps;
1280
+ if (budgetDeps === undefined) {
1281
+ budgetDeps = {};
1282
+ try {
1283
+ const bg = await import("../../lib/budget-guard.mjs");
1284
+ const st = bg.dailyStatus({ agentRoot });
1285
+ if (!st.posture.selfDirected && enforcement === "active") {
1286
+ say(
1287
+ "warn",
1288
+ `[goal-steward] budget ${st.mode} (${st.pct}% of $${st.capUSD}/day, ${st.capSource}) — self-directed work STOPS: ` +
1289
+ "measuring continues, creating does not"
1290
+ );
1291
+ enforcement = "observe";
1292
+ }
1293
+ budgetDeps = {
1294
+ budgetStatus: st,
1295
+ maxRung: st.posture.maxRung ?? undefined,
1296
+ // The seat's remaining envelope for the period, in cents — the same
1297
+ // envelope hq funded, not a local guess. Null envelope ⇒ undefined ⇒
1298
+ // routeRung's budget gate stays inert rather than inventing a ceiling.
1299
+ remainingCents: st.envelope && Number.isFinite(st.envelope.remainingUSD)
1300
+ ? Math.max(0, Math.round(st.envelope.remainingUSD * 100))
1301
+ : undefined,
1302
+ };
1303
+
1304
+ // THE 125% RUNG, on the plan. Reconcile the compiled plan against the
1305
+ // live band: OUTCOME obligations (and the measure/review cadences that
1306
+ // serve them) are suspended, one `budget_breach` drift row each, on the
1307
+ // band EDGE only. The plan file itself is not rewritten — see
1308
+ // lib/plan/budget-runtime.mjs for why a schedule that flaps with spend
1309
+ // would be worse than the breach.
1310
+ try {
1311
+ const { reconcileBudgetPosture } = await import("../../lib/plan/budget-runtime.mjs");
1312
+
1313
+ // The return leg. A suspension only this laptop knows about is
1314
+ // invisible to the person who funds the seat — and funding is the only
1315
+ // thing that clears it. hq dedupes an open row per
1316
+ // (memberId, kind, key), so re-reporting is safe.
1317
+ let mirrorDrift;
1318
+ if (conn && conn.base && conn.token) {
1319
+ const { call } = await import("../../lib/org/client.mjs");
1320
+ mirrorDrift = async ({ kind, key, detail, resolve }) =>
1321
+ call(
1322
+ "mandate.reportDrift",
1323
+ { memberId, kind, key, detail: detail || null, ...(resolve ? { resolve: true } : {}) },
1324
+ { ...conn, fetchImpl: opts.fetchImpl }
1325
+ );
1326
+ } else {
1327
+ say("warn", "[goal-steward] no org credential — a budget suspension will be recorded LOCALLY ONLY and hq will not show it");
1328
+ }
1329
+
1330
+ const rec = await reconcileBudgetPosture(
1331
+ { agentRoot, posture: st.posture, status: st },
1332
+ { now: opts.now, log: (level, msg) => say(level, `[goal-steward] ${msg}`), mirrorDrift }
1333
+ );
1334
+ if (rec.recorded > 0) {
1335
+ say("warn", `[goal-steward] recorded ${rec.recorded} budget_breach drift row(s) at band ${rec.band}% (${rec.mirrored} mirrored to hq)`);
1336
+ }
1337
+ } catch (err) {
1338
+ say("error", `[goal-steward] budget/plan reconcile failed (${err && err.message ? err.message : err}) — obligations are NOT suspended this tick`);
1339
+ }
1340
+ } catch (err) {
1341
+ say("warn", `[goal-steward] could not read the budget posture (${err && err.message ? err.message : err}) — running UNMETERED this tick`);
1342
+ }
1343
+ }
1344
+
1201
1345
  const run = opts.runImpl || (await import("../../lib/goals/loop.mjs")).runGoalSteward;
1202
1346
  const report = await run({
1203
1347
  agentRoot,
@@ -1207,6 +1351,7 @@ async function guardGoalSteward({ event, agentRoot, log }, opts = {}) {
1207
1351
  sensors,
1208
1352
  reachable,
1209
1353
  memberId,
1354
+ ...budgetDeps,
1210
1355
  ...(conn || {}),
1211
1356
  ...(mirrorSample ? { mirrorSample } : {}),
1212
1357
  ...(opts.runDeps || {}),
@@ -1371,6 +1516,11 @@ export const CADENCE_REGISTRY = {
1371
1516
 
1372
1517
  // Economics / scheduling cadences (WS-economics). Deterministic → inline,
1373
1518
  // except dynamic-jobs (guarded escalate-when-due).
1519
+ "nightly-backup": {
1520
+ mode: "inline",
1521
+ handler: handleNightlyBackup,
1522
+ description: "Nightly DR backup — tar.gz of durable state to local restore points outside the repo, plus the offsite copy when configured. Inline (no LLM); a failed run is reported, never swallowed.",
1523
+ },
1374
1524
  "nightly-cost-reconcile": {
1375
1525
  mode: "inline",
1376
1526
  handler: handleNightlyCostReconcile,
@@ -1525,6 +1675,13 @@ export function getCadenceDef(name) {
1525
1675
  if (Number.isFinite(cfg.budgetCents)) def.budgetCents = cfg.budgetCents;
1526
1676
  else _warnUnmetered(name, cfg.obligationKey);
1527
1677
  if (cfg.obligationKey) def.obligationKey = cfg.obligationKey;
1678
+ // Provenance the RUNTIME budget gate needs (lib/plan/emit.toRegistry): which
1679
+ // obligation this cadence serves, and whether it is free to keep running at
1680
+ // a breached band. Without them the consumer cannot tell a funded
1681
+ // objective's measure-cadence from the seat's own heartbeat.
1682
+ if (cfg.scope) def.scope = cfg.scope;
1683
+ if (cfg.objectiveId) def.objectiveId = cfg.objectiveId;
1684
+ def.offlineSafe = cfg.offlineSafe === true;
1528
1685
  return def;
1529
1686
  }
1530
1687
  return null;
@@ -1548,6 +1705,7 @@ export { guardSkillCurator, maybeSpawnConsolidation };
1548
1705
  // them via CADENCE_REGISTRY). Each takes an optional second arg of injectable
1549
1706
  // seams (lib fn / clock / cfg) so tests never touch the network or real config.
1550
1707
  export {
1708
+ handleNightlyBackup,
1551
1709
  handleNightlyCostReconcile,
1552
1710
  handleFleetCostDigest,
1553
1711
  handleOrgCostSync,
@@ -20,6 +20,7 @@ import { tmpdir } from "node:os";
20
20
  import { join } from "node:path";
21
21
 
22
22
  import { guardSkillCurator, maybeSpawnConsolidation, CADENCE_REGISTRY,
23
+ handleNightlyBackup,
23
24
  handleNightlyCostReconcile, handleFleetCostDigest, handleOrgCostSync, guardDynamicJobs,
24
25
  guardMessagingInbound, guardDirectoryHygiene, guardBrandSteward } from "./cadence-handlers.mjs";
25
26
  import { writeSkill } from "../../lib/learning/skill-writer.mjs";
@@ -181,6 +182,69 @@ test("CADENCE_REGISTRY wires the economics/scheduling cadences with the right mo
181
182
  assert.ok(CADENCE_REGISTRY["dynamic-jobs"].prompt);
182
183
  });
183
184
 
185
+ // ---------------------------------------------------------------------------
186
+ // nightly-backup (DR) — the whole point is that a failed run is never silent
187
+ // ---------------------------------------------------------------------------
188
+
189
+ test("CADENCE_REGISTRY wires nightly-backup as an inline handler", () => {
190
+ assert.equal(CADENCE_REGISTRY["nightly-backup"].mode, "inline");
191
+ assert.equal(typeof CADENCE_REGISTRY["nightly-backup"].handler, "function");
192
+ // Inline cadences must NOT carry a prompt — a prompt would let the consumer
193
+ // spawn a Claude session for a tar job.
194
+ assert.ok(!CADENCE_REGISTRY["nightly-backup"].prompt);
195
+ });
196
+
197
+ test("nightly-backup: a clean run is ok and reports no warnings", async () => {
198
+ const res = await handleNightlyBackup(ev("nightly-backup"), {
199
+ runImpl: async () => ({ code: 0, stdout: "[info] backup complete (tier=offsite).\n", stderr: "" }),
200
+ });
201
+ assert.equal(res.ok, true);
202
+ assert.equal(res.decision, "inline");
203
+ assert.equal(res.cadence, "nightly-backup");
204
+ assert.equal(res.warnings, 0);
205
+ });
206
+
207
+ test("nightly-backup: a non-zero exit is REPORTED, not swallowed", async () => {
208
+ const logs = [];
209
+ const res = await handleNightlyBackup(
210
+ { event: { cadence: "nightly-backup" }, agentRoot: tmpRoot(), log: (lvl, m) => logs.push([lvl, m]) },
211
+ { runImpl: async () => ({ code: 1, stdout: "", stderr: "[error] REFUSING: no .maestro/backup-config.yaml\n" }) }
212
+ );
213
+ assert.equal(res.ok, false);
214
+ assert.match(res.error, /exited 1/);
215
+ assert.ok(logs.some(([lvl, m]) => lvl === "error" && /REFUSING/.test(m)), "the refusal must reach the daemon log");
216
+ });
217
+
218
+ test("nightly-backup: a degraded-but-successful run surfaces its warnings in the daemon log", async () => {
219
+ const logs = [];
220
+ const res = await handleNightlyBackup(
221
+ { event: { cadence: "nightly-backup" }, agentRoot: tmpRoot(), log: (lvl, m) => logs.push([lvl, m]) },
222
+ {
223
+ runImpl: async () => ({
224
+ code: 0,
225
+ stdout:
226
+ "[info] plan: tier=local\n" +
227
+ "[warn] offsite is NOT configured — these restore points do not survive losing this machine.\n" +
228
+ "[warn] include path DROPPED by the credential deny-list: .env (matched .env)\n",
229
+ stderr: "",
230
+ }),
231
+ }
232
+ );
233
+ assert.equal(res.ok, true);
234
+ assert.equal(res.warnings, 2);
235
+ assert.ok(logs.some(([, m]) => /offsite is NOT configured/.test(m)));
236
+ assert.ok(logs.some(([, m]) => /credential deny-list/.test(m)));
237
+ assert.ok(!logs.some(([, m]) => /\[info\]/.test(m)), "info noise stays in the backup log, not the daemon log");
238
+ });
239
+
240
+ test("nightly-backup: a throwing runner degrades to ok:false, never an unhandled rejection", async () => {
241
+ const res = await handleNightlyBackup(ev("nightly-backup"), {
242
+ runImpl: async () => { throw new Error("spawn ENOENT"); },
243
+ });
244
+ assert.equal(res.ok, false);
245
+ assert.match(res.error, /spawn ENOENT/);
246
+ });
247
+
184
248
  test("nightly-cost-reconcile: inline, surfaces driftPct, passes NO fetchImpl (fails open)", async () => {
185
249
  let sawFetch = "unset";
186
250
  const fake = async (o) => {
@@ -25,21 +25,30 @@
25
25
 
26
26
  import { test } from "node:test";
27
27
  import assert from "node:assert/strict";
28
- import { createRequire } from "node:module";
28
+ import { readFileSync } from "node:fs";
29
29
 
30
30
  import * as classifier from "./classifier.mjs";
31
31
 
32
32
  // ── 1. Imports cleanly with no openai package present ───────────────────────
33
33
 
34
- test("classifier.mjs imports cleanly with no openai package installed", () => {
34
+ test("classifier.mjs imports cleanly and does not depend on openai", () => {
35
35
  // The module-level import above already succeeded (this file loaded), which
36
- // is the load-bearing assertion. Double-check openai is genuinely absent so
37
- // this test fails loudly if someone reintroduces the hard dependency.
38
- const require = createRequire(import.meta.url);
39
- assert.throws(
40
- () => require.resolve("openai"),
41
- /Cannot find module 'openai'|Cannot find package 'openai'/,
42
- "openai must NOT be resolvable — the classifier must not depend on it"
36
+ // is the load-bearing assertion.
37
+ //
38
+ // The invariant is about THIS MODULE's source, not the ambient node_modules
39
+ // tree. An earlier version asserted `require.resolve("openai")` throws, but
40
+ // that is a proxy that only holds inside the SDK's own checkout: every agent
41
+ // repo that pulls openai in transitively made this fail while the real
42
+ // invariant stayed true. A guard that goes red on every downstream machine
43
+ // just teaches operators to ignore red suites, so assert on the source.
44
+ const src = readFileSync(new URL("./classifier.mjs", import.meta.url), "utf-8");
45
+ const importsOpenai = /(?:^|\n)\s*import[^\n]*["']openai["']/.test(src)
46
+ || /\bimport\(\s*["']openai["']\s*\)/.test(src)
47
+ || /\brequire\(\s*["']openai["']\s*\)/.test(src);
48
+ assert.equal(
49
+ importsOpenai,
50
+ false,
51
+ "classifier.mjs must not import openai — a hard dependency here broke the whole module on agents that lack it"
43
52
  );
44
53
  assert.equal(typeof classifier.classifyItem, "function", "classifyItem is exported");
45
54
  });