@cohortapp/agent-sdk 2.5.1 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/bin/maestro.mjs +185 -88
  2. package/bin/maestro.test.mjs +175 -48
  3. package/docs/runbooks/backup-restore.md +65 -33
  4. package/framework-features.json +4 -4
  5. package/lib/backup/policy.mjs +710 -0
  6. package/lib/backup/policy.test.mjs +305 -0
  7. package/lib/budget-escalate.mjs +133 -0
  8. package/lib/budget-escalate.test.mjs +232 -0
  9. package/lib/budget-guard.envelope.test.mjs +476 -0
  10. package/lib/budget-guard.mjs +853 -75
  11. package/lib/budget-guard.test.mjs +91 -42
  12. package/lib/cadences.mjs +33 -0
  13. package/lib/channels/orgmail/adapter.mjs +88 -3
  14. package/lib/channels/orgmail/adapter.test.mjs +137 -0
  15. package/lib/channels/repeat-suppressor.mjs +198 -0
  16. package/lib/channels/repeat-suppressor.test.mjs +134 -0
  17. package/lib/comms/receipts.mjs +297 -0
  18. package/lib/cost/ledger-row.mjs +333 -0
  19. package/lib/cost/ledger-row.test.mjs +183 -0
  20. package/lib/execution/drive.mjs +28 -1
  21. package/lib/execution/effects.mjs +191 -12
  22. package/lib/execution/effects.test.mjs +50 -11
  23. package/lib/goals/admission.mjs +13 -1
  24. package/lib/goals/admission.test.mjs +26 -1
  25. package/lib/goals/loop.mjs +13 -0
  26. package/lib/kpi-sensors.test.mjs +3 -0
  27. package/lib/mandate/cache.mjs +13 -5
  28. package/lib/mandate/derive.mjs +146 -21
  29. package/lib/mandate/derive.test.mjs +50 -6
  30. package/lib/mandate/model.mjs +32 -4
  31. package/lib/mandate/refresh.test.mjs +16 -2
  32. package/lib/mcp/server.test.mjs +12 -3
  33. package/lib/model-router/economics.mjs +107 -76
  34. package/lib/model-router/economics.test.mjs +64 -46
  35. package/lib/model-router/integration-coverage.test.mjs +39 -37
  36. package/lib/model-router/ledger.mjs +75 -22
  37. package/lib/model-router/ledger.test.mjs +35 -2
  38. package/lib/org/client.mjs +14 -0
  39. package/lib/org/cost-sync.mjs +16 -2
  40. package/lib/org/doctor.mjs +62 -1
  41. package/lib/org/doctor.test.mjs +36 -3
  42. package/lib/org/email-remedy.mjs +49 -0
  43. package/lib/org/engagement-ledger.mjs +376 -0
  44. package/lib/org/engagement-ledger.test.mjs +112 -0
  45. package/lib/org/engagement.mjs +1056 -0
  46. package/lib/org/engagement.test.mjs +739 -0
  47. package/lib/org/messaging.mjs +230 -3
  48. package/lib/org/messaging.test.mjs +110 -1
  49. package/lib/org/param-contract.mjs +56 -2
  50. package/lib/org/param-contract.test.mjs +26 -0
  51. package/lib/org/protocol.checksum +1 -1
  52. package/lib/org/protocol.mjs +5 -0
  53. package/lib/org/protocol.test.mjs +7 -1
  54. package/lib/org/tool-surface.mjs +506 -10
  55. package/lib/org/tool-surface.test.mjs +191 -7
  56. package/lib/org/ui-parity.mjs +333 -6
  57. package/lib/org/ui-parity.test.mjs +96 -3
  58. package/lib/org/work-ledger.mjs +241 -0
  59. package/lib/org/work-ledger.test.mjs +237 -0
  60. package/lib/plan/adoption-e2e.test.mjs +366 -0
  61. package/lib/plan/budget-enforcement.test.mjs +400 -0
  62. package/lib/plan/budget-runtime.mjs +215 -0
  63. package/lib/plan/compile.mjs +201 -5
  64. package/lib/plan/compile.test.mjs +19 -5
  65. package/lib/plan/emit.mjs +8 -0
  66. package/lib/plan/emit.test.mjs +18 -0
  67. package/lib/resource-governor.mjs +58 -12
  68. package/lib/resource-governor.test.mjs +41 -1
  69. package/lib/security/audit-engine.mjs +45 -8
  70. package/lib/security/audit-engine.test.mjs +35 -0
  71. package/lib/setup/enroll-from-cohort.mjs +14 -1
  72. package/lib/setup/sections/mandate.mjs +48 -7
  73. package/lib/setup/sections/mandate.test.mjs +17 -2
  74. package/lib/setup/sections/orgmail.mjs +10 -2
  75. package/lib/setup/state.mjs +83 -2
  76. package/lib/telemetry/collect.mjs +360 -20
  77. package/lib/telemetry/collect.test.mjs +266 -0
  78. package/package.json +1 -1
  79. package/scripts/cost/track-claude-usage.mjs +207 -48
  80. package/scripts/cost/track-claude-usage.test.mjs +148 -0
  81. package/scripts/daemon/agent-daemon.mjs +315 -17
  82. package/scripts/daemon/assurance-e2e.test.mjs +421 -0
  83. package/scripts/daemon/assurance.mjs +944 -0
  84. package/scripts/daemon/assurance.test.mjs +668 -0
  85. package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
  86. package/scripts/daemon/cadence-consumer.mjs +147 -9
  87. package/scripts/daemon/cadence-consumer.test.mjs +6 -0
  88. package/scripts/daemon/cadence-handlers.mjs +158 -0
  89. package/scripts/daemon/cadence-handlers.test.mjs +64 -0
  90. package/scripts/daemon/deliver.mjs +314 -0
  91. package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
  92. package/scripts/daemon/dispatcher.mjs +64 -6
  93. package/scripts/daemon/responder-cost.test.mjs +68 -0
  94. package/scripts/daemon/responder.mjs +351 -298
  95. package/scripts/local-triggers/generate-plists.test.mjs +7 -4
  96. package/scripts/maintenance/backup-run.mjs +415 -0
  97. package/scripts/maintenance/backup-to-cloud.sh +16 -116
  98. package/scripts/org/send-orgmail.mjs +16 -0
  99. package/scripts/record-receipt.sh +63 -0
  100. package/scripts/restore-from-backup.sh +14 -3
  101. package/scripts/restore-from-backup.test.mjs +8 -5
  102. package/scripts/send-email-threaded.py +47 -0
  103. package/scripts/send-sms.sh +4 -0
  104. package/scripts/send-whatsapp.sh +4 -0
  105. package/scripts/setup/init-backup.mjs +93 -38
  106. package/scripts/slack-send.sh +12 -0
@@ -107,7 +107,7 @@ function listPlists(agentRoot) {
107
107
  // Tests
108
108
  // ---------------------------------------------------------------------------
109
109
 
110
- test("generator emits 22 standard plists with the agent's first name", async () => {
110
+ test("generator emits 23 standard plists with the agent's first name", async () => {
111
111
  // Standard inventory (cadence-bus v1 + slack-socket-mode v1), with NO
112
112
  // config/.cadence-plists.tsv present (no archetype cadences):
113
113
  // daemon, poll-relay, slack-socket (3 infra), PLUS one trigger plist per
@@ -119,7 +119,8 @@ test("generator emits 22 standard plists with the agent's first name", async ()
119
119
  // org-cost-sync, monitoring-alerts, dynamic-jobs (5), PLUS the SP10
120
120
  // messaging-inbound cadence (1), PLUS the 2026-07 Directory & Design
121
121
  // stewards directory-hygiene + brand-steward (2), PLUS the D3 self-directed
122
- // loop goal-steward (1). = 3 infra + 19 trigger = 22 total.
122
+ // loop goal-steward (1), PLUS the 2026-08 DR cadence nightly-backup (1).
123
+ // = 3 infra + 20 trigger = 23 total.
123
124
  // org-pulse used to be declared standard but was OMITTED by the old hardcoded
124
125
  // list (it never got a plist); deriving from the SoT fixes that drift.
125
126
  // The former weekly-* cadences are still ARCHETYPE-DRIVEN (function × altitude),
@@ -129,9 +130,11 @@ test("generator emits 22 standard plists with the agent's first name", async ()
129
130
  const r = runGenerator(root);
130
131
  assert.equal(r.status, 0, r.stderr);
131
132
  const plists = listPlists(root);
132
- assert.equal(plists.length, 22, `expected 22 standard plists; got ${plists.join(",")}`);
133
+ assert.equal(plists.length, 23, `expected 23 standard plists; got ${plists.join(",")}`);
133
134
  assert.ok(plists.includes("ai.maestro.alice-messaging-inbound.plist"),
134
135
  "messaging-inbound (SP10 standard cadence) must get a SoT-derived plist");
136
+ assert.ok(plists.includes("ai.maestro.alice-nightly-backup.plist"),
137
+ "nightly-backup (the DR cadence) must be scheduled, not left to a human to remember");
135
138
  for (const id of ["directory-hygiene", "brand-steward"]) {
136
139
  assert.ok(plists.includes(`ai.maestro.alice-${id}.plist`),
137
140
  `Directory/Design steward "${id}" must get a SoT-derived plist; got ${plists.join(",")}`);
@@ -200,7 +203,7 @@ test("archetype cadences from config/.cadence-plists.tsv emit extra trigger plis
200
203
  const r = runGenerator(root);
201
204
  assert.equal(r.status, 0, r.stderr);
202
205
  const plists = listPlists(root);
203
- assert.equal(plists.length, 24, `expected 22 standard + 2 archetype; got ${plists.join(",")}`);
206
+ assert.equal(plists.length, 25, `expected 23 standard + 2 archetype; got ${plists.join(",")}`);
204
207
  const dir = join(root, "scripts/local-triggers/plists");
205
208
  const eng = readFileSync(join(dir, "ai.maestro.erin-engineering-health.plist"), "utf-8");
206
209
  assert.match(eng, /<key>Weekday<\/key>\s*<integer>3<\/integer>/); // base64 schedule decoded
@@ -0,0 +1,415 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * backup-run.mjs — the nightly DR job.
4
+ *
5
+ * Replaces the brains of scripts/maintenance/backup-to-cloud.sh (which is now a
6
+ * thin wrapper so any already-installed plist keeps working). The shell driver
7
+ * parsed YAML with awk, had no deny-list, and — the part that mattered —
8
+ * `exit 0`'d silently when unconfigured, which is how a fleet ran for months
9
+ * with zero agents backed up and nothing red anywhere.
10
+ *
11
+ * What it does, in order:
12
+ * 1. Resolve the plan (lib/backup/policy.mjs — the single source of truth
13
+ * shared with doctor and the nightly-backup cadence).
14
+ * 2. REFUSE to run if the plan is unusable, loudly, with a non-zero exit.
15
+ * Never a silent success.
16
+ * 3. tar.gz the include paths with the hard credential deny-list applied as
17
+ * --exclude patterns (belt) on top of the plan's already-filtered include
18
+ * list (braces).
19
+ * 4. VERIFY the archive does not contain a denied path before it goes
20
+ * anywhere. A backup that ships `.env` to a bucket is a credential leak
21
+ * that looks exactly like a successful backup, so the check happens on the
22
+ * real archive listing, not on the intent.
23
+ * 5. Place it in the local restore-point directory (Tier 1), prune to
24
+ * retention, then push offsite (Tier 2) when configured.
25
+ * 6. Stamp .maestro/last-backup.json — the marker doctor and
26
+ * lib/diagnostics/backup-freshness.mjs read.
27
+ *
28
+ * Exit codes: 0 success (including "offsite skipped because unconfigured" —
29
+ * that is a WARN state doctor owns, not a job failure), 1 refused/failed.
30
+ *
31
+ * Usage:
32
+ * node scripts/maintenance/backup-run.mjs [--dry-run] [--agent-dir <path>]
33
+ */
34
+
35
+ import { execFileSync } from "node:child_process";
36
+ import {
37
+ existsSync,
38
+ mkdirSync,
39
+ readdirSync,
40
+ rmSync,
41
+ statSync,
42
+ writeFileSync,
43
+ appendFileSync,
44
+ } from "node:fs";
45
+ import { tmpdir } from "node:os";
46
+ import { dirname, join, resolve } from "node:path";
47
+ import { fileURLToPath } from "node:url";
48
+
49
+ const __dirname = dirname(fileURLToPath(import.meta.url));
50
+
51
+ const argv = process.argv.slice(2);
52
+ const DRY_RUN = argv.includes("--dry-run");
53
+ const agentDirArg = (() => {
54
+ const i = argv.indexOf("--agent-dir");
55
+ return i >= 0 ? argv[i + 1] : null;
56
+ })();
57
+
58
+ const AGENT_DIR = resolve(
59
+ agentDirArg || process.env.AGENT_ROOT || process.env.AGENT_DIR || join(__dirname, "..", "..")
60
+ );
61
+
62
+ const LOG_DIR = join(AGENT_DIR, "logs", "maintenance");
63
+ const LOG_FILE = join(LOG_DIR, "backup.log");
64
+
65
+ function log(level, msg) {
66
+ const line = `[${new Date().toISOString()}] [${level}] ${msg}`;
67
+ process.stdout.write(`${line}\n`);
68
+ try {
69
+ mkdirSync(LOG_DIR, { recursive: true });
70
+ appendFileSync(LOG_FILE, `${line}\n`);
71
+ } catch {
72
+ /* logging must never be the thing that fails the backup */
73
+ }
74
+ }
75
+
76
+ /**
77
+ * Resolve lib/backup/policy.mjs from the agent's own lib/ first (an agent repo
78
+ * carries its own framework copy), then the installed SDK, then this checkout —
79
+ * the same resolution ladder generate-plists.sh uses.
80
+ */
81
+ async function loadPolicy() {
82
+ const candidates = [
83
+ join(AGENT_DIR, "lib", "backup", "policy.mjs"),
84
+ join(AGENT_DIR, "node_modules", "@cohortapp", "agent-sdk", "lib", "backup", "policy.mjs"),
85
+ join(__dirname, "..", "..", "lib", "backup", "policy.mjs"),
86
+ ];
87
+ for (const c of candidates) {
88
+ if (existsSync(c)) return import(`file://${c}`);
89
+ }
90
+ throw new Error(`lib/backup/policy.mjs not found (looked in: ${candidates.join(", ")})`);
91
+ }
92
+
93
+ async function main() {
94
+ const policy = await loadPolicy();
95
+ const plan = policy.resolveBackupPlan({ agentRoot: AGENT_DIR });
96
+
97
+ // ── Refuse loudly rather than exit 0 silently ────────────────────────────
98
+ if (!plan.present) {
99
+ log("error", "REFUSING: no .maestro/backup-config.yaml. Fix: maestro init backup-replication --apply");
100
+ process.exit(1);
101
+ }
102
+ if (!plan.readable) {
103
+ log("error", "REFUSING: .maestro/backup-config.yaml is unparseable YAML.");
104
+ process.exit(1);
105
+ }
106
+ if (!plan.enabled) {
107
+ log("error", "REFUSING: backup-config.yaml has enabled != true — this machine has no restore points.");
108
+ process.exit(1);
109
+ }
110
+ for (const v of plan.violations) {
111
+ log("warn", `include path DROPPED by the credential deny-list: ${v.path} (matched ${v.pattern})`);
112
+ }
113
+ if (!plan.include.length) {
114
+ log("error", "REFUSING: every include path was rejected — the archive would be empty.");
115
+ process.exit(1);
116
+ }
117
+
118
+ // Only archive what actually exists; a missing optional tree (outputs/ on a
119
+ // fresh agent) must not fail the whole run.
120
+ const present = plan.include.filter((p) => existsSync(join(AGENT_DIR, p)));
121
+ const absent = plan.include.filter((p) => !present.includes(p));
122
+ if (absent.length) log("info", `skipping absent include path(s): ${absent.join(", ")}`);
123
+ if (!present.length) {
124
+ log("error", "REFUSING: none of the configured include paths exist on disk.");
125
+ process.exit(1);
126
+ }
127
+
128
+ const stamp = new Date().toISOString().replace(/[:.]/g, "-").replace(/Z$/, "Z");
129
+ const archiveName = `${plan.prefix}-${stamp}.tar.gz`;
130
+ // Tier 1 off + Tier 2 on is a legitimate (if unusual) posture: stage in the
131
+ // OS temp dir, never inside the agent repo. An archive that lives in the tree
132
+ // it protects is not a backup, and it would also end up inside the NEXT one.
133
+ const localDir = plan.local.enabled ? plan.local.dir : join(tmpdir(), "maestro-backup-staging");
134
+ const archivePath = join(localDir, archiveName);
135
+
136
+ log("info", `plan: tier=${plan.tier} include=[${present.join(", ")}] → ${archivePath}`);
137
+ if (DRY_RUN) {
138
+ log("info", "--dry-run: stopping before tar.");
139
+ return;
140
+ }
141
+
142
+ mkdirSync(localDir, { recursive: true });
143
+
144
+ // ── 2b. CONTENT deny-list ───────────────────────────────────────────────
145
+ // The path deny-list cannot see a credential stored INSIDE an included file,
146
+ // and that is not hypothetical: state/setup/progress.json carried this agent's
147
+ // live COHORT_API_KEY in plaintext while the archive listing verified clean.
148
+ // Scan the real bytes on disk and turn every hit into a tar --exclude, so the
149
+ // secret never enters the tarball in the first place.
150
+ //
151
+ // A hit DROPS THE FILE; it does not fail the run. A false positive must cost
152
+ // one file in a restore point, never the machine's whole DR posture.
153
+ const contentScan = policy.scanIncludesForSecrets(plan, AGENT_DIR);
154
+ const contentExcludes = [];
155
+ for (const leak of contentScan.leaks) {
156
+ contentExcludes.push(`--exclude=${leak.path}`);
157
+ log("warn", `file DROPPED by the CONTENT deny-list: ${leak.path} (matched ${leak.patterns.join(", ")}) — a credential inside an archived file is the same leak as archiving .env`);
158
+ }
159
+ for (const big of contentScan.oversize) {
160
+ log("warn", `NOT content-scanned (over the size limit): ${big} — it is still archived; the path deny-list is its only cover`);
161
+ }
162
+ for (const e of contentScan.errors.slice(0, 10)) log("warn", `content scan could not read ${e}`);
163
+ if (contentScan.truncated) {
164
+ log("error", "REFUSING: the content scan hit its file budget before finishing — an unscanned tree may contain credentials.");
165
+ process.exit(1);
166
+ }
167
+ log("info", `content scan: ${contentScan.scanned} file(s) read, ${contentScan.leaks.length} dropped, ${contentScan.skippedBinary} binary skipped`);
168
+
169
+ // ── 3. build the archive ────────────────────────────────────────────────
170
+ const excludeArgs = [...policy.tarExcludeArgs(plan), ...contentExcludes];
171
+ try {
172
+ execFileSync("tar", [...excludeArgs, "-czf", archivePath, "-C", AGENT_DIR, ...present], {
173
+ stdio: ["ignore", "pipe", "pipe"],
174
+ maxBuffer: 32 * 1024 * 1024,
175
+ });
176
+ } catch (err) {
177
+ log("error", `tar failed: ${err && err.message}`);
178
+ try { rmSync(archivePath, { force: true }); } catch { /* best effort */ }
179
+ process.exit(1);
180
+ }
181
+
182
+ // ── 4. verify no denied path made it in ─────────────────────────────────
183
+ // This is the check that turns the deny-list from an intention into a
184
+ // guarantee. It reads the REAL archive listing: a symlink, a nested copy, or
185
+ // a tar flag that behaved differently than expected all surface here.
186
+ let listing = "";
187
+ try {
188
+ listing = execFileSync("tar", ["-tzf", archivePath], {
189
+ encoding: "utf-8",
190
+ maxBuffer: 128 * 1024 * 1024,
191
+ });
192
+ } catch (err) {
193
+ log("error", `could not list the archive to verify it: ${err && err.message}`);
194
+ rmSync(archivePath, { force: true });
195
+ process.exit(1);
196
+ }
197
+ const leaked = [];
198
+ const contentDenied = new Set(contentScan.leaks.map((l) => l.path));
199
+ for (const entry of listing.split("\n")) {
200
+ const rel = entry.trim().replace(/^\.\//, "");
201
+ if (!rel || rel.endsWith("/")) continue;
202
+ const hit = policy.denyMatch(rel);
203
+ if (hit) leaked.push(`${rel} (${hit})`);
204
+ // A file the content scan flagged must not be in the listing. If tar kept it
205
+ // anyway (an --exclude that did not match the stored path form), that is
206
+ // exactly the silent leak this whole check exists to catch.
207
+ else if (contentDenied.has(rel)) leaked.push(`${rel} (content deny-list — the --exclude did not take)`);
208
+ if (leaked.length >= 10) break;
209
+ }
210
+ if (leaked.length) {
211
+ rmSync(archivePath, { force: true });
212
+ log("error", `REFUSING to keep an archive containing denied paths — DELETED it. Offenders: ${leaked.join("; ")}`);
213
+ process.exit(1);
214
+ }
215
+
216
+ const size = statSync(archivePath).size;
217
+ // Say exactly what was verified. "verified clean" over a path-only check is the
218
+ // false assurance that let a live API key ship.
219
+ log(
220
+ "info",
221
+ `archive verified: ${archiveName} (${size} bytes, ${listing.split("\n").length - 1} entries) — ` +
222
+ `no denied PATHS, and ${contentScan.leaks.length} content-flagged file(s) confirmed absent ` +
223
+ `(${contentScan.scanned} file(s) content-scanned, ${contentScan.oversize.length} too large to scan)`
224
+ );
225
+
226
+ // ── 5a. prune local restore points ──────────────────────────────────────
227
+ let pruned = 0;
228
+ if (plan.local.enabled) {
229
+ const cutoff = Date.now() - plan.local.retentionDays * 86_400_000;
230
+ for (const name of safeReaddir(localDir)) {
231
+ if (!name.startsWith(`${plan.prefix}-`) || !name.endsWith(".tar.gz")) continue;
232
+ const full = join(localDir, name);
233
+ if (full === archivePath) continue;
234
+ try {
235
+ if (statSync(full).mtimeMs < cutoff) {
236
+ rmSync(full, { force: true });
237
+ pruned++;
238
+ }
239
+ } catch { /* a file that vanished under us is already pruned */ }
240
+ }
241
+ if (pruned) log("info", `pruned ${pruned} local restore point(s) older than ${plan.local.retentionDays}d`);
242
+ }
243
+
244
+ // ── 5b. offsite ─────────────────────────────────────────────────────────
245
+ let offsiteOk = false;
246
+ let offsiteDetail = "not configured";
247
+ if (plan.offsite.configured) {
248
+ const key = `${plan.prefix}/${archiveName}`;
249
+ try {
250
+ if (plan.offsite.provider === "gcs") {
251
+ requireBinary("gsutil", "install the gcloud SDK, or switch offsite.provider");
252
+ execFileSync("gsutil", ["cp", archivePath, `gs://${stripScheme(plan.offsite.bucket)}/${key}`], { stdio: "pipe" });
253
+ } else if (plan.offsite.provider === "s3") {
254
+ requireBinary("aws", "install the AWS CLI, or switch offsite.provider");
255
+ execFileSync("aws", ["s3", "cp", archivePath, `s3://${stripScheme(plan.offsite.bucket)}/${key}`], { stdio: "pipe" });
256
+ } else if (plan.offsite.provider === "rsync") {
257
+ requireBinary("rsync", "install rsync, or switch offsite.provider");
258
+ execFileSync("rsync", ["-az", archivePath, `${plan.offsite.bucket.replace(/\/+$/, "")}/${plan.prefix}/`], { stdio: "pipe" });
259
+ }
260
+ offsiteOk = true;
261
+ offsiteDetail = `${plan.offsite.provider}://${plan.offsite.bucket}/${key}`;
262
+ log("info", `offsite copy complete → ${offsiteDetail}`);
263
+ } catch (err) {
264
+ // A failed offsite push does NOT discard the local restore point — we are
265
+ // strictly better off than before. But it is never silent: the marker
266
+ // records the failure and doctor reads the marker.
267
+ offsiteDetail = `FAILED: ${err && err.message}`;
268
+ log("error", `offsite copy failed (local restore point kept): ${err && err.message}`);
269
+ }
270
+ } else {
271
+ log("warn", "offsite is NOT configured — these restore points do not survive losing this machine. Set offsite.provider + offsite.bucket in .maestro/backup-config.yaml.");
272
+ }
273
+
274
+ // ── 5c. prune OFFSITE to retention ──────────────────────────────────────
275
+ // `offsite.retention_days` was parsed into the plan, shipped in the default
276
+ // config beside a comment describing Tier 2 as working, and consumed by
277
+ // NOTHING: the only pruning was local, on `local.retentionDays`. An operator
278
+ // who set `retention_days: 30` got a bucket that grew forever and was never
279
+ // told. Prune with the same CLI that uploaded, over our OWN key prefix only.
280
+ let offsitePruned = 0;
281
+ let offsitePruneDetail = "not attempted";
282
+ if (offsiteOk) {
283
+ try {
284
+ offsitePruned = pruneOffsite(plan, log);
285
+ offsitePruneDetail = `${offsitePruned} archive(s) older than ${plan.offsite.retentionDays}d removed`;
286
+ } catch (err) {
287
+ // Never fatal: the restore points exist, which is the point of the job.
288
+ offsitePruneDetail = `FAILED: ${err && err.message}`;
289
+ log("error", `offsite retention prune failed (the upload succeeded; old archives remain): ${err && err.message}`);
290
+ }
291
+ }
292
+
293
+ // Staging dir (Tier 1 off) is not a restore point — clear it once pushed.
294
+ if (!plan.local.enabled && offsiteOk) {
295
+ try { rmSync(archivePath, { force: true }); } catch { /* best effort */ }
296
+ }
297
+
298
+ // ── 6. stamp the marker doctor reads ────────────────────────────────────
299
+ const marker = {
300
+ completed_at: new Date().toISOString(),
301
+ tier: offsiteOk ? "offsite" : "local",
302
+ provider: offsiteOk ? plan.offsite.provider : "local",
303
+ bucket: offsiteOk ? plan.offsite.bucket : localDir,
304
+ prefix: plan.prefix,
305
+ archive: archiveName,
306
+ size_bytes: size,
307
+ include: present,
308
+ local_dir: localDir,
309
+ local_pruned: pruned,
310
+ offsite: offsiteDetail,
311
+ offsite_retention_days: plan.offsite.configured ? plan.offsite.retentionDays : null,
312
+ offsite_pruned: offsitePruneDetail,
313
+ denied_include_paths: plan.violations,
314
+ // The CONTENT deny-list's findings, on the record. A leak that is dropped
315
+ // silently teaches nobody; the operator needs to know a credential was
316
+ // sitting in an archived file and rotate it.
317
+ denied_content_paths: contentScan.leaks,
318
+ content_scanned_files: contentScan.scanned,
319
+ content_unscanned_oversize: contentScan.oversize,
320
+ };
321
+ writeFileSync(join(AGENT_DIR, ".maestro", "last-backup.json"), `${JSON.stringify(marker, null, 2)}\n`);
322
+ log("info", `backup complete (tier=${marker.tier}).`);
323
+ }
324
+
325
+ function safeReaddir(dir) {
326
+ try {
327
+ return readdirSync(dir);
328
+ } catch {
329
+ return [];
330
+ }
331
+ }
332
+
333
+ function stripScheme(bucket) {
334
+ return String(bucket).replace(/^(gs|s3):\/\//, "").replace(/\/+$/, "");
335
+ }
336
+
337
+ /**
338
+ * Delete offsite archives older than `offsite.retentionDays`.
339
+ *
340
+ * Scoped THREE ways so this can never delete something that is not ours:
341
+ * - only under `<bucket>/<prefix>/`
342
+ * - only names matching `<prefix>-<ISO stamp>.tar.gz`
343
+ * - only stamps older than the retention window (parsed from the NAME, not
344
+ * from remote metadata, so a re-upload cannot resurrect an object)
345
+ *
346
+ * rsync targets are deliberately NOT pruned: doing so means running a remote
347
+ * `find -delete` over ssh against a path from a config file, which is a much
348
+ * bigger gun than a bucket delete. The operator is TOLD rather than left to
349
+ * assume — the honest half of "fail-open is fine, silent is not".
350
+ *
351
+ * @returns {number} objects removed
352
+ */
353
+ function pruneOffsite(plan, log) {
354
+ const days = plan.offsite.retentionDays;
355
+ if (!Number.isFinite(days) || days <= 0) return 0;
356
+ const cutoff = Date.now() - days * 86_400_000;
357
+ const prefix = plan.prefix;
358
+ const bucket = stripScheme(plan.offsite.bucket);
359
+ const nameRe = new RegExp(`^${prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}-(.+)\\.tar\\.gz$`);
360
+
361
+ const ageOf = (name) => {
362
+ const m = nameRe.exec(name);
363
+ if (!m) return null;
364
+ // `2026-08-12T08-26-30-411Z` → restore the ISO punctuation.
365
+ const iso = m[1].replace(/T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, "T$1:$2:$3.$4Z");
366
+ const t = Date.parse(iso);
367
+ return Number.isFinite(t) ? t : null;
368
+ };
369
+
370
+ if (plan.offsite.provider === "rsync") {
371
+ log(
372
+ "warn",
373
+ `offsite.retention_days: ${days} is NOT enforced for the rsync provider — maestro will not run a remote ` +
374
+ "delete over ssh against a path from a config file. Age out archives on the receiving host (a cron " +
375
+ "`find <path> -name '*.tar.gz' -mtime +N -delete`), or switch offsite.provider to gcs/s3."
376
+ );
377
+ return 0;
378
+ }
379
+
380
+ const list = plan.offsite.provider === "gcs"
381
+ ? execFileSync("gsutil", ["ls", `gs://${bucket}/${prefix}/`], { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] })
382
+ : execFileSync("aws", ["s3", "ls", `s3://${bucket}/${prefix}/`], { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] });
383
+
384
+ let removed = 0;
385
+ for (const raw of list.split("\n")) {
386
+ const line = raw.trim();
387
+ if (!line) continue;
388
+ const name = line.split(/[\s/]+/).pop();
389
+ if (!name || !name.endsWith(".tar.gz")) continue;
390
+ const t = ageOf(name);
391
+ if (t === null || t >= cutoff) continue;
392
+ const key = `${prefix}/${name}`;
393
+ if (plan.offsite.provider === "gcs") {
394
+ execFileSync("gsutil", ["rm", `gs://${bucket}/${key}`], { stdio: "pipe" });
395
+ } else {
396
+ execFileSync("aws", ["s3", "rm", `s3://${bucket}/${key}`], { stdio: "pipe" });
397
+ }
398
+ removed += 1;
399
+ }
400
+ if (removed) log("info", `offsite prune: removed ${removed} archive(s) older than ${days}d from ${plan.offsite.provider}://${bucket}/${prefix}/`);
401
+ return removed;
402
+ }
403
+
404
+ function requireBinary(bin, hint) {
405
+ try {
406
+ execFileSync("/usr/bin/which", [bin], { stdio: "ignore" });
407
+ } catch {
408
+ throw new Error(`${bin} not found in PATH — ${hint}`);
409
+ }
410
+ }
411
+
412
+ main().catch((err) => {
413
+ log("error", `unhandled: ${err && err.stack ? err.stack : err}`);
414
+ process.exit(1);
415
+ });
@@ -1,124 +1,24 @@
1
1
  #!/bin/bash
2
- # backup-to-cloud.sh — Off-machine state backup driver.
2
+ # backup-to-cloud.sh — COMPATIBILITY WRAPPER.
3
3
  #
4
- # Reads .maestro/backup-config.yaml and pushes the configured paths to GCS
5
- # or S3 (or rsyncs to a remote host). Designed to be invoked daily via a
6
- # launchd plist. Idempotent; safe to re-run.
4
+ # The backup driver moved to scripts/maintenance/backup-run.mjs. This shell
5
+ # entrypoint stays because installed launchd plists and the runbook reference
6
+ # it by name; it now just execs the Node runner.
7
7
  #
8
- # If .maestro/backup-config.yaml is missing or `enabled: false`, exits 0
9
- # silently. Errors during upload are logged to logs/maintenance/backup.log
10
- # and surface in `maestro doctor`.
8
+ # Why the move: this script parsed YAML with awk, had no credential deny-list,
9
+ # required a cloud bucket before it would do anything, and — the part that
10
+ # actually cost us — `exit 0`'d SILENTLY when unconfigured. A fleet can run for
11
+ # months with zero agents backed up and nothing red anywhere. backup-run.mjs
12
+ # refuses loudly with a non-zero exit instead, and writes local restore points
13
+ # with no credentials at all so the default posture is "something", not "nothing".
14
+ #
15
+ # See lib/backup/policy.mjs for the posture (what/where/never) and
16
+ # docs/runbooks/backup-restore.md for the restore path.
11
17
 
12
18
  set -e
13
19
 
14
20
  SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
15
- AGENT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
16
- CFG="$AGENT_DIR/.maestro/backup-config.yaml"
17
- LOG_DIR="$AGENT_DIR/logs/maintenance"
18
- LOG="$LOG_DIR/backup.log"
19
- mkdir -p "$LOG_DIR"
20
-
21
- log() { echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] $1" | tee -a "$LOG"; }
22
-
23
- if [ ! -f "$CFG" ]; then
24
- log "backup-config.yaml not found — backup not configured. Run: maestro init backup-replication --apply"
25
- exit 0
26
- fi
27
-
28
- # Tiny YAML reader (only the fields we need).
29
- get() {
30
- awk -v k="$1" '
31
- $1 == k":" { sub(/^[^:]+:[[:space:]]*/, "", $0); gsub(/^"|"$/, "", $0); print; exit }
32
- ' "$CFG"
33
- }
34
-
35
- ENABLED=$(get enabled)
36
- if [ "$ENABLED" != "true" ]; then
37
- log "Backup not enabled (.maestro/backup-config.yaml: enabled: false). Skipping."
38
- exit 0
39
- fi
40
-
41
- PROVIDER=$(get provider)
42
- BUCKET=$(get bucket)
43
- PREFIX=$(get prefix)
44
- RETENTION=$(get retention_days)
45
- [ -z "$RETENTION" ] && RETENTION=30
46
-
47
- if [ -z "$BUCKET" ] || [ -z "$PROVIDER" ] || [ -z "$PREFIX" ]; then
48
- log "ERROR: backup-config.yaml missing required fields (provider / bucket / prefix)."
49
- exit 1
50
- fi
51
-
52
- DATE=$(date -u +"%Y-%m-%d")
53
- log "Starting backup → $PROVIDER://$BUCKET/$PREFIX/$DATE/"
54
-
55
- # Default include paths if the YAML doesn't override them.
56
- INCLUDE_DEFAULT=(state knowledge outputs config/agent.json)
57
- INCLUDE_PATHS=()
58
- in_include=0
59
- while IFS= read -r line; do
60
- if echo "$line" | grep -qE "^include:"; then in_include=1; continue; fi
61
- if [ "$in_include" = "1" ]; then
62
- if echo "$line" | grep -qE "^[a-z]"; then in_include=0; continue; fi
63
- path=$(echo "$line" | sed 's/^[[:space:]]*-[[:space:]]*//' | sed 's/[[:space:]]*$//')
64
- [ -n "$path" ] && INCLUDE_PATHS+=("$path")
65
- fi
66
- done < "$CFG"
67
- [ ${#INCLUDE_PATHS[@]} -eq 0 ] && INCLUDE_PATHS=("${INCLUDE_DEFAULT[@]}")
68
-
69
- # Build the archive
70
- TMP_DIR=$(mktemp -d)
71
- trap "rm -rf $TMP_DIR" EXIT
72
- TARBALL="$TMP_DIR/$PREFIX-$DATE.tar.gz"
73
-
74
- log "Packaging: ${INCLUDE_PATHS[*]}"
75
- tar --exclude="state/tmp" --exclude="state/rag/index" --exclude=".DS_Store" \
76
- -czf "$TARBALL" -C "$AGENT_DIR" "${INCLUDE_PATHS[@]}" 2>>"$LOG" || {
77
- log "ERROR: tar failed"
78
- exit 1
79
- }
80
- SIZE=$(stat -f%z "$TARBALL")
81
- log "Packaged: $TARBALL ($SIZE bytes)"
82
-
83
- # Upload — choose provider.
84
- case "$PROVIDER" in
85
- gcs)
86
- if ! command -v gsutil >/dev/null 2>&1; then
87
- log "ERROR: gsutil not found in PATH. Install gcloud SDK or change provider."
88
- exit 1
89
- fi
90
- gsutil cp "$TARBALL" "gs://$BUCKET/$PREFIX/$DATE/$(basename "$TARBALL")" >>"$LOG" 2>&1
91
- ;;
92
- s3)
93
- if ! command -v aws >/dev/null 2>&1; then
94
- log "ERROR: aws CLI not found in PATH. Install awscli or change provider."
95
- exit 1
96
- fi
97
- aws s3 cp "$TARBALL" "s3://$BUCKET/$PREFIX/$DATE/$(basename "$TARBALL")" >>"$LOG" 2>&1
98
- ;;
99
- rsync)
100
- # bucket field carries the rsync target (user@host:/path)
101
- rsync -avz "$TARBALL" "$BUCKET/$PREFIX/$DATE/" >>"$LOG" 2>&1
102
- ;;
103
- *)
104
- log "ERROR: unknown provider '$PROVIDER'"
105
- exit 1
106
- ;;
107
- esac
108
-
109
- log "Upload complete: $PROVIDER://$BUCKET/$PREFIX/$DATE/"
110
-
111
- # Update .maestro/last-backup.json so doctor can verify freshness.
112
- mkdir -p "$AGENT_DIR/.maestro"
113
- cat > "$AGENT_DIR/.maestro/last-backup.json" <<JSON
114
- {
115
- "completed_at": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
116
- "provider": "$PROVIDER",
117
- "bucket": "$BUCKET",
118
- "prefix": "$PREFIX",
119
- "size_bytes": $SIZE,
120
- "tarball": "$(basename "$TARBALL")"
121
- }
122
- JSON
21
+ AGENT_DIR="${AGENT_ROOT:-${AGENT_DIR:-$(cd "$SCRIPT_DIR/../.." && pwd)}}"
123
22
 
124
- log "Backup complete."
23
+ NODE_BIN="$(command -v node || echo /opt/homebrew/bin/node)"
24
+ exec "$NODE_BIN" "$SCRIPT_DIR/backup-run.mjs" --agent-dir "$AGENT_DIR" "$@"
@@ -214,6 +214,22 @@ async function main() {
214
214
 
215
215
  const status = frame.result?.status;
216
216
  if (status === "queued" || status === "deduped") {
217
+ // Witness the send for the daemon's answer-assurance ledger. This CLI is the
218
+ // sanctioned orgmail lane for a spawned session, and a session is a separate
219
+ // process: without a receipt the daemon reads a delivered reply as silence
220
+ // and apologises to a human who already has their answer. Attribution rides
221
+ // in on the dispatcher's env. Best-effort — never fails the send.
222
+ try {
223
+ const { recordOutbound } = await import("../../lib/comms/receipts.mjs");
224
+ recordOutbound({
225
+ service: "orgmail",
226
+ channel: frame.result.threadId || flags.thread || flags.to[0],
227
+ kind: "session",
228
+ via: "send-orgmail.mjs",
229
+ chars: String(text).length,
230
+ agentRoot: cfg.agentRoot,
231
+ });
232
+ } catch { /* bookkeeping must never break a send */ }
217
233
  console.log(`send-orgmail: ${status} — messageId ${frame.result.messageId} threadId ${frame.result.threadId}`);
218
234
  process.exit(0);
219
235
  }