@cohortapp/agent-sdk 2.5.1 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/bin/maestro.mjs +185 -88
  2. package/bin/maestro.test.mjs +175 -48
  3. package/docs/runbooks/backup-restore.md +65 -33
  4. package/framework-features.json +4 -4
  5. package/lib/backup/policy.mjs +710 -0
  6. package/lib/backup/policy.test.mjs +305 -0
  7. package/lib/budget-escalate.mjs +133 -0
  8. package/lib/budget-escalate.test.mjs +232 -0
  9. package/lib/budget-guard.envelope.test.mjs +476 -0
  10. package/lib/budget-guard.mjs +853 -75
  11. package/lib/budget-guard.test.mjs +91 -42
  12. package/lib/cadences.mjs +33 -0
  13. package/lib/channels/orgmail/adapter.mjs +88 -3
  14. package/lib/channels/orgmail/adapter.test.mjs +137 -0
  15. package/lib/channels/repeat-suppressor.mjs +198 -0
  16. package/lib/channels/repeat-suppressor.test.mjs +134 -0
  17. package/lib/comms/receipts.mjs +297 -0
  18. package/lib/cost/ledger-row.mjs +333 -0
  19. package/lib/cost/ledger-row.test.mjs +183 -0
  20. package/lib/execution/drive.mjs +28 -1
  21. package/lib/execution/effects.mjs +191 -12
  22. package/lib/execution/effects.test.mjs +50 -11
  23. package/lib/goals/admission.mjs +13 -1
  24. package/lib/goals/admission.test.mjs +26 -1
  25. package/lib/goals/loop.mjs +13 -0
  26. package/lib/kpi-sensors.test.mjs +3 -0
  27. package/lib/mandate/cache.mjs +13 -5
  28. package/lib/mandate/derive.mjs +146 -21
  29. package/lib/mandate/derive.test.mjs +50 -6
  30. package/lib/mandate/model.mjs +32 -4
  31. package/lib/mandate/refresh.test.mjs +16 -2
  32. package/lib/mcp/server.test.mjs +12 -3
  33. package/lib/model-router/economics.mjs +107 -76
  34. package/lib/model-router/economics.test.mjs +64 -46
  35. package/lib/model-router/integration-coverage.test.mjs +39 -37
  36. package/lib/model-router/ledger.mjs +75 -22
  37. package/lib/model-router/ledger.test.mjs +35 -2
  38. package/lib/org/client.mjs +14 -0
  39. package/lib/org/cost-sync.mjs +16 -2
  40. package/lib/org/doctor.mjs +62 -1
  41. package/lib/org/doctor.test.mjs +36 -3
  42. package/lib/org/email-remedy.mjs +49 -0
  43. package/lib/org/engagement-ledger.mjs +376 -0
  44. package/lib/org/engagement-ledger.test.mjs +112 -0
  45. package/lib/org/engagement.mjs +1056 -0
  46. package/lib/org/engagement.test.mjs +739 -0
  47. package/lib/org/messaging.mjs +230 -3
  48. package/lib/org/messaging.test.mjs +110 -1
  49. package/lib/org/param-contract.mjs +56 -2
  50. package/lib/org/param-contract.test.mjs +26 -0
  51. package/lib/org/protocol.checksum +1 -1
  52. package/lib/org/protocol.mjs +5 -0
  53. package/lib/org/protocol.test.mjs +7 -1
  54. package/lib/org/tool-surface.mjs +506 -10
  55. package/lib/org/tool-surface.test.mjs +191 -7
  56. package/lib/org/ui-parity.mjs +333 -6
  57. package/lib/org/ui-parity.test.mjs +96 -3
  58. package/lib/org/work-ledger.mjs +241 -0
  59. package/lib/org/work-ledger.test.mjs +237 -0
  60. package/lib/plan/adoption-e2e.test.mjs +366 -0
  61. package/lib/plan/budget-enforcement.test.mjs +400 -0
  62. package/lib/plan/budget-runtime.mjs +215 -0
  63. package/lib/plan/compile.mjs +201 -5
  64. package/lib/plan/compile.test.mjs +19 -5
  65. package/lib/plan/emit.mjs +8 -0
  66. package/lib/plan/emit.test.mjs +18 -0
  67. package/lib/resource-governor.mjs +58 -12
  68. package/lib/resource-governor.test.mjs +41 -1
  69. package/lib/security/audit-engine.mjs +45 -8
  70. package/lib/security/audit-engine.test.mjs +35 -0
  71. package/lib/setup/enroll-from-cohort.mjs +14 -1
  72. package/lib/setup/sections/mandate.mjs +48 -7
  73. package/lib/setup/sections/mandate.test.mjs +17 -2
  74. package/lib/setup/sections/orgmail.mjs +10 -2
  75. package/lib/setup/state.mjs +83 -2
  76. package/lib/telemetry/collect.mjs +360 -20
  77. package/lib/telemetry/collect.test.mjs +266 -0
  78. package/package.json +1 -1
  79. package/scripts/cost/track-claude-usage.mjs +207 -48
  80. package/scripts/cost/track-claude-usage.test.mjs +148 -0
  81. package/scripts/daemon/agent-daemon.mjs +315 -17
  82. package/scripts/daemon/assurance-e2e.test.mjs +421 -0
  83. package/scripts/daemon/assurance.mjs +944 -0
  84. package/scripts/daemon/assurance.test.mjs +668 -0
  85. package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
  86. package/scripts/daemon/cadence-consumer.mjs +147 -9
  87. package/scripts/daemon/cadence-consumer.test.mjs +6 -0
  88. package/scripts/daemon/cadence-handlers.mjs +158 -0
  89. package/scripts/daemon/cadence-handlers.test.mjs +64 -0
  90. package/scripts/daemon/deliver.mjs +314 -0
  91. package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
  92. package/scripts/daemon/dispatcher.mjs +64 -6
  93. package/scripts/daemon/responder-cost.test.mjs +68 -0
  94. package/scripts/daemon/responder.mjs +351 -298
  95. package/scripts/local-triggers/generate-plists.test.mjs +7 -4
  96. package/scripts/maintenance/backup-run.mjs +415 -0
  97. package/scripts/maintenance/backup-to-cloud.sh +16 -116
  98. package/scripts/org/send-orgmail.mjs +16 -0
  99. package/scripts/record-receipt.sh +63 -0
  100. package/scripts/restore-from-backup.sh +14 -3
  101. package/scripts/restore-from-backup.test.mjs +8 -5
  102. package/scripts/send-email-threaded.py +47 -0
  103. package/scripts/send-sms.sh +4 -0
  104. package/scripts/send-whatsapp.sh +4 -0
  105. package/scripts/setup/init-backup.mjs +93 -38
  106. package/scripts/slack-send.sh +12 -0
@@ -358,60 +358,123 @@ test("setup --status in an agent dir lists section detect() statuses", async ()
358
358
  // doctor cost-telemetry tripwire — pure predicate (observability F2)
359
359
  // ---------------------------------------------------------------------------
360
360
 
361
- test("summariseLedgerRows sums estimated_usd and counts rows; tolerates junk", () => {
361
+ /** A measured session row in the shape the tracker now writes. */
362
+ function measuredRow(over = {}) {
363
+ return {
364
+ ts: new Date().toISOString(), cadence: "inbox", source: "dispatcher",
365
+ model: "sonnet", input_tokens: 16, output_tokens: 5000,
366
+ cache_read_input_tokens: 553181, measurement: "measured",
367
+ estimated_usd: 0.241, total_cost_usd: 0.533412, ...over,
368
+ };
369
+ }
370
+ /** A session that ran but whose usage could not be read. */
371
+ function unmeasuredRow(over = {}) {
372
+ return {
373
+ ts: new Date().toISOString(), cadence: "inbox", source: "dispatcher",
374
+ model: "sonnet", input_tokens: null, output_tokens: null,
375
+ measurement: "unknown", unmeasured_reason: "no-usage-field",
376
+ estimated_usd: null, total_cost_usd: null, ...over,
377
+ };
378
+ }
379
+ /** A message send: no model ran, so $0 is a fact rather than a gap. */
380
+ function nonLlmRow(over = {}) {
381
+ return {
382
+ ts: new Date().toISOString(), source: "messaging", task_class: "messaging.send",
383
+ model: null, input_tokens: 0, output_tokens: 0, estimated_usd: 0, ...over,
384
+ };
385
+ }
386
+
387
+ test("summariseLedgerRows bills the authoritative cost, not the cache-blind estimate", () => {
388
+ const s = summariseLedgerRows([measuredRow(), measuredRow({ total_cost_usd: 1 })]);
389
+ assert.equal(s.rowCount, 2);
390
+ assert.equal(s.totalUsd, 1.533412, "total_cost_usd wins over estimated_usd");
391
+ });
392
+
393
+ test("summariseLedgerRows counts sessions, not attribution rows; tolerates junk", () => {
362
394
  const s = summariseLedgerRows([
363
- { estimated_usd: 0.5 },
364
- { estimated_usd: 1.25 },
365
- { estimated_usd: "0.25" }, // numeric string coerced
366
- { estimated_usd: NaN }, // ignored as 0
367
- {}, // counts as a row, $0
368
- null, // skipped entirely
369
- "garbage", // skipped entirely
395
+ measuredRow(), unmeasuredRow(), nonLlmRow(), null, "garbage",
370
396
  ]);
371
- assert.equal(s.rowCount, 5, "only object rows count");
372
- assert.equal(s.totalUsd, 2, "0.5 + 1.25 + 0.25 = 2.00");
397
+ assert.equal(s.rowCount, 2, "1 measured + 1 unmeasured; the send is not a session");
398
+ assert.equal(s.measured, 1);
399
+ assert.equal(s.unmeasured, 1);
400
+ assert.equal(s.nonLlmRows, 1);
373
401
  });
374
402
 
375
403
  test("summariseLedgerRows on empty input is zeroed", () => {
376
- assert.deepEqual(summariseLedgerRows([]), { rowCount: 0, totalUsd: 0 });
377
- assert.deepEqual(summariseLedgerRows(undefined), { rowCount: 0, totalUsd: 0 });
404
+ assert.equal(summariseLedgerRows([]).rowCount, 0);
405
+ assert.equal(summariseLedgerRows([]).totalUsd, 0);
406
+ assert.equal(summariseLedgerRows(undefined).rowCount, 0);
407
+ });
408
+
409
+ test("evaluateCostTripwire: unmeasured rows → RED (the 0-vs-unknown distinction)", () => {
410
+ // The defect doctor exists to catch: a session ran and nobody priced it.
411
+ const t = evaluateCostTripwire({ rows: [measuredRow(), unmeasuredRow()], sessionCount: 2 });
412
+ assert.equal(t.red, true);
413
+ assert.equal(t.unmeasured, 1);
414
+ assert.match(t.reason, /NO token counts/);
415
+ assert.match(t.fix, /unmeasured_reason/);
378
416
  });
379
417
 
380
- test("evaluateCostTripwire: rows exist but spend is $0 → RED (telemetry blind)", () => {
418
+ test("evaluateCostTripwire: a genuine zero-token session is NOT unmeasured", () => {
419
+ // A model ran and emitted nothing. That is a measurement; it must stay GREEN
420
+ // as long as the day bills above $0 — otherwise doctor cries wolf.
381
421
  const t = evaluateCostTripwire({
382
- rows: [{ estimated_usd: 0 }, { estimated_usd: 0 }, { estimated_usd: 0 }],
383
- sessionCount: 0,
422
+ rows: [measuredRow(), measuredRow({ input_tokens: 40, output_tokens: 0, total_cost_usd: 0 })],
423
+ sessionCount: 2,
384
424
  });
425
+ assert.equal(t.red, false);
426
+ assert.equal(t.unmeasured, 0);
427
+ });
428
+
429
+ test("evaluateCostTripwire: sessions logged but no ledger rows → RED (unrecorded)", () => {
430
+ const t = evaluateCostTripwire({ rows: [], sessionCount: 3 });
385
431
  assert.equal(t.red, true);
386
- assert.equal(t.status, "red");
387
- assert.equal(t.rowCount, 3);
388
- assert.equal(t.totalUsd, 0);
389
- assert.match(t.reason, /blind/i);
432
+ assert.equal(t.unrecorded, 3);
433
+ assert.match(t.reason, /only 0 dispatcher row\(s\) reached the cost ledger/);
434
+ });
435
+
436
+ test("evaluateCostTripwire: rows from ANOTHER writer do not mask an unrecorded dispatcher", () => {
437
+ // The population bug this replaced. `sessionCount` comes from the dispatcher's
438
+ // own session log; `rowCount` counted rows from every writer. With the
439
+ // responder now writing rows too, a dispatcher that had stopped recording
440
+ // entirely read as unrecorded:0 / GREEN — 35 real sessions unpriced, and the
441
+ // one detector meant to catch exactly that said nothing.
442
+ const responderRows = Array.from({ length: 40 }, () => measuredRow({ source: "responder" }));
443
+ const t = evaluateCostTripwire({ rows: responderRows, sessionCount: 35 });
444
+ assert.equal(t.red, true, "35 dispatcher sessions with 0 dispatcher rows is RED");
445
+ assert.equal(t.unrecorded, 35);
446
+ assert.equal(t.sourceRowCount, 0);
447
+ assert.match(t.reason, /dispatcher logged 35/);
390
448
  });
391
449
 
392
- test("evaluateCostTripwire: no rows but heartbeat-active session → RED", () => {
393
- // Daemon is live (heartbeat) but the ledger is empty → still blind.
394
- const t = evaluateCostTripwire({ rows: [], sessionCount: 1 });
450
+ test("evaluateCostTripwire: measured rows billing $0 → RED (pricing broken)", () => {
451
+ const t = evaluateCostTripwire({
452
+ rows: [measuredRow({ total_cost_usd: 0, estimated_usd: 0 })],
453
+ sessionCount: 1,
454
+ });
395
455
  assert.equal(t.red, true);
396
- assert.equal(t.status, "red");
397
- assert.match(t.reason, /blind/i);
456
+ assert.match(t.reason, /bill to \$0/);
457
+ assert.match(t.fix, /--cache-read-tokens/);
398
458
  });
399
459
 
400
460
  test("evaluateCostTripwire: no sessions and no rows → GREEN (nothing to measure)", () => {
401
461
  const t = evaluateCostTripwire({ rows: [], sessionCount: 0 });
402
462
  assert.equal(t.red, false);
403
- assert.equal(t.status, "green");
404
463
  assert.match(t.reason, /nothing to measure/i);
405
464
  });
406
465
 
407
- test("evaluateCostTripwire: spend > 0 → GREEN even with sessions", () => {
408
- const t = evaluateCostTripwire({
409
- rows: [{ estimated_usd: 0 }, { estimated_usd: 0.42 }],
410
- sessionCount: 1,
411
- });
466
+ test("evaluateCostTripwire: attribution-only day is quiet, not blind", () => {
467
+ // Regression guard for the false alarm this replaced: 41 message-send rows on
468
+ // 2026-08-11 would have armed the old tripwire all by themselves.
469
+ const t = evaluateCostTripwire({ rows: [nonLlmRow(), nonLlmRow()], sessionCount: 0 });
412
470
  assert.equal(t.red, false);
413
- assert.equal(t.status, "green");
414
- assert.equal(t.totalUsd, 0.42);
471
+ assert.match(t.reason, /nothing to measure/i);
472
+ });
473
+
474
+ test("evaluateCostTripwire: spend > 0 with every session measured → GREEN", () => {
475
+ const t = evaluateCostTripwire({ rows: [measuredRow()], sessionCount: 1 });
476
+ assert.equal(t.red, false);
477
+ assert.equal(t.totalUsd, 0.533412);
415
478
  assert.match(t.reason, /live/i);
416
479
  });
417
480
 
@@ -442,19 +505,19 @@ async function makeBareAgent() {
442
505
 
443
506
  function utcDate() { return new Date().toISOString().slice(0, 10); }
444
507
 
445
- test("doctor RED when today's ledger has rows but $0 spend", async () => {
508
+ test("doctor RED when a session ran with no token counts recorded", async () => {
446
509
  const root = await makeBareAgent();
447
510
  try {
448
511
  mkdirSync(join(root, "state/cost-tracking"), { recursive: true });
449
512
  const ledger = join(root, "state/cost-tracking", `${utcDate()}.jsonl`);
450
513
  writeFileSync(ledger,
451
- JSON.stringify({ ts: new Date().toISOString(), cadence: "inbox-processor", model: "sonnet", input_tokens: 0, output_tokens: 0, estimated_usd: 0 }) + "\n" +
452
- JSON.stringify({ ts: new Date().toISOString(), cadence: "backlog-executor", model: "opus", input_tokens: 0, output_tokens: 0, estimated_usd: 0 }) + "\n"
514
+ JSON.stringify(unmeasuredRow({ unmeasured_reason: "no-usage-field" })) + "\n" +
515
+ JSON.stringify(unmeasuredRow({ cadence: "backlog", unmeasured_reason: "empty-stdout" })) + "\n"
453
516
  );
454
517
  const r = runCli(["doctor"], root);
455
518
  const out = (r.stdout || "") + (r.stderr || "");
456
519
  assert.match(out, /Cost telemetry BLIND/);
457
- assert.match(out, /track-claude-usage\.mjs/);
520
+ assert.match(out, /NO token counts/);
458
521
  } finally { await rmRoot(root); }
459
522
  });
460
523
 
@@ -463,12 +526,10 @@ test("doctor GREEN when today's ledger records real spend", async () => {
463
526
  try {
464
527
  mkdirSync(join(root, "state/cost-tracking"), { recursive: true });
465
528
  const ledger = join(root, "state/cost-tracking", `${utcDate()}.jsonl`);
466
- writeFileSync(ledger,
467
- JSON.stringify({ ts: new Date().toISOString(), cadence: "inbox-processor", model: "sonnet", input_tokens: 1500, output_tokens: 320, estimated_usd: 0.0093 }) + "\n"
468
- );
529
+ writeFileSync(ledger, JSON.stringify(measuredRow()) + "\n");
469
530
  const r = runCli(["doctor"], root);
470
531
  const out = (r.stdout || "") + (r.stderr || "");
471
- assert.match(out, /Cost telemetry live/);
532
+ assert.match(out, /Cost telemetry: spend telemetry live/);
472
533
  assert.ok(!out.includes("Cost telemetry BLIND"), "must not flag blind when spend>0");
473
534
  } finally { await rmRoot(root); }
474
535
  });
@@ -478,7 +539,23 @@ test("doctor GREEN (nothing to measure) when no ledger exists today", async () =
478
539
  try {
479
540
  const r = runCli(["doctor"], root);
480
541
  const out = (r.stdout || "") + (r.stderr || "");
481
- assert.match(out, /no sessions recorded today/);
542
+ assert.match(out, /nothing to measure/);
543
+ } finally { await rmRoot(root); }
544
+ });
545
+
546
+ test("doctor does NOT cry wolf on an empty ledger while the daemon is live", async () => {
547
+ // The regression this replaced: doctor keyed "sessions ran" off a fresh
548
+ // heartbeat, so every UTC day went RED between midnight and the first session
549
+ // and printed a fix that had already shipped.
550
+ const root = await makeBareAgent();
551
+ try {
552
+ mkdirSync(join(root, "state/cadence-bus"), { recursive: true });
553
+ writeFileSync(join(root, "state/cadence-bus/health.json"),
554
+ JSON.stringify({ ts: new Date().toISOString() }));
555
+ const r = runCli(["doctor"], root);
556
+ const out = (r.stdout || "") + (r.stderr || "");
557
+ assert.ok(!out.includes("Cost telemetry BLIND"), "a live daemon with no sessions yet is not blind");
558
+ assert.match(out, /nothing to measure/);
482
559
  } finally { await rmRoot(root); }
483
560
  });
484
561
 
@@ -860,30 +937,80 @@ test("secrets sync without a configured broker exits 1", async () => {
860
937
  // doctor — backup-config DR check (gaps-enterprise-ops G16)
861
938
  // ---------------------------------------------------------------------------
862
939
 
863
- test("doctor RED when no backup target is configured", async () => {
940
+ test("doctor RED when there is no DR posture at all, and names the one command that fixes it", async () => {
864
941
  const root = await makeBareAgent();
865
942
  try {
866
943
  const r = runCli(["doctor"], root);
867
944
  const out = (r.stdout || "") + (r.stderr || "");
868
- assert.match(out, /No backup target configured/);
945
+ assert.match(out, /No DR posture at all/);
869
946
  assert.match(out, /backup-config\.yaml/);
947
+ assert.match(out, /maestro init backup-replication --apply/);
870
948
  // doctor exits 1 whenever any issue is found; the missing backup is one.
871
949
  assert.equal(r.status, 1, out);
872
950
  } finally { await rmRoot(root); }
873
951
  });
874
952
 
875
- test("doctor GREEN for backup when configured + a fresh successful run is recorded", async () => {
953
+ test("doctor RED when backup is enabled but has never actually run", async () => {
954
+ const root = await makeBareAgent();
955
+ try {
956
+ mkdirSync(join(root, ".maestro"), { recursive: true });
957
+ writeFileSync(join(root, ".maestro/backup-config.yaml"), "enabled: true\nprefix: t\n");
958
+ const r = runCli(["doctor"], root);
959
+ const out = (r.stdout || "") + (r.stderr || "");
960
+ assert.match(out, /NEVER completed a run/);
961
+ assert.match(out, /backup-run\.mjs/);
962
+ } finally { await rmRoot(root); }
963
+ });
964
+
965
+ test("doctor WARNS (not GREEN) when restore points are fresh but local-only", async () => {
876
966
  const root = await makeBareAgent();
877
967
  try {
878
968
  mkdirSync(join(root, ".maestro"), { recursive: true });
879
- writeFileSync(join(root, ".maestro/backup-config.yaml"), "enabled: true\nprovider: s3\nbucket: agent-backups\n");
969
+ writeFileSync(join(root, ".maestro/backup-config.yaml"), "enabled: true\nprefix: t\nlocal:\n enabled: true\n");
970
+ writeFileSync(join(root, ".maestro/last-backup.json"), JSON.stringify({
971
+ completed_at: new Date().toISOString(), tier: "local", provider: "local",
972
+ }));
973
+ const r = runCli(["doctor"], root);
974
+ const out = (r.stdout || "") + (r.stderr || "");
975
+ assert.match(out, /LOCAL-ONLY/);
976
+ assert.match(out, /offsite\.provider/);
977
+ assert.ok(!out.includes("No DR posture at all"), "must not claim the config is missing");
978
+ } finally { await rmRoot(root); }
979
+ });
980
+
981
+ test("doctor GREEN for backup only when it is fresh AND off-machine", async () => {
982
+ const root = await makeBareAgent();
983
+ try {
984
+ mkdirSync(join(root, ".maestro"), { recursive: true });
985
+ writeFileSync(
986
+ join(root, ".maestro/backup-config.yaml"),
987
+ "enabled: true\nprefix: t\noffsite:\n provider: s3\n bucket: agent-backups\n"
988
+ );
989
+ writeFileSync(join(root, ".maestro/last-backup.json"), JSON.stringify({
990
+ completed_at: new Date().toISOString(), tier: "offsite", provider: "s3", bucket: "agent-backups",
991
+ }));
992
+ const r = runCli(["doctor"], root);
993
+ const out = (r.stdout || "") + (r.stderr || "");
994
+ assert.match(out, /Backup fresh \+ off-machine/);
995
+ assert.ok(!out.includes("No DR posture at all"), "must not flag missing backup when fresh");
996
+ assert.ok(!out.includes("LOCAL-ONLY"), "offsite is configured");
997
+ } finally { await rmRoot(root); }
998
+ });
999
+
1000
+ test("doctor reports an include path that the credential deny-list drops", async () => {
1001
+ const root = await makeBareAgent();
1002
+ try {
1003
+ mkdirSync(join(root, ".maestro"), { recursive: true });
1004
+ writeFileSync(
1005
+ join(root, ".maestro/backup-config.yaml"),
1006
+ "enabled: true\nprefix: t\ninclude:\n - state\n - .env\n"
1007
+ );
880
1008
  writeFileSync(join(root, ".maestro/last-backup.json"), JSON.stringify({
881
- completed_at: new Date().toISOString(), provider: "s3", bucket: "agent-backups",
1009
+ completed_at: new Date().toISOString(), tier: "local",
882
1010
  }));
883
1011
  const r = runCli(["doctor"], root);
884
1012
  const out = (r.stdout || "") + (r.stderr || "");
885
- assert.match(out, /Backup configured \+ fresh/);
886
- assert.ok(!out.includes("No backup target configured"), "must not flag missing backup when fresh");
1013
+ assert.match(out, /DROPPED by the credential deny-list/);
887
1014
  } finally { await rmRoot(root); }
888
1015
  });
889
1016
 
@@ -12,27 +12,47 @@ This complements `docs/runbooks/mac-mini-bootstrap.md` (fresh provisioning) and
12
12
  compromised machine, do NOT restore onto the same hardware — rebuild on a clean
13
13
  box.
14
14
 
15
- ## What is backed up, and the honest gaps
16
-
17
- Off-machine backup is driven by `scripts/maintenance/backup-to-cloud.sh`, reading
18
- `.maestro/backup-config.yaml`, run daily via launchd. Configure it with
19
- `maestro init backup-replication --apply` (`scripts/setup/init-backup.mjs`).
20
-
21
- Know these caveats before you rely on it:
22
-
23
- - **Backup is opt-in and silent when off.** If `.maestro/backup-config.yaml` is
24
- missing or `enabled: false`, the script exits 0 and does nothing. An agent with
25
- no backup configured looks identical to a healthy one in casual inspection —
26
- check explicitly (below).
27
- - **Coverage historically excluded `logs/` and the audit records.** Confirm your
28
- config's include paths cover `state/`, `logs/`, and `.maestro/`. If `logs/` is
29
- excluded, a dead machine loses that agent's local compliance history — the
30
- org-server export (next section) is your compensating control.
31
- - **`retention_days` may not be enforced** by older builds; do not assume old
32
- archives are pruned.
33
- - **Archives may be unencrypted** unless you enabled client-side encryption.
34
- Treat the backup bucket as sensitive, restrict access, and prefer an encrypted
35
- bucket and encrypted archives.
15
+ ## The posture: two tiers
16
+
17
+ Since 2026-08 the DR posture is **restore points first, offsite second**, driven
18
+ by `scripts/maintenance/backup-run.mjs` (the `nightly-backup` cadence, 03:10
19
+ local) reading `.maestro/backup-config.yaml`. `scripts/maintenance/backup-to-cloud.sh`
20
+ is now a thin compatibility wrapper around the same runner.
21
+
22
+ | Tier | Default | Where | Survives |
23
+ |---|---|---|---|
24
+ | 1 — local restore points | **ON**, zero credentials | `~/Library/Application Support/Maestro/backups/<prefix>/`, **outside** the agent repo | corrupted queue, bad state write, botched checkout, `rm -rf` of the repo |
25
+ | 2 — offsite copy | opt-in (`offsite.provider` + `offsite.bucket`) | gcs / s3 / rsync | machine loss, disk death, fire |
26
+
27
+ `maestro init backup-replication --apply` writes the Tier-1 default, and it is
28
+ `auto: true` — so a **new agent is born with a DR posture instead of a red
29
+ doctor line**. Tier 2 stays a deliberate act (it needs a bucket and credentials);
30
+ `maestro doctor` WARNs until it is set, naming the residual risk in words.
31
+
32
+ **Archived by default:** `state/`, `knowledge/`, `memory/`, `outputs/`,
33
+ `config/`, `.maestro/`. `logs/` is deliberately excluded — largest tree, least
34
+ value, and the authoritative audit trail is already the org event ledger (next
35
+ section). Add it explicitly to `include:` if you want it.
36
+
37
+ **NEVER archived, enforced in code and not overridable from the config**
38
+ (`lib/backup/policy.mjs` → `DENY_PATTERNS`): `.env`, `.env.*`,
39
+ `.cohort-key.json`, `.claude/.credentials.json`, `*.pem` / `*.key` / SSH keys,
40
+ `node_modules`, `.git`, `state/tmp`, the RAG index, sqlite WAL/SHM files.
41
+ `.env` carries `COHORT_API_KEY` (which authenticates as this member) and
42
+ `.cohort-key.json` is the device identity — a backup that shipped either to a
43
+ bucket would be a credential-exfiltration channel that looks *exactly* like a
44
+ successful backup. The deny-list is applied three times: it drops offending
45
+ `include:` entries at plan time (reporting them), it becomes `tar --exclude`
46
+ arguments, and the runner **lists the finished archive and deletes it** if a
47
+ denied path got in anyway. A restore re-pairs for a fresh token; it never
48
+ restores a dead identity.
49
+
50
+ Remaining honest gaps:
51
+
52
+ - **Archives are unencrypted at rest** unless you point Tier 2 at an encrypted
53
+ bucket. Treat the backup destination as sensitive and restrict access.
54
+ - **Tier 1 alone is not DR.** A local restore point on a dead disk is nothing.
55
+ Doctor says so explicitly; do not read the absence of a red line as safety.
36
56
 
37
57
  The org server is your fleet-as-DR layer for the audit record: every
38
58
  side-effecting action is a hash-chained `org_events` row at Cohort, exportable
@@ -46,18 +66,26 @@ non-repudiable record.
46
66
  ```bash
47
67
  cd ~/<agent>-ai
48
68
 
49
- # 1. Is backup configured and enabled?
50
- test -f .maestro/backup-config.yaml && grep -E '^enabled:' .maestro/backup-config.yaml
69
+ # 1. Is backup configured and enabled, and at which tier?
70
+ test -f .maestro/backup-config.yaml && grep -E '^enabled:|provider:|bucket:' .maestro/backup-config.yaml
51
71
 
52
- # 2. Did the last run succeed, and when?
72
+ # 2. Did the last run succeed, when, and to where?
73
+ cat .maestro/last-backup.json
53
74
  tail -n 20 logs/maintenance/backup.log
54
75
 
55
- # 3. Does the configured destination actually have today's archive?
56
- # (provider-specific; e.g. for GCS:)
57
- # gcloud storage ls "gs://<bucket>/<prefix>/$(date -u +%F)/"
76
+ # 3. Do the local restore points actually exist?
77
+ ls -lh ~/Library/Application\ Support/Maestro/backups/*/
58
78
 
59
- # 4. doctor surfaces backup freshness in the governance posture.
79
+ # 4. Does the offsite destination have the archive? (provider-specific)
80
+ # gcloud storage ls "gs://<bucket>/<prefix>/"
81
+
82
+ # 5. doctor's verdict — fail (no restore point) vs warn (local-only / stale)
83
+ # vs ok (fresh AND off-machine).
60
84
  maestro doctor
85
+
86
+ # 6. Force a run now, without waiting for 03:10.
87
+ node scripts/maintenance/backup-run.mjs
88
+ node scripts/maintenance/backup-run.mjs --dry-run # print the plan only
61
89
  ```
62
90
 
63
91
  If `maestro doctor` does not flag a missing/stale backup red, do not assume the
@@ -163,8 +191,10 @@ A rebuilt machine with no backup is the next dead machine.
163
191
  ```bash
164
192
  cd ~/<agent>-ai
165
193
  maestro init backup-replication --apply # if .maestro/backup-config.yaml absent
166
- # confirm include paths cover state/, logs/, .maestro/; confirm encryption on.
167
- maestro doctor # backup freshness should clear within a day
194
+ # Restore the OFFSITE block from the dead machine's config (Tier 1 is on by
195
+ # default; Tier 2 is the one that saves you next time).
196
+ node scripts/maintenance/backup-run.mjs # don't wait for 03:10 — prove it works now
197
+ maestro doctor # expect "Backup fresh + off-machine"
168
198
  ```
169
199
 
170
200
  ---
@@ -195,9 +225,11 @@ tested restore.
195
225
 
196
226
  | Need | Command |
197
227
  |---|---|
198
- | Is backup on? | `grep -E '^enabled:' .maestro/backup-config.yaml` |
199
- | Last backup status | `tail logs/maintenance/backup.log` |
200
- | Configure backup | `maestro init backup-replication --apply` |
228
+ | Is backup on, and at which tier? | `grep -E '^enabled:\|provider:\|bucket:' .maestro/backup-config.yaml` |
229
+ | Last backup status | `cat .maestro/last-backup.json` / `tail logs/maintenance/backup.log` |
230
+ | Configure backup (Tier 1 default) | `maestro init backup-replication --apply` |
231
+ | Run a backup now | `node scripts/maintenance/backup-run.mjs` |
232
+ | Add the offsite tier | set `offsite.provider` + `offsite.bucket` in `.maestro/backup-config.yaml` |
201
233
  | Governance posture incl. backup freshness | `maestro doctor` |
202
234
  | Re-mint agent identity | `cohort pair approve <agentId> --scopes <...>` |
203
235
  | Recover audit history | `cohort export-audit --format ocsf` |
@@ -94,13 +94,13 @@
94
94
  }
95
95
  },
96
96
  "backup-replication": {
97
- "version": "1",
97
+ "version": "2",
98
98
  "since": "1.9.0",
99
- "title": "Off-machine state backup",
99
+ "title": "Disaster-recovery backup (local restore points + optional offsite)",
100
100
  "init": {
101
- "auto": false,
101
+ "auto": true,
102
102
  "command": "node scripts/setup/init-backup.mjs",
103
- "description": "Configure off-machine backup of state/, knowledge/, outputs/, and rotated logs to GCS or S3. Requires bucket name + credentials. Doctor flags if last successful backup is >24h old once configured."
103
+ "description": "Write .maestro/backup-config.yaml with the default DR posture so a new agent is not born failing doctor. TIER 1 (on, zero credentials): nightly tar.gz of state/, knowledge/, memory/, outputs/, config/ and .maestro/ into ~/Library/Application Support/Maestro/backups/<prefix>/ — outside the agent repo — via the nightly-backup cadence at 03:10, 14-day retention. TIER 2 (opt-in): set offsite.provider (gcs|s3|rsync) + offsite.bucket to get a copy off the machine; doctor WARNS until you do, because Tier 1 alone does not survive losing the box. NEVER archived, enforced in code (lib/backup/policy.mjs DENY_PATTERNS), not overridable from the config: .env, .cohort-key.json, private keys, .claude/.credentials.json, node_modules, .git, state/tmp, the RAG index."
104
104
  }
105
105
  },
106
106
  "slack-socket-mode": {