@cohortapp/agent-sdk 2.5.1 → 2.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/bin/maestro.mjs +305 -89
  2. package/bin/maestro.test.mjs +357 -48
  3. package/docs/runbooks/backup-restore.md +65 -33
  4. package/framework-features.json +4 -4
  5. package/lib/backup/policy.mjs +710 -0
  6. package/lib/backup/policy.test.mjs +305 -0
  7. package/lib/budget-escalate.mjs +133 -0
  8. package/lib/budget-escalate.test.mjs +232 -0
  9. package/lib/budget-guard.envelope.test.mjs +476 -0
  10. package/lib/budget-guard.mjs +853 -75
  11. package/lib/budget-guard.test.mjs +91 -42
  12. package/lib/cadences.mjs +33 -0
  13. package/lib/channels/orgmail/adapter.mjs +88 -3
  14. package/lib/channels/orgmail/adapter.test.mjs +137 -0
  15. package/lib/channels/repeat-suppressor.mjs +198 -0
  16. package/lib/channels/repeat-suppressor.test.mjs +134 -0
  17. package/lib/comms/receipts.mjs +297 -0
  18. package/lib/cost/ledger-row.mjs +333 -0
  19. package/lib/cost/ledger-row.test.mjs +183 -0
  20. package/lib/execution/drive.mjs +28 -1
  21. package/lib/execution/effects.mjs +191 -12
  22. package/lib/execution/effects.test.mjs +50 -11
  23. package/lib/goals/admission.mjs +13 -1
  24. package/lib/goals/admission.test.mjs +26 -1
  25. package/lib/goals/loop.mjs +13 -0
  26. package/lib/kpi-sensors.test.mjs +3 -0
  27. package/lib/mandate/cache.mjs +13 -5
  28. package/lib/mandate/derive.mjs +146 -21
  29. package/lib/mandate/derive.test.mjs +50 -6
  30. package/lib/mandate/model.mjs +32 -4
  31. package/lib/mandate/refresh.test.mjs +16 -2
  32. package/lib/mcp/server.test.mjs +12 -3
  33. package/lib/model-router/economics.mjs +107 -76
  34. package/lib/model-router/economics.test.mjs +64 -46
  35. package/lib/model-router/integration-coverage.test.mjs +39 -37
  36. package/lib/model-router/ledger.mjs +75 -22
  37. package/lib/model-router/ledger.test.mjs +35 -2
  38. package/lib/org/client.mjs +14 -0
  39. package/lib/org/cost-sync.mjs +16 -2
  40. package/lib/org/doctor.mjs +62 -1
  41. package/lib/org/doctor.test.mjs +36 -3
  42. package/lib/org/email-remedy.mjs +49 -0
  43. package/lib/org/engagement-ledger.mjs +376 -0
  44. package/lib/org/engagement-ledger.test.mjs +112 -0
  45. package/lib/org/engagement.mjs +1056 -0
  46. package/lib/org/engagement.test.mjs +739 -0
  47. package/lib/org/messaging.mjs +230 -3
  48. package/lib/org/messaging.test.mjs +110 -1
  49. package/lib/org/param-contract.mjs +56 -2
  50. package/lib/org/param-contract.test.mjs +26 -0
  51. package/lib/org/protocol.checksum +1 -1
  52. package/lib/org/protocol.mjs +5 -0
  53. package/lib/org/protocol.test.mjs +7 -1
  54. package/lib/org/tool-surface.mjs +506 -10
  55. package/lib/org/tool-surface.test.mjs +191 -7
  56. package/lib/org/ui-parity.mjs +333 -6
  57. package/lib/org/ui-parity.test.mjs +96 -3
  58. package/lib/org/work-ledger.mjs +241 -0
  59. package/lib/org/work-ledger.test.mjs +237 -0
  60. package/lib/plan/adoption-e2e.test.mjs +366 -0
  61. package/lib/plan/budget-enforcement.test.mjs +400 -0
  62. package/lib/plan/budget-runtime.mjs +215 -0
  63. package/lib/plan/compile.mjs +201 -5
  64. package/lib/plan/compile.test.mjs +19 -5
  65. package/lib/plan/emit.mjs +8 -0
  66. package/lib/plan/emit.test.mjs +18 -0
  67. package/lib/resource-governor.mjs +58 -12
  68. package/lib/resource-governor.test.mjs +41 -1
  69. package/lib/security/audit-engine.mjs +45 -8
  70. package/lib/security/audit-engine.test.mjs +35 -0
  71. package/lib/setup/enroll-from-cohort.mjs +14 -1
  72. package/lib/setup/sections/mandate.mjs +48 -7
  73. package/lib/setup/sections/mandate.test.mjs +17 -2
  74. package/lib/setup/sections/orgmail.mjs +10 -2
  75. package/lib/setup/state.mjs +83 -2
  76. package/lib/telemetry/collect.mjs +360 -20
  77. package/lib/telemetry/collect.test.mjs +266 -0
  78. package/package.json +1 -1
  79. package/scripts/cost/track-claude-usage.mjs +207 -48
  80. package/scripts/cost/track-claude-usage.test.mjs +148 -0
  81. package/scripts/daemon/agent-daemon.mjs +315 -17
  82. package/scripts/daemon/assurance-e2e.test.mjs +421 -0
  83. package/scripts/daemon/assurance.mjs +944 -0
  84. package/scripts/daemon/assurance.test.mjs +668 -0
  85. package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
  86. package/scripts/daemon/cadence-consumer.mjs +147 -9
  87. package/scripts/daemon/cadence-consumer.test.mjs +6 -0
  88. package/scripts/daemon/cadence-handlers.mjs +158 -0
  89. package/scripts/daemon/cadence-handlers.test.mjs +64 -0
  90. package/scripts/daemon/classifier.test.mjs +18 -9
  91. package/scripts/daemon/deliver.mjs +314 -0
  92. package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
  93. package/scripts/daemon/dispatcher.mjs +64 -6
  94. package/scripts/daemon/responder-cost.test.mjs +68 -0
  95. package/scripts/daemon/responder.mjs +351 -298
  96. package/scripts/local-triggers/generate-plists.test.mjs +7 -4
  97. package/scripts/maintenance/backup-run.mjs +415 -0
  98. package/scripts/maintenance/backup-to-cloud.sh +16 -116
  99. package/scripts/org/send-orgmail.mjs +16 -0
  100. package/scripts/record-receipt.sh +63 -0
  101. package/scripts/restore-from-backup.sh +14 -3
  102. package/scripts/restore-from-backup.test.mjs +8 -5
  103. package/scripts/send-email-threaded.py +47 -0
  104. package/scripts/send-sms.sh +4 -0
  105. package/scripts/send-whatsapp.sh +4 -0
  106. package/scripts/setup/init-backup.mjs +93 -38
  107. package/scripts/slack-send.sh +12 -0
@@ -358,60 +358,123 @@ test("setup --status in an agent dir lists section detect() statuses", async ()
358
358
  // doctor cost-telemetry tripwire — pure predicate (observability F2)
359
359
  // ---------------------------------------------------------------------------
360
360
 
361
- test("summariseLedgerRows sums estimated_usd and counts rows; tolerates junk", () => {
361
+ /** A measured session row in the shape the tracker now writes. */
362
+ function measuredRow(over = {}) {
363
+ return {
364
+ ts: new Date().toISOString(), cadence: "inbox", source: "dispatcher",
365
+ model: "sonnet", input_tokens: 16, output_tokens: 5000,
366
+ cache_read_input_tokens: 553181, measurement: "measured",
367
+ estimated_usd: 0.241, total_cost_usd: 0.533412, ...over,
368
+ };
369
+ }
370
+ /** A session that ran but whose usage could not be read. */
371
+ function unmeasuredRow(over = {}) {
372
+ return {
373
+ ts: new Date().toISOString(), cadence: "inbox", source: "dispatcher",
374
+ model: "sonnet", input_tokens: null, output_tokens: null,
375
+ measurement: "unknown", unmeasured_reason: "no-usage-field",
376
+ estimated_usd: null, total_cost_usd: null, ...over,
377
+ };
378
+ }
379
+ /** A message send: no model ran, so $0 is a fact rather than a gap. */
380
+ function nonLlmRow(over = {}) {
381
+ return {
382
+ ts: new Date().toISOString(), source: "messaging", task_class: "messaging.send",
383
+ model: null, input_tokens: 0, output_tokens: 0, estimated_usd: 0, ...over,
384
+ };
385
+ }
386
+
387
+ test("summariseLedgerRows bills the authoritative cost, not the cache-blind estimate", () => {
388
+ const s = summariseLedgerRows([measuredRow(), measuredRow({ total_cost_usd: 1 })]);
389
+ assert.equal(s.rowCount, 2);
390
+ assert.equal(s.totalUsd, 1.533412, "total_cost_usd wins over estimated_usd");
391
+ });
392
+
393
+ test("summariseLedgerRows counts sessions, not attribution rows; tolerates junk", () => {
362
394
  const s = summariseLedgerRows([
363
- { estimated_usd: 0.5 },
364
- { estimated_usd: 1.25 },
365
- { estimated_usd: "0.25" }, // numeric string coerced
366
- { estimated_usd: NaN }, // ignored as 0
367
- {}, // counts as a row, $0
368
- null, // skipped entirely
369
- "garbage", // skipped entirely
395
+ measuredRow(), unmeasuredRow(), nonLlmRow(), null, "garbage",
370
396
  ]);
371
- assert.equal(s.rowCount, 5, "only object rows count");
372
- assert.equal(s.totalUsd, 2, "0.5 + 1.25 + 0.25 = 2.00");
397
+ assert.equal(s.rowCount, 2, "1 measured + 1 unmeasured; the send is not a session");
398
+ assert.equal(s.measured, 1);
399
+ assert.equal(s.unmeasured, 1);
400
+ assert.equal(s.nonLlmRows, 1);
373
401
  });
374
402
 
375
403
  test("summariseLedgerRows on empty input is zeroed", () => {
376
- assert.deepEqual(summariseLedgerRows([]), { rowCount: 0, totalUsd: 0 });
377
- assert.deepEqual(summariseLedgerRows(undefined), { rowCount: 0, totalUsd: 0 });
404
+ assert.equal(summariseLedgerRows([]).rowCount, 0);
405
+ assert.equal(summariseLedgerRows([]).totalUsd, 0);
406
+ assert.equal(summariseLedgerRows(undefined).rowCount, 0);
378
407
  });
379
408
 
380
- test("evaluateCostTripwire: rows exist but spend is $0 → RED (telemetry blind)", () => {
409
+ test("evaluateCostTripwire: unmeasured rows → RED (the 0-vs-unknown distinction)", () => {
410
+ // The defect doctor exists to catch: a session ran and nobody priced it.
411
+ const t = evaluateCostTripwire({ rows: [measuredRow(), unmeasuredRow()], sessionCount: 2 });
412
+ assert.equal(t.red, true);
413
+ assert.equal(t.unmeasured, 1);
414
+ assert.match(t.reason, /NO token counts/);
415
+ assert.match(t.fix, /unmeasured_reason/);
416
+ });
417
+
418
+ test("evaluateCostTripwire: a genuine zero-token session is NOT unmeasured", () => {
419
+ // A model ran and emitted nothing. That is a measurement; it must stay GREEN
420
+ // as long as the day bills above $0 — otherwise doctor cries wolf.
381
421
  const t = evaluateCostTripwire({
382
- rows: [{ estimated_usd: 0 }, { estimated_usd: 0 }, { estimated_usd: 0 }],
383
- sessionCount: 0,
422
+ rows: [measuredRow(), measuredRow({ input_tokens: 40, output_tokens: 0, total_cost_usd: 0 })],
423
+ sessionCount: 2,
384
424
  });
425
+ assert.equal(t.red, false);
426
+ assert.equal(t.unmeasured, 0);
427
+ });
428
+
429
+ test("evaluateCostTripwire: sessions logged but no ledger rows → RED (unrecorded)", () => {
430
+ const t = evaluateCostTripwire({ rows: [], sessionCount: 3 });
385
431
  assert.equal(t.red, true);
386
- assert.equal(t.status, "red");
387
- assert.equal(t.rowCount, 3);
388
- assert.equal(t.totalUsd, 0);
389
- assert.match(t.reason, /blind/i);
432
+ assert.equal(t.unrecorded, 3);
433
+ assert.match(t.reason, /only 0 dispatcher row\(s\) reached the cost ledger/);
390
434
  });
391
435
 
392
- test("evaluateCostTripwire: no rows but heartbeat-active session → RED", () => {
393
- // Daemon is live (heartbeat) but the ledger is empty → still blind.
394
- const t = evaluateCostTripwire({ rows: [], sessionCount: 1 });
436
+ test("evaluateCostTripwire: rows from ANOTHER writer do not mask an unrecorded dispatcher", () => {
437
+ // The population bug this replaced. `sessionCount` comes from the dispatcher's
438
+ // own session log; `rowCount` counted rows from every writer. With the
439
+ // responder now writing rows too, a dispatcher that had stopped recording
440
+ // entirely read as unrecorded:0 / GREEN — 35 real sessions unpriced, and the
441
+ // one detector meant to catch exactly that said nothing.
442
+ const responderRows = Array.from({ length: 40 }, () => measuredRow({ source: "responder" }));
443
+ const t = evaluateCostTripwire({ rows: responderRows, sessionCount: 35 });
444
+ assert.equal(t.red, true, "35 dispatcher sessions with 0 dispatcher rows is RED");
445
+ assert.equal(t.unrecorded, 35);
446
+ assert.equal(t.sourceRowCount, 0);
447
+ assert.match(t.reason, /dispatcher logged 35/);
448
+ });
449
+
450
+ test("evaluateCostTripwire: measured rows billing $0 → RED (pricing broken)", () => {
451
+ const t = evaluateCostTripwire({
452
+ rows: [measuredRow({ total_cost_usd: 0, estimated_usd: 0 })],
453
+ sessionCount: 1,
454
+ });
395
455
  assert.equal(t.red, true);
396
- assert.equal(t.status, "red");
397
- assert.match(t.reason, /blind/i);
456
+ assert.match(t.reason, /bill to \$0/);
457
+ assert.match(t.fix, /--cache-read-tokens/);
398
458
  });
399
459
 
400
460
  test("evaluateCostTripwire: no sessions and no rows → GREEN (nothing to measure)", () => {
401
461
  const t = evaluateCostTripwire({ rows: [], sessionCount: 0 });
402
462
  assert.equal(t.red, false);
403
- assert.equal(t.status, "green");
404
463
  assert.match(t.reason, /nothing to measure/i);
405
464
  });
406
465
 
407
- test("evaluateCostTripwire: spend > 0 → GREEN even with sessions", () => {
408
- const t = evaluateCostTripwire({
409
- rows: [{ estimated_usd: 0 }, { estimated_usd: 0.42 }],
410
- sessionCount: 1,
411
- });
466
+ test("evaluateCostTripwire: attribution-only day is quiet, not blind", () => {
467
+ // Regression guard for the false alarm this replaced: 41 message-send rows on
468
+ // 2026-08-11 would have armed the old tripwire all by themselves.
469
+ const t = evaluateCostTripwire({ rows: [nonLlmRow(), nonLlmRow()], sessionCount: 0 });
412
470
  assert.equal(t.red, false);
413
- assert.equal(t.status, "green");
414
- assert.equal(t.totalUsd, 0.42);
471
+ assert.match(t.reason, /nothing to measure/i);
472
+ });
473
+
474
+ test("evaluateCostTripwire: spend > 0 with every session measured → GREEN", () => {
475
+ const t = evaluateCostTripwire({ rows: [measuredRow()], sessionCount: 1 });
476
+ assert.equal(t.red, false);
477
+ assert.equal(t.totalUsd, 0.533412);
415
478
  assert.match(t.reason, /live/i);
416
479
  });
417
480
 
@@ -442,19 +505,19 @@ async function makeBareAgent() {
442
505
 
443
506
  function utcDate() { return new Date().toISOString().slice(0, 10); }
444
507
 
445
- test("doctor RED when today's ledger has rows but $0 spend", async () => {
508
+ test("doctor RED when a session ran with no token counts recorded", async () => {
446
509
  const root = await makeBareAgent();
447
510
  try {
448
511
  mkdirSync(join(root, "state/cost-tracking"), { recursive: true });
449
512
  const ledger = join(root, "state/cost-tracking", `${utcDate()}.jsonl`);
450
513
  writeFileSync(ledger,
451
- JSON.stringify({ ts: new Date().toISOString(), cadence: "inbox-processor", model: "sonnet", input_tokens: 0, output_tokens: 0, estimated_usd: 0 }) + "\n" +
452
- JSON.stringify({ ts: new Date().toISOString(), cadence: "backlog-executor", model: "opus", input_tokens: 0, output_tokens: 0, estimated_usd: 0 }) + "\n"
514
+ JSON.stringify(unmeasuredRow({ unmeasured_reason: "no-usage-field" })) + "\n" +
515
+ JSON.stringify(unmeasuredRow({ cadence: "backlog", unmeasured_reason: "empty-stdout" })) + "\n"
453
516
  );
454
517
  const r = runCli(["doctor"], root);
455
518
  const out = (r.stdout || "") + (r.stderr || "");
456
519
  assert.match(out, /Cost telemetry BLIND/);
457
- assert.match(out, /track-claude-usage\.mjs/);
520
+ assert.match(out, /NO token counts/);
458
521
  } finally { await rmRoot(root); }
459
522
  });
460
523
 
@@ -463,12 +526,10 @@ test("doctor GREEN when today's ledger records real spend", async () => {
463
526
  try {
464
527
  mkdirSync(join(root, "state/cost-tracking"), { recursive: true });
465
528
  const ledger = join(root, "state/cost-tracking", `${utcDate()}.jsonl`);
466
- writeFileSync(ledger,
467
- JSON.stringify({ ts: new Date().toISOString(), cadence: "inbox-processor", model: "sonnet", input_tokens: 1500, output_tokens: 320, estimated_usd: 0.0093 }) + "\n"
468
- );
529
+ writeFileSync(ledger, JSON.stringify(measuredRow()) + "\n");
469
530
  const r = runCli(["doctor"], root);
470
531
  const out = (r.stdout || "") + (r.stderr || "");
471
- assert.match(out, /Cost telemetry live/);
532
+ assert.match(out, /Cost telemetry: spend telemetry live/);
472
533
  assert.ok(!out.includes("Cost telemetry BLIND"), "must not flag blind when spend>0");
473
534
  } finally { await rmRoot(root); }
474
535
  });
@@ -478,7 +539,23 @@ test("doctor GREEN (nothing to measure) when no ledger exists today", async () =
478
539
  try {
479
540
  const r = runCli(["doctor"], root);
480
541
  const out = (r.stdout || "") + (r.stderr || "");
481
- assert.match(out, /no sessions recorded today/);
542
+ assert.match(out, /nothing to measure/);
543
+ } finally { await rmRoot(root); }
544
+ });
545
+
546
+ test("doctor does NOT cry wolf on an empty ledger while the daemon is live", async () => {
547
+ // The regression this replaced: doctor keyed "sessions ran" off a fresh
548
+ // heartbeat, so every UTC day went RED between midnight and the first session
549
+ // and printed a fix that had already shipped.
550
+ const root = await makeBareAgent();
551
+ try {
552
+ mkdirSync(join(root, "state/cadence-bus"), { recursive: true });
553
+ writeFileSync(join(root, "state/cadence-bus/health.json"),
554
+ JSON.stringify({ ts: new Date().toISOString() }));
555
+ const r = runCli(["doctor"], root);
556
+ const out = (r.stdout || "") + (r.stderr || "");
557
+ assert.ok(!out.includes("Cost telemetry BLIND"), "a live daemon with no sessions yet is not blind");
558
+ assert.match(out, /nothing to measure/);
482
559
  } finally { await rmRoot(root); }
483
560
  });
484
561
 
@@ -860,30 +937,80 @@ test("secrets sync without a configured broker exits 1", async () => {
860
937
  // doctor — backup-config DR check (gaps-enterprise-ops G16)
861
938
  // ---------------------------------------------------------------------------
862
939
 
863
- test("doctor RED when no backup target is configured", async () => {
940
+ test("doctor RED when there is no DR posture at all, and names the one command that fixes it", async () => {
864
941
  const root = await makeBareAgent();
865
942
  try {
866
943
  const r = runCli(["doctor"], root);
867
944
  const out = (r.stdout || "") + (r.stderr || "");
868
- assert.match(out, /No backup target configured/);
945
+ assert.match(out, /No DR posture at all/);
869
946
  assert.match(out, /backup-config\.yaml/);
947
+ assert.match(out, /maestro init backup-replication --apply/);
870
948
  // doctor exits 1 whenever any issue is found; the missing backup is one.
871
949
  assert.equal(r.status, 1, out);
872
950
  } finally { await rmRoot(root); }
873
951
  });
874
952
 
875
- test("doctor GREEN for backup when configured + a fresh successful run is recorded", async () => {
953
+ test("doctor RED when backup is enabled but has never actually run", async () => {
954
+ const root = await makeBareAgent();
955
+ try {
956
+ mkdirSync(join(root, ".maestro"), { recursive: true });
957
+ writeFileSync(join(root, ".maestro/backup-config.yaml"), "enabled: true\nprefix: t\n");
958
+ const r = runCli(["doctor"], root);
959
+ const out = (r.stdout || "") + (r.stderr || "");
960
+ assert.match(out, /NEVER completed a run/);
961
+ assert.match(out, /backup-run\.mjs/);
962
+ } finally { await rmRoot(root); }
963
+ });
964
+
965
+ test("doctor WARNS (not GREEN) when restore points are fresh but local-only", async () => {
966
+ const root = await makeBareAgent();
967
+ try {
968
+ mkdirSync(join(root, ".maestro"), { recursive: true });
969
+ writeFileSync(join(root, ".maestro/backup-config.yaml"), "enabled: true\nprefix: t\nlocal:\n enabled: true\n");
970
+ writeFileSync(join(root, ".maestro/last-backup.json"), JSON.stringify({
971
+ completed_at: new Date().toISOString(), tier: "local", provider: "local",
972
+ }));
973
+ const r = runCli(["doctor"], root);
974
+ const out = (r.stdout || "") + (r.stderr || "");
975
+ assert.match(out, /LOCAL-ONLY/);
976
+ assert.match(out, /offsite\.provider/);
977
+ assert.ok(!out.includes("No DR posture at all"), "must not claim the config is missing");
978
+ } finally { await rmRoot(root); }
979
+ });
980
+
981
+ test("doctor GREEN for backup only when it is fresh AND off-machine", async () => {
876
982
  const root = await makeBareAgent();
877
983
  try {
878
984
  mkdirSync(join(root, ".maestro"), { recursive: true });
879
- writeFileSync(join(root, ".maestro/backup-config.yaml"), "enabled: true\nprovider: s3\nbucket: agent-backups\n");
985
+ writeFileSync(
986
+ join(root, ".maestro/backup-config.yaml"),
987
+ "enabled: true\nprefix: t\noffsite:\n provider: s3\n bucket: agent-backups\n"
988
+ );
880
989
  writeFileSync(join(root, ".maestro/last-backup.json"), JSON.stringify({
881
- completed_at: new Date().toISOString(), provider: "s3", bucket: "agent-backups",
990
+ completed_at: new Date().toISOString(), tier: "offsite", provider: "s3", bucket: "agent-backups",
882
991
  }));
883
992
  const r = runCli(["doctor"], root);
884
993
  const out = (r.stdout || "") + (r.stderr || "");
885
- assert.match(out, /Backup configured \+ fresh/);
886
- assert.ok(!out.includes("No backup target configured"), "must not flag missing backup when fresh");
994
+ assert.match(out, /Backup fresh \+ off-machine/);
995
+ assert.ok(!out.includes("No DR posture at all"), "must not flag missing backup when fresh");
996
+ assert.ok(!out.includes("LOCAL-ONLY"), "offsite is configured");
997
+ } finally { await rmRoot(root); }
998
+ });
999
+
1000
+ test("doctor reports an include path that the credential deny-list drops", async () => {
1001
+ const root = await makeBareAgent();
1002
+ try {
1003
+ mkdirSync(join(root, ".maestro"), { recursive: true });
1004
+ writeFileSync(
1005
+ join(root, ".maestro/backup-config.yaml"),
1006
+ "enabled: true\nprefix: t\ninclude:\n - state\n - .env\n"
1007
+ );
1008
+ writeFileSync(join(root, ".maestro/last-backup.json"), JSON.stringify({
1009
+ completed_at: new Date().toISOString(), tier: "local",
1010
+ }));
1011
+ const r = runCli(["doctor"], root);
1012
+ const out = (r.stdout || "") + (r.stderr || "");
1013
+ assert.match(out, /DROPPED by the credential deny-list/);
887
1014
  } finally { await rmRoot(root); }
888
1015
  });
889
1016
 
@@ -1013,3 +1140,185 @@ test("doctor surfaces the new framework systems (org / secrets / diagnostics)",
1013
1140
  await rmRoot(shim);
1014
1141
  }
1015
1142
  });
1143
+
1144
+ // ---------------------------------------------------------------------------
1145
+ // upgrade — prune (framework files deleted upstream)
1146
+ //
1147
+ // The copy loop only ever adds and overwrites, so before this every file the
1148
+ // SDK had ever shipped stayed resident on an agent machine forever. Orphaned
1149
+ // *modules* keep importing symbols their collaborators no longer export, so
1150
+ // they fail permanently and drag the agent's own suite red — found in the
1151
+ // field on Isla's machine, where a stale scripts/daemon/responder.test.mjs
1152
+ // survived the transport refactor still importing a deleted export.
1153
+ //
1154
+ // Prune's contract is narrow on purpose: it removes a file only when git
1155
+ // proves it is tracked-and-clean, i.e. pristine framework material the
1156
+ // operator never touched. Everything else is kept and reported.
1157
+ // ---------------------------------------------------------------------------
1158
+
1159
+ /**
1160
+ * Agent repo carrying `orphans` — paths upstream does not ship. Written before
1161
+ * the commit so git sees them as tracked and clean (prune's precondition).
1162
+ */
1163
+ async function makeAgentWithOrphans(orphans) {
1164
+ const root = await tmpRoot("maestro-orphan");
1165
+ mkdirSync(join(root, "config"), { recursive: true });
1166
+ writeFileSync(join(root, "CLAUDE.md"), "# agent\n");
1167
+ writeFileSync(join(root, "config/agent.ts"), "export const AGENT = { firstName: 'orphan' };\n");
1168
+ writeFileSync(join(root, "package.json"), '{"name":"orphan-agent","version":"1.0.0"}\n');
1169
+ for (const [rel, body] of Object.entries(orphans)) {
1170
+ mkdirSync(dirname(join(root, rel)), { recursive: true });
1171
+ writeFileSync(join(root, rel), body);
1172
+ }
1173
+ execFileSync("git", ["init", "-q"], { cwd: root });
1174
+ execFileSync("git", ["add", "-A"], { cwd: root });
1175
+ execFileSync(
1176
+ "git",
1177
+ ["-c", "user.email=test@test", "-c", "user.name=test", "commit", "-q", "-m", "init"],
1178
+ { cwd: root },
1179
+ );
1180
+ return root;
1181
+ }
1182
+
1183
+ const ORPHAN = "lib/zz-deleted-upstream-fixture.mjs";
1184
+
1185
+ test("upgrade prunes a pristine framework file deleted upstream, with a backup", async () => {
1186
+ const root = await makeAgentWithOrphans({ [ORPHAN]: "export const gone = 1;\n" });
1187
+ try {
1188
+ const r = runCli(["upgrade"], root);
1189
+ assert.equal(r.status, 0, r.stderr);
1190
+ assert.ok(!existsSync(join(root, ORPHAN)), "orphan should have been pruned");
1191
+ assert.ok(
1192
+ existsSync(join(root, ".maestro/backup", ORPHAN)),
1193
+ "prune must be reversible — a backup is required",
1194
+ );
1195
+ assert.equal(
1196
+ readFileSync(join(root, ".maestro/backup", ORPHAN), "utf-8"),
1197
+ "export const gone = 1;\n",
1198
+ "backup must hold the original bytes",
1199
+ );
1200
+ } finally {
1201
+ await fsp.rm(root, { recursive: true, force: true });
1202
+ }
1203
+ });
1204
+
1205
+ test("upgrade never prunes a file the operator edited", async () => {
1206
+ const root = await makeAgentWithOrphans({ [ORPHAN]: "export const gone = 1;\n" });
1207
+ try {
1208
+ writeFileSync(join(root, ORPHAN), "export const gone = 2; // my change\n");
1209
+ const r = runCli(["upgrade"], root);
1210
+ assert.equal(r.status, 0, r.stderr);
1211
+ assert.ok(existsSync(join(root, ORPHAN)), "an edited orphan is the operator's, not ours to delete");
1212
+ assert.match(
1213
+ readFileSync(join(root, ORPHAN), "utf-8"),
1214
+ /my change/,
1215
+ "the operator's bytes must survive verbatim",
1216
+ );
1217
+ assert.match(r.stdout, /kept/i, "the kept orphan must be reported, not silently left");
1218
+ } finally {
1219
+ await fsp.rm(root, { recursive: true, force: true });
1220
+ }
1221
+ });
1222
+
1223
+ test("upgrade never prunes an untracked file", async () => {
1224
+ const root = await makeAgentWithOrphans({});
1225
+ try {
1226
+ // Written after the commit → untracked → indistinguishable from operator work.
1227
+ mkdirSync(join(root, "lib"), { recursive: true });
1228
+ writeFileSync(join(root, ORPHAN), "export const mine = 1;\n");
1229
+ const r = runCli(["upgrade"], root);
1230
+ assert.equal(r.status, 0, r.stderr);
1231
+ assert.ok(existsSync(join(root, ORPHAN)), "untracked files are never prune candidates");
1232
+ } finally {
1233
+ await fsp.rm(root, { recursive: true, force: true });
1234
+ }
1235
+ });
1236
+
1237
+ test("upgrade --no-prune leaves upstream-deleted files in place", async () => {
1238
+ const root = await makeAgentWithOrphans({ [ORPHAN]: "export const gone = 1;\n" });
1239
+ try {
1240
+ const r = runCli(["upgrade", "--no-prune"], root);
1241
+ assert.equal(r.status, 0, r.stderr);
1242
+ assert.ok(existsSync(join(root, ORPHAN)), "--no-prune must suppress removal entirely");
1243
+ } finally {
1244
+ await fsp.rm(root, { recursive: true, force: true });
1245
+ }
1246
+ });
1247
+
1248
+ test("upgrade --dry-run reports a prune without performing it", async () => {
1249
+ const root = await makeAgentWithOrphans({ [ORPHAN]: "export const gone = 1;\n" });
1250
+ try {
1251
+ const r = runCli(["upgrade", "--dry-run"], root);
1252
+ assert.equal(r.status, 0, r.stderr);
1253
+ assert.ok(existsSync(join(root, ORPHAN)), "dry run must not delete");
1254
+ assert.ok(!existsSync(join(root, ".maestro/backup", ORPHAN)), "dry run must not write backups");
1255
+ assert.match(r.stdout, /prune/i, "dry run must still tell the operator what it would remove");
1256
+ } finally {
1257
+ await fsp.rm(root, { recursive: true, force: true });
1258
+ }
1259
+ });
1260
+
1261
+ test("upgrade honours .maestroignore over prune", async () => {
1262
+ const root = await makeAgentWithOrphans({
1263
+ [ORPHAN]: "export const gone = 1;\n",
1264
+ ".maestroignore": `${ORPHAN}\n`,
1265
+ });
1266
+ try {
1267
+ const r = runCli(["upgrade"], root);
1268
+ assert.equal(r.status, 0, r.stderr);
1269
+ assert.ok(existsSync(join(root, ORPHAN)), ".maestroignore must win over prune, as it does over overwrite");
1270
+ } finally {
1271
+ await fsp.rm(root, { recursive: true, force: true });
1272
+ }
1273
+ });
1274
+
1275
+ test("upgrade prune leaves files that upstream still ships", async () => {
1276
+ const root = await makeAgentWithOrphans({});
1277
+ try {
1278
+ const r = runCli(["upgrade"], root);
1279
+ assert.equal(r.status, 0, r.stderr);
1280
+ // A file the SDK genuinely ships must survive — the guard against a prune
1281
+ // that mistakes "present upstream" for "absent" and empties the repo.
1282
+ assert.ok(
1283
+ existsSync(join(root, "lib/identity/persona.mjs")),
1284
+ "currently-shipped framework files must never be pruned",
1285
+ );
1286
+ assert.ok(existsSync(join(root, "scripts/daemon/responder.mjs")));
1287
+ } finally {
1288
+ await fsp.rm(root, { recursive: true, force: true });
1289
+ }
1290
+ });
1291
+
1292
+ test("upgrade prune never touches merge-mode agents/", async () => {
1293
+ const custom = "agents/my-private-agent.md";
1294
+ const root = await makeAgentWithOrphans({ [custom]: "# mine\n" });
1295
+ try {
1296
+ const r = runCli(["upgrade"], root);
1297
+ assert.equal(r.status, 0, r.stderr);
1298
+ assert.ok(
1299
+ existsSync(join(root, custom)),
1300
+ "agents/ is merge-mode: local-only files there are the point, not orphans",
1301
+ );
1302
+ } finally {
1303
+ await fsp.rm(root, { recursive: true, force: true });
1304
+ }
1305
+ });
1306
+
1307
+ test("upgrade prune never removes machine-generated launchd plists", async () => {
1308
+ // A committed plist is tracked, clean, and absent upstream — it satisfies
1309
+ // every safety check prune has, yet it is this machine's schedule, not
1310
+ // framework material. Regression: an early prune deleted the very plist the
1311
+ // cadence-bus migration was about to back up and regenerate.
1312
+ const plist = "scripts/local-triggers/plists/ai.adaptic.some-trigger.plist";
1313
+ const root = await makeAgentWithOrphans({ [plist]: "<?xml version=\"1.0\"?><plist/>" });
1314
+ try {
1315
+ const r = runCli(["upgrade"], root);
1316
+ assert.equal(r.status, 0, r.stderr);
1317
+ assert.ok(
1318
+ existsSync(join(root, plist)),
1319
+ "generated plists are machine config; prune must leave them alone",
1320
+ );
1321
+ } finally {
1322
+ await fsp.rm(root, { recursive: true, force: true });
1323
+ }
1324
+ });
@@ -12,27 +12,47 @@ This complements `docs/runbooks/mac-mini-bootstrap.md` (fresh provisioning) and
12
12
  compromised machine, do NOT restore onto the same hardware — rebuild on a clean
13
13
  box.
14
14
 
15
- ## What is backed up, and the honest gaps
16
-
17
- Off-machine backup is driven by `scripts/maintenance/backup-to-cloud.sh`, reading
18
- `.maestro/backup-config.yaml`, run daily via launchd. Configure it with
19
- `maestro init backup-replication --apply` (`scripts/setup/init-backup.mjs`).
20
-
21
- Know these caveats before you rely on it:
22
-
23
- - **Backup is opt-in and silent when off.** If `.maestro/backup-config.yaml` is
24
- missing or `enabled: false`, the script exits 0 and does nothing. An agent with
25
- no backup configured looks identical to a healthy one in casual inspection —
26
- check explicitly (below).
27
- - **Coverage historically excluded `logs/` and the audit records.** Confirm your
28
- config's include paths cover `state/`, `logs/`, and `.maestro/`. If `logs/` is
29
- excluded, a dead machine loses that agent's local compliance history — the
30
- org-server export (next section) is your compensating control.
31
- - **`retention_days` may not be enforced** by older builds; do not assume old
32
- archives are pruned.
33
- - **Archives may be unencrypted** unless you enabled client-side encryption.
34
- Treat the backup bucket as sensitive, restrict access, and prefer an encrypted
35
- bucket and encrypted archives.
15
+ ## The posture: two tiers
16
+
17
+ Since 2026-08 the DR posture is **restore points first, offsite second**, driven
18
+ by `scripts/maintenance/backup-run.mjs` (the `nightly-backup` cadence, 03:10
19
+ local) reading `.maestro/backup-config.yaml`. `scripts/maintenance/backup-to-cloud.sh`
20
+ is now a thin compatibility wrapper around the same runner.
21
+
22
+ | Tier | Default | Where | Survives |
23
+ |---|---|---|---|
24
+ | 1 — local restore points | **ON**, zero credentials | `~/Library/Application Support/Maestro/backups/<prefix>/`, **outside** the agent repo | corrupted queue, bad state write, botched checkout, `rm -rf` of the repo |
25
+ | 2 — offsite copy | opt-in (`offsite.provider` + `offsite.bucket`) | gcs / s3 / rsync | machine loss, disk death, fire |
26
+
27
+ `maestro init backup-replication --apply` writes the Tier-1 default, and it is
28
+ `auto: true` — so a **new agent is born with a DR posture instead of a red
29
+ doctor line**. Tier 2 stays a deliberate act (it needs a bucket and credentials);
30
+ `maestro doctor` WARNs until it is set, naming the residual risk in words.
31
+
32
+ **Archived by default:** `state/`, `knowledge/`, `memory/`, `outputs/`,
33
+ `config/`, `.maestro/`. `logs/` is deliberately excluded — largest tree, least
34
+ value, and the authoritative audit trail is already the org event ledger (next
35
+ section). Add it explicitly to `include:` if you want it.
36
+
37
+ **NEVER archived, enforced in code and not overridable from the config**
38
+ (`lib/backup/policy.mjs` → `DENY_PATTERNS`): `.env`, `.env.*`,
39
+ `.cohort-key.json`, `.claude/.credentials.json`, `*.pem` / `*.key` / SSH keys,
40
+ `node_modules`, `.git`, `state/tmp`, the RAG index, sqlite WAL/SHM files.
41
+ `.env` carries `COHORT_API_KEY` (which authenticates as this member) and
42
+ `.cohort-key.json` is the device identity — a backup that shipped either to a
43
+ bucket would be a credential-exfiltration channel that looks *exactly* like a
44
+ successful backup. The deny-list is applied three times: it drops offending
45
+ `include:` entries at plan time (reporting them), it becomes `tar --exclude`
46
+ arguments, and the runner **lists the finished archive and deletes it** if a
47
+ denied path got in anyway. A restore re-pairs for a fresh token; it never
48
+ restores a dead identity.
49
+
50
+ Remaining honest gaps:
51
+
52
+ - **Archives are unencrypted at rest** unless you point Tier 2 at an encrypted
53
+ bucket. Treat the backup destination as sensitive and restrict access.
54
+ - **Tier 1 alone is not DR.** A local restore point on a dead disk is nothing.
55
+ Doctor says so explicitly; do not read the absence of a red line as safety.
36
56
 
37
57
  The org server is your fleet-as-DR layer for the audit record: every
38
58
  side-effecting action is a hash-chained `org_events` row at Cohort, exportable
@@ -46,18 +66,26 @@ non-repudiable record.
46
66
  ```bash
47
67
  cd ~/<agent>-ai
48
68
 
49
- # 1. Is backup configured and enabled?
50
- test -f .maestro/backup-config.yaml && grep -E '^enabled:' .maestro/backup-config.yaml
69
+ # 1. Is backup configured and enabled, and at which tier?
70
+ test -f .maestro/backup-config.yaml && grep -E '^enabled:|provider:|bucket:' .maestro/backup-config.yaml
51
71
 
52
- # 2. Did the last run succeed, and when?
72
+ # 2. Did the last run succeed, when, and to where?
73
+ cat .maestro/last-backup.json
53
74
  tail -n 20 logs/maintenance/backup.log
54
75
 
55
- # 3. Does the configured destination actually have today's archive?
56
- # (provider-specific; e.g. for GCS:)
57
- # gcloud storage ls "gs://<bucket>/<prefix>/$(date -u +%F)/"
76
+ # 3. Do the local restore points actually exist?
77
+ ls -lh ~/Library/Application\ Support/Maestro/backups/*/
58
78
 
59
- # 4. doctor surfaces backup freshness in the governance posture.
79
+ # 4. Does the offsite destination have the archive? (provider-specific)
80
+ # gcloud storage ls "gs://<bucket>/<prefix>/"
81
+
82
+ # 5. doctor's verdict — fail (no restore point) vs warn (local-only / stale)
83
+ # vs ok (fresh AND off-machine).
60
84
  maestro doctor
85
+
86
+ # 6. Force a run now, without waiting for 03:10.
87
+ node scripts/maintenance/backup-run.mjs
88
+ node scripts/maintenance/backup-run.mjs --dry-run # print the plan only
61
89
  ```
62
90
 
63
91
  If `maestro doctor` does not flag a missing/stale backup red, do not assume the
@@ -163,8 +191,10 @@ A rebuilt machine with no backup is the next dead machine.
163
191
  ```bash
164
192
  cd ~/<agent>-ai
165
193
  maestro init backup-replication --apply # if .maestro/backup-config.yaml absent
166
- # confirm include paths cover state/, logs/, .maestro/; confirm encryption on.
167
- maestro doctor # backup freshness should clear within a day
194
+ # Restore the OFFSITE block from the dead machine's config (Tier 1 is on by
195
+ # default; Tier 2 is the one that saves you next time).
196
+ node scripts/maintenance/backup-run.mjs # don't wait for 03:10 — prove it works now
197
+ maestro doctor # expect "Backup fresh + off-machine"
168
198
  ```
169
199
 
170
200
  ---
@@ -195,9 +225,11 @@ tested restore.
195
225
 
196
226
  | Need | Command |
197
227
  |---|---|
198
- | Is backup on? | `grep -E '^enabled:' .maestro/backup-config.yaml` |
199
- | Last backup status | `tail logs/maintenance/backup.log` |
200
- | Configure backup | `maestro init backup-replication --apply` |
228
+ | Is backup on, and at which tier? | `grep -E '^enabled:\|provider:\|bucket:' .maestro/backup-config.yaml` |
229
+ | Last backup status | `cat .maestro/last-backup.json` / `tail logs/maintenance/backup.log` |
230
+ | Configure backup (Tier 1 default) | `maestro init backup-replication --apply` |
231
+ | Run a backup now | `node scripts/maintenance/backup-run.mjs` |
232
+ | Add the offsite tier | set `offsite.provider` + `offsite.bucket` in `.maestro/backup-config.yaml` |
201
233
  | Governance posture incl. backup freshness | `maestro doctor` |
202
234
  | Re-mint agent identity | `cohort pair approve <agentId> --scopes <...>` |
203
235
  | Recover audit history | `cohort export-audit --format ocsf` |
@@ -94,13 +94,13 @@
94
94
  }
95
95
  },
96
96
  "backup-replication": {
97
- "version": "1",
97
+ "version": "2",
98
98
  "since": "1.9.0",
99
- "title": "Off-machine state backup",
99
+ "title": "Disaster-recovery backup (local restore points + optional offsite)",
100
100
  "init": {
101
- "auto": false,
101
+ "auto": true,
102
102
  "command": "node scripts/setup/init-backup.mjs",
103
- "description": "Configure off-machine backup of state/, knowledge/, outputs/, and rotated logs to GCS or S3. Requires bucket name + credentials. Doctor flags if last successful backup is >24h old once configured."
103
+ "description": "Write .maestro/backup-config.yaml with the default DR posture so a new agent is not born failing doctor. TIER 1 (on, zero credentials): nightly tar.gz of state/, knowledge/, memory/, outputs/, config/ and .maestro/ into ~/Library/Application Support/Maestro/backups/<prefix>/ — outside the agent repo — via the nightly-backup cadence at 03:10, 14-day retention. TIER 2 (opt-in): set offsite.provider (gcs|s3|rsync) + offsite.bucket to get a copy off the machine; doctor WARNS until you do, because Tier 1 alone does not survive losing the box. NEVER archived, enforced in code (lib/backup/policy.mjs DENY_PATTERNS), not overridable from the config: .env, .cohort-key.json, private keys, .claude/.credentials.json, node_modules, .git, state/tmp, the RAG index."
104
104
  }
105
105
  },
106
106
  "slack-socket-mode": {