@cohortapp/agent-sdk 2.5.1 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +185 -88
- package/bin/maestro.test.mjs +175 -48
- package/docs/runbooks/backup-restore.md +65 -33
- package/framework-features.json +4 -4
- package/lib/backup/policy.mjs +710 -0
- package/lib/backup/policy.test.mjs +305 -0
- package/lib/budget-escalate.mjs +133 -0
- package/lib/budget-escalate.test.mjs +232 -0
- package/lib/budget-guard.envelope.test.mjs +476 -0
- package/lib/budget-guard.mjs +853 -75
- package/lib/budget-guard.test.mjs +91 -42
- package/lib/cadences.mjs +33 -0
- package/lib/channels/orgmail/adapter.mjs +88 -3
- package/lib/channels/orgmail/adapter.test.mjs +137 -0
- package/lib/channels/repeat-suppressor.mjs +198 -0
- package/lib/channels/repeat-suppressor.test.mjs +134 -0
- package/lib/comms/receipts.mjs +297 -0
- package/lib/cost/ledger-row.mjs +333 -0
- package/lib/cost/ledger-row.test.mjs +183 -0
- package/lib/execution/drive.mjs +28 -1
- package/lib/execution/effects.mjs +191 -12
- package/lib/execution/effects.test.mjs +50 -11
- package/lib/goals/admission.mjs +13 -1
- package/lib/goals/admission.test.mjs +26 -1
- package/lib/goals/loop.mjs +13 -0
- package/lib/kpi-sensors.test.mjs +3 -0
- package/lib/mandate/cache.mjs +13 -5
- package/lib/mandate/derive.mjs +146 -21
- package/lib/mandate/derive.test.mjs +50 -6
- package/lib/mandate/model.mjs +32 -4
- package/lib/mandate/refresh.test.mjs +16 -2
- package/lib/mcp/server.test.mjs +12 -3
- package/lib/model-router/economics.mjs +107 -76
- package/lib/model-router/economics.test.mjs +64 -46
- package/lib/model-router/integration-coverage.test.mjs +39 -37
- package/lib/model-router/ledger.mjs +75 -22
- package/lib/model-router/ledger.test.mjs +35 -2
- package/lib/org/client.mjs +14 -0
- package/lib/org/cost-sync.mjs +16 -2
- package/lib/org/doctor.mjs +62 -1
- package/lib/org/doctor.test.mjs +36 -3
- package/lib/org/email-remedy.mjs +49 -0
- package/lib/org/engagement-ledger.mjs +376 -0
- package/lib/org/engagement-ledger.test.mjs +112 -0
- package/lib/org/engagement.mjs +1056 -0
- package/lib/org/engagement.test.mjs +739 -0
- package/lib/org/messaging.mjs +230 -3
- package/lib/org/messaging.test.mjs +110 -1
- package/lib/org/param-contract.mjs +56 -2
- package/lib/org/param-contract.test.mjs +26 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +5 -0
- package/lib/org/protocol.test.mjs +7 -1
- package/lib/org/tool-surface.mjs +506 -10
- package/lib/org/tool-surface.test.mjs +191 -7
- package/lib/org/ui-parity.mjs +333 -6
- package/lib/org/ui-parity.test.mjs +96 -3
- package/lib/org/work-ledger.mjs +241 -0
- package/lib/org/work-ledger.test.mjs +237 -0
- package/lib/plan/adoption-e2e.test.mjs +366 -0
- package/lib/plan/budget-enforcement.test.mjs +400 -0
- package/lib/plan/budget-runtime.mjs +215 -0
- package/lib/plan/compile.mjs +201 -5
- package/lib/plan/compile.test.mjs +19 -5
- package/lib/plan/emit.mjs +8 -0
- package/lib/plan/emit.test.mjs +18 -0
- package/lib/resource-governor.mjs +58 -12
- package/lib/resource-governor.test.mjs +41 -1
- package/lib/security/audit-engine.mjs +45 -8
- package/lib/security/audit-engine.test.mjs +35 -0
- package/lib/setup/enroll-from-cohort.mjs +14 -1
- package/lib/setup/sections/mandate.mjs +48 -7
- package/lib/setup/sections/mandate.test.mjs +17 -2
- package/lib/setup/sections/orgmail.mjs +10 -2
- package/lib/setup/state.mjs +83 -2
- package/lib/telemetry/collect.mjs +360 -20
- package/lib/telemetry/collect.test.mjs +266 -0
- package/package.json +1 -1
- package/scripts/cost/track-claude-usage.mjs +207 -48
- package/scripts/cost/track-claude-usage.test.mjs +148 -0
- package/scripts/daemon/agent-daemon.mjs +315 -17
- package/scripts/daemon/assurance-e2e.test.mjs +421 -0
- package/scripts/daemon/assurance.mjs +944 -0
- package/scripts/daemon/assurance.test.mjs +668 -0
- package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
- package/scripts/daemon/cadence-consumer.mjs +147 -9
- package/scripts/daemon/cadence-consumer.test.mjs +6 -0
- package/scripts/daemon/cadence-handlers.mjs +158 -0
- package/scripts/daemon/cadence-handlers.test.mjs +64 -0
- package/scripts/daemon/deliver.mjs +314 -0
- package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
- package/scripts/daemon/dispatcher.mjs +64 -6
- package/scripts/daemon/responder-cost.test.mjs +68 -0
- package/scripts/daemon/responder.mjs +351 -298
- package/scripts/local-triggers/generate-plists.test.mjs +7 -4
- package/scripts/maintenance/backup-run.mjs +415 -0
- package/scripts/maintenance/backup-to-cloud.sh +16 -116
- package/scripts/org/send-orgmail.mjs +16 -0
- package/scripts/record-receipt.sh +63 -0
- package/scripts/restore-from-backup.sh +14 -3
- package/scripts/restore-from-backup.test.mjs +8 -5
- package/scripts/send-email-threaded.py +47 -0
- package/scripts/send-sms.sh +4 -0
- package/scripts/send-whatsapp.sh +4 -0
- package/scripts/setup/init-backup.mjs +93 -38
- package/scripts/slack-send.sh +12 -0
package/bin/maestro.test.mjs
CHANGED
|
@@ -358,60 +358,123 @@ test("setup --status in an agent dir lists section detect() statuses", async ()
|
|
|
358
358
|
// doctor cost-telemetry tripwire — pure predicate (observability F2)
|
|
359
359
|
// ---------------------------------------------------------------------------
|
|
360
360
|
|
|
361
|
-
|
|
361
|
+
/** A measured session row in the shape the tracker now writes. */
|
|
362
|
+
function measuredRow(over = {}) {
|
|
363
|
+
return {
|
|
364
|
+
ts: new Date().toISOString(), cadence: "inbox", source: "dispatcher",
|
|
365
|
+
model: "sonnet", input_tokens: 16, output_tokens: 5000,
|
|
366
|
+
cache_read_input_tokens: 553181, measurement: "measured",
|
|
367
|
+
estimated_usd: 0.241, total_cost_usd: 0.533412, ...over,
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
/** A session that ran but whose usage could not be read. */
|
|
371
|
+
function unmeasuredRow(over = {}) {
|
|
372
|
+
return {
|
|
373
|
+
ts: new Date().toISOString(), cadence: "inbox", source: "dispatcher",
|
|
374
|
+
model: "sonnet", input_tokens: null, output_tokens: null,
|
|
375
|
+
measurement: "unknown", unmeasured_reason: "no-usage-field",
|
|
376
|
+
estimated_usd: null, total_cost_usd: null, ...over,
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
/** A message send: no model ran, so $0 is a fact rather than a gap. */
|
|
380
|
+
function nonLlmRow(over = {}) {
|
|
381
|
+
return {
|
|
382
|
+
ts: new Date().toISOString(), source: "messaging", task_class: "messaging.send",
|
|
383
|
+
model: null, input_tokens: 0, output_tokens: 0, estimated_usd: 0, ...over,
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
test("summariseLedgerRows bills the authoritative cost, not the cache-blind estimate", () => {
|
|
388
|
+
const s = summariseLedgerRows([measuredRow(), measuredRow({ total_cost_usd: 1 })]);
|
|
389
|
+
assert.equal(s.rowCount, 2);
|
|
390
|
+
assert.equal(s.totalUsd, 1.533412, "total_cost_usd wins over estimated_usd");
|
|
391
|
+
});
|
|
392
|
+
|
|
393
|
+
test("summariseLedgerRows counts sessions, not attribution rows; tolerates junk", () => {
|
|
362
394
|
const s = summariseLedgerRows([
|
|
363
|
-
|
|
364
|
-
{ estimated_usd: 1.25 },
|
|
365
|
-
{ estimated_usd: "0.25" }, // numeric string coerced
|
|
366
|
-
{ estimated_usd: NaN }, // ignored as 0
|
|
367
|
-
{}, // counts as a row, $0
|
|
368
|
-
null, // skipped entirely
|
|
369
|
-
"garbage", // skipped entirely
|
|
395
|
+
measuredRow(), unmeasuredRow(), nonLlmRow(), null, "garbage",
|
|
370
396
|
]);
|
|
371
|
-
assert.equal(s.rowCount,
|
|
372
|
-
assert.equal(s.
|
|
397
|
+
assert.equal(s.rowCount, 2, "1 measured + 1 unmeasured; the send is not a session");
|
|
398
|
+
assert.equal(s.measured, 1);
|
|
399
|
+
assert.equal(s.unmeasured, 1);
|
|
400
|
+
assert.equal(s.nonLlmRows, 1);
|
|
373
401
|
});
|
|
374
402
|
|
|
375
403
|
test("summariseLedgerRows on empty input is zeroed", () => {
|
|
376
|
-
assert.
|
|
377
|
-
assert.
|
|
404
|
+
assert.equal(summariseLedgerRows([]).rowCount, 0);
|
|
405
|
+
assert.equal(summariseLedgerRows([]).totalUsd, 0);
|
|
406
|
+
assert.equal(summariseLedgerRows(undefined).rowCount, 0);
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
test("evaluateCostTripwire: unmeasured rows → RED (the 0-vs-unknown distinction)", () => {
|
|
410
|
+
// The defect doctor exists to catch: a session ran and nobody priced it.
|
|
411
|
+
const t = evaluateCostTripwire({ rows: [measuredRow(), unmeasuredRow()], sessionCount: 2 });
|
|
412
|
+
assert.equal(t.red, true);
|
|
413
|
+
assert.equal(t.unmeasured, 1);
|
|
414
|
+
assert.match(t.reason, /NO token counts/);
|
|
415
|
+
assert.match(t.fix, /unmeasured_reason/);
|
|
378
416
|
});
|
|
379
417
|
|
|
380
|
-
test("evaluateCostTripwire:
|
|
418
|
+
test("evaluateCostTripwire: a genuine zero-token session is NOT unmeasured", () => {
|
|
419
|
+
// A model ran and emitted nothing. That is a measurement; it must stay GREEN
|
|
420
|
+
// as long as the day bills above $0 — otherwise doctor cries wolf.
|
|
381
421
|
const t = evaluateCostTripwire({
|
|
382
|
-
rows: [{
|
|
383
|
-
sessionCount:
|
|
422
|
+
rows: [measuredRow(), measuredRow({ input_tokens: 40, output_tokens: 0, total_cost_usd: 0 })],
|
|
423
|
+
sessionCount: 2,
|
|
384
424
|
});
|
|
425
|
+
assert.equal(t.red, false);
|
|
426
|
+
assert.equal(t.unmeasured, 0);
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
test("evaluateCostTripwire: sessions logged but no ledger rows → RED (unrecorded)", () => {
|
|
430
|
+
const t = evaluateCostTripwire({ rows: [], sessionCount: 3 });
|
|
385
431
|
assert.equal(t.red, true);
|
|
386
|
-
assert.equal(t.
|
|
387
|
-
assert.
|
|
388
|
-
|
|
389
|
-
|
|
432
|
+
assert.equal(t.unrecorded, 3);
|
|
433
|
+
assert.match(t.reason, /only 0 dispatcher row\(s\) reached the cost ledger/);
|
|
434
|
+
});
|
|
435
|
+
|
|
436
|
+
test("evaluateCostTripwire: rows from ANOTHER writer do not mask an unrecorded dispatcher", () => {
|
|
437
|
+
// The population bug this replaced. `sessionCount` comes from the dispatcher's
|
|
438
|
+
// own session log; `rowCount` counted rows from every writer. With the
|
|
439
|
+
// responder now writing rows too, a dispatcher that had stopped recording
|
|
440
|
+
// entirely read as unrecorded:0 / GREEN — 35 real sessions unpriced, and the
|
|
441
|
+
// one detector meant to catch exactly that said nothing.
|
|
442
|
+
const responderRows = Array.from({ length: 40 }, () => measuredRow({ source: "responder" }));
|
|
443
|
+
const t = evaluateCostTripwire({ rows: responderRows, sessionCount: 35 });
|
|
444
|
+
assert.equal(t.red, true, "35 dispatcher sessions with 0 dispatcher rows is RED");
|
|
445
|
+
assert.equal(t.unrecorded, 35);
|
|
446
|
+
assert.equal(t.sourceRowCount, 0);
|
|
447
|
+
assert.match(t.reason, /dispatcher logged 35/);
|
|
390
448
|
});
|
|
391
449
|
|
|
392
|
-
test("evaluateCostTripwire:
|
|
393
|
-
|
|
394
|
-
|
|
450
|
+
test("evaluateCostTripwire: measured rows billing $0 → RED (pricing broken)", () => {
|
|
451
|
+
const t = evaluateCostTripwire({
|
|
452
|
+
rows: [measuredRow({ total_cost_usd: 0, estimated_usd: 0 })],
|
|
453
|
+
sessionCount: 1,
|
|
454
|
+
});
|
|
395
455
|
assert.equal(t.red, true);
|
|
396
|
-
assert.
|
|
397
|
-
assert.match(t.
|
|
456
|
+
assert.match(t.reason, /bill to \$0/);
|
|
457
|
+
assert.match(t.fix, /--cache-read-tokens/);
|
|
398
458
|
});
|
|
399
459
|
|
|
400
460
|
test("evaluateCostTripwire: no sessions and no rows → GREEN (nothing to measure)", () => {
|
|
401
461
|
const t = evaluateCostTripwire({ rows: [], sessionCount: 0 });
|
|
402
462
|
assert.equal(t.red, false);
|
|
403
|
-
assert.equal(t.status, "green");
|
|
404
463
|
assert.match(t.reason, /nothing to measure/i);
|
|
405
464
|
});
|
|
406
465
|
|
|
407
|
-
test("evaluateCostTripwire:
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
});
|
|
466
|
+
test("evaluateCostTripwire: attribution-only day is quiet, not blind", () => {
|
|
467
|
+
// Regression guard for the false alarm this replaced: 41 message-send rows on
|
|
468
|
+
// 2026-08-11 would have armed the old tripwire all by themselves.
|
|
469
|
+
const t = evaluateCostTripwire({ rows: [nonLlmRow(), nonLlmRow()], sessionCount: 0 });
|
|
412
470
|
assert.equal(t.red, false);
|
|
413
|
-
assert.
|
|
414
|
-
|
|
471
|
+
assert.match(t.reason, /nothing to measure/i);
|
|
472
|
+
});
|
|
473
|
+
|
|
474
|
+
test("evaluateCostTripwire: spend > 0 with every session measured → GREEN", () => {
|
|
475
|
+
const t = evaluateCostTripwire({ rows: [measuredRow()], sessionCount: 1 });
|
|
476
|
+
assert.equal(t.red, false);
|
|
477
|
+
assert.equal(t.totalUsd, 0.533412);
|
|
415
478
|
assert.match(t.reason, /live/i);
|
|
416
479
|
});
|
|
417
480
|
|
|
@@ -442,19 +505,19 @@ async function makeBareAgent() {
|
|
|
442
505
|
|
|
443
506
|
function utcDate() { return new Date().toISOString().slice(0, 10); }
|
|
444
507
|
|
|
445
|
-
test("doctor RED when
|
|
508
|
+
test("doctor RED when a session ran with no token counts recorded", async () => {
|
|
446
509
|
const root = await makeBareAgent();
|
|
447
510
|
try {
|
|
448
511
|
mkdirSync(join(root, "state/cost-tracking"), { recursive: true });
|
|
449
512
|
const ledger = join(root, "state/cost-tracking", `${utcDate()}.jsonl`);
|
|
450
513
|
writeFileSync(ledger,
|
|
451
|
-
JSON.stringify({
|
|
452
|
-
JSON.stringify({
|
|
514
|
+
JSON.stringify(unmeasuredRow({ unmeasured_reason: "no-usage-field" })) + "\n" +
|
|
515
|
+
JSON.stringify(unmeasuredRow({ cadence: "backlog", unmeasured_reason: "empty-stdout" })) + "\n"
|
|
453
516
|
);
|
|
454
517
|
const r = runCli(["doctor"], root);
|
|
455
518
|
const out = (r.stdout || "") + (r.stderr || "");
|
|
456
519
|
assert.match(out, /Cost telemetry BLIND/);
|
|
457
|
-
assert.match(out, /
|
|
520
|
+
assert.match(out, /NO token counts/);
|
|
458
521
|
} finally { await rmRoot(root); }
|
|
459
522
|
});
|
|
460
523
|
|
|
@@ -463,12 +526,10 @@ test("doctor GREEN when today's ledger records real spend", async () => {
|
|
|
463
526
|
try {
|
|
464
527
|
mkdirSync(join(root, "state/cost-tracking"), { recursive: true });
|
|
465
528
|
const ledger = join(root, "state/cost-tracking", `${utcDate()}.jsonl`);
|
|
466
|
-
writeFileSync(ledger,
|
|
467
|
-
JSON.stringify({ ts: new Date().toISOString(), cadence: "inbox-processor", model: "sonnet", input_tokens: 1500, output_tokens: 320, estimated_usd: 0.0093 }) + "\n"
|
|
468
|
-
);
|
|
529
|
+
writeFileSync(ledger, JSON.stringify(measuredRow()) + "\n");
|
|
469
530
|
const r = runCli(["doctor"], root);
|
|
470
531
|
const out = (r.stdout || "") + (r.stderr || "");
|
|
471
|
-
assert.match(out, /Cost telemetry live/);
|
|
532
|
+
assert.match(out, /Cost telemetry: spend telemetry live/);
|
|
472
533
|
assert.ok(!out.includes("Cost telemetry BLIND"), "must not flag blind when spend>0");
|
|
473
534
|
} finally { await rmRoot(root); }
|
|
474
535
|
});
|
|
@@ -478,7 +539,23 @@ test("doctor GREEN (nothing to measure) when no ledger exists today", async () =
|
|
|
478
539
|
try {
|
|
479
540
|
const r = runCli(["doctor"], root);
|
|
480
541
|
const out = (r.stdout || "") + (r.stderr || "");
|
|
481
|
-
assert.match(out, /
|
|
542
|
+
assert.match(out, /nothing to measure/);
|
|
543
|
+
} finally { await rmRoot(root); }
|
|
544
|
+
});
|
|
545
|
+
|
|
546
|
+
test("doctor does NOT cry wolf on an empty ledger while the daemon is live", async () => {
|
|
547
|
+
// The regression this replaced: doctor keyed "sessions ran" off a fresh
|
|
548
|
+
// heartbeat, so every UTC day went RED between midnight and the first session
|
|
549
|
+
// and printed a fix that had already shipped.
|
|
550
|
+
const root = await makeBareAgent();
|
|
551
|
+
try {
|
|
552
|
+
mkdirSync(join(root, "state/cadence-bus"), { recursive: true });
|
|
553
|
+
writeFileSync(join(root, "state/cadence-bus/health.json"),
|
|
554
|
+
JSON.stringify({ ts: new Date().toISOString() }));
|
|
555
|
+
const r = runCli(["doctor"], root);
|
|
556
|
+
const out = (r.stdout || "") + (r.stderr || "");
|
|
557
|
+
assert.ok(!out.includes("Cost telemetry BLIND"), "a live daemon with no sessions yet is not blind");
|
|
558
|
+
assert.match(out, /nothing to measure/);
|
|
482
559
|
} finally { await rmRoot(root); }
|
|
483
560
|
});
|
|
484
561
|
|
|
@@ -860,30 +937,80 @@ test("secrets sync without a configured broker exits 1", async () => {
|
|
|
860
937
|
// doctor — backup-config DR check (gaps-enterprise-ops G16)
|
|
861
938
|
// ---------------------------------------------------------------------------
|
|
862
939
|
|
|
863
|
-
test("doctor RED when no
|
|
940
|
+
test("doctor RED when there is no DR posture at all, and names the one command that fixes it", async () => {
|
|
864
941
|
const root = await makeBareAgent();
|
|
865
942
|
try {
|
|
866
943
|
const r = runCli(["doctor"], root);
|
|
867
944
|
const out = (r.stdout || "") + (r.stderr || "");
|
|
868
|
-
assert.match(out, /No
|
|
945
|
+
assert.match(out, /No DR posture at all/);
|
|
869
946
|
assert.match(out, /backup-config\.yaml/);
|
|
947
|
+
assert.match(out, /maestro init backup-replication --apply/);
|
|
870
948
|
// doctor exits 1 whenever any issue is found; the missing backup is one.
|
|
871
949
|
assert.equal(r.status, 1, out);
|
|
872
950
|
} finally { await rmRoot(root); }
|
|
873
951
|
});
|
|
874
952
|
|
|
875
|
-
test("doctor
|
|
953
|
+
test("doctor RED when backup is enabled but has never actually run", async () => {
|
|
954
|
+
const root = await makeBareAgent();
|
|
955
|
+
try {
|
|
956
|
+
mkdirSync(join(root, ".maestro"), { recursive: true });
|
|
957
|
+
writeFileSync(join(root, ".maestro/backup-config.yaml"), "enabled: true\nprefix: t\n");
|
|
958
|
+
const r = runCli(["doctor"], root);
|
|
959
|
+
const out = (r.stdout || "") + (r.stderr || "");
|
|
960
|
+
assert.match(out, /NEVER completed a run/);
|
|
961
|
+
assert.match(out, /backup-run\.mjs/);
|
|
962
|
+
} finally { await rmRoot(root); }
|
|
963
|
+
});
|
|
964
|
+
|
|
965
|
+
test("doctor WARNS (not GREEN) when restore points are fresh but local-only", async () => {
|
|
876
966
|
const root = await makeBareAgent();
|
|
877
967
|
try {
|
|
878
968
|
mkdirSync(join(root, ".maestro"), { recursive: true });
|
|
879
|
-
writeFileSync(join(root, ".maestro/backup-config.yaml"), "enabled: true\
|
|
969
|
+
writeFileSync(join(root, ".maestro/backup-config.yaml"), "enabled: true\nprefix: t\nlocal:\n enabled: true\n");
|
|
970
|
+
writeFileSync(join(root, ".maestro/last-backup.json"), JSON.stringify({
|
|
971
|
+
completed_at: new Date().toISOString(), tier: "local", provider: "local",
|
|
972
|
+
}));
|
|
973
|
+
const r = runCli(["doctor"], root);
|
|
974
|
+
const out = (r.stdout || "") + (r.stderr || "");
|
|
975
|
+
assert.match(out, /LOCAL-ONLY/);
|
|
976
|
+
assert.match(out, /offsite\.provider/);
|
|
977
|
+
assert.ok(!out.includes("No DR posture at all"), "must not claim the config is missing");
|
|
978
|
+
} finally { await rmRoot(root); }
|
|
979
|
+
});
|
|
980
|
+
|
|
981
|
+
test("doctor GREEN for backup only when it is fresh AND off-machine", async () => {
|
|
982
|
+
const root = await makeBareAgent();
|
|
983
|
+
try {
|
|
984
|
+
mkdirSync(join(root, ".maestro"), { recursive: true });
|
|
985
|
+
writeFileSync(
|
|
986
|
+
join(root, ".maestro/backup-config.yaml"),
|
|
987
|
+
"enabled: true\nprefix: t\noffsite:\n provider: s3\n bucket: agent-backups\n"
|
|
988
|
+
);
|
|
989
|
+
writeFileSync(join(root, ".maestro/last-backup.json"), JSON.stringify({
|
|
990
|
+
completed_at: new Date().toISOString(), tier: "offsite", provider: "s3", bucket: "agent-backups",
|
|
991
|
+
}));
|
|
992
|
+
const r = runCli(["doctor"], root);
|
|
993
|
+
const out = (r.stdout || "") + (r.stderr || "");
|
|
994
|
+
assert.match(out, /Backup fresh \+ off-machine/);
|
|
995
|
+
assert.ok(!out.includes("No DR posture at all"), "must not flag missing backup when fresh");
|
|
996
|
+
assert.ok(!out.includes("LOCAL-ONLY"), "offsite is configured");
|
|
997
|
+
} finally { await rmRoot(root); }
|
|
998
|
+
});
|
|
999
|
+
|
|
1000
|
+
test("doctor reports an include path that the credential deny-list drops", async () => {
|
|
1001
|
+
const root = await makeBareAgent();
|
|
1002
|
+
try {
|
|
1003
|
+
mkdirSync(join(root, ".maestro"), { recursive: true });
|
|
1004
|
+
writeFileSync(
|
|
1005
|
+
join(root, ".maestro/backup-config.yaml"),
|
|
1006
|
+
"enabled: true\nprefix: t\ninclude:\n - state\n - .env\n"
|
|
1007
|
+
);
|
|
880
1008
|
writeFileSync(join(root, ".maestro/last-backup.json"), JSON.stringify({
|
|
881
|
-
completed_at: new Date().toISOString(),
|
|
1009
|
+
completed_at: new Date().toISOString(), tier: "local",
|
|
882
1010
|
}));
|
|
883
1011
|
const r = runCli(["doctor"], root);
|
|
884
1012
|
const out = (r.stdout || "") + (r.stderr || "");
|
|
885
|
-
assert.match(out, /
|
|
886
|
-
assert.ok(!out.includes("No backup target configured"), "must not flag missing backup when fresh");
|
|
1013
|
+
assert.match(out, /DROPPED by the credential deny-list/);
|
|
887
1014
|
} finally { await rmRoot(root); }
|
|
888
1015
|
});
|
|
889
1016
|
|
|
@@ -12,27 +12,47 @@ This complements `docs/runbooks/mac-mini-bootstrap.md` (fresh provisioning) and
|
|
|
12
12
|
compromised machine, do NOT restore onto the same hardware — rebuild on a clean
|
|
13
13
|
box.
|
|
14
14
|
|
|
15
|
-
##
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
15
|
+
## The posture: two tiers
|
|
16
|
+
|
|
17
|
+
Since 2026-08 the DR posture is **restore points first, offsite second**, driven
|
|
18
|
+
by `scripts/maintenance/backup-run.mjs` (the `nightly-backup` cadence, 03:10
|
|
19
|
+
local) reading `.maestro/backup-config.yaml`. `scripts/maintenance/backup-to-cloud.sh`
|
|
20
|
+
is now a thin compatibility wrapper around the same runner.
|
|
21
|
+
|
|
22
|
+
| Tier | Default | Where | Survives |
|
|
23
|
+
|---|---|---|---|
|
|
24
|
+
| 1 — local restore points | **ON**, zero credentials | `~/Library/Application Support/Maestro/backups/<prefix>/`, **outside** the agent repo | corrupted queue, bad state write, botched checkout, `rm -rf` of the repo |
|
|
25
|
+
| 2 — offsite copy | opt-in (`offsite.provider` + `offsite.bucket`) | gcs / s3 / rsync | machine loss, disk death, fire |
|
|
26
|
+
|
|
27
|
+
`maestro init backup-replication --apply` writes the Tier-1 default, and it is
|
|
28
|
+
`auto: true` — so a **new agent is born with a DR posture instead of a red
|
|
29
|
+
doctor line**. Tier 2 stays a deliberate act (it needs a bucket and credentials);
|
|
30
|
+
`maestro doctor` WARNs until it is set, naming the residual risk in words.
|
|
31
|
+
|
|
32
|
+
**Archived by default:** `state/`, `knowledge/`, `memory/`, `outputs/`,
|
|
33
|
+
`config/`, `.maestro/`. `logs/` is deliberately excluded — largest tree, least
|
|
34
|
+
value, and the authoritative audit trail is already the org event ledger (next
|
|
35
|
+
section). Add it explicitly to `include:` if you want it.
|
|
36
|
+
|
|
37
|
+
**NEVER archived, enforced in code and not overridable from the config**
|
|
38
|
+
(`lib/backup/policy.mjs` → `DENY_PATTERNS`): `.env`, `.env.*`,
|
|
39
|
+
`.cohort-key.json`, `.claude/.credentials.json`, `*.pem` / `*.key` / SSH keys,
|
|
40
|
+
`node_modules`, `.git`, `state/tmp`, the RAG index, sqlite WAL/SHM files.
|
|
41
|
+
`.env` carries `COHORT_API_KEY` (which authenticates as this member) and
|
|
42
|
+
`.cohort-key.json` is the device identity — a backup that shipped either to a
|
|
43
|
+
bucket would be a credential-exfiltration channel that looks *exactly* like a
|
|
44
|
+
successful backup. The deny-list is applied three times: it drops offending
|
|
45
|
+
`include:` entries at plan time (reporting them), it becomes `tar --exclude`
|
|
46
|
+
arguments, and the runner **lists the finished archive and deletes it** if a
|
|
47
|
+
denied path got in anyway. A restore re-pairs for a fresh token; it never
|
|
48
|
+
restores a dead identity.
|
|
49
|
+
|
|
50
|
+
Remaining honest gaps:
|
|
51
|
+
|
|
52
|
+
- **Archives are unencrypted at rest** unless you point Tier 2 at an encrypted
|
|
53
|
+
bucket. Treat the backup destination as sensitive and restrict access.
|
|
54
|
+
- **Tier 1 alone is not DR.** A local restore point on a dead disk is nothing.
|
|
55
|
+
Doctor says so explicitly; do not read the absence of a red line as safety.
|
|
36
56
|
|
|
37
57
|
The org server is your fleet-as-DR layer for the audit record: every
|
|
38
58
|
side-effecting action is a hash-chained `org_events` row at Cohort, exportable
|
|
@@ -46,18 +66,26 @@ non-repudiable record.
|
|
|
46
66
|
```bash
|
|
47
67
|
cd ~/<agent>-ai
|
|
48
68
|
|
|
49
|
-
# 1. Is backup configured and enabled?
|
|
50
|
-
test -f .maestro/backup-config.yaml && grep -E '^enabled:' .maestro/backup-config.yaml
|
|
69
|
+
# 1. Is backup configured and enabled, and at which tier?
|
|
70
|
+
test -f .maestro/backup-config.yaml && grep -E '^enabled:|provider:|bucket:' .maestro/backup-config.yaml
|
|
51
71
|
|
|
52
|
-
# 2. Did the last run succeed, and
|
|
72
|
+
# 2. Did the last run succeed, when, and to where?
|
|
73
|
+
cat .maestro/last-backup.json
|
|
53
74
|
tail -n 20 logs/maintenance/backup.log
|
|
54
75
|
|
|
55
|
-
# 3.
|
|
56
|
-
|
|
57
|
-
# gcloud storage ls "gs://<bucket>/<prefix>/$(date -u +%F)/"
|
|
76
|
+
# 3. Do the local restore points actually exist?
|
|
77
|
+
ls -lh ~/Library/Application\ Support/Maestro/backups/*/
|
|
58
78
|
|
|
59
|
-
# 4.
|
|
79
|
+
# 4. Does the offsite destination have the archive? (provider-specific)
|
|
80
|
+
# gcloud storage ls "gs://<bucket>/<prefix>/"
|
|
81
|
+
|
|
82
|
+
# 5. doctor's verdict — fail (no restore point) vs warn (local-only / stale)
|
|
83
|
+
# vs ok (fresh AND off-machine).
|
|
60
84
|
maestro doctor
|
|
85
|
+
|
|
86
|
+
# 6. Force a run now, without waiting for 03:10.
|
|
87
|
+
node scripts/maintenance/backup-run.mjs
|
|
88
|
+
node scripts/maintenance/backup-run.mjs --dry-run # print the plan only
|
|
61
89
|
```
|
|
62
90
|
|
|
63
91
|
If `maestro doctor` does not flag a missing/stale backup red, do not assume the
|
|
@@ -163,8 +191,10 @@ A rebuilt machine with no backup is the next dead machine.
|
|
|
163
191
|
```bash
|
|
164
192
|
cd ~/<agent>-ai
|
|
165
193
|
maestro init backup-replication --apply # if .maestro/backup-config.yaml absent
|
|
166
|
-
#
|
|
167
|
-
|
|
194
|
+
# Restore the OFFSITE block from the dead machine's config (Tier 1 is on by
|
|
195
|
+
# default; Tier 2 is the one that saves you next time).
|
|
196
|
+
node scripts/maintenance/backup-run.mjs # don't wait for 03:10 — prove it works now
|
|
197
|
+
maestro doctor # expect "Backup fresh + off-machine"
|
|
168
198
|
```
|
|
169
199
|
|
|
170
200
|
---
|
|
@@ -195,9 +225,11 @@ tested restore.
|
|
|
195
225
|
|
|
196
226
|
| Need | Command |
|
|
197
227
|
|---|---|
|
|
198
|
-
| Is backup on? | `grep -E '^enabled:' .maestro/backup-config.yaml` |
|
|
199
|
-
| Last backup status | `tail logs/maintenance/backup.log` |
|
|
200
|
-
| Configure backup | `maestro init backup-replication --apply` |
|
|
228
|
+
| Is backup on, and at which tier? | `grep -E '^enabled:\|provider:\|bucket:' .maestro/backup-config.yaml` |
|
|
229
|
+
| Last backup status | `cat .maestro/last-backup.json` / `tail logs/maintenance/backup.log` |
|
|
230
|
+
| Configure backup (Tier 1 default) | `maestro init backup-replication --apply` |
|
|
231
|
+
| Run a backup now | `node scripts/maintenance/backup-run.mjs` |
|
|
232
|
+
| Add the offsite tier | set `offsite.provider` + `offsite.bucket` in `.maestro/backup-config.yaml` |
|
|
201
233
|
| Governance posture incl. backup freshness | `maestro doctor` |
|
|
202
234
|
| Re-mint agent identity | `cohort pair approve <agentId> --scopes <...>` |
|
|
203
235
|
| Recover audit history | `cohort export-audit --format ocsf` |
|
package/framework-features.json
CHANGED
|
@@ -94,13 +94,13 @@
|
|
|
94
94
|
}
|
|
95
95
|
},
|
|
96
96
|
"backup-replication": {
|
|
97
|
-
"version": "
|
|
97
|
+
"version": "2",
|
|
98
98
|
"since": "1.9.0",
|
|
99
|
-
"title": "
|
|
99
|
+
"title": "Disaster-recovery backup (local restore points + optional offsite)",
|
|
100
100
|
"init": {
|
|
101
|
-
"auto":
|
|
101
|
+
"auto": true,
|
|
102
102
|
"command": "node scripts/setup/init-backup.mjs",
|
|
103
|
-
"description": "
|
|
103
|
+
"description": "Write .maestro/backup-config.yaml with the default DR posture so a new agent is not born failing doctor. TIER 1 (on, zero credentials): nightly tar.gz of state/, knowledge/, memory/, outputs/, config/ and .maestro/ into ~/Library/Application Support/Maestro/backups/<prefix>/ — outside the agent repo — via the nightly-backup cadence at 03:10, 14-day retention. TIER 2 (opt-in): set offsite.provider (gcs|s3|rsync) + offsite.bucket to get a copy off the machine; doctor WARNS until you do, because Tier 1 alone does not survive losing the box. NEVER archived, enforced in code (lib/backup/policy.mjs DENY_PATTERNS), not overridable from the config: .env, .cohort-key.json, private keys, .claude/.credentials.json, node_modules, .git, state/tmp, the RAG index."
|
|
104
104
|
}
|
|
105
105
|
},
|
|
106
106
|
"slack-socket-mode": {
|