@cohortapp/agent-sdk 2.5.1 → 2.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +305 -89
- package/bin/maestro.test.mjs +357 -48
- package/docs/runbooks/backup-restore.md +65 -33
- package/framework-features.json +4 -4
- package/lib/backup/policy.mjs +710 -0
- package/lib/backup/policy.test.mjs +305 -0
- package/lib/budget-escalate.mjs +133 -0
- package/lib/budget-escalate.test.mjs +232 -0
- package/lib/budget-guard.envelope.test.mjs +476 -0
- package/lib/budget-guard.mjs +853 -75
- package/lib/budget-guard.test.mjs +91 -42
- package/lib/cadences.mjs +33 -0
- package/lib/channels/orgmail/adapter.mjs +88 -3
- package/lib/channels/orgmail/adapter.test.mjs +137 -0
- package/lib/channels/repeat-suppressor.mjs +198 -0
- package/lib/channels/repeat-suppressor.test.mjs +134 -0
- package/lib/comms/receipts.mjs +297 -0
- package/lib/cost/ledger-row.mjs +333 -0
- package/lib/cost/ledger-row.test.mjs +183 -0
- package/lib/execution/drive.mjs +28 -1
- package/lib/execution/effects.mjs +191 -12
- package/lib/execution/effects.test.mjs +50 -11
- package/lib/goals/admission.mjs +13 -1
- package/lib/goals/admission.test.mjs +26 -1
- package/lib/goals/loop.mjs +13 -0
- package/lib/kpi-sensors.test.mjs +3 -0
- package/lib/mandate/cache.mjs +13 -5
- package/lib/mandate/derive.mjs +146 -21
- package/lib/mandate/derive.test.mjs +50 -6
- package/lib/mandate/model.mjs +32 -4
- package/lib/mandate/refresh.test.mjs +16 -2
- package/lib/mcp/server.test.mjs +12 -3
- package/lib/model-router/economics.mjs +107 -76
- package/lib/model-router/economics.test.mjs +64 -46
- package/lib/model-router/integration-coverage.test.mjs +39 -37
- package/lib/model-router/ledger.mjs +75 -22
- package/lib/model-router/ledger.test.mjs +35 -2
- package/lib/org/client.mjs +14 -0
- package/lib/org/cost-sync.mjs +16 -2
- package/lib/org/doctor.mjs +62 -1
- package/lib/org/doctor.test.mjs +36 -3
- package/lib/org/email-remedy.mjs +49 -0
- package/lib/org/engagement-ledger.mjs +376 -0
- package/lib/org/engagement-ledger.test.mjs +112 -0
- package/lib/org/engagement.mjs +1056 -0
- package/lib/org/engagement.test.mjs +739 -0
- package/lib/org/messaging.mjs +230 -3
- package/lib/org/messaging.test.mjs +110 -1
- package/lib/org/param-contract.mjs +56 -2
- package/lib/org/param-contract.test.mjs +26 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +5 -0
- package/lib/org/protocol.test.mjs +7 -1
- package/lib/org/tool-surface.mjs +506 -10
- package/lib/org/tool-surface.test.mjs +191 -7
- package/lib/org/ui-parity.mjs +333 -6
- package/lib/org/ui-parity.test.mjs +96 -3
- package/lib/org/work-ledger.mjs +241 -0
- package/lib/org/work-ledger.test.mjs +237 -0
- package/lib/plan/adoption-e2e.test.mjs +366 -0
- package/lib/plan/budget-enforcement.test.mjs +400 -0
- package/lib/plan/budget-runtime.mjs +215 -0
- package/lib/plan/compile.mjs +201 -5
- package/lib/plan/compile.test.mjs +19 -5
- package/lib/plan/emit.mjs +8 -0
- package/lib/plan/emit.test.mjs +18 -0
- package/lib/resource-governor.mjs +58 -12
- package/lib/resource-governor.test.mjs +41 -1
- package/lib/security/audit-engine.mjs +45 -8
- package/lib/security/audit-engine.test.mjs +35 -0
- package/lib/setup/enroll-from-cohort.mjs +14 -1
- package/lib/setup/sections/mandate.mjs +48 -7
- package/lib/setup/sections/mandate.test.mjs +17 -2
- package/lib/setup/sections/orgmail.mjs +10 -2
- package/lib/setup/state.mjs +83 -2
- package/lib/telemetry/collect.mjs +360 -20
- package/lib/telemetry/collect.test.mjs +266 -0
- package/package.json +1 -1
- package/scripts/cost/track-claude-usage.mjs +207 -48
- package/scripts/cost/track-claude-usage.test.mjs +148 -0
- package/scripts/daemon/agent-daemon.mjs +315 -17
- package/scripts/daemon/assurance-e2e.test.mjs +421 -0
- package/scripts/daemon/assurance.mjs +944 -0
- package/scripts/daemon/assurance.test.mjs +668 -0
- package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
- package/scripts/daemon/cadence-consumer.mjs +147 -9
- package/scripts/daemon/cadence-consumer.test.mjs +6 -0
- package/scripts/daemon/cadence-handlers.mjs +158 -0
- package/scripts/daemon/cadence-handlers.test.mjs +64 -0
- package/scripts/daemon/classifier.test.mjs +18 -9
- package/scripts/daemon/deliver.mjs +314 -0
- package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
- package/scripts/daemon/dispatcher.mjs +64 -6
- package/scripts/daemon/responder-cost.test.mjs +68 -0
- package/scripts/daemon/responder.mjs +351 -298
- package/scripts/local-triggers/generate-plists.test.mjs +7 -4
- package/scripts/maintenance/backup-run.mjs +415 -0
- package/scripts/maintenance/backup-to-cloud.sh +16 -116
- package/scripts/org/send-orgmail.mjs +16 -0
- package/scripts/record-receipt.sh +63 -0
- package/scripts/restore-from-backup.sh +14 -3
- package/scripts/restore-from-backup.test.mjs +8 -5
- package/scripts/send-email-threaded.py +47 -0
- package/scripts/send-sms.sh +4 -0
- package/scripts/send-whatsapp.sh +4 -0
- package/scripts/setup/init-backup.mjs +93 -38
- package/scripts/slack-send.sh +12 -0
|
@@ -107,7 +107,7 @@ function listPlists(agentRoot) {
|
|
|
107
107
|
// Tests
|
|
108
108
|
// ---------------------------------------------------------------------------
|
|
109
109
|
|
|
110
|
-
test("generator emits
|
|
110
|
+
test("generator emits 23 standard plists with the agent's first name", async () => {
|
|
111
111
|
// Standard inventory (cadence-bus v1 + slack-socket-mode v1), with NO
|
|
112
112
|
// config/.cadence-plists.tsv present (no archetype cadences):
|
|
113
113
|
// daemon, poll-relay, slack-socket (3 infra), PLUS one trigger plist per
|
|
@@ -119,7 +119,8 @@ test("generator emits 22 standard plists with the agent's first name", async ()
|
|
|
119
119
|
// org-cost-sync, monitoring-alerts, dynamic-jobs (5), PLUS the SP10
|
|
120
120
|
// messaging-inbound cadence (1), PLUS the 2026-07 Directory & Design
|
|
121
121
|
// stewards directory-hygiene + brand-steward (2), PLUS the D3 self-directed
|
|
122
|
-
// loop goal-steward (1)
|
|
122
|
+
// loop goal-steward (1), PLUS the 2026-08 DR cadence nightly-backup (1).
|
|
123
|
+
// = 3 infra + 20 trigger = 23 total.
|
|
123
124
|
// org-pulse used to be declared standard but was OMITTED by the old hardcoded
|
|
124
125
|
// list (it never got a plist); deriving from the SoT fixes that drift.
|
|
125
126
|
// The former weekly-* cadences are still ARCHETYPE-DRIVEN (function × altitude),
|
|
@@ -129,9 +130,11 @@ test("generator emits 22 standard plists with the agent's first name", async ()
|
|
|
129
130
|
const r = runGenerator(root);
|
|
130
131
|
assert.equal(r.status, 0, r.stderr);
|
|
131
132
|
const plists = listPlists(root);
|
|
132
|
-
assert.equal(plists.length,
|
|
133
|
+
assert.equal(plists.length, 23, `expected 23 standard plists; got ${plists.join(",")}`);
|
|
133
134
|
assert.ok(plists.includes("ai.maestro.alice-messaging-inbound.plist"),
|
|
134
135
|
"messaging-inbound (SP10 standard cadence) must get a SoT-derived plist");
|
|
136
|
+
assert.ok(plists.includes("ai.maestro.alice-nightly-backup.plist"),
|
|
137
|
+
"nightly-backup (the DR cadence) must be scheduled, not left to a human to remember");
|
|
135
138
|
for (const id of ["directory-hygiene", "brand-steward"]) {
|
|
136
139
|
assert.ok(plists.includes(`ai.maestro.alice-${id}.plist`),
|
|
137
140
|
`Directory/Design steward "${id}" must get a SoT-derived plist; got ${plists.join(",")}`);
|
|
@@ -200,7 +203,7 @@ test("archetype cadences from config/.cadence-plists.tsv emit extra trigger plis
|
|
|
200
203
|
const r = runGenerator(root);
|
|
201
204
|
assert.equal(r.status, 0, r.stderr);
|
|
202
205
|
const plists = listPlists(root);
|
|
203
|
-
assert.equal(plists.length,
|
|
206
|
+
assert.equal(plists.length, 25, `expected 23 standard + 2 archetype; got ${plists.join(",")}`);
|
|
204
207
|
const dir = join(root, "scripts/local-triggers/plists");
|
|
205
208
|
const eng = readFileSync(join(dir, "ai.maestro.erin-engineering-health.plist"), "utf-8");
|
|
206
209
|
assert.match(eng, /<key>Weekday<\/key>\s*<integer>3<\/integer>/); // base64 schedule decoded
|
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* backup-run.mjs — the nightly DR job.
|
|
4
|
+
*
|
|
5
|
+
* Replaces the brains of scripts/maintenance/backup-to-cloud.sh (which is now a
|
|
6
|
+
* thin wrapper so any already-installed plist keeps working). The shell driver
|
|
7
|
+
* parsed YAML with awk, had no deny-list, and — the part that mattered —
|
|
8
|
+
* `exit 0`'d silently when unconfigured, which is how a fleet ran for months
|
|
9
|
+
* with zero agents backed up and nothing red anywhere.
|
|
10
|
+
*
|
|
11
|
+
* What it does, in order:
|
|
12
|
+
* 1. Resolve the plan (lib/backup/policy.mjs — the single source of truth
|
|
13
|
+
* shared with doctor and the nightly-backup cadence).
|
|
14
|
+
* 2. REFUSE to run if the plan is unusable, loudly, with a non-zero exit.
|
|
15
|
+
* Never a silent success.
|
|
16
|
+
* 3. tar.gz the include paths with the hard credential deny-list applied as
|
|
17
|
+
* --exclude patterns (belt) on top of the plan's already-filtered include
|
|
18
|
+
* list (braces).
|
|
19
|
+
* 4. VERIFY the archive does not contain a denied path before it goes
|
|
20
|
+
* anywhere. A backup that ships `.env` to a bucket is a credential leak
|
|
21
|
+
* that looks exactly like a successful backup, so the check happens on the
|
|
22
|
+
* real archive listing, not on the intent.
|
|
23
|
+
* 5. Place it in the local restore-point directory (Tier 1), prune to
|
|
24
|
+
* retention, then push offsite (Tier 2) when configured.
|
|
25
|
+
* 6. Stamp .maestro/last-backup.json — the marker doctor and
|
|
26
|
+
* lib/diagnostics/backup-freshness.mjs read.
|
|
27
|
+
*
|
|
28
|
+
* Exit codes: 0 success (including "offsite skipped because unconfigured" —
|
|
29
|
+
* that is a WARN state doctor owns, not a job failure), 1 refused/failed.
|
|
30
|
+
*
|
|
31
|
+
* Usage:
|
|
32
|
+
* node scripts/maintenance/backup-run.mjs [--dry-run] [--agent-dir <path>]
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { execFileSync } from "node:child_process";
|
|
36
|
+
import {
|
|
37
|
+
existsSync,
|
|
38
|
+
mkdirSync,
|
|
39
|
+
readdirSync,
|
|
40
|
+
rmSync,
|
|
41
|
+
statSync,
|
|
42
|
+
writeFileSync,
|
|
43
|
+
appendFileSync,
|
|
44
|
+
} from "node:fs";
|
|
45
|
+
import { tmpdir } from "node:os";
|
|
46
|
+
import { dirname, join, resolve } from "node:path";
|
|
47
|
+
import { fileURLToPath } from "node:url";
|
|
48
|
+
|
|
49
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
50
|
+
|
|
51
|
+
const argv = process.argv.slice(2);
|
|
52
|
+
const DRY_RUN = argv.includes("--dry-run");
|
|
53
|
+
const agentDirArg = (() => {
|
|
54
|
+
const i = argv.indexOf("--agent-dir");
|
|
55
|
+
return i >= 0 ? argv[i + 1] : null;
|
|
56
|
+
})();
|
|
57
|
+
|
|
58
|
+
const AGENT_DIR = resolve(
|
|
59
|
+
agentDirArg || process.env.AGENT_ROOT || process.env.AGENT_DIR || join(__dirname, "..", "..")
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
const LOG_DIR = join(AGENT_DIR, "logs", "maintenance");
|
|
63
|
+
const LOG_FILE = join(LOG_DIR, "backup.log");
|
|
64
|
+
|
|
65
|
+
function log(level, msg) {
|
|
66
|
+
const line = `[${new Date().toISOString()}] [${level}] ${msg}`;
|
|
67
|
+
process.stdout.write(`${line}\n`);
|
|
68
|
+
try {
|
|
69
|
+
mkdirSync(LOG_DIR, { recursive: true });
|
|
70
|
+
appendFileSync(LOG_FILE, `${line}\n`);
|
|
71
|
+
} catch {
|
|
72
|
+
/* logging must never be the thing that fails the backup */
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Resolve lib/backup/policy.mjs from the agent's own lib/ first (an agent repo
|
|
78
|
+
* carries its own framework copy), then the installed SDK, then this checkout —
|
|
79
|
+
* the same resolution ladder generate-plists.sh uses.
|
|
80
|
+
*/
|
|
81
|
+
async function loadPolicy() {
|
|
82
|
+
const candidates = [
|
|
83
|
+
join(AGENT_DIR, "lib", "backup", "policy.mjs"),
|
|
84
|
+
join(AGENT_DIR, "node_modules", "@cohortapp", "agent-sdk", "lib", "backup", "policy.mjs"),
|
|
85
|
+
join(__dirname, "..", "..", "lib", "backup", "policy.mjs"),
|
|
86
|
+
];
|
|
87
|
+
for (const c of candidates) {
|
|
88
|
+
if (existsSync(c)) return import(`file://${c}`);
|
|
89
|
+
}
|
|
90
|
+
throw new Error(`lib/backup/policy.mjs not found (looked in: ${candidates.join(", ")})`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function main() {
|
|
94
|
+
const policy = await loadPolicy();
|
|
95
|
+
const plan = policy.resolveBackupPlan({ agentRoot: AGENT_DIR });
|
|
96
|
+
|
|
97
|
+
// ── Refuse loudly rather than exit 0 silently ────────────────────────────
|
|
98
|
+
if (!plan.present) {
|
|
99
|
+
log("error", "REFUSING: no .maestro/backup-config.yaml. Fix: maestro init backup-replication --apply");
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
if (!plan.readable) {
|
|
103
|
+
log("error", "REFUSING: .maestro/backup-config.yaml is unparseable YAML.");
|
|
104
|
+
process.exit(1);
|
|
105
|
+
}
|
|
106
|
+
if (!plan.enabled) {
|
|
107
|
+
log("error", "REFUSING: backup-config.yaml has enabled != true — this machine has no restore points.");
|
|
108
|
+
process.exit(1);
|
|
109
|
+
}
|
|
110
|
+
for (const v of plan.violations) {
|
|
111
|
+
log("warn", `include path DROPPED by the credential deny-list: ${v.path} (matched ${v.pattern})`);
|
|
112
|
+
}
|
|
113
|
+
if (!plan.include.length) {
|
|
114
|
+
log("error", "REFUSING: every include path was rejected — the archive would be empty.");
|
|
115
|
+
process.exit(1);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Only archive what actually exists; a missing optional tree (outputs/ on a
|
|
119
|
+
// fresh agent) must not fail the whole run.
|
|
120
|
+
const present = plan.include.filter((p) => existsSync(join(AGENT_DIR, p)));
|
|
121
|
+
const absent = plan.include.filter((p) => !present.includes(p));
|
|
122
|
+
if (absent.length) log("info", `skipping absent include path(s): ${absent.join(", ")}`);
|
|
123
|
+
if (!present.length) {
|
|
124
|
+
log("error", "REFUSING: none of the configured include paths exist on disk.");
|
|
125
|
+
process.exit(1);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-").replace(/Z$/, "Z");
|
|
129
|
+
const archiveName = `${plan.prefix}-${stamp}.tar.gz`;
|
|
130
|
+
// Tier 1 off + Tier 2 on is a legitimate (if unusual) posture: stage in the
|
|
131
|
+
// OS temp dir, never inside the agent repo. An archive that lives in the tree
|
|
132
|
+
// it protects is not a backup, and it would also end up inside the NEXT one.
|
|
133
|
+
const localDir = plan.local.enabled ? plan.local.dir : join(tmpdir(), "maestro-backup-staging");
|
|
134
|
+
const archivePath = join(localDir, archiveName);
|
|
135
|
+
|
|
136
|
+
log("info", `plan: tier=${plan.tier} include=[${present.join(", ")}] → ${archivePath}`);
|
|
137
|
+
if (DRY_RUN) {
|
|
138
|
+
log("info", "--dry-run: stopping before tar.");
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
mkdirSync(localDir, { recursive: true });
|
|
143
|
+
|
|
144
|
+
// ── 2b. CONTENT deny-list ───────────────────────────────────────────────
|
|
145
|
+
// The path deny-list cannot see a credential stored INSIDE an included file,
|
|
146
|
+
// and that is not hypothetical: state/setup/progress.json carried this agent's
|
|
147
|
+
// live COHORT_API_KEY in plaintext while the archive listing verified clean.
|
|
148
|
+
// Scan the real bytes on disk and turn every hit into a tar --exclude, so the
|
|
149
|
+
// secret never enters the tarball in the first place.
|
|
150
|
+
//
|
|
151
|
+
// A hit DROPS THE FILE; it does not fail the run. A false positive must cost
|
|
152
|
+
// one file in a restore point, never the machine's whole DR posture.
|
|
153
|
+
const contentScan = policy.scanIncludesForSecrets(plan, AGENT_DIR);
|
|
154
|
+
const contentExcludes = [];
|
|
155
|
+
for (const leak of contentScan.leaks) {
|
|
156
|
+
contentExcludes.push(`--exclude=${leak.path}`);
|
|
157
|
+
log("warn", `file DROPPED by the CONTENT deny-list: ${leak.path} (matched ${leak.patterns.join(", ")}) — a credential inside an archived file is the same leak as archiving .env`);
|
|
158
|
+
}
|
|
159
|
+
for (const big of contentScan.oversize) {
|
|
160
|
+
log("warn", `NOT content-scanned (over the size limit): ${big} — it is still archived; the path deny-list is its only cover`);
|
|
161
|
+
}
|
|
162
|
+
for (const e of contentScan.errors.slice(0, 10)) log("warn", `content scan could not read ${e}`);
|
|
163
|
+
if (contentScan.truncated) {
|
|
164
|
+
log("error", "REFUSING: the content scan hit its file budget before finishing — an unscanned tree may contain credentials.");
|
|
165
|
+
process.exit(1);
|
|
166
|
+
}
|
|
167
|
+
log("info", `content scan: ${contentScan.scanned} file(s) read, ${contentScan.leaks.length} dropped, ${contentScan.skippedBinary} binary skipped`);
|
|
168
|
+
|
|
169
|
+
// ── 3. build the archive ────────────────────────────────────────────────
|
|
170
|
+
const excludeArgs = [...policy.tarExcludeArgs(plan), ...contentExcludes];
|
|
171
|
+
try {
|
|
172
|
+
execFileSync("tar", [...excludeArgs, "-czf", archivePath, "-C", AGENT_DIR, ...present], {
|
|
173
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
174
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
175
|
+
});
|
|
176
|
+
} catch (err) {
|
|
177
|
+
log("error", `tar failed: ${err && err.message}`);
|
|
178
|
+
try { rmSync(archivePath, { force: true }); } catch { /* best effort */ }
|
|
179
|
+
process.exit(1);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ── 4. verify no denied path made it in ─────────────────────────────────
|
|
183
|
+
// This is the check that turns the deny-list from an intention into a
|
|
184
|
+
// guarantee. It reads the REAL archive listing: a symlink, a nested copy, or
|
|
185
|
+
// a tar flag that behaved differently than expected all surface here.
|
|
186
|
+
let listing = "";
|
|
187
|
+
try {
|
|
188
|
+
listing = execFileSync("tar", ["-tzf", archivePath], {
|
|
189
|
+
encoding: "utf-8",
|
|
190
|
+
maxBuffer: 128 * 1024 * 1024,
|
|
191
|
+
});
|
|
192
|
+
} catch (err) {
|
|
193
|
+
log("error", `could not list the archive to verify it: ${err && err.message}`);
|
|
194
|
+
rmSync(archivePath, { force: true });
|
|
195
|
+
process.exit(1);
|
|
196
|
+
}
|
|
197
|
+
const leaked = [];
|
|
198
|
+
const contentDenied = new Set(contentScan.leaks.map((l) => l.path));
|
|
199
|
+
for (const entry of listing.split("\n")) {
|
|
200
|
+
const rel = entry.trim().replace(/^\.\//, "");
|
|
201
|
+
if (!rel || rel.endsWith("/")) continue;
|
|
202
|
+
const hit = policy.denyMatch(rel);
|
|
203
|
+
if (hit) leaked.push(`${rel} (${hit})`);
|
|
204
|
+
// A file the content scan flagged must not be in the listing. If tar kept it
|
|
205
|
+
// anyway (an --exclude that did not match the stored path form), that is
|
|
206
|
+
// exactly the silent leak this whole check exists to catch.
|
|
207
|
+
else if (contentDenied.has(rel)) leaked.push(`${rel} (content deny-list — the --exclude did not take)`);
|
|
208
|
+
if (leaked.length >= 10) break;
|
|
209
|
+
}
|
|
210
|
+
if (leaked.length) {
|
|
211
|
+
rmSync(archivePath, { force: true });
|
|
212
|
+
log("error", `REFUSING to keep an archive containing denied paths — DELETED it. Offenders: ${leaked.join("; ")}`);
|
|
213
|
+
process.exit(1);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const size = statSync(archivePath).size;
|
|
217
|
+
// Say exactly what was verified. "verified clean" over a path-only check is the
|
|
218
|
+
// false assurance that let a live API key ship.
|
|
219
|
+
log(
|
|
220
|
+
"info",
|
|
221
|
+
`archive verified: ${archiveName} (${size} bytes, ${listing.split("\n").length - 1} entries) — ` +
|
|
222
|
+
`no denied PATHS, and ${contentScan.leaks.length} content-flagged file(s) confirmed absent ` +
|
|
223
|
+
`(${contentScan.scanned} file(s) content-scanned, ${contentScan.oversize.length} too large to scan)`
|
|
224
|
+
);
|
|
225
|
+
|
|
226
|
+
// ── 5a. prune local restore points ──────────────────────────────────────
|
|
227
|
+
let pruned = 0;
|
|
228
|
+
if (plan.local.enabled) {
|
|
229
|
+
const cutoff = Date.now() - plan.local.retentionDays * 86_400_000;
|
|
230
|
+
for (const name of safeReaddir(localDir)) {
|
|
231
|
+
if (!name.startsWith(`${plan.prefix}-`) || !name.endsWith(".tar.gz")) continue;
|
|
232
|
+
const full = join(localDir, name);
|
|
233
|
+
if (full === archivePath) continue;
|
|
234
|
+
try {
|
|
235
|
+
if (statSync(full).mtimeMs < cutoff) {
|
|
236
|
+
rmSync(full, { force: true });
|
|
237
|
+
pruned++;
|
|
238
|
+
}
|
|
239
|
+
} catch { /* a file that vanished under us is already pruned */ }
|
|
240
|
+
}
|
|
241
|
+
if (pruned) log("info", `pruned ${pruned} local restore point(s) older than ${plan.local.retentionDays}d`);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// ── 5b. offsite ─────────────────────────────────────────────────────────
|
|
245
|
+
let offsiteOk = false;
|
|
246
|
+
let offsiteDetail = "not configured";
|
|
247
|
+
if (plan.offsite.configured) {
|
|
248
|
+
const key = `${plan.prefix}/${archiveName}`;
|
|
249
|
+
try {
|
|
250
|
+
if (plan.offsite.provider === "gcs") {
|
|
251
|
+
requireBinary("gsutil", "install the gcloud SDK, or switch offsite.provider");
|
|
252
|
+
execFileSync("gsutil", ["cp", archivePath, `gs://${stripScheme(plan.offsite.bucket)}/${key}`], { stdio: "pipe" });
|
|
253
|
+
} else if (plan.offsite.provider === "s3") {
|
|
254
|
+
requireBinary("aws", "install the AWS CLI, or switch offsite.provider");
|
|
255
|
+
execFileSync("aws", ["s3", "cp", archivePath, `s3://${stripScheme(plan.offsite.bucket)}/${key}`], { stdio: "pipe" });
|
|
256
|
+
} else if (plan.offsite.provider === "rsync") {
|
|
257
|
+
requireBinary("rsync", "install rsync, or switch offsite.provider");
|
|
258
|
+
execFileSync("rsync", ["-az", archivePath, `${plan.offsite.bucket.replace(/\/+$/, "")}/${plan.prefix}/`], { stdio: "pipe" });
|
|
259
|
+
}
|
|
260
|
+
offsiteOk = true;
|
|
261
|
+
offsiteDetail = `${plan.offsite.provider}://${plan.offsite.bucket}/${key}`;
|
|
262
|
+
log("info", `offsite copy complete → ${offsiteDetail}`);
|
|
263
|
+
} catch (err) {
|
|
264
|
+
// A failed offsite push does NOT discard the local restore point — we are
|
|
265
|
+
// strictly better off than before. But it is never silent: the marker
|
|
266
|
+
// records the failure and doctor reads the marker.
|
|
267
|
+
offsiteDetail = `FAILED: ${err && err.message}`;
|
|
268
|
+
log("error", `offsite copy failed (local restore point kept): ${err && err.message}`);
|
|
269
|
+
}
|
|
270
|
+
} else {
|
|
271
|
+
log("warn", "offsite is NOT configured — these restore points do not survive losing this machine. Set offsite.provider + offsite.bucket in .maestro/backup-config.yaml.");
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// ── 5c. prune OFFSITE to retention ──────────────────────────────────────
|
|
275
|
+
// `offsite.retention_days` was parsed into the plan, shipped in the default
|
|
276
|
+
// config beside a comment describing Tier 2 as working, and consumed by
|
|
277
|
+
// NOTHING: the only pruning was local, on `local.retentionDays`. An operator
|
|
278
|
+
// who set `retention_days: 30` got a bucket that grew forever and was never
|
|
279
|
+
// told. Prune with the same CLI that uploaded, over our OWN key prefix only.
|
|
280
|
+
let offsitePruned = 0;
|
|
281
|
+
let offsitePruneDetail = "not attempted";
|
|
282
|
+
if (offsiteOk) {
|
|
283
|
+
try {
|
|
284
|
+
offsitePruned = pruneOffsite(plan, log);
|
|
285
|
+
offsitePruneDetail = `${offsitePruned} archive(s) older than ${plan.offsite.retentionDays}d removed`;
|
|
286
|
+
} catch (err) {
|
|
287
|
+
// Never fatal: the restore points exist, which is the point of the job.
|
|
288
|
+
offsitePruneDetail = `FAILED: ${err && err.message}`;
|
|
289
|
+
log("error", `offsite retention prune failed (the upload succeeded; old archives remain): ${err && err.message}`);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// Staging dir (Tier 1 off) is not a restore point — clear it once pushed.
|
|
294
|
+
if (!plan.local.enabled && offsiteOk) {
|
|
295
|
+
try { rmSync(archivePath, { force: true }); } catch { /* best effort */ }
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// ── 6. stamp the marker doctor reads ────────────────────────────────────
|
|
299
|
+
const marker = {
|
|
300
|
+
completed_at: new Date().toISOString(),
|
|
301
|
+
tier: offsiteOk ? "offsite" : "local",
|
|
302
|
+
provider: offsiteOk ? plan.offsite.provider : "local",
|
|
303
|
+
bucket: offsiteOk ? plan.offsite.bucket : localDir,
|
|
304
|
+
prefix: plan.prefix,
|
|
305
|
+
archive: archiveName,
|
|
306
|
+
size_bytes: size,
|
|
307
|
+
include: present,
|
|
308
|
+
local_dir: localDir,
|
|
309
|
+
local_pruned: pruned,
|
|
310
|
+
offsite: offsiteDetail,
|
|
311
|
+
offsite_retention_days: plan.offsite.configured ? plan.offsite.retentionDays : null,
|
|
312
|
+
offsite_pruned: offsitePruneDetail,
|
|
313
|
+
denied_include_paths: plan.violations,
|
|
314
|
+
// The CONTENT deny-list's findings, on the record. A leak that is dropped
|
|
315
|
+
// silently teaches nobody; the operator needs to know a credential was
|
|
316
|
+
// sitting in an archived file and rotate it.
|
|
317
|
+
denied_content_paths: contentScan.leaks,
|
|
318
|
+
content_scanned_files: contentScan.scanned,
|
|
319
|
+
content_unscanned_oversize: contentScan.oversize,
|
|
320
|
+
};
|
|
321
|
+
writeFileSync(join(AGENT_DIR, ".maestro", "last-backup.json"), `${JSON.stringify(marker, null, 2)}\n`);
|
|
322
|
+
log("info", `backup complete (tier=${marker.tier}).`);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function safeReaddir(dir) {
|
|
326
|
+
try {
|
|
327
|
+
return readdirSync(dir);
|
|
328
|
+
} catch {
|
|
329
|
+
return [];
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
function stripScheme(bucket) {
|
|
334
|
+
return String(bucket).replace(/^(gs|s3):\/\//, "").replace(/\/+$/, "");
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Delete offsite archives older than `offsite.retentionDays`.
|
|
339
|
+
*
|
|
340
|
+
* Scoped THREE ways so this can never delete something that is not ours:
|
|
341
|
+
* - only under `<bucket>/<prefix>/`
|
|
342
|
+
* - only names matching `<prefix>-<ISO stamp>.tar.gz`
|
|
343
|
+
* - only stamps older than the retention window (parsed from the NAME, not
|
|
344
|
+
* from remote metadata, so a re-upload cannot resurrect an object)
|
|
345
|
+
*
|
|
346
|
+
* rsync targets are deliberately NOT pruned: doing so means running a remote
|
|
347
|
+
* `find -delete` over ssh against a path from a config file, which is a much
|
|
348
|
+
* bigger gun than a bucket delete. The operator is TOLD rather than left to
|
|
349
|
+
* assume — the honest half of "fail-open is fine, silent is not".
|
|
350
|
+
*
|
|
351
|
+
* @returns {number} objects removed
|
|
352
|
+
*/
|
|
353
|
+
function pruneOffsite(plan, log) {
|
|
354
|
+
const days = plan.offsite.retentionDays;
|
|
355
|
+
if (!Number.isFinite(days) || days <= 0) return 0;
|
|
356
|
+
const cutoff = Date.now() - days * 86_400_000;
|
|
357
|
+
const prefix = plan.prefix;
|
|
358
|
+
const bucket = stripScheme(plan.offsite.bucket);
|
|
359
|
+
const nameRe = new RegExp(`^${prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}-(.+)\\.tar\\.gz$`);
|
|
360
|
+
|
|
361
|
+
const ageOf = (name) => {
|
|
362
|
+
const m = nameRe.exec(name);
|
|
363
|
+
if (!m) return null;
|
|
364
|
+
// `2026-08-12T08-26-30-411Z` → restore the ISO punctuation.
|
|
365
|
+
const iso = m[1].replace(/T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, "T$1:$2:$3.$4Z");
|
|
366
|
+
const t = Date.parse(iso);
|
|
367
|
+
return Number.isFinite(t) ? t : null;
|
|
368
|
+
};
|
|
369
|
+
|
|
370
|
+
if (plan.offsite.provider === "rsync") {
|
|
371
|
+
log(
|
|
372
|
+
"warn",
|
|
373
|
+
`offsite.retention_days: ${days} is NOT enforced for the rsync provider — maestro will not run a remote ` +
|
|
374
|
+
"delete over ssh against a path from a config file. Age out archives on the receiving host (a cron " +
|
|
375
|
+
"`find <path> -name '*.tar.gz' -mtime +N -delete`), or switch offsite.provider to gcs/s3."
|
|
376
|
+
);
|
|
377
|
+
return 0;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const list = plan.offsite.provider === "gcs"
|
|
381
|
+
? execFileSync("gsutil", ["ls", `gs://${bucket}/${prefix}/`], { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] })
|
|
382
|
+
: execFileSync("aws", ["s3", "ls", `s3://${bucket}/${prefix}/`], { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] });
|
|
383
|
+
|
|
384
|
+
let removed = 0;
|
|
385
|
+
for (const raw of list.split("\n")) {
|
|
386
|
+
const line = raw.trim();
|
|
387
|
+
if (!line) continue;
|
|
388
|
+
const name = line.split(/[\s/]+/).pop();
|
|
389
|
+
if (!name || !name.endsWith(".tar.gz")) continue;
|
|
390
|
+
const t = ageOf(name);
|
|
391
|
+
if (t === null || t >= cutoff) continue;
|
|
392
|
+
const key = `${prefix}/${name}`;
|
|
393
|
+
if (plan.offsite.provider === "gcs") {
|
|
394
|
+
execFileSync("gsutil", ["rm", `gs://${bucket}/${key}`], { stdio: "pipe" });
|
|
395
|
+
} else {
|
|
396
|
+
execFileSync("aws", ["s3", "rm", `s3://${bucket}/${key}`], { stdio: "pipe" });
|
|
397
|
+
}
|
|
398
|
+
removed += 1;
|
|
399
|
+
}
|
|
400
|
+
if (removed) log("info", `offsite prune: removed ${removed} archive(s) older than ${days}d from ${plan.offsite.provider}://${bucket}/${prefix}/`);
|
|
401
|
+
return removed;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function requireBinary(bin, hint) {
|
|
405
|
+
try {
|
|
406
|
+
execFileSync("/usr/bin/which", [bin], { stdio: "ignore" });
|
|
407
|
+
} catch {
|
|
408
|
+
throw new Error(`${bin} not found in PATH — ${hint}`);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
main().catch((err) => {
|
|
413
|
+
log("error", `unhandled: ${err && err.stack ? err.stack : err}`);
|
|
414
|
+
process.exit(1);
|
|
415
|
+
});
|
|
@@ -1,124 +1,24 @@
|
|
|
1
1
|
#!/bin/bash
|
|
2
|
-
# backup-to-cloud.sh —
|
|
2
|
+
# backup-to-cloud.sh — COMPATIBILITY WRAPPER.
|
|
3
3
|
#
|
|
4
|
-
#
|
|
5
|
-
#
|
|
6
|
-
#
|
|
4
|
+
# The backup driver moved to scripts/maintenance/backup-run.mjs. This shell
|
|
5
|
+
# entrypoint stays because installed launchd plists and the runbook reference
|
|
6
|
+
# it by name; it now just execs the Node runner.
|
|
7
7
|
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
8
|
+
# Why the move: this script parsed YAML with awk, had no credential deny-list,
|
|
9
|
+
# required a cloud bucket before it would do anything, and — the part that
|
|
10
|
+
# actually cost us — `exit 0`'d SILENTLY when unconfigured. A fleet can run for
|
|
11
|
+
# months with zero agents backed up and nothing red anywhere. backup-run.mjs
|
|
12
|
+
# refuses loudly with a non-zero exit instead, and writes local restore points
|
|
13
|
+
# with no credentials at all so the default posture is "something", not "nothing".
|
|
14
|
+
#
|
|
15
|
+
# See lib/backup/policy.mjs for the posture (what/where/never) and
|
|
16
|
+
# docs/runbooks/backup-restore.md for the restore path.
|
|
11
17
|
|
|
12
18
|
set -e
|
|
13
19
|
|
|
14
20
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
15
|
-
AGENT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
16
|
-
CFG="$AGENT_DIR/.maestro/backup-config.yaml"
|
|
17
|
-
LOG_DIR="$AGENT_DIR/logs/maintenance"
|
|
18
|
-
LOG="$LOG_DIR/backup.log"
|
|
19
|
-
mkdir -p "$LOG_DIR"
|
|
20
|
-
|
|
21
|
-
log() { echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] $1" | tee -a "$LOG"; }
|
|
22
|
-
|
|
23
|
-
if [ ! -f "$CFG" ]; then
|
|
24
|
-
log "backup-config.yaml not found — backup not configured. Run: maestro init backup-replication --apply"
|
|
25
|
-
exit 0
|
|
26
|
-
fi
|
|
27
|
-
|
|
28
|
-
# Tiny YAML reader (only the fields we need).
|
|
29
|
-
get() {
|
|
30
|
-
awk -v k="$1" '
|
|
31
|
-
$1 == k":" { sub(/^[^:]+:[[:space:]]*/, "", $0); gsub(/^"|"$/, "", $0); print; exit }
|
|
32
|
-
' "$CFG"
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
ENABLED=$(get enabled)
|
|
36
|
-
if [ "$ENABLED" != "true" ]; then
|
|
37
|
-
log "Backup not enabled (.maestro/backup-config.yaml: enabled: false). Skipping."
|
|
38
|
-
exit 0
|
|
39
|
-
fi
|
|
40
|
-
|
|
41
|
-
PROVIDER=$(get provider)
|
|
42
|
-
BUCKET=$(get bucket)
|
|
43
|
-
PREFIX=$(get prefix)
|
|
44
|
-
RETENTION=$(get retention_days)
|
|
45
|
-
[ -z "$RETENTION" ] && RETENTION=30
|
|
46
|
-
|
|
47
|
-
if [ -z "$BUCKET" ] || [ -z "$PROVIDER" ] || [ -z "$PREFIX" ]; then
|
|
48
|
-
log "ERROR: backup-config.yaml missing required fields (provider / bucket / prefix)."
|
|
49
|
-
exit 1
|
|
50
|
-
fi
|
|
51
|
-
|
|
52
|
-
DATE=$(date -u +"%Y-%m-%d")
|
|
53
|
-
log "Starting backup → $PROVIDER://$BUCKET/$PREFIX/$DATE/"
|
|
54
|
-
|
|
55
|
-
# Default include paths if the YAML doesn't override them.
|
|
56
|
-
INCLUDE_DEFAULT=(state knowledge outputs config/agent.json)
|
|
57
|
-
INCLUDE_PATHS=()
|
|
58
|
-
in_include=0
|
|
59
|
-
while IFS= read -r line; do
|
|
60
|
-
if echo "$line" | grep -qE "^include:"; then in_include=1; continue; fi
|
|
61
|
-
if [ "$in_include" = "1" ]; then
|
|
62
|
-
if echo "$line" | grep -qE "^[a-z]"; then in_include=0; continue; fi
|
|
63
|
-
path=$(echo "$line" | sed 's/^[[:space:]]*-[[:space:]]*//' | sed 's/[[:space:]]*$//')
|
|
64
|
-
[ -n "$path" ] && INCLUDE_PATHS+=("$path")
|
|
65
|
-
fi
|
|
66
|
-
done < "$CFG"
|
|
67
|
-
[ ${#INCLUDE_PATHS[@]} -eq 0 ] && INCLUDE_PATHS=("${INCLUDE_DEFAULT[@]}")
|
|
68
|
-
|
|
69
|
-
# Build the archive
|
|
70
|
-
TMP_DIR=$(mktemp -d)
|
|
71
|
-
trap "rm -rf $TMP_DIR" EXIT
|
|
72
|
-
TARBALL="$TMP_DIR/$PREFIX-$DATE.tar.gz"
|
|
73
|
-
|
|
74
|
-
log "Packaging: ${INCLUDE_PATHS[*]}"
|
|
75
|
-
tar --exclude="state/tmp" --exclude="state/rag/index" --exclude=".DS_Store" \
|
|
76
|
-
-czf "$TARBALL" -C "$AGENT_DIR" "${INCLUDE_PATHS[@]}" 2>>"$LOG" || {
|
|
77
|
-
log "ERROR: tar failed"
|
|
78
|
-
exit 1
|
|
79
|
-
}
|
|
80
|
-
SIZE=$(stat -f%z "$TARBALL")
|
|
81
|
-
log "Packaged: $TARBALL ($SIZE bytes)"
|
|
82
|
-
|
|
83
|
-
# Upload — choose provider.
|
|
84
|
-
case "$PROVIDER" in
|
|
85
|
-
gcs)
|
|
86
|
-
if ! command -v gsutil >/dev/null 2>&1; then
|
|
87
|
-
log "ERROR: gsutil not found in PATH. Install gcloud SDK or change provider."
|
|
88
|
-
exit 1
|
|
89
|
-
fi
|
|
90
|
-
gsutil cp "$TARBALL" "gs://$BUCKET/$PREFIX/$DATE/$(basename "$TARBALL")" >>"$LOG" 2>&1
|
|
91
|
-
;;
|
|
92
|
-
s3)
|
|
93
|
-
if ! command -v aws >/dev/null 2>&1; then
|
|
94
|
-
log "ERROR: aws CLI not found in PATH. Install awscli or change provider."
|
|
95
|
-
exit 1
|
|
96
|
-
fi
|
|
97
|
-
aws s3 cp "$TARBALL" "s3://$BUCKET/$PREFIX/$DATE/$(basename "$TARBALL")" >>"$LOG" 2>&1
|
|
98
|
-
;;
|
|
99
|
-
rsync)
|
|
100
|
-
# bucket field carries the rsync target (user@host:/path)
|
|
101
|
-
rsync -avz "$TARBALL" "$BUCKET/$PREFIX/$DATE/" >>"$LOG" 2>&1
|
|
102
|
-
;;
|
|
103
|
-
*)
|
|
104
|
-
log "ERROR: unknown provider '$PROVIDER'"
|
|
105
|
-
exit 1
|
|
106
|
-
;;
|
|
107
|
-
esac
|
|
108
|
-
|
|
109
|
-
log "Upload complete: $PROVIDER://$BUCKET/$PREFIX/$DATE/"
|
|
110
|
-
|
|
111
|
-
# Update .maestro/last-backup.json so doctor can verify freshness.
|
|
112
|
-
mkdir -p "$AGENT_DIR/.maestro"
|
|
113
|
-
cat > "$AGENT_DIR/.maestro/last-backup.json" <<JSON
|
|
114
|
-
{
|
|
115
|
-
"completed_at": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
|
|
116
|
-
"provider": "$PROVIDER",
|
|
117
|
-
"bucket": "$BUCKET",
|
|
118
|
-
"prefix": "$PREFIX",
|
|
119
|
-
"size_bytes": $SIZE,
|
|
120
|
-
"tarball": "$(basename "$TARBALL")"
|
|
121
|
-
}
|
|
122
|
-
JSON
|
|
21
|
+
AGENT_DIR="${AGENT_ROOT:-${AGENT_DIR:-$(cd "$SCRIPT_DIR/../.." && pwd)}}"
|
|
123
22
|
|
|
124
|
-
|
|
23
|
+
NODE_BIN="$(command -v node || echo /opt/homebrew/bin/node)"
|
|
24
|
+
exec "$NODE_BIN" "$SCRIPT_DIR/backup-run.mjs" --agent-dir "$AGENT_DIR" "$@"
|
|
@@ -214,6 +214,22 @@ async function main() {
|
|
|
214
214
|
|
|
215
215
|
const status = frame.result?.status;
|
|
216
216
|
if (status === "queued" || status === "deduped") {
|
|
217
|
+
// Witness the send for the daemon's answer-assurance ledger. This CLI is the
|
|
218
|
+
// sanctioned orgmail lane for a spawned session, and a session is a separate
|
|
219
|
+
// process: without a receipt the daemon reads a delivered reply as silence
|
|
220
|
+
// and apologises to a human who already has their answer. Attribution rides
|
|
221
|
+
// in on the dispatcher's env. Best-effort — never fails the send.
|
|
222
|
+
try {
|
|
223
|
+
const { recordOutbound } = await import("../../lib/comms/receipts.mjs");
|
|
224
|
+
recordOutbound({
|
|
225
|
+
service: "orgmail",
|
|
226
|
+
channel: frame.result.threadId || flags.thread || flags.to[0],
|
|
227
|
+
kind: "session",
|
|
228
|
+
via: "send-orgmail.mjs",
|
|
229
|
+
chars: String(text).length,
|
|
230
|
+
agentRoot: cfg.agentRoot,
|
|
231
|
+
});
|
|
232
|
+
} catch { /* bookkeeping must never break a send */ }
|
|
217
233
|
console.log(`send-orgmail: ${status} — messageId ${frame.result.messageId} threadId ${frame.result.threadId}`);
|
|
218
234
|
process.exit(0);
|
|
219
235
|
}
|