@cohortapp/agent-sdk 2.5.1 → 2.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +305 -89
- package/bin/maestro.test.mjs +357 -48
- package/docs/runbooks/backup-restore.md +65 -33
- package/framework-features.json +4 -4
- package/lib/backup/policy.mjs +710 -0
- package/lib/backup/policy.test.mjs +305 -0
- package/lib/budget-escalate.mjs +133 -0
- package/lib/budget-escalate.test.mjs +232 -0
- package/lib/budget-guard.envelope.test.mjs +476 -0
- package/lib/budget-guard.mjs +853 -75
- package/lib/budget-guard.test.mjs +91 -42
- package/lib/cadences.mjs +33 -0
- package/lib/channels/orgmail/adapter.mjs +88 -3
- package/lib/channels/orgmail/adapter.test.mjs +137 -0
- package/lib/channels/repeat-suppressor.mjs +198 -0
- package/lib/channels/repeat-suppressor.test.mjs +134 -0
- package/lib/comms/receipts.mjs +297 -0
- package/lib/cost/ledger-row.mjs +333 -0
- package/lib/cost/ledger-row.test.mjs +183 -0
- package/lib/execution/drive.mjs +28 -1
- package/lib/execution/effects.mjs +191 -12
- package/lib/execution/effects.test.mjs +50 -11
- package/lib/goals/admission.mjs +13 -1
- package/lib/goals/admission.test.mjs +26 -1
- package/lib/goals/loop.mjs +13 -0
- package/lib/kpi-sensors.test.mjs +3 -0
- package/lib/mandate/cache.mjs +13 -5
- package/lib/mandate/derive.mjs +146 -21
- package/lib/mandate/derive.test.mjs +50 -6
- package/lib/mandate/model.mjs +32 -4
- package/lib/mandate/refresh.test.mjs +16 -2
- package/lib/mcp/server.test.mjs +12 -3
- package/lib/model-router/economics.mjs +107 -76
- package/lib/model-router/economics.test.mjs +64 -46
- package/lib/model-router/integration-coverage.test.mjs +39 -37
- package/lib/model-router/ledger.mjs +75 -22
- package/lib/model-router/ledger.test.mjs +35 -2
- package/lib/org/client.mjs +14 -0
- package/lib/org/cost-sync.mjs +16 -2
- package/lib/org/doctor.mjs +62 -1
- package/lib/org/doctor.test.mjs +36 -3
- package/lib/org/email-remedy.mjs +49 -0
- package/lib/org/engagement-ledger.mjs +376 -0
- package/lib/org/engagement-ledger.test.mjs +112 -0
- package/lib/org/engagement.mjs +1056 -0
- package/lib/org/engagement.test.mjs +739 -0
- package/lib/org/messaging.mjs +230 -3
- package/lib/org/messaging.test.mjs +110 -1
- package/lib/org/param-contract.mjs +56 -2
- package/lib/org/param-contract.test.mjs +26 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +5 -0
- package/lib/org/protocol.test.mjs +7 -1
- package/lib/org/tool-surface.mjs +506 -10
- package/lib/org/tool-surface.test.mjs +191 -7
- package/lib/org/ui-parity.mjs +333 -6
- package/lib/org/ui-parity.test.mjs +96 -3
- package/lib/org/work-ledger.mjs +241 -0
- package/lib/org/work-ledger.test.mjs +237 -0
- package/lib/plan/adoption-e2e.test.mjs +366 -0
- package/lib/plan/budget-enforcement.test.mjs +400 -0
- package/lib/plan/budget-runtime.mjs +215 -0
- package/lib/plan/compile.mjs +201 -5
- package/lib/plan/compile.test.mjs +19 -5
- package/lib/plan/emit.mjs +8 -0
- package/lib/plan/emit.test.mjs +18 -0
- package/lib/resource-governor.mjs +58 -12
- package/lib/resource-governor.test.mjs +41 -1
- package/lib/security/audit-engine.mjs +45 -8
- package/lib/security/audit-engine.test.mjs +35 -0
- package/lib/setup/enroll-from-cohort.mjs +14 -1
- package/lib/setup/sections/mandate.mjs +48 -7
- package/lib/setup/sections/mandate.test.mjs +17 -2
- package/lib/setup/sections/orgmail.mjs +10 -2
- package/lib/setup/state.mjs +83 -2
- package/lib/telemetry/collect.mjs +360 -20
- package/lib/telemetry/collect.test.mjs +266 -0
- package/package.json +1 -1
- package/scripts/cost/track-claude-usage.mjs +207 -48
- package/scripts/cost/track-claude-usage.test.mjs +148 -0
- package/scripts/daemon/agent-daemon.mjs +315 -17
- package/scripts/daemon/assurance-e2e.test.mjs +421 -0
- package/scripts/daemon/assurance.mjs +944 -0
- package/scripts/daemon/assurance.test.mjs +668 -0
- package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
- package/scripts/daemon/cadence-consumer.mjs +147 -9
- package/scripts/daemon/cadence-consumer.test.mjs +6 -0
- package/scripts/daemon/cadence-handlers.mjs +158 -0
- package/scripts/daemon/cadence-handlers.test.mjs +64 -0
- package/scripts/daemon/classifier.test.mjs +18 -9
- package/scripts/daemon/deliver.mjs +314 -0
- package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
- package/scripts/daemon/dispatcher.mjs +64 -6
- package/scripts/daemon/responder-cost.test.mjs +68 -0
- package/scripts/daemon/responder.mjs +351 -298
- package/scripts/local-triggers/generate-plists.test.mjs +7 -4
- package/scripts/maintenance/backup-run.mjs +415 -0
- package/scripts/maintenance/backup-to-cloud.sh +16 -116
- package/scripts/org/send-orgmail.mjs +16 -0
- package/scripts/record-receipt.sh +63 -0
- package/scripts/restore-from-backup.sh +14 -3
- package/scripts/restore-from-backup.test.mjs +8 -5
- package/scripts/send-email-threaded.py +47 -0
- package/scripts/send-sms.sh +4 -0
- package/scripts/send-whatsapp.sh +4 -0
- package/scripts/setup/init-backup.mjs +93 -38
- package/scripts/slack-send.sh +12 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* policy.test.mjs — coverage for lib/backup/policy.mjs.
|
|
3
|
+
*
|
|
4
|
+
* The load-bearing cases are the two that a silent bug would make invisible:
|
|
5
|
+
* 1. the hard deny-list actually drops `.env` / `.cohort-key.json` even when
|
|
6
|
+
* the config asks for them (credential exfiltration via backup), and
|
|
7
|
+
* 2. the severity ladder distinguishes "no restore point at all" (fail) from
|
|
8
|
+
* "restore points that would not survive a house fire" (warn).
|
|
9
|
+
*
|
|
10
|
+
* Hermetic: tmp agent roots, injected env/home. No tar, no network, no launchd.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { test } from "node:test";
|
|
14
|
+
import assert from "node:assert/strict";
|
|
15
|
+
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
|
16
|
+
import { tmpdir } from "node:os";
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
|
|
19
|
+
import {
|
|
20
|
+
DEFAULT_INCLUDE,
|
|
21
|
+
DENY_PATTERNS,
|
|
22
|
+
backupVerdict,
|
|
23
|
+
defaultConfigYaml,
|
|
24
|
+
defaultLocalRoot,
|
|
25
|
+
denyMatch,
|
|
26
|
+
parseBackupConfig,
|
|
27
|
+
resolveBackupPlan,
|
|
28
|
+
sanitisePrefix,
|
|
29
|
+
tarExcludeArgs,
|
|
30
|
+
scanTextForSecrets,
|
|
31
|
+
scanIncludesForSecrets,
|
|
32
|
+
} from "./policy.mjs";
|
|
33
|
+
|
|
34
|
+
function makeAgent(configBody) {
|
|
35
|
+
const root = mkdtempSync(join(tmpdir(), "bkpol-"));
|
|
36
|
+
mkdirSync(join(root, ".maestro"), { recursive: true });
|
|
37
|
+
if (configBody !== undefined) {
|
|
38
|
+
writeFileSync(join(root, ".maestro", "backup-config.yaml"), configBody);
|
|
39
|
+
}
|
|
40
|
+
return root;
|
|
41
|
+
}
|
|
42
|
+
const cleanup = (root) => rmSync(root, { recursive: true, force: true });
|
|
43
|
+
|
|
44
|
+
// ── deny-list ──────────────────────────────────────────────────────────────
|
|
45
|
+
|
|
46
|
+
test("deny-list rejects live credentials by exact name and by glob", () => {
|
|
47
|
+
assert.equal(denyMatch(".env"), ".env");
|
|
48
|
+
assert.equal(denyMatch(".env.local"), ".env.*");
|
|
49
|
+
assert.equal(denyMatch(".cohort-key.json"), ".cohort-key.json");
|
|
50
|
+
assert.equal(denyMatch(".claude/.credentials.json"), ".claude/.credentials.json");
|
|
51
|
+
assert.ok(denyMatch("config/tls/server.pem"));
|
|
52
|
+
assert.ok(denyMatch("secrets/id_rsa"));
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("deny-list applies to nested segments (node_modules at any depth)", () => {
|
|
56
|
+
assert.equal(denyMatch("node_modules"), "node_modules");
|
|
57
|
+
assert.equal(denyMatch("plugins/x/node_modules/y"), "node_modules");
|
|
58
|
+
assert.equal(denyMatch("state/rag/index/vectors.bin"), "state/rag/index");
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("deny-list leaves the durable state we DO want alone", () => {
|
|
62
|
+
for (const p of ["state", "state/queues/action-stack.yaml", "knowledge", "memory", "outputs", "config/agent.json", ".maestro/last-backup.json"]) {
|
|
63
|
+
assert.equal(denyMatch(p), null, `${p} must be backupable`);
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test("a config that asks for .env gets it DROPPED and reported, not obeyed", () => {
|
|
68
|
+
const root = makeAgent(`enabled: true
|
|
69
|
+
prefix: test
|
|
70
|
+
include:
|
|
71
|
+
- state
|
|
72
|
+
- .env
|
|
73
|
+
- .cohort-key.json
|
|
74
|
+
- ../../etc
|
|
75
|
+
`);
|
|
76
|
+
const plan = resolveBackupPlan({ agentRoot: root, env: { MAESTRO_BACKUP_DIR: "/tmp/b" } });
|
|
77
|
+
cleanup(root);
|
|
78
|
+
assert.deepEqual(plan.include, ["state"]);
|
|
79
|
+
const denied = plan.violations.map((v) => v.path).sort();
|
|
80
|
+
assert.deepEqual(denied, ["../../etc", ".cohort-key.json", ".env"]);
|
|
81
|
+
assert.ok(plan.violations.some((v) => v.pattern === "outside-agent-root"));
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test("tar exclude args always carry the hard deny-list first", () => {
|
|
85
|
+
const args = tarExcludeArgs({ exclude: ["outputs/scratch"] });
|
|
86
|
+
assert.ok(args.includes("--exclude=.env"));
|
|
87
|
+
assert.ok(args.includes("--exclude=.cohort-key.json"));
|
|
88
|
+
assert.ok(args.includes("--exclude=node_modules"));
|
|
89
|
+
assert.ok(args.includes("--exclude=*/node_modules"));
|
|
90
|
+
assert.ok(args.includes("--exclude=outputs/scratch"));
|
|
91
|
+
assert.ok(args.indexOf("--exclude=.env") < args.indexOf("--exclude=outputs/scratch"));
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("every deny pattern is non-empty and lowercase-stable", () => {
|
|
95
|
+
for (const p of DENY_PATTERNS) assert.ok(typeof p === "string" && p.length > 0);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
// ── config parsing / plan resolution ───────────────────────────────────────
|
|
99
|
+
|
|
100
|
+
test("missing config → tier none, nothing enabled", () => {
|
|
101
|
+
const root = makeAgent();
|
|
102
|
+
const plan = resolveBackupPlan({ agentRoot: root });
|
|
103
|
+
cleanup(root);
|
|
104
|
+
assert.equal(plan.present, false);
|
|
105
|
+
assert.equal(plan.tier, "none");
|
|
106
|
+
assert.equal(plan.enabled, false);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("malformed YAML → present but unreadable (a DIFFERENT problem from absent)", () => {
|
|
110
|
+
const root = makeAgent("enabled: true\n : : broken\n\t- x");
|
|
111
|
+
const plan = resolveBackupPlan({ agentRoot: root });
|
|
112
|
+
cleanup(root);
|
|
113
|
+
assert.equal(plan.present, true);
|
|
114
|
+
assert.equal(plan.readable, false);
|
|
115
|
+
assert.equal(plan.tier, "unreadable");
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test("the shipped default config resolves to the LOCAL tier with no credentials", () => {
|
|
119
|
+
const root = makeAgent(defaultConfigYaml({ prefix: "astra-ai" }));
|
|
120
|
+
const plan = resolveBackupPlan({ agentRoot: root, home: "/Users/fake" });
|
|
121
|
+
cleanup(root);
|
|
122
|
+
assert.equal(plan.enabled, true);
|
|
123
|
+
assert.equal(plan.tier, "local");
|
|
124
|
+
assert.equal(plan.prefix, "astra-ai");
|
|
125
|
+
assert.equal(plan.local.enabled, true);
|
|
126
|
+
assert.equal(plan.offsite.configured, false);
|
|
127
|
+
assert.deepEqual(plan.include, [...DEFAULT_INCLUDE]);
|
|
128
|
+
assert.deepEqual(plan.violations, []);
|
|
129
|
+
assert.ok(plan.local.dir.endsWith("/Maestro/backups/astra-ai"));
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test("offsite provider + bucket promotes the tier to offsite", () => {
|
|
133
|
+
const root = makeAgent(`enabled: true
|
|
134
|
+
prefix: astra-ai
|
|
135
|
+
offsite:
|
|
136
|
+
provider: s3
|
|
137
|
+
bucket: acme-agent-backups
|
|
138
|
+
`);
|
|
139
|
+
const plan = resolveBackupPlan({ agentRoot: root, home: "/Users/fake" });
|
|
140
|
+
cleanup(root);
|
|
141
|
+
assert.equal(plan.tier, "offsite");
|
|
142
|
+
assert.equal(plan.offsite.provider, "s3");
|
|
143
|
+
assert.equal(plan.offsite.bucket, "acme-agent-backups");
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("an unknown offsite provider does not count as configured", () => {
|
|
147
|
+
const root = makeAgent(`enabled: true
|
|
148
|
+
offsite:
|
|
149
|
+
provider: dropbox
|
|
150
|
+
bucket: nope
|
|
151
|
+
`);
|
|
152
|
+
const plan = resolveBackupPlan({ agentRoot: root, home: "/Users/fake" });
|
|
153
|
+
cleanup(root);
|
|
154
|
+
assert.equal(plan.offsite.configured, false);
|
|
155
|
+
assert.equal(plan.tier, "local");
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
test("enabled:false → tier disabled even with a full config", () => {
|
|
159
|
+
const root = makeAgent(`enabled: false
|
|
160
|
+
offsite:
|
|
161
|
+
provider: gcs
|
|
162
|
+
bucket: b
|
|
163
|
+
`);
|
|
164
|
+
const plan = resolveBackupPlan({ agentRoot: root, home: "/Users/fake" });
|
|
165
|
+
cleanup(root);
|
|
166
|
+
assert.equal(plan.tier, "disabled");
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
test("MAESTRO_BACKUP_DIR overrides the local root", () => {
|
|
170
|
+
assert.equal(defaultLocalRoot({ env: { MAESTRO_BACKUP_DIR: "/mnt/dr" } }), "/mnt/dr");
|
|
171
|
+
assert.equal(
|
|
172
|
+
defaultLocalRoot({ env: {}, home: "/Users/x" }),
|
|
173
|
+
"/Users/x/Library/Application Support/Maestro/backups"
|
|
174
|
+
);
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
test("parseBackupConfig never throws and rejects non-objects", () => {
|
|
178
|
+
assert.equal(parseBackupConfig(""), null);
|
|
179
|
+
assert.equal(parseBackupConfig("- a\n- b"), null);
|
|
180
|
+
assert.equal(parseBackupConfig("just a string"), null);
|
|
181
|
+
assert.deepEqual(parseBackupConfig("enabled: true"), { enabled: true });
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
test("prefixes are sanitised into safe archive/bucket keys", () => {
|
|
185
|
+
assert.equal(sanitisePrefix("Astra AI"), "astra-ai");
|
|
186
|
+
assert.equal(sanitisePrefix("../../etc/passwd"), "etc-passwd");
|
|
187
|
+
assert.equal(sanitisePrefix(""), "agent");
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
// ── severity ladder ────────────────────────────────────────────────────────
|
|
191
|
+
|
|
192
|
+
const FRESH = { configured: true, lastBackupAt: "2026-08-12T03:10:00Z", ageHours: 6, stale: false };
|
|
193
|
+
|
|
194
|
+
test("no config → FAIL naming the one command that fixes it", () => {
|
|
195
|
+
const v = backupVerdict({ present: false }, {});
|
|
196
|
+
assert.equal(v.level, "fail");
|
|
197
|
+
assert.match(v.fix, /maestro init backup-replication --apply/);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test("configured + enabled but never run → FAIL (there is no restore point)", () => {
|
|
201
|
+
const root = makeAgent(defaultConfigYaml({ prefix: "a" }));
|
|
202
|
+
const plan = resolveBackupPlan({ agentRoot: root, home: "/Users/fake" });
|
|
203
|
+
cleanup(root);
|
|
204
|
+
const v = backupVerdict(plan, { configured: true, lastBackupAt: null, ageHours: null, stale: true });
|
|
205
|
+
assert.equal(v.level, "fail");
|
|
206
|
+
assert.match(v.msg, /NEVER completed/);
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
test("local-only but fresh → WARN that names the residual machine-loss risk", () => {
|
|
210
|
+
const root = makeAgent(defaultConfigYaml({ prefix: "a" }));
|
|
211
|
+
const plan = resolveBackupPlan({ agentRoot: root, home: "/Users/fake" });
|
|
212
|
+
cleanup(root);
|
|
213
|
+
const v = backupVerdict(plan, FRESH);
|
|
214
|
+
assert.equal(v.level, "warn");
|
|
215
|
+
assert.match(v.msg, /LOCAL-ONLY/);
|
|
216
|
+
assert.match(v.fix, /offsite\.provider/);
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
test("offsite + fresh → the only OK", () => {
|
|
220
|
+
const root = makeAgent(`enabled: true
|
|
221
|
+
prefix: a
|
|
222
|
+
offsite:
|
|
223
|
+
provider: gcs
|
|
224
|
+
bucket: agent-backups
|
|
225
|
+
`);
|
|
226
|
+
const plan = resolveBackupPlan({ agentRoot: root, home: "/Users/fake" });
|
|
227
|
+
cleanup(root);
|
|
228
|
+
const v = backupVerdict(plan, FRESH);
|
|
229
|
+
assert.equal(v.level, "ok");
|
|
230
|
+
assert.match(v.msg, /gcs:\/\/agent-backups/);
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
test("stale beats local-only: a stale offsite backup still WARNs", () => {
|
|
234
|
+
const root = makeAgent(`enabled: true
|
|
235
|
+
offsite:
|
|
236
|
+
provider: s3
|
|
237
|
+
bucket: b
|
|
238
|
+
`);
|
|
239
|
+
const plan = resolveBackupPlan({ agentRoot: root, home: "/Users/fake" });
|
|
240
|
+
cleanup(root);
|
|
241
|
+
const v = backupVerdict(plan, { configured: true, lastBackupAt: "2026-08-01T00:00:00Z", ageHours: 200, stale: true });
|
|
242
|
+
assert.equal(v.level, "warn");
|
|
243
|
+
assert.match(v.msg, /stale/);
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
test("an include list that is entirely denied FAILs rather than shipping an empty archive", () => {
|
|
247
|
+
const root = makeAgent(`enabled: true
|
|
248
|
+
include:
|
|
249
|
+
- .env
|
|
250
|
+
`);
|
|
251
|
+
const plan = resolveBackupPlan({ agentRoot: root, home: "/Users/fake" });
|
|
252
|
+
cleanup(root);
|
|
253
|
+
const v = backupVerdict(plan, FRESH);
|
|
254
|
+
assert.equal(v.level, "fail");
|
|
255
|
+
assert.match(v.msg, /every include path was rejected/);
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
// ---------------------------------------------------------------------------
|
|
259
|
+
// THE CONTENT DENY-LIST — the hole a path deny-list cannot see
|
|
260
|
+
// ---------------------------------------------------------------------------
|
|
261
|
+
|
|
262
|
+
test("scanTextForSecrets catches the credential shapes that authenticate AS this agent", () => {
|
|
263
|
+
// The real leak: state/setup/progress.json carried "orgToken":"nlk_…" — the
|
|
264
|
+
// byte-identical value of COHORT_API_KEY in .env — inside an included tree,
|
|
265
|
+
// while the runner logged "archive verified clean".
|
|
266
|
+
assert.deepEqual(scanTextForSecrets('{"orgToken":"nlk_7bc80020aa11bb22cc33dd44ee55ff66"}'), ["cohort-api-key"]);
|
|
267
|
+
assert.deepEqual(scanTextForSecrets("sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAA"), ["anthropic-key"]);
|
|
268
|
+
assert.deepEqual(scanTextForSecrets("-----BEGIN OPENSSH PRIVATE KEY-----"), ["private-key-block"]);
|
|
269
|
+
assert.deepEqual(scanTextForSecrets("ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"), ["github-token"]);
|
|
270
|
+
// And does not fire on ordinary prose or config.
|
|
271
|
+
assert.deepEqual(scanTextForSecrets('{"orgAuthMethod":"apikey","orgBase":"https://hq.example"}'), []);
|
|
272
|
+
assert.deepEqual(scanTextForSecrets("the token lives in .env"), []);
|
|
273
|
+
});
|
|
274
|
+
|
|
275
|
+
test("scanIncludesForSecrets finds a credential INSIDE an included file", () => {
|
|
276
|
+
const root = mkdtempSync(join(tmpdir(), "backup-content-"));
|
|
277
|
+
try {
|
|
278
|
+
mkdirSync(join(root, "state", "setup"), { recursive: true });
|
|
279
|
+
mkdirSync(join(root, "knowledge"), { recursive: true });
|
|
280
|
+
writeFileSync(join(root, "state", "setup", "progress.json"),
|
|
281
|
+
JSON.stringify({ answers: { orgId: "acme", orgToken: "nlk_7bc80020aa11bb22cc33dd44ee55ff66" } }));
|
|
282
|
+
writeFileSync(join(root, "state", "notes.md"), "no secrets here");
|
|
283
|
+
writeFileSync(join(root, "knowledge", "faq.md"), "the api key is in .env");
|
|
284
|
+
|
|
285
|
+
const plan = { include: ["state", "knowledge"], exclude: [] };
|
|
286
|
+
const r = scanIncludesForSecrets(plan, root);
|
|
287
|
+
assert.deepEqual(r.leaks.map((l) => l.path), ["state/setup/progress.json"]);
|
|
288
|
+
assert.deepEqual(r.leaks[0].patterns, ["cohort-api-key"]);
|
|
289
|
+
assert.equal(r.truncated, false);
|
|
290
|
+
assert.ok(r.scanned >= 3);
|
|
291
|
+
} finally { rmSync(root, { recursive: true, force: true }); }
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
test("the content scan skips path-denied trees and binary files", () => {
|
|
295
|
+
const root = mkdtempSync(join(tmpdir(), "backup-content-"));
|
|
296
|
+
try {
|
|
297
|
+
// A dependency full of example keys must not cost us the whole DR posture.
|
|
298
|
+
mkdirSync(join(root, "state", "node_modules", "pkg"), { recursive: true });
|
|
299
|
+
writeFileSync(join(root, "state", "node_modules", "pkg", "readme.md"), "sk-ant-api03-BBBBBBBBBBBBBBBBBBBBBBBB");
|
|
300
|
+
writeFileSync(join(root, "state", "blob.bin"), Buffer.from([0x00, 0x01, 0x02, 0x00]));
|
|
301
|
+
const r = scanIncludesForSecrets({ include: ["state"], exclude: [] }, root);
|
|
302
|
+
assert.deepEqual(r.leaks, [], "node_modules is already path-denied");
|
|
303
|
+
assert.equal(r.skippedBinary, 1);
|
|
304
|
+
} finally { rmSync(root, { recursive: true, force: true }); }
|
|
305
|
+
});
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/budget-escalate.mjs — the 80 % rung's delivery leg.
|
|
3
|
+
*
|
|
4
|
+
* `lib/budget-guard.maybeNotify` decides WHETHER a notice fires, at which band,
|
|
5
|
+
* and WHO it is addressed to (the seat's REVIEWER collaborator —
|
|
6
|
+
* `WorkforceMember.supervisorId` as hq published it on the mandate body). This
|
|
7
|
+
* module is the only thing that puts that notice on the wire.
|
|
8
|
+
*
|
|
9
|
+
* NEVER TO THE OWNER. The seat is not told its own budget band. An agent that
|
|
10
|
+
* knows it is running out of money has every incentive to optimise the meter
|
|
11
|
+
* rather than the work, and the meter is the one number it must not be able to
|
|
12
|
+
* move. When there is no supervisor edge the notice is LOGGED as an error and
|
|
13
|
+
* delivered nowhere — an unsupervised seat is a provisioning gap, and the fix is
|
|
14
|
+
* to set `supervisorId`, not to hand the agent its own alarm.
|
|
15
|
+
*
|
|
16
|
+
* DELIVERY, with no new protocol surface:
|
|
17
|
+
* channel.resolveOrCreateDm(memberId) → the deterministic 1:1 DM channel
|
|
18
|
+
* messaging.send(channelId, body) → the notice
|
|
19
|
+
* Both are existing, idempotent methods on `messaging.write`; `resolveOrCreateDm`
|
|
20
|
+
* is replay-safe by construction (deterministic `dm-<sorted slugs>` slug), so a
|
|
21
|
+
* retried escalation reuses the same channel rather than littering DMs.
|
|
22
|
+
*
|
|
23
|
+
* FAIL-OPEN, NEVER FAIL-SILENT. Every failure path returns `{ok:false, reason}`
|
|
24
|
+
* AND logs: a spend escalation that quietly does not arrive is worse than no
|
|
25
|
+
* escalation, because the operator believes the channel works.
|
|
26
|
+
*
|
|
27
|
+
* @module lib/budget-escalate
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
"use strict";
|
|
31
|
+
|
|
32
|
+
import { maybeNotify } from "./budget-guard.mjs";
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Fire the period's budget notice, if one is due, to the supervisor.
|
|
36
|
+
*
|
|
37
|
+
* @param {object} o
|
|
38
|
+
* @param {string} o.agentRoot
|
|
39
|
+
* @param {object} [o.conn] { base, token, orgId } — the org credential
|
|
40
|
+
* @param {object} [deps] { now?, log?, callImpl?, notifyImpl?, sendImpl? }
|
|
41
|
+
* @returns {Promise<{ok:boolean, notified:boolean, band:number, delivered:boolean,
|
|
42
|
+
* recipient:object|null, reason:string|null, status?:object}>}
|
|
43
|
+
*/
|
|
44
|
+
export async function escalateBudget(o = {}, deps = {}) {
|
|
45
|
+
const log = typeof deps.log === "function"
|
|
46
|
+
? deps.log
|
|
47
|
+
: (level, msg) => { (level === "error" ? console.error : console.warn)(`[budget-escalate] ${msg}`); };
|
|
48
|
+
|
|
49
|
+
const notify = deps.notifyImpl || maybeNotify;
|
|
50
|
+
let notice;
|
|
51
|
+
try {
|
|
52
|
+
// `retryMs` rides through: the guard records an ATTEMPT and backs off between
|
|
53
|
+
// them, and this timer's own period is what that backoff is calibrated to.
|
|
54
|
+
notice = notify({ agentRoot: o.agentRoot, now: deps.now, retryMs: deps.retryMs, log });
|
|
55
|
+
} catch (err) {
|
|
56
|
+
log("error", `the budget guard threw while deciding whether to escalate (${err && err.message ? err.message : err}) — NO notice was sent`);
|
|
57
|
+
return { ok: false, notified: false, band: 0, delivered: false, recipient: null, reason: "guard-threw" };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (!notice || !notice.notified) {
|
|
61
|
+
return { ok: true, notified: false, band: (notice && notice.band) || 0, delivered: false, recipient: null, reason: null, status: notice && notice.status };
|
|
62
|
+
}
|
|
63
|
+
// maybeNotify has already logged the "no supervisor edge" error loudly; the
|
|
64
|
+
// band is recorded either way so it does not re-fire every tick.
|
|
65
|
+
if (!notice.recipient) {
|
|
66
|
+
return { ok: true, notified: true, band: notice.band, delivered: false, recipient: null, reason: "no-supervisor-edge", status: notice.status };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const conn = o.conn || {};
|
|
70
|
+
if (!conn.base || !conn.token) {
|
|
71
|
+
log("error", `band ${notice.band}% notice for supervisor ${notice.recipient.memberId} could not be sent: no org credential on this seat`);
|
|
72
|
+
return { ok: false, notified: true, band: notice.band, delivered: false, recipient: notice.recipient, reason: "no-credential", status: notice.status };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const call = deps.callImpl || (await import("./org/client.mjs")).call;
|
|
76
|
+
let channelId = null;
|
|
77
|
+
try {
|
|
78
|
+
const frame = await call("channel.resolveOrCreateDm", { memberId: notice.recipient.memberId }, conn);
|
|
79
|
+
if (frame && frame.ok && frame.result) {
|
|
80
|
+
channelId = frame.result.channelId || frame.result.id || (frame.result.channel && frame.result.channel.id) || null;
|
|
81
|
+
}
|
|
82
|
+
if (!channelId) {
|
|
83
|
+
const code = (frame && frame.error && frame.error.code) || "unknown";
|
|
84
|
+
log("error", `band ${notice.band}% notice undeliverable: could not resolve a DM channel with supervisor ${notice.recipient.memberId} (${code})`);
|
|
85
|
+
return { ok: false, notified: true, band: notice.band, delivered: false, recipient: notice.recipient, reason: `dm-resolve-failed:${code}`, status: notice.status };
|
|
86
|
+
}
|
|
87
|
+
} catch (err) {
|
|
88
|
+
log("error", `band ${notice.band}% notice undeliverable: channel.resolveOrCreateDm threw (${err && err.message ? err.message : err})`);
|
|
89
|
+
return { ok: false, notified: true, band: notice.band, delivered: false, recipient: notice.recipient, reason: "dm-resolve-threw", status: notice.status };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// The body names the seat, the band, the money and the enforcement now in
|
|
93
|
+
// force — a supervisor must be able to act on it without opening a dashboard.
|
|
94
|
+
const body = `**Seat budget — band ${notice.band}%**\n\n${notice.message}\n\n_This notice goes to you rather than to the seat: an agent must not be handed its own meter. Raise the envelope on the member's Employee record (payBasis.meteredBudget) to clear it._`;
|
|
95
|
+
|
|
96
|
+
try {
|
|
97
|
+
const send = deps.sendImpl || (await import("./org/messaging.mjs")).sendMessage;
|
|
98
|
+
const res = await send(
|
|
99
|
+
{
|
|
100
|
+
channel: channelId,
|
|
101
|
+
body,
|
|
102
|
+
// Stable per (band, period): a retried escalation dedupes server-side
|
|
103
|
+
// instead of DMing the supervisor twice about the same rung.
|
|
104
|
+
idempotencyId: `budget-${notice.status.date}-${notice.band}`,
|
|
105
|
+
},
|
|
106
|
+
{ cfg: o.cfg, agentRoot: o.agentRoot, base: conn.base, token: conn.token, fetchImpl: deps.fetchImpl }
|
|
107
|
+
);
|
|
108
|
+
if (!res || res.ok === false) {
|
|
109
|
+
const code = (res && res.error && res.error.code) || "unknown";
|
|
110
|
+
log("error", `band ${notice.band}% notice REJECTED by messaging.send (${code}) — the supervisor was not told`);
|
|
111
|
+
return { ok: false, notified: true, band: notice.band, delivered: false, recipient: notice.recipient, reason: `send-failed:${code}`, status: notice.status };
|
|
112
|
+
}
|
|
113
|
+
} catch (err) {
|
|
114
|
+
log("error", `band ${notice.band}% notice threw on send (${err && err.message ? err.message : err}) — the supervisor was not told`);
|
|
115
|
+
return { ok: false, notified: true, band: notice.band, delivered: false, recipient: notice.recipient, reason: "send-threw", status: notice.status };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// ── COMMIT ONLY NOW ────────────────────────────────────────────────────────
|
|
119
|
+
// `maybeNotify` records an ATTEMPT; the band is consumed here, after the send
|
|
120
|
+
// actually succeeded. Before this, the band was written to disk before the
|
|
121
|
+
// first byte went out, so a single transient failure (hq restarting, the org
|
|
122
|
+
// credential not yet resolved) permanently dropped the period's escalation —
|
|
123
|
+
// the timer polled forever and never retried. Every failure path above returns
|
|
124
|
+
// WITHOUT committing, which is what makes the retry real.
|
|
125
|
+
if (typeof notice.commit === "function") {
|
|
126
|
+
try { notice.commit(); }
|
|
127
|
+
catch (err) { log("error", `band ${notice.band}% notice was delivered but could not be recorded (${err && err.message ? err.message : err}) — it may be re-sent; messaging.send is idempotent on the same key`); }
|
|
128
|
+
}
|
|
129
|
+
log("warn", `band ${notice.band}% notice delivered to supervisor ${notice.recipient.memberId}`);
|
|
130
|
+
return { ok: true, notified: true, band: notice.band, delivered: true, recipient: notice.recipient, reason: null, status: notice.status };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export default { escalateBudget };
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* budget-escalate.test.mjs — the 80% rung reaches a HUMAN, or says why not.
|
|
3
|
+
*
|
|
4
|
+
* The rule under test is the anti-Goodhart one: the notice goes to the
|
|
5
|
+
* supervisor and NEVER to the seat that is spending the money. Everything else
|
|
6
|
+
* here is about the second rule — a notice that cannot be delivered is an ERROR
|
|
7
|
+
* in the log, never a silent no-op.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { test } from "node:test";
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
|
|
13
|
+
import { escalateBudget } from "./budget-escalate.mjs";
|
|
14
|
+
|
|
15
|
+
const STATUS = { date: "2026-06-15", spentUSD: 27, capUSD: 30, pct: 90, capSource: "seat-envelope" };
|
|
16
|
+
const CONN = { base: "https://hq.example", token: "t0ken", orgId: "org_1" };
|
|
17
|
+
|
|
18
|
+
function notice(over = {}) {
|
|
19
|
+
return {
|
|
20
|
+
notified: true,
|
|
21
|
+
band: 80,
|
|
22
|
+
message: "Seat spend at 90% ($27.00 / $30.00 today) — heads-up only.",
|
|
23
|
+
recipient: { memberId: "M-SUPERVISOR", role: "REVIEWER" },
|
|
24
|
+
status: STATUS,
|
|
25
|
+
...over,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
test("a due notice is DM'd to the supervisor over resolveOrCreateDm + messaging.send", async () => {
|
|
30
|
+
const calls = [];
|
|
31
|
+
const sent = [];
|
|
32
|
+
const r = await escalateBudget(
|
|
33
|
+
{ agentRoot: "/tmp/x", conn: CONN },
|
|
34
|
+
{
|
|
35
|
+
log: () => {},
|
|
36
|
+
notifyImpl: () => notice(),
|
|
37
|
+
callImpl: async (method, params) => {
|
|
38
|
+
calls.push([method, params]);
|
|
39
|
+
return { ok: true, result: { channelId: "chan_dm_1", created: false } };
|
|
40
|
+
},
|
|
41
|
+
sendImpl: async (params) => { sent.push(params); return { ok: true, result: { id: "msg_1" } }; },
|
|
42
|
+
}
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
assert.equal(r.ok, true);
|
|
46
|
+
assert.equal(r.delivered, true);
|
|
47
|
+
assert.deepEqual(calls[0], ["channel.resolveOrCreateDm", { memberId: "M-SUPERVISOR" }]);
|
|
48
|
+
assert.equal(sent[0].channel, "chan_dm_1");
|
|
49
|
+
assert.match(sent[0].body, /band 80%/i);
|
|
50
|
+
assert.match(sent[0].body, /an agent must not be handed its own meter/);
|
|
51
|
+
// Idempotent per (period, band): a retry does not DM the supervisor twice.
|
|
52
|
+
assert.equal(sent[0].idempotencyId, "budget-2026-06-15-80");
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("no supervisor edge → nothing is sent ANYWHERE, least of all to the owner", async () => {
|
|
56
|
+
const calls = [];
|
|
57
|
+
const sent = [];
|
|
58
|
+
const r = await escalateBudget(
|
|
59
|
+
{ agentRoot: "/tmp/x", conn: CONN },
|
|
60
|
+
{
|
|
61
|
+
log: () => {},
|
|
62
|
+
notifyImpl: () => notice({ recipient: null }),
|
|
63
|
+
callImpl: async (...a) => { calls.push(a); return { ok: true, result: {} }; },
|
|
64
|
+
sendImpl: async (p) => { sent.push(p); return { ok: true }; },
|
|
65
|
+
}
|
|
66
|
+
);
|
|
67
|
+
assert.equal(r.notified, true);
|
|
68
|
+
assert.equal(r.delivered, false);
|
|
69
|
+
assert.equal(r.reason, "no-supervisor-edge");
|
|
70
|
+
assert.equal(calls.length, 0, "no channel is opened");
|
|
71
|
+
assert.equal(sent.length, 0, "and no message is sent — the seat is never told its own band");
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test("an undeliverable notice is an ERROR in the log, never a silent no-op", async () => {
|
|
75
|
+
for (const [label, deps, expected] of [
|
|
76
|
+
[
|
|
77
|
+
"no credential",
|
|
78
|
+
{ notifyImpl: () => notice(), callImpl: async () => ({ ok: true, result: { channelId: "c" } }), sendImpl: async () => ({ ok: true }) },
|
|
79
|
+
"no-credential",
|
|
80
|
+
],
|
|
81
|
+
[
|
|
82
|
+
"DM resolve rejected",
|
|
83
|
+
{
|
|
84
|
+
notifyImpl: () => notice(),
|
|
85
|
+
callImpl: async () => ({ ok: false, error: { code: "FORBIDDEN_SCOPE" } }),
|
|
86
|
+
sendImpl: async () => ({ ok: true }),
|
|
87
|
+
},
|
|
88
|
+
"dm-resolve-failed:FORBIDDEN_SCOPE",
|
|
89
|
+
],
|
|
90
|
+
[
|
|
91
|
+
"send rejected",
|
|
92
|
+
{
|
|
93
|
+
notifyImpl: () => notice(),
|
|
94
|
+
callImpl: async () => ({ ok: true, result: { channelId: "c" } }),
|
|
95
|
+
sendImpl: async () => ({ ok: false, error: { code: "BAD_REQUEST" } }),
|
|
96
|
+
},
|
|
97
|
+
"send-failed:BAD_REQUEST",
|
|
98
|
+
],
|
|
99
|
+
]) {
|
|
100
|
+
const logged = [];
|
|
101
|
+
const conn = label === "no credential" ? {} : CONN;
|
|
102
|
+
const r = await escalateBudget({ agentRoot: "/tmp/x", conn }, { ...deps, log: (l, m) => logged.push(`${l}:${m}`) });
|
|
103
|
+
assert.equal(r.delivered, false, label);
|
|
104
|
+
assert.equal(r.reason, expected, label);
|
|
105
|
+
assert.ok(logged.some((l) => l.startsWith("error:")), `${label} must log an error`);
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("a throwing transport degrades to a logged failure, never an exception", async () => {
|
|
110
|
+
const logged = [];
|
|
111
|
+
const r = await escalateBudget(
|
|
112
|
+
{ agentRoot: "/tmp/x", conn: CONN },
|
|
113
|
+
{
|
|
114
|
+
log: (l, m) => logged.push(`${l}:${m}`),
|
|
115
|
+
notifyImpl: () => notice(),
|
|
116
|
+
callImpl: async () => { throw new Error("socket hang up"); },
|
|
117
|
+
}
|
|
118
|
+
);
|
|
119
|
+
assert.equal(r.ok, false);
|
|
120
|
+
assert.equal(r.reason, "dm-resolve-threw");
|
|
121
|
+
assert.ok(logged.some((l) => /socket hang up/.test(l)));
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
test("no band due → no traffic at all", async () => {
|
|
125
|
+
let calls = 0;
|
|
126
|
+
const r = await escalateBudget(
|
|
127
|
+
{ agentRoot: "/tmp/x", conn: CONN },
|
|
128
|
+
{ log: () => {}, notifyImpl: () => ({ notified: false, band: 0, status: STATUS }), callImpl: async () => { calls++; return { ok: true }; } }
|
|
129
|
+
);
|
|
130
|
+
assert.equal(r.notified, false);
|
|
131
|
+
assert.equal(calls, 0);
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("a throwing budget guard does not take the daemon down", async () => {
|
|
135
|
+
const logged = [];
|
|
136
|
+
const r = await escalateBudget(
|
|
137
|
+
{ agentRoot: "/tmp/x", conn: CONN },
|
|
138
|
+
{ log: (l, m) => logged.push(`${l}:${m}`), notifyImpl: () => { throw new Error("ledger unreadable"); } }
|
|
139
|
+
);
|
|
140
|
+
assert.equal(r.ok, false);
|
|
141
|
+
assert.equal(r.reason, "guard-threw");
|
|
142
|
+
assert.ok(logged.some((l) => l.startsWith("error:") && /NO notice was sent/.test(l)));
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
// ---------------------------------------------------------------------------
|
|
146
|
+
// THE REAL maybeNotify — write-AFTER-delivery, not write-then-hope
|
|
147
|
+
// ---------------------------------------------------------------------------
|
|
148
|
+
// Every test above stubs `notifyImpl`, i.e. it mocks the component that owns the
|
|
149
|
+
// ordering under test. These use the real guard against a real temp agent root,
|
|
150
|
+
// which is the only way to catch "the band was consumed before the first byte
|
|
151
|
+
// went out".
|
|
152
|
+
|
|
153
|
+
import { promises as fsp } from "node:fs";
|
|
154
|
+
import { writeFileSync, mkdirSync, readFileSync, existsSync } from "node:fs";
|
|
155
|
+
import { tmpdir } from "node:os";
|
|
156
|
+
import { join } from "node:path";
|
|
157
|
+
|
|
158
|
+
const FIXED = Date.UTC(2026, 5, 15, 12, 0, 0);
|
|
159
|
+
const DAY = "2026-06-15";
|
|
160
|
+
|
|
161
|
+
async function escalateRoot({ spentUsd = 27, seatBudgetCents = 3_000 } = {}) {
|
|
162
|
+
const dir = join(tmpdir(), `bg-escalate-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
|
|
163
|
+
mkdirSync(join(dir, "state", "cost-tracking"), { recursive: true });
|
|
164
|
+
mkdirSync(join(dir, "state", "mandate"), { recursive: true });
|
|
165
|
+
writeFileSync(
|
|
166
|
+
join(dir, "state", "mandate", "cache.json"),
|
|
167
|
+
JSON.stringify({
|
|
168
|
+
version: 1, checksum: "x", source: "server", fetchedAt: new Date(FIXED).toISOString(),
|
|
169
|
+
body: {
|
|
170
|
+
contractVersion: 1, memberId: "A016", objectives: [], seatBudgetCents,
|
|
171
|
+
budgetPeriod: "monthly", budgetSource: "employee.payBasis.meteredBudget",
|
|
172
|
+
collaborators: [{ memberId: "M-SUPERVISOR", role: "REVIEWER", source: "workforceMember.supervisorId" }],
|
|
173
|
+
degradations: [],
|
|
174
|
+
},
|
|
175
|
+
})
|
|
176
|
+
);
|
|
177
|
+
writeFileSync(
|
|
178
|
+
join(dir, "state", "cost-tracking", `${DAY}.jsonl`),
|
|
179
|
+
JSON.stringify({ ts: `${DAY}T10:00:00Z`, model: "opus", source: "dispatcher", input_tokens: 10, output_tokens: 10, total_cost_usd: spentUsd, estimated_usd: spentUsd }) + "\n"
|
|
180
|
+
);
|
|
181
|
+
return dir;
|
|
182
|
+
}
|
|
183
|
+
const notifiedBands = (dir) => {
|
|
184
|
+
const p = join(dir, "state", "cost-tracking", `${DAY}.budget.json`);
|
|
185
|
+
return existsSync(p) ? (JSON.parse(readFileSync(p, "utf-8")).notified_bands || []) : [];
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
test("a transient delivery failure does NOT burn the period's band — it is retried", async () => {
|
|
189
|
+
// The defect: `maybeNotify` wrote `notified_bands` BEFORE any delivery was
|
|
190
|
+
// attempted, and nothing could un-mark it. One ECONNREFUSED (or a daemon that
|
|
191
|
+
// started before the org credential resolved) permanently dropped the 80% rung
|
|
192
|
+
// for the whole period, and the 5-minute timer then polled forever without
|
|
193
|
+
// ever retrying. The rung's entire purpose is to tell the supervisor.
|
|
194
|
+
const dir = await escalateRoot();
|
|
195
|
+
try {
|
|
196
|
+
const deps = (callImpl) => ({
|
|
197
|
+
log: () => {}, now: () => FIXED, retryMs: 0, callImpl,
|
|
198
|
+
sendImpl: async () => ({ ok: true, result: { id: "m1" } }),
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
const t1 = await escalateBudget({ agentRoot: dir, conn: CONN }, deps(async () => { throw new Error("ECONNREFUSED"); }));
|
|
202
|
+
assert.equal(t1.notified, true);
|
|
203
|
+
assert.equal(t1.delivered, false);
|
|
204
|
+
assert.deepEqual(notifiedBands(dir), [], "the band is NOT consumed by a failed attempt");
|
|
205
|
+
|
|
206
|
+
// hq recovers. The next tick must actually deliver.
|
|
207
|
+
const sent = [];
|
|
208
|
+
const t2 = await escalateBudget({ agentRoot: dir, conn: CONN }, {
|
|
209
|
+
...deps(async () => ({ ok: true, result: { channelId: "chan_1" } })),
|
|
210
|
+
sendImpl: async (p) => { sent.push(p); return { ok: true, result: { id: "m1" } }; },
|
|
211
|
+
});
|
|
212
|
+
assert.equal(t2.delivered, true, "the retry lands");
|
|
213
|
+
assert.equal(sent.length, 1);
|
|
214
|
+
assert.ok(notifiedBands(dir).includes(80), "and NOW the band is consumed");
|
|
215
|
+
|
|
216
|
+
// And it does not re-send afterwards.
|
|
217
|
+
const t3 = await escalateBudget({ agentRoot: dir, conn: CONN }, deps(async () => ({ ok: true, result: { channelId: "chan_1" } })));
|
|
218
|
+
assert.equal(t3.notified, false, "at most one notice per band per period");
|
|
219
|
+
} finally { await fsp.rm(dir, { recursive: true, force: true }); }
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
test("a persistent outage gives up LOUDLY rather than retrying forever in silence", async () => {
|
|
223
|
+
const dir = await escalateRoot();
|
|
224
|
+
try {
|
|
225
|
+
const logged = [];
|
|
226
|
+
const deps = { log: (lvl, msg) => logged.push(`${lvl} ${msg}`), now: () => FIXED, retryMs: 0,
|
|
227
|
+
callImpl: async () => { throw new Error("ECONNREFUSED"); }, sendImpl: async () => ({ ok: true }) };
|
|
228
|
+
for (let i = 0; i < 8; i++) await escalateBudget({ agentRoot: dir, conn: CONN }, deps);
|
|
229
|
+
assert.ok(notifiedBands(dir).includes(80), "the band is finally consumed so the log stops spinning");
|
|
230
|
+
assert.ok(logged.some((l) => /ABANDONED after/.test(l)), "and giving up is an ERROR, not a shrug");
|
|
231
|
+
} finally { await fsp.rm(dir, { recursive: true, force: true }); }
|
|
232
|
+
});
|