@cohortapp/agent-sdk 2.5.1 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +185 -88
- package/bin/maestro.test.mjs +175 -48
- package/docs/runbooks/backup-restore.md +65 -33
- package/framework-features.json +4 -4
- package/lib/backup/policy.mjs +710 -0
- package/lib/backup/policy.test.mjs +305 -0
- package/lib/budget-escalate.mjs +133 -0
- package/lib/budget-escalate.test.mjs +232 -0
- package/lib/budget-guard.envelope.test.mjs +476 -0
- package/lib/budget-guard.mjs +853 -75
- package/lib/budget-guard.test.mjs +91 -42
- package/lib/cadences.mjs +33 -0
- package/lib/channels/orgmail/adapter.mjs +88 -3
- package/lib/channels/orgmail/adapter.test.mjs +137 -0
- package/lib/channels/repeat-suppressor.mjs +198 -0
- package/lib/channels/repeat-suppressor.test.mjs +134 -0
- package/lib/comms/receipts.mjs +297 -0
- package/lib/cost/ledger-row.mjs +333 -0
- package/lib/cost/ledger-row.test.mjs +183 -0
- package/lib/execution/drive.mjs +28 -1
- package/lib/execution/effects.mjs +191 -12
- package/lib/execution/effects.test.mjs +50 -11
- package/lib/goals/admission.mjs +13 -1
- package/lib/goals/admission.test.mjs +26 -1
- package/lib/goals/loop.mjs +13 -0
- package/lib/kpi-sensors.test.mjs +3 -0
- package/lib/mandate/cache.mjs +13 -5
- package/lib/mandate/derive.mjs +146 -21
- package/lib/mandate/derive.test.mjs +50 -6
- package/lib/mandate/model.mjs +32 -4
- package/lib/mandate/refresh.test.mjs +16 -2
- package/lib/mcp/server.test.mjs +12 -3
- package/lib/model-router/economics.mjs +107 -76
- package/lib/model-router/economics.test.mjs +64 -46
- package/lib/model-router/integration-coverage.test.mjs +39 -37
- package/lib/model-router/ledger.mjs +75 -22
- package/lib/model-router/ledger.test.mjs +35 -2
- package/lib/org/client.mjs +14 -0
- package/lib/org/cost-sync.mjs +16 -2
- package/lib/org/doctor.mjs +62 -1
- package/lib/org/doctor.test.mjs +36 -3
- package/lib/org/email-remedy.mjs +49 -0
- package/lib/org/engagement-ledger.mjs +376 -0
- package/lib/org/engagement-ledger.test.mjs +112 -0
- package/lib/org/engagement.mjs +1056 -0
- package/lib/org/engagement.test.mjs +739 -0
- package/lib/org/messaging.mjs +230 -3
- package/lib/org/messaging.test.mjs +110 -1
- package/lib/org/param-contract.mjs +56 -2
- package/lib/org/param-contract.test.mjs +26 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +5 -0
- package/lib/org/protocol.test.mjs +7 -1
- package/lib/org/tool-surface.mjs +506 -10
- package/lib/org/tool-surface.test.mjs +191 -7
- package/lib/org/ui-parity.mjs +333 -6
- package/lib/org/ui-parity.test.mjs +96 -3
- package/lib/org/work-ledger.mjs +241 -0
- package/lib/org/work-ledger.test.mjs +237 -0
- package/lib/plan/adoption-e2e.test.mjs +366 -0
- package/lib/plan/budget-enforcement.test.mjs +400 -0
- package/lib/plan/budget-runtime.mjs +215 -0
- package/lib/plan/compile.mjs +201 -5
- package/lib/plan/compile.test.mjs +19 -5
- package/lib/plan/emit.mjs +8 -0
- package/lib/plan/emit.test.mjs +18 -0
- package/lib/resource-governor.mjs +58 -12
- package/lib/resource-governor.test.mjs +41 -1
- package/lib/security/audit-engine.mjs +45 -8
- package/lib/security/audit-engine.test.mjs +35 -0
- package/lib/setup/enroll-from-cohort.mjs +14 -1
- package/lib/setup/sections/mandate.mjs +48 -7
- package/lib/setup/sections/mandate.test.mjs +17 -2
- package/lib/setup/sections/orgmail.mjs +10 -2
- package/lib/setup/state.mjs +83 -2
- package/lib/telemetry/collect.mjs +360 -20
- package/lib/telemetry/collect.test.mjs +266 -0
- package/package.json +1 -1
- package/scripts/cost/track-claude-usage.mjs +207 -48
- package/scripts/cost/track-claude-usage.test.mjs +148 -0
- package/scripts/daemon/agent-daemon.mjs +315 -17
- package/scripts/daemon/assurance-e2e.test.mjs +421 -0
- package/scripts/daemon/assurance.mjs +944 -0
- package/scripts/daemon/assurance.test.mjs +668 -0
- package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
- package/scripts/daemon/cadence-consumer.mjs +147 -9
- package/scripts/daemon/cadence-consumer.test.mjs +6 -0
- package/scripts/daemon/cadence-handlers.mjs +158 -0
- package/scripts/daemon/cadence-handlers.test.mjs +64 -0
- package/scripts/daemon/deliver.mjs +314 -0
- package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
- package/scripts/daemon/dispatcher.mjs +64 -6
- package/scripts/daemon/responder-cost.test.mjs +68 -0
- package/scripts/daemon/responder.mjs +351 -298
- package/scripts/local-triggers/generate-plists.test.mjs +7 -4
- package/scripts/maintenance/backup-run.mjs +415 -0
- package/scripts/maintenance/backup-to-cloud.sh +16 -116
- package/scripts/org/send-orgmail.mjs +16 -0
- package/scripts/record-receipt.sh +63 -0
- package/scripts/restore-from-backup.sh +14 -3
- package/scripts/restore-from-backup.test.mjs +8 -5
- package/scripts/send-email-threaded.py +47 -0
- package/scripts/send-sms.sh +4 -0
- package/scripts/send-whatsapp.sh +4 -0
- package/scripts/setup/init-backup.mjs +93 -38
- package/scripts/slack-send.sh +12 -0
|
@@ -31,9 +31,26 @@ async function makeLedgerDir() {
|
|
|
31
31
|
}
|
|
32
32
|
async function rm(dir) { try { await fsp.rm(dir, { recursive: true, force: true }); } catch { /* */ } }
|
|
33
33
|
|
|
34
|
-
/**
|
|
34
|
+
/**
|
|
35
|
+
* Write `rows` (array of {usd}) into today's ledger file as MEASURED sessions.
|
|
36
|
+
*
|
|
37
|
+
* `total_cost_usd` + real token counts, because that is what a row from a
|
|
38
|
+
* correctly-instrumented caller looks like and it is the column the governor
|
|
39
|
+
* now bills against (lib/cost/ledger-row.mjs). A row carrying only
|
|
40
|
+
* `estimated_usd` and no tokens classifies as `unknown` — deliberately, so an
|
|
41
|
+
* unmeasured session can never read as a free one — and would be imputed rather
|
|
42
|
+
* than summed, which is not what these band tests are about.
|
|
43
|
+
*/
|
|
35
44
|
function seedLedger(dir, rows) {
|
|
36
|
-
const body = rows.map((r) => JSON.stringify({
|
|
45
|
+
const body = rows.map((r) => JSON.stringify({
|
|
46
|
+
ts: `${DATE}T10:00:00.000Z`,
|
|
47
|
+
total_cost_usd: r.usd,
|
|
48
|
+
estimated_usd: r.usd,
|
|
49
|
+
model: "sonnet",
|
|
50
|
+
cadence: "x",
|
|
51
|
+
input_tokens: 1000,
|
|
52
|
+
output_tokens: 200,
|
|
53
|
+
})).join("\n") + "\n";
|
|
37
54
|
writeFileSync(join(dir, `${DATE}.jsonl`), body);
|
|
38
55
|
}
|
|
39
56
|
|
|
@@ -41,12 +58,13 @@ function seedLedger(dir, rows) {
|
|
|
41
58
|
// banding
|
|
42
59
|
// ---------------------------------------------------------------------------
|
|
43
60
|
|
|
44
|
-
test("dailyStatus: below
|
|
61
|
+
test("dailyStatus: below 80% is band 0, nothing degraded", async () => {
|
|
45
62
|
const dir = await makeLedgerDir();
|
|
46
63
|
try {
|
|
47
64
|
seedLedger(dir, [{ usd: 4 }]); // $4 of $10 = 40%
|
|
48
65
|
const s = dailyStatus({ ledgerDir: dir, now: clk, capUSD: 10 });
|
|
49
66
|
assert.equal(s.band, 0);
|
|
67
|
+
assert.equal(s.mode, "normal");
|
|
50
68
|
assert.equal(s.essentialOnly, false);
|
|
51
69
|
assert.equal(s.spentUSD, 4);
|
|
52
70
|
assert.equal(s.pct, 40);
|
|
@@ -54,7 +72,8 @@ test("dailyStatus: below 50% is band 0, not essential-only", async () => {
|
|
|
54
72
|
});
|
|
55
73
|
|
|
56
74
|
test("dailyStatus: crossing each threshold lands in the right band", async () => {
|
|
57
|
-
|
|
75
|
+
// The governance ladder: 80 notify · 100 degrade · 125 suspend · 150 refuse.
|
|
76
|
+
for (const [usd, expected] of [[5, 0], [8, 80], [9.9, 80], [10, 100], [12.5, 125], [15, 150], [40, 150]]) {
|
|
58
77
|
const dir = await makeLedgerDir();
|
|
59
78
|
try {
|
|
60
79
|
seedLedger(dir, [{ usd }]);
|
|
@@ -91,7 +110,8 @@ test("dailyStatus: iteration cap can be the binding constraint", async () => {
|
|
|
91
110
|
const s = dailyStatus({ ledgerDir: dir, now: clk, capUSD: 1000, iterationCap: 10 });
|
|
92
111
|
assert.equal(s.sessions, 10);
|
|
93
112
|
assert.equal(s.band, 100, "iteration cap should drive band to 100");
|
|
94
|
-
assert.equal(s.
|
|
113
|
+
assert.equal(s.mode, "degraded");
|
|
114
|
+
assert.equal(s.posture.selfDirected, false, "D3 stops at the degrade rung");
|
|
95
115
|
} finally { await rm(dir); }
|
|
96
116
|
});
|
|
97
117
|
|
|
@@ -99,12 +119,15 @@ test("dailyStatus: iteration cap can be the binding constraint", async () => {
|
|
|
99
119
|
// essential-only at 100%
|
|
100
120
|
// ---------------------------------------------------------------------------
|
|
101
121
|
|
|
102
|
-
test("essentialOnly
|
|
122
|
+
test("essentialOnly (the legacy flag) now means the SUSPEND rung at 125%", async () => {
|
|
103
123
|
const dir = await makeLedgerDir();
|
|
104
124
|
try {
|
|
105
|
-
|
|
106
|
-
|
|
125
|
+
// 100% degrades — inbox + obligations continue on the cheapest class — so
|
|
126
|
+
// the legacy all-or-nothing flag must NOT be set there. That rung was the
|
|
127
|
+
// one the old binary gate collapsed.
|
|
107
128
|
seedLedger(dir, [{ usd: 10 }]);
|
|
129
|
+
assert.equal(dailyStatus({ ledgerDir: dir, now: clk, capUSD: 10 }).essentialOnly, false);
|
|
130
|
+
seedLedger(dir, [{ usd: 12.5 }]);
|
|
108
131
|
assert.equal(dailyStatus({ ledgerDir: dir, now: clk, capUSD: 10 }).essentialOnly, true);
|
|
109
132
|
} finally { await rm(dir); }
|
|
110
133
|
});
|
|
@@ -119,24 +142,29 @@ test("maybeNotify fires once per band and never repeats within the day", async (
|
|
|
119
142
|
const notices = [];
|
|
120
143
|
const notify = (n) => notices.push(n.band);
|
|
121
144
|
|
|
122
|
-
//
|
|
123
|
-
|
|
124
|
-
|
|
145
|
+
// A supervisor must exist or the notice is logged and sent nowhere by
|
|
146
|
+
// design — see maybeNotify. Inject one.
|
|
147
|
+
const seat = { seatEnvelope: { seatBudgetCents: null, supervisorMemberId: "M-SUP" } };
|
|
148
|
+
|
|
149
|
+
// 85% → notify band 80 once.
|
|
150
|
+
seedLedger(dir, [{ usd: 8.5 }]);
|
|
151
|
+
let r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify, ...seat });
|
|
125
152
|
assert.equal(r.notified, true);
|
|
126
|
-
assert.equal(r.band,
|
|
153
|
+
assert.equal(r.band, 80);
|
|
154
|
+
assert.deepEqual(r.recipient, { memberId: "M-SUP", role: "REVIEWER" });
|
|
127
155
|
|
|
128
|
-
// Still
|
|
129
|
-
r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify });
|
|
156
|
+
// Still 85% → no repeat.
|
|
157
|
+
r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify, ...seat });
|
|
130
158
|
assert.equal(r.notified, false);
|
|
131
159
|
|
|
132
|
-
// Climb to
|
|
133
|
-
seedLedger(dir, [{ usd:
|
|
134
|
-
r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify });
|
|
160
|
+
// Climb to 105% → notify band 100 once.
|
|
161
|
+
seedLedger(dir, [{ usd: 10.5 }]);
|
|
162
|
+
r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify, ...seat });
|
|
135
163
|
assert.equal(r.notified, true);
|
|
136
|
-
assert.equal(r.band,
|
|
164
|
+
assert.equal(r.band, 100);
|
|
137
165
|
|
|
138
|
-
assert.deepEqual(notices, [
|
|
139
|
-
assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [
|
|
166
|
+
assert.deepEqual(notices, [80, 100]);
|
|
167
|
+
assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [80, 100]);
|
|
140
168
|
} finally { await rm(dir); }
|
|
141
169
|
});
|
|
142
170
|
|
|
@@ -144,42 +172,63 @@ test("maybeNotify on a big jump notifies once at the highest crossed band", asyn
|
|
|
144
172
|
const dir = await makeLedgerDir();
|
|
145
173
|
try {
|
|
146
174
|
const notices = [];
|
|
147
|
-
// Jump straight from $0 to
|
|
148
|
-
//
|
|
149
|
-
|
|
150
|
-
|
|
175
|
+
// Jump straight from $0 to 130% — ONE notice at band 125, with 80/100/125
|
|
176
|
+
// all recorded so they never re-fire.
|
|
177
|
+
const seat = { seatEnvelope: { seatBudgetCents: null, supervisorMemberId: "M-SUP" } };
|
|
178
|
+
seedLedger(dir, [{ usd: 13 }]);
|
|
179
|
+
const r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n.band), ...seat });
|
|
151
180
|
assert.equal(r.notified, true);
|
|
152
|
-
assert.equal(r.band,
|
|
153
|
-
assert.deepEqual(notices, [
|
|
154
|
-
assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [
|
|
181
|
+
assert.equal(r.band, 125);
|
|
182
|
+
assert.deepEqual(notices, [125], "exactly one notice surfaced");
|
|
183
|
+
assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [80, 100, 125]);
|
|
155
184
|
|
|
156
185
|
// A later check at the same level does nothing.
|
|
157
|
-
const r2 = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n.band) });
|
|
186
|
+
const r2 = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n.band), ...seat });
|
|
158
187
|
assert.equal(r2.notified, false);
|
|
159
|
-
assert.deepEqual(notices, [
|
|
188
|
+
assert.deepEqual(notices, [125]);
|
|
160
189
|
} finally { await rm(dir); }
|
|
161
190
|
});
|
|
162
191
|
|
|
163
|
-
test("maybeNotify
|
|
192
|
+
test("maybeNotify past 150% surfaces the refuse notice once", async () => {
|
|
164
193
|
const dir = await makeLedgerDir();
|
|
165
194
|
try {
|
|
166
195
|
const notices = [];
|
|
167
|
-
|
|
168
|
-
|
|
196
|
+
const seat = { seatEnvelope: { seatBudgetCents: null, supervisorMemberId: "M-SUP" } };
|
|
197
|
+
seedLedger(dir, [{ usd: 16 }]); // 160%
|
|
198
|
+
const r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n), ...seat });
|
|
169
199
|
assert.equal(r.notified, true);
|
|
170
|
-
assert.equal(r.band,
|
|
171
|
-
assert.match(r.message, /
|
|
172
|
-
assert.equal(r.status.
|
|
200
|
+
assert.equal(r.band, 150);
|
|
201
|
+
assert.match(r.message, /REFUSING new sessions that are not a direct human reply/);
|
|
202
|
+
assert.equal(r.status.posture.refuseNonHumanSpawn, true);
|
|
173
203
|
// All bands recorded; a repeat is a no-op.
|
|
174
|
-
assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [
|
|
175
|
-
const r2 = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n) });
|
|
204
|
+
assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [80, 100, 125, 150]);
|
|
205
|
+
const r2 = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n), ...seat });
|
|
176
206
|
assert.equal(r2.notified, false);
|
|
177
207
|
assert.equal(notices.length, 1);
|
|
178
208
|
} finally { await rm(dir); }
|
|
179
209
|
});
|
|
180
210
|
|
|
181
|
-
test("
|
|
182
|
-
|
|
211
|
+
test("a seat with NO supervisor edge is never told its own budget band", async () => {
|
|
212
|
+
const dir = await makeLedgerDir();
|
|
213
|
+
try {
|
|
214
|
+
const notices = [];
|
|
215
|
+
const logged = [];
|
|
216
|
+
seedLedger(dir, [{ usd: 9 }]); // 90% → band 80
|
|
217
|
+
const r = maybeNotify({
|
|
218
|
+
ledgerDir: dir, now: clk, capUSD: 10,
|
|
219
|
+
notify: (n) => notices.push(n),
|
|
220
|
+
log: (level, msg) => logged.push(`${level}:${msg}`),
|
|
221
|
+
});
|
|
222
|
+
assert.equal(r.band, 80);
|
|
223
|
+
assert.equal(r.recipient, null);
|
|
224
|
+
assert.equal(notices.length, 0, "no notice may be delivered to the owner");
|
|
225
|
+
assert.ok(logged.some((l) => l.startsWith("error:") && /NO supervisor edge/.test(l)),
|
|
226
|
+
"the missing edge must be LOUD, not silent");
|
|
227
|
+
} finally { await rm(dir); }
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test("BANDS is the canonical 80/100/125/150 governance ladder", () => {
|
|
231
|
+
assert.deepEqual(BANDS, [80, 100, 125, 150]);
|
|
183
232
|
});
|
|
184
233
|
|
|
185
234
|
// ---------------------------------------------------------------------------
|
|
@@ -204,7 +253,7 @@ test("dailyStatus reads the cap from config/recovery.yaml when present", async (
|
|
|
204
253
|
assert.equal(s.capUSD, 40, "cap should come from recovery.yaml, not the 25 default");
|
|
205
254
|
assert.equal(s.spentUSD, 20);
|
|
206
255
|
assert.equal(s.pct, 50, "20/40 = 50%");
|
|
207
|
-
assert.equal(s.band, 50);
|
|
256
|
+
assert.equal(s.band, 0, "50% of the ceiling degrades nothing");
|
|
208
257
|
} finally { await rm(dir); }
|
|
209
258
|
});
|
|
210
259
|
|
|
@@ -218,8 +267,8 @@ test("recovery.yaml cap takes precedence over DAILY_SPEND_CAP_USD env", async ()
|
|
|
218
267
|
seedLedger(join(dir, "state", "cost-tracking"), [{ usd: 10 }]);
|
|
219
268
|
const s = dailyStatus({ agentRoot: dir, now: clk });
|
|
220
269
|
assert.equal(s.capUSD, 10, "yaml cap must win over the env cap");
|
|
221
|
-
assert.equal(s.band, 100, "10/10 spent under the yaml cap is 100%
|
|
222
|
-
assert.equal(s.
|
|
270
|
+
assert.equal(s.band, 100, "10/10 spent under the yaml cap is the 100% degrade rung");
|
|
271
|
+
assert.equal(s.mode, "degraded");
|
|
223
272
|
} finally {
|
|
224
273
|
if (prevEnv === undefined) delete process.env.DAILY_SPEND_CAP_USD;
|
|
225
274
|
else process.env.DAILY_SPEND_CAP_USD = prevEnv;
|
package/lib/cadences.mjs
CHANGED
|
@@ -48,6 +48,11 @@ export const STANDARD_CADENCES = [
|
|
|
48
48
|
// job is actually due, so the consumer spawns a session to action it.
|
|
49
49
|
// nightly-cost-reconcile: three-way spend reconcile vs the Admin Cost API
|
|
50
50
|
// (fails open with no fetchImpl); flags drift > 10%. Inline, no prompt.
|
|
51
|
+
// nightly-backup: the DR job. Inline (tar + optional upload, no LLM). 03:10 —
|
|
52
|
+
// quiet hours, and ahead of nightly-cost-reconcile so the archive captures the
|
|
53
|
+
// day's ledger before anything rewrites it. See lib/backup/policy.mjs for what
|
|
54
|
+
// is archived, where, and what is never archived at all.
|
|
55
|
+
{ id: "nightly-backup", scope: "standard", mode: "inline", calendar: { hour: 3, minute: 10 } },
|
|
51
56
|
{ id: "nightly-cost-reconcile", scope: "standard", mode: "inline", calendar: { hour: 3, minute: 30 } },
|
|
52
57
|
// fleet-cost-digest: build the daily fleet LLM-spend digest. Inline, no prompt.
|
|
53
58
|
{ id: "fleet-cost-digest", scope: "standard", mode: "inline", calendar: { hour: 7, minute: 0 } },
|
|
@@ -113,6 +118,34 @@ export const ALTITUDE_CADENCES = {
|
|
|
113
118
|
],
|
|
114
119
|
};
|
|
115
120
|
|
|
121
|
+
/**
|
|
122
|
+
* THE HUMAN LANE. Cadences whose job is getting a human's message to the agent
|
|
123
|
+
* and the agent's answer back — the work Invariant #1 says never goes dark.
|
|
124
|
+
*
|
|
125
|
+
* This exists because the budget ladder's top rung ("refuse anything that is not
|
|
126
|
+
* a direct human reply") was implemented as `obligation.kind !== "REACT"`, and
|
|
127
|
+
* every cadence the consumer evaluates is a SCHEDULE. At band 150 that blocked
|
|
128
|
+
* 19 of 26 obligations including `inbox-processor` and `messaging-inbound` — the
|
|
129
|
+
* cadence that PULLS Cohort DMs and @mentions into the inbox pipeline at all. It
|
|
130
|
+
* only looked harmless because both guards default to `decision:"inline"` and
|
|
131
|
+
* return before the gate; set `INBOX_CADENCE_ESCALATE=1` or
|
|
132
|
+
* `MESSAGING_INBOUND_ESCALATE=1` (the documented cadence-only mode for operators
|
|
133
|
+
* running without the reactive daemon) and the seat stopped answering its inbox
|
|
134
|
+
* at 150 %, which is the exact failure the rung was written to avoid.
|
|
135
|
+
*
|
|
136
|
+
* Membership test: does a human wait on the other end of this cadence?
|
|
137
|
+
* inbox-processor — drains the inbox a human wrote into.
|
|
138
|
+
* messaging-inbound — the only puller of org DMs/@mentions.
|
|
139
|
+
* dynamic-jobs — DELIBERATELY NOT here: those are the agent's own
|
|
140
|
+
* reminders to itself, not somebody waiting on a reply.
|
|
141
|
+
*/
|
|
142
|
+
export const HUMAN_LANE_CADENCES = Object.freeze(new Set(["inbox-processor", "messaging-inbound"]));
|
|
143
|
+
|
|
144
|
+
/** Is this cadence part of the human-reply lane? See {@link HUMAN_LANE_CADENCES}. */
|
|
145
|
+
export function isHumanLaneCadence(id) {
|
|
146
|
+
return HUMAN_LANE_CADENCES.has(String(id || ""));
|
|
147
|
+
}
|
|
148
|
+
|
|
116
149
|
/** Cadence keywords that map to a recurring schedule (event-driven is excluded). */
|
|
117
150
|
export const RECURRING = new Set(["daily", "weekly", "monthly", "quarterly", "continuous"]);
|
|
118
151
|
|
|
@@ -50,6 +50,10 @@ import { BaseAdapter, CHANNEL_COUNTERS } from "../base-adapter.mjs";
|
|
|
50
50
|
import { randomUUID } from "node:crypto";
|
|
51
51
|
import { loadOrgConfig, configFromAgent } from "../../org/client.mjs";
|
|
52
52
|
import { emailSend, emailMarkRead, emailInbox, emailMessage } from "../../org/ui-parity.mjs";
|
|
53
|
+
import { RepeatSuppressor } from "../repeat-suppressor.mjs";
|
|
54
|
+
// One classifier for every surface that reports an email.* failure (this poll
|
|
55
|
+
// loop, `maestro doctor`, `maestro setup`) — see lib/org/email-remedy.mjs.
|
|
56
|
+
import { remedyFor } from "../../org/email-remedy.mjs";
|
|
53
57
|
|
|
54
58
|
export const ORGMAIL_CAPABILITIES = [
|
|
55
59
|
CHANNEL_CAPABILITIES.SEND,
|
|
@@ -60,6 +64,17 @@ export const ORGMAIL_CAPABILITIES = [
|
|
|
60
64
|
/** Default poll cadence — the messaging-inbound rhythm. */
|
|
61
65
|
export const DEFAULT_POLL_MS = 45_000;
|
|
62
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Poll cadence while a TERMINAL condition stands (no mailbox assigned, key not
|
|
69
|
+
* paired, key lacks the email scope). These need a human act in Cohort; the
|
|
70
|
+
* 45s rhythm cannot fix them and only produces log noise and pointless load, so
|
|
71
|
+
* the loop drops to 10 minutes and self-heals the moment the condition clears.
|
|
72
|
+
*/
|
|
73
|
+
export const DEGRADED_POLL_MS = 10 * 60_000;
|
|
74
|
+
|
|
75
|
+
/** Condition key for the standing-inbox-failure suppressor. */
|
|
76
|
+
const INBOX_CONDITION = "email.inbox";
|
|
77
|
+
|
|
63
78
|
export class OrgMailAdapter extends BaseAdapter {
|
|
64
79
|
/**
|
|
65
80
|
* @param {object} opts
|
|
@@ -88,6 +103,13 @@ export class OrgMailAdapter extends BaseAdapter {
|
|
|
88
103
|
this._pollMs = resolvePollMs(opts.pollMs, gate, process.env);
|
|
89
104
|
this._timer = null;
|
|
90
105
|
this._polling = false;
|
|
106
|
+
// Standing-condition tracker: a mailbox that does not exist is one FACT,
|
|
107
|
+
// not one fact per 45 seconds. See lib/channels/repeat-suppressor.mjs.
|
|
108
|
+
this._suppressor = opts.suppressor || new RepeatSuppressor({ restateMs: opts.restateMs });
|
|
109
|
+
// Set while a terminal condition stands, so healthCheck and the daemon can
|
|
110
|
+
// report WHY the lane is down without re-probing.
|
|
111
|
+
this._standingFault = null;
|
|
112
|
+
this._degraded = false;
|
|
91
113
|
}
|
|
92
114
|
|
|
93
115
|
/** Resolve {base, token, orgId} from config/org.yaml (the enrollment SoT). */
|
|
@@ -122,10 +144,31 @@ export class OrgMailAdapter extends BaseAdapter {
|
|
|
122
144
|
}
|
|
123
145
|
|
|
124
146
|
await tick(); // first sweep immediately so start() sees a live adapter
|
|
125
|
-
this.
|
|
147
|
+
this._startTimer(tick);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* (Re)arm the poll timer at the cadence the CURRENT state deserves: the
|
|
152
|
+
* configured rhythm normally, DEGRADED_POLL_MS while a terminal condition
|
|
153
|
+
* stands. Called on every transition so recovery is immediate — the first
|
|
154
|
+
* successful sweep after an admin assigns the mailbox restores 45s polling.
|
|
155
|
+
*/
|
|
156
|
+
_startTimer(tick) {
|
|
157
|
+
if (this._timer) this._clearInterval(this._timer);
|
|
158
|
+
const ms = this._degraded ? DEGRADED_POLL_MS : this._pollMs;
|
|
159
|
+
this._tick = tick;
|
|
160
|
+
this._timer = this._setInterval(tick, ms);
|
|
126
161
|
if (this._timer && typeof this._timer.unref === "function") this._timer.unref();
|
|
127
162
|
}
|
|
128
163
|
|
|
164
|
+
/** Enter/leave degraded cadence, re-arming the timer only on a transition. */
|
|
165
|
+
_setDegraded(next, reason) {
|
|
166
|
+
if (this._degraded === next) return;
|
|
167
|
+
this._degraded = next;
|
|
168
|
+
this._standingFault = next ? reason : null;
|
|
169
|
+
if (this._tick) this._startTimer(this._tick);
|
|
170
|
+
}
|
|
171
|
+
|
|
129
172
|
/**
|
|
130
173
|
* One inbox sweep: list unread summaries, fetch each full DTO (newest LAST
|
|
131
174
|
* so downstream sees chronological order), hand off, then markRead ONLY on a
|
|
@@ -137,9 +180,30 @@ export class OrgMailAdapter extends BaseAdapter {
|
|
|
137
180
|
if (!o.base || !o.token) return; // not enrolled — dormant, never throws
|
|
138
181
|
const inbox = await emailInbox({ unreadOnly: true, limit: 50 }, o);
|
|
139
182
|
if (!inbox.ok) {
|
|
140
|
-
|
|
183
|
+
// A standing failure is ONE fact, not one fact per tick. Before this, a
|
|
184
|
+
// seat with no mailbox wrote the same NOT_FOUND line 478 times in a day —
|
|
185
|
+
// true every time, useful once, and loud enough to bury everything else.
|
|
186
|
+
// Now: full detail on the first occurrence, a re-statement with the
|
|
187
|
+
// suppressed count once an hour, an explicit RESOLVED line on recovery,
|
|
188
|
+
// and a 10-minute poll while the condition needs a human to clear it.
|
|
189
|
+
const code = inbox.error?.code || "?";
|
|
190
|
+
const seen = this._suppressor.observe(INBOX_CONDITION, { code });
|
|
191
|
+
if (seen.emit) {
|
|
192
|
+
this.log(
|
|
193
|
+
seen.terminal ? "error" : "warn",
|
|
194
|
+
`email.inbox ${code}: ${inbox.error?.message || "no detail"}` +
|
|
195
|
+
(seen.terminal ? ` — ${remedyFor(code)}` : "") +
|
|
196
|
+
seen.suffix
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
this._setDegraded(seen.terminal, seen.terminal ? { code, message: inbox.error?.message || "", since: new Date().toISOString() } : null);
|
|
141
200
|
return;
|
|
142
201
|
}
|
|
202
|
+
const recovered = this._suppressor.clear(INBOX_CONDITION);
|
|
203
|
+
if (recovered.wasActive) {
|
|
204
|
+
this.log("info", `email.inbox ${recovered.message} — the workspace mailbox lane is live again`);
|
|
205
|
+
}
|
|
206
|
+
this._setDegraded(false, null);
|
|
143
207
|
const summaries = Array.isArray(inbox.result?.messages) ? inbox.result.messages : [];
|
|
144
208
|
// Server returns newest-first (keyset createdAt desc); we deliver newest-LAST.
|
|
145
209
|
const ordered = [...summaries].reverse();
|
|
@@ -289,7 +353,28 @@ export class OrgMailAdapter extends BaseAdapter {
|
|
|
289
353
|
const addr = r.result?.mailbox?.address || "";
|
|
290
354
|
return { ok: true, detail: `orgmail mailbox ${addr || "reachable"}` };
|
|
291
355
|
}
|
|
292
|
-
|
|
356
|
+
// Suppressing the POLL log must not suppress the health ANSWER: the whole
|
|
357
|
+
// point of throttling repeats is that the standing condition stays legible
|
|
358
|
+
// somewhere. It lives here, and in `standingFault()`, so anything that asks
|
|
359
|
+
// gets the full story including the remedy.
|
|
360
|
+
const code = r.error?.code || "?";
|
|
361
|
+
const remedy = remedyFor(code);
|
|
362
|
+
return {
|
|
363
|
+
ok: false,
|
|
364
|
+
detail:
|
|
365
|
+
`orgmail: ${code} ${r.error?.message || "unreachable"}` +
|
|
366
|
+
(remedy ? ` — ${remedy}` : "") +
|
|
367
|
+
(this._degraded ? " (poll is degraded to 10m until this clears)" : ""),
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* The standing terminal condition, if any — `{code, message, since}`. Read by
|
|
373
|
+
* the daemon/doctor so "this lane is down and here is why" survives the log
|
|
374
|
+
* throttle. Null when the lane is healthy.
|
|
375
|
+
*/
|
|
376
|
+
standingFault() {
|
|
377
|
+
return this._standingFault;
|
|
293
378
|
}
|
|
294
379
|
|
|
295
380
|
async stop() {
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
resolvePollMs,
|
|
21
21
|
htmlToText,
|
|
22
22
|
DEFAULT_POLL_MS,
|
|
23
|
+
DEGRADED_POLL_MS,
|
|
23
24
|
ORGMAIL_CAPABILITIES,
|
|
24
25
|
} from "./adapter.mjs";
|
|
25
26
|
import { getPlatform, isDaemonManagedPlatform } from "../contract.mjs";
|
|
@@ -309,3 +310,139 @@ test("healthCheck: missing enrollment is an honest unhealthy, not a throw", asyn
|
|
|
309
310
|
assert.equal(h.ok, false);
|
|
310
311
|
assert.match(h.detail, /enrollment missing/);
|
|
311
312
|
});
|
|
313
|
+
|
|
314
|
+
// ── standing-fault throttle (the 478-lines-a-day incident) ─────────────────
|
|
315
|
+
//
|
|
316
|
+
// A member with no mailbox made every 45s poll write the same NOT_FOUND line:
|
|
317
|
+
// 478 of them on 2026-08-11, 164 the day before. The lane was dead and the
|
|
318
|
+
// only signal was noise. These tests pin the fix: full detail once, silence in
|
|
319
|
+
// between, a degraded poll while it needs a human, and an explicit recovery.
|
|
320
|
+
|
|
321
|
+
/** A fake hq whose email.inbox fails with `code` until `heal()` is called. */
|
|
322
|
+
function failingInbox(code, message = "no active mailbox is assigned to this agent") {
|
|
323
|
+
const state = { healed: false };
|
|
324
|
+
const fn = async (url) => {
|
|
325
|
+
const u = String(url);
|
|
326
|
+
const respond = (payload, status = 200, ok = true) => ({ ok, status, json: async () => payload, headers: { get: () => undefined } });
|
|
327
|
+
if (u.endsWith("/api/v1/email.inbox")) {
|
|
328
|
+
if (state.healed) {
|
|
329
|
+
return respond({ ok: true, result: { mailbox: { address: "astra@agents.example" }, messages: [], nextCursor: null } });
|
|
330
|
+
}
|
|
331
|
+
return respond({ ok: false, error: { code, message } }, 404, false);
|
|
332
|
+
}
|
|
333
|
+
return respond({ ok: true, result: {} });
|
|
334
|
+
};
|
|
335
|
+
fn.heal = () => { state.healed = true; };
|
|
336
|
+
return fn;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/** Build an adapter that captures its log lines and lets tests drive ticks. */
|
|
340
|
+
function makeLoggingAdapter(fetchImpl, over = {}) {
|
|
341
|
+
const logs = [];
|
|
342
|
+
const intervals = [];
|
|
343
|
+
const a = new OrgMailAdapter({
|
|
344
|
+
agentRoot: "/tmp/orgmail-test",
|
|
345
|
+
orgConfig: ORG_CFG,
|
|
346
|
+
config: { orgmail: { enabled: true, poll_seconds: 45, mark_read: true } },
|
|
347
|
+
fetchImpl,
|
|
348
|
+
supervise: false,
|
|
349
|
+
sendGate: false,
|
|
350
|
+
setInterval: (fn, ms) => { intervals.push({ fn, ms }); return { unref() {} }; },
|
|
351
|
+
clearInterval: () => {},
|
|
352
|
+
log: (level, msg) => logs.push({ level, msg }),
|
|
353
|
+
...over,
|
|
354
|
+
});
|
|
355
|
+
return { a, logs, intervals };
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
test("standing NOT_FOUND: the first poll logs it IN FULL with the remedy", async () => {
|
|
359
|
+
const { a, logs } = makeLoggingAdapter(failingInbox("NOT_FOUND"));
|
|
360
|
+
await a.start({ onInbound: () => {} });
|
|
361
|
+
await settle();
|
|
362
|
+
const line = logs.find((l) => /email\.inbox NOT_FOUND/.test(l.msg));
|
|
363
|
+
assert.ok(line, "the first occurrence must be logged");
|
|
364
|
+
assert.equal(line.level, "error", "a dead lane is not a warning");
|
|
365
|
+
assert.match(line.msg, /workspace ADMIN/, "an ACTOR");
|
|
366
|
+
assert.match(line.msg, /Settings → Email → Mailboxes/, "and an ACT");
|
|
367
|
+
await a.stop();
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
test("standing NOT_FOUND: 40 further polls add ZERO lines (the incident, prevented)", async () => {
|
|
371
|
+
const { a, logs, intervals } = makeLoggingAdapter(failingInbox("NOT_FOUND"));
|
|
372
|
+
await a.start({ onInbound: () => {} });
|
|
373
|
+
await settle();
|
|
374
|
+
const afterFirst = logs.length;
|
|
375
|
+
const tick = intervals[intervals.length - 1].fn;
|
|
376
|
+
for (let i = 0; i < 40; i += 1) { await tick(); await settle(); }
|
|
377
|
+
assert.equal(logs.length, afterFirst, `expected no repeats, got ${logs.length - afterFirst}`);
|
|
378
|
+
await a.stop();
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
test("standing NOT_FOUND degrades the poll to 10m — no point hammering a human-gated fault", async () => {
|
|
382
|
+
const { a, intervals } = makeLoggingAdapter(failingInbox("NOT_FOUND"));
|
|
383
|
+
await a.start({ onInbound: () => {} });
|
|
384
|
+
await settle();
|
|
385
|
+
assert.equal(intervals[intervals.length - 1].ms, DEGRADED_POLL_MS);
|
|
386
|
+
await a.stop();
|
|
387
|
+
});
|
|
388
|
+
|
|
389
|
+
test("recovery: the mailbox appearing logs RESOLVED and restores the normal cadence", async () => {
|
|
390
|
+
const fetchImpl = failingInbox("NOT_FOUND");
|
|
391
|
+
const { a, logs, intervals } = makeLoggingAdapter(fetchImpl);
|
|
392
|
+
await a.start({ onInbound: () => {} });
|
|
393
|
+
await settle();
|
|
394
|
+
const tick = intervals[intervals.length - 1].fn;
|
|
395
|
+
for (let i = 0; i < 5; i += 1) { await tick(); await settle(); }
|
|
396
|
+
|
|
397
|
+
fetchImpl.heal(); // an admin assigns the mailbox
|
|
398
|
+
await tick();
|
|
399
|
+
await settle();
|
|
400
|
+
|
|
401
|
+
const resolved = logs.find((l) => /RESOLVED/.test(l.msg));
|
|
402
|
+
assert.ok(resolved, "recovery must be stated explicitly, not inferred from silence");
|
|
403
|
+
assert.match(resolved.msg, /6 failed attempt/);
|
|
404
|
+
assert.match(resolved.msg, /live again/);
|
|
405
|
+
assert.equal(intervals[intervals.length - 1].ms, 45_000, "normal cadence restored");
|
|
406
|
+
assert.equal(a.standingFault(), null);
|
|
407
|
+
await a.stop();
|
|
408
|
+
});
|
|
409
|
+
|
|
410
|
+
test("a NON-terminal error keeps the normal cadence and stays a warn", async () => {
|
|
411
|
+
const { a, logs, intervals } = makeLoggingAdapter(failingInbox("INTERNAL", "boom"));
|
|
412
|
+
await a.start({ onInbound: () => {} });
|
|
413
|
+
await settle();
|
|
414
|
+
const line = logs.find((l) => /email\.inbox INTERNAL/.test(l.msg));
|
|
415
|
+
assert.equal(line.level, "warn", "a transient server error is not a dead lane");
|
|
416
|
+
assert.equal(intervals[intervals.length - 1].ms, 45_000);
|
|
417
|
+
assert.equal(a.standingFault(), null);
|
|
418
|
+
await a.stop();
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
test("throttling the LOG never throttles the ANSWER: healthCheck still states the fault + remedy", async () => {
|
|
422
|
+
const { a, intervals } = makeLoggingAdapter(failingInbox("NOT_FOUND"));
|
|
423
|
+
await a.start({ onInbound: () => {} });
|
|
424
|
+
await settle();
|
|
425
|
+
const tick = intervals[intervals.length - 1].fn;
|
|
426
|
+
for (let i = 0; i < 20; i += 1) { await tick(); await settle(); }
|
|
427
|
+
const h = await a.healthCheck();
|
|
428
|
+
assert.equal(h.ok, false);
|
|
429
|
+
assert.match(h.detail, /NOT_FOUND/);
|
|
430
|
+
assert.match(h.detail, /workspace ADMIN/);
|
|
431
|
+
assert.match(h.detail, /degraded to 10m/);
|
|
432
|
+
// And the machine-readable form, for doctor / state files.
|
|
433
|
+
const fault = a.standingFault();
|
|
434
|
+
assert.equal(fault.code, "NOT_FOUND");
|
|
435
|
+
assert.match(fault.since, /^\d{4}-/);
|
|
436
|
+
await a.stop();
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
test("FORBIDDEN_SCOPE gets the KEY remedy, not the mailbox one", async () => {
|
|
440
|
+
const { a, logs } = makeLoggingAdapter(failingInbox("FORBIDDEN_SCOPE", "not paired"));
|
|
441
|
+
await a.start({ onInbound: () => {} });
|
|
442
|
+
await settle();
|
|
443
|
+
const line = logs.find((l) => /email\.inbox FORBIDDEN_SCOPE/.test(l.msg));
|
|
444
|
+
assert.equal(line.level, "error");
|
|
445
|
+
assert.match(line.msg, /Settings → API keys/);
|
|
446
|
+
assert.ok(!/Settings → Email → Mailboxes/.test(line.msg), "wrong remedy for this cause");
|
|
447
|
+
await a.stop();
|
|
448
|
+
});
|