@cohortapp/agent-sdk 2.5.1 → 2.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/bin/maestro.mjs +305 -89
  2. package/bin/maestro.test.mjs +357 -48
  3. package/docs/runbooks/backup-restore.md +65 -33
  4. package/framework-features.json +4 -4
  5. package/lib/backup/policy.mjs +710 -0
  6. package/lib/backup/policy.test.mjs +305 -0
  7. package/lib/budget-escalate.mjs +133 -0
  8. package/lib/budget-escalate.test.mjs +232 -0
  9. package/lib/budget-guard.envelope.test.mjs +476 -0
  10. package/lib/budget-guard.mjs +853 -75
  11. package/lib/budget-guard.test.mjs +91 -42
  12. package/lib/cadences.mjs +33 -0
  13. package/lib/channels/orgmail/adapter.mjs +88 -3
  14. package/lib/channels/orgmail/adapter.test.mjs +137 -0
  15. package/lib/channels/repeat-suppressor.mjs +198 -0
  16. package/lib/channels/repeat-suppressor.test.mjs +134 -0
  17. package/lib/comms/receipts.mjs +297 -0
  18. package/lib/cost/ledger-row.mjs +333 -0
  19. package/lib/cost/ledger-row.test.mjs +183 -0
  20. package/lib/execution/drive.mjs +28 -1
  21. package/lib/execution/effects.mjs +191 -12
  22. package/lib/execution/effects.test.mjs +50 -11
  23. package/lib/goals/admission.mjs +13 -1
  24. package/lib/goals/admission.test.mjs +26 -1
  25. package/lib/goals/loop.mjs +13 -0
  26. package/lib/kpi-sensors.test.mjs +3 -0
  27. package/lib/mandate/cache.mjs +13 -5
  28. package/lib/mandate/derive.mjs +146 -21
  29. package/lib/mandate/derive.test.mjs +50 -6
  30. package/lib/mandate/model.mjs +32 -4
  31. package/lib/mandate/refresh.test.mjs +16 -2
  32. package/lib/mcp/server.test.mjs +12 -3
  33. package/lib/model-router/economics.mjs +107 -76
  34. package/lib/model-router/economics.test.mjs +64 -46
  35. package/lib/model-router/integration-coverage.test.mjs +39 -37
  36. package/lib/model-router/ledger.mjs +75 -22
  37. package/lib/model-router/ledger.test.mjs +35 -2
  38. package/lib/org/client.mjs +14 -0
  39. package/lib/org/cost-sync.mjs +16 -2
  40. package/lib/org/doctor.mjs +62 -1
  41. package/lib/org/doctor.test.mjs +36 -3
  42. package/lib/org/email-remedy.mjs +49 -0
  43. package/lib/org/engagement-ledger.mjs +376 -0
  44. package/lib/org/engagement-ledger.test.mjs +112 -0
  45. package/lib/org/engagement.mjs +1056 -0
  46. package/lib/org/engagement.test.mjs +739 -0
  47. package/lib/org/messaging.mjs +230 -3
  48. package/lib/org/messaging.test.mjs +110 -1
  49. package/lib/org/param-contract.mjs +56 -2
  50. package/lib/org/param-contract.test.mjs +26 -0
  51. package/lib/org/protocol.checksum +1 -1
  52. package/lib/org/protocol.mjs +5 -0
  53. package/lib/org/protocol.test.mjs +7 -1
  54. package/lib/org/tool-surface.mjs +506 -10
  55. package/lib/org/tool-surface.test.mjs +191 -7
  56. package/lib/org/ui-parity.mjs +333 -6
  57. package/lib/org/ui-parity.test.mjs +96 -3
  58. package/lib/org/work-ledger.mjs +241 -0
  59. package/lib/org/work-ledger.test.mjs +237 -0
  60. package/lib/plan/adoption-e2e.test.mjs +366 -0
  61. package/lib/plan/budget-enforcement.test.mjs +400 -0
  62. package/lib/plan/budget-runtime.mjs +215 -0
  63. package/lib/plan/compile.mjs +201 -5
  64. package/lib/plan/compile.test.mjs +19 -5
  65. package/lib/plan/emit.mjs +8 -0
  66. package/lib/plan/emit.test.mjs +18 -0
  67. package/lib/resource-governor.mjs +58 -12
  68. package/lib/resource-governor.test.mjs +41 -1
  69. package/lib/security/audit-engine.mjs +45 -8
  70. package/lib/security/audit-engine.test.mjs +35 -0
  71. package/lib/setup/enroll-from-cohort.mjs +14 -1
  72. package/lib/setup/sections/mandate.mjs +48 -7
  73. package/lib/setup/sections/mandate.test.mjs +17 -2
  74. package/lib/setup/sections/orgmail.mjs +10 -2
  75. package/lib/setup/state.mjs +83 -2
  76. package/lib/telemetry/collect.mjs +360 -20
  77. package/lib/telemetry/collect.test.mjs +266 -0
  78. package/package.json +1 -1
  79. package/scripts/cost/track-claude-usage.mjs +207 -48
  80. package/scripts/cost/track-claude-usage.test.mjs +148 -0
  81. package/scripts/daemon/agent-daemon.mjs +315 -17
  82. package/scripts/daemon/assurance-e2e.test.mjs +421 -0
  83. package/scripts/daemon/assurance.mjs +944 -0
  84. package/scripts/daemon/assurance.test.mjs +668 -0
  85. package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
  86. package/scripts/daemon/cadence-consumer.mjs +147 -9
  87. package/scripts/daemon/cadence-consumer.test.mjs +6 -0
  88. package/scripts/daemon/cadence-handlers.mjs +158 -0
  89. package/scripts/daemon/cadence-handlers.test.mjs +64 -0
  90. package/scripts/daemon/classifier.test.mjs +18 -9
  91. package/scripts/daemon/deliver.mjs +314 -0
  92. package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
  93. package/scripts/daemon/dispatcher.mjs +64 -6
  94. package/scripts/daemon/responder-cost.test.mjs +68 -0
  95. package/scripts/daemon/responder.mjs +351 -298
  96. package/scripts/local-triggers/generate-plists.test.mjs +7 -4
  97. package/scripts/maintenance/backup-run.mjs +415 -0
  98. package/scripts/maintenance/backup-to-cloud.sh +16 -116
  99. package/scripts/org/send-orgmail.mjs +16 -0
  100. package/scripts/record-receipt.sh +63 -0
  101. package/scripts/restore-from-backup.sh +14 -3
  102. package/scripts/restore-from-backup.test.mjs +8 -5
  103. package/scripts/send-email-threaded.py +47 -0
  104. package/scripts/send-sms.sh +4 -0
  105. package/scripts/send-whatsapp.sh +4 -0
  106. package/scripts/setup/init-backup.mjs +93 -38
  107. package/scripts/slack-send.sh +12 -0
@@ -31,9 +31,26 @@ async function makeLedgerDir() {
31
31
  }
32
32
  async function rm(dir) { try { await fsp.rm(dir, { recursive: true, force: true }); } catch { /* */ } }
33
33
 
34
- /** Write `rows` (array of {estimated_usd}) into today's ledger file. */
34
+ /**
35
+ * Write `rows` (array of {usd}) into today's ledger file as MEASURED sessions.
36
+ *
37
+ * `total_cost_usd` + real token counts, because that is what a row from a
38
+ * correctly-instrumented caller looks like and it is the column the governor
39
+ * now bills against (lib/cost/ledger-row.mjs). A row carrying only
40
+ * `estimated_usd` and no tokens classifies as `unknown` — deliberately, so an
41
+ * unmeasured session can never read as a free one — and would be imputed rather
42
+ * than summed, which is not what these band tests are about.
43
+ */
35
44
  function seedLedger(dir, rows) {
36
- const body = rows.map((r) => JSON.stringify({ ts: `${DATE}T10:00:00.000Z`, estimated_usd: r.usd, model: "sonnet", cadence: "x" })).join("\n") + "\n";
45
+ const body = rows.map((r) => JSON.stringify({
46
+ ts: `${DATE}T10:00:00.000Z`,
47
+ total_cost_usd: r.usd,
48
+ estimated_usd: r.usd,
49
+ model: "sonnet",
50
+ cadence: "x",
51
+ input_tokens: 1000,
52
+ output_tokens: 200,
53
+ })).join("\n") + "\n";
37
54
  writeFileSync(join(dir, `${DATE}.jsonl`), body);
38
55
  }
39
56
 
@@ -41,12 +58,13 @@ function seedLedger(dir, rows) {
41
58
  // banding
42
59
  // ---------------------------------------------------------------------------
43
60
 
44
- test("dailyStatus: below 50% is band 0, not essential-only", async () => {
61
+ test("dailyStatus: below 80% is band 0, nothing degraded", async () => {
45
62
  const dir = await makeLedgerDir();
46
63
  try {
47
64
  seedLedger(dir, [{ usd: 4 }]); // $4 of $10 = 40%
48
65
  const s = dailyStatus({ ledgerDir: dir, now: clk, capUSD: 10 });
49
66
  assert.equal(s.band, 0);
67
+ assert.equal(s.mode, "normal");
50
68
  assert.equal(s.essentialOnly, false);
51
69
  assert.equal(s.spentUSD, 4);
52
70
  assert.equal(s.pct, 40);
@@ -54,7 +72,8 @@ test("dailyStatus: below 50% is band 0, not essential-only", async () => {
54
72
  });
55
73
 
56
74
  test("dailyStatus: crossing each threshold lands in the right band", async () => {
57
- for (const [usd, expected] of [[5, 50], [7.5, 75], [9, 90], [10, 100], [12, 100]]) {
75
+ // The governance ladder: 80 notify · 100 degrade · 125 suspend · 150 refuse.
76
+ for (const [usd, expected] of [[5, 0], [8, 80], [9.9, 80], [10, 100], [12.5, 125], [15, 150], [40, 150]]) {
58
77
  const dir = await makeLedgerDir();
59
78
  try {
60
79
  seedLedger(dir, [{ usd }]);
@@ -91,7 +110,8 @@ test("dailyStatus: iteration cap can be the binding constraint", async () => {
91
110
  const s = dailyStatus({ ledgerDir: dir, now: clk, capUSD: 1000, iterationCap: 10 });
92
111
  assert.equal(s.sessions, 10);
93
112
  assert.equal(s.band, 100, "iteration cap should drive band to 100");
94
- assert.equal(s.essentialOnly, true);
113
+ assert.equal(s.mode, "degraded");
114
+ assert.equal(s.posture.selfDirected, false, "D3 stops at the degrade rung");
95
115
  } finally { await rm(dir); }
96
116
  });
97
117
 
@@ -99,12 +119,15 @@ test("dailyStatus: iteration cap can be the binding constraint", async () => {
99
119
  // essential-only at 100%
100
120
  // ---------------------------------------------------------------------------
101
121
 
102
- test("essentialOnly is true only at/after 100%", async () => {
122
+ test("essentialOnly (the legacy flag) now means the SUSPEND rung at 125%", async () => {
103
123
  const dir = await makeLedgerDir();
104
124
  try {
105
- seedLedger(dir, [{ usd: 9.99 }]);
106
- assert.equal(dailyStatus({ ledgerDir: dir, now: clk, capUSD: 10 }).essentialOnly, false);
125
+ // 100% degrades — inbox + obligations continue on the cheapest class — so
126
+ // the legacy all-or-nothing flag must NOT be set there. That rung was the
127
+ // one the old binary gate collapsed.
107
128
  seedLedger(dir, [{ usd: 10 }]);
129
+ assert.equal(dailyStatus({ ledgerDir: dir, now: clk, capUSD: 10 }).essentialOnly, false);
130
+ seedLedger(dir, [{ usd: 12.5 }]);
108
131
  assert.equal(dailyStatus({ ledgerDir: dir, now: clk, capUSD: 10 }).essentialOnly, true);
109
132
  } finally { await rm(dir); }
110
133
  });
@@ -119,24 +142,29 @@ test("maybeNotify fires once per band and never repeats within the day", async (
119
142
  const notices = [];
120
143
  const notify = (n) => notices.push(n.band);
121
144
 
122
- // 60% → notify band 50 once.
123
- seedLedger(dir, [{ usd: 6 }]);
124
- let r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify });
145
+ // A supervisor must exist or the notice is logged and sent nowhere by
146
+ // design — see maybeNotify. Inject one.
147
+ const seat = { seatEnvelope: { seatBudgetCents: null, supervisorMemberId: "M-SUP" } };
148
+
149
+ // 85% → notify band 80 once.
150
+ seedLedger(dir, [{ usd: 8.5 }]);
151
+ let r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify, ...seat });
125
152
  assert.equal(r.notified, true);
126
- assert.equal(r.band, 50);
153
+ assert.equal(r.band, 80);
154
+ assert.deepEqual(r.recipient, { memberId: "M-SUP", role: "REVIEWER" });
127
155
 
128
- // Still 60% → no repeat.
129
- r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify });
156
+ // Still 85% → no repeat.
157
+ r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify, ...seat });
130
158
  assert.equal(r.notified, false);
131
159
 
132
- // Climb to 80% → notify band 75 once.
133
- seedLedger(dir, [{ usd: 8 }]);
134
- r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify });
160
+ // Climb to 105% → notify band 100 once.
161
+ seedLedger(dir, [{ usd: 10.5 }]);
162
+ r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify, ...seat });
135
163
  assert.equal(r.notified, true);
136
- assert.equal(r.band, 75);
164
+ assert.equal(r.band, 100);
137
165
 
138
- assert.deepEqual(notices, [50, 75]);
139
- assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [50, 75]);
166
+ assert.deepEqual(notices, [80, 100]);
167
+ assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [80, 100]);
140
168
  } finally { await rm(dir); }
141
169
  });
142
170
 
@@ -144,42 +172,63 @@ test("maybeNotify on a big jump notifies once at the highest crossed band", asyn
144
172
  const dir = await makeLedgerDir();
145
173
  try {
146
174
  const notices = [];
147
- // Jump straight from $0 to 95% — should surface ONE notice at band 90,
148
- // and record 50/75/90 as all notified so they never re-fire.
149
- seedLedger(dir, [{ usd: 9.5 }]);
150
- const r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n.band) });
175
+ // Jump straight from $0 to 130% — ONE notice at band 125, with 80/100/125
176
+ // all recorded so they never re-fire.
177
+ const seat = { seatEnvelope: { seatBudgetCents: null, supervisorMemberId: "M-SUP" } };
178
+ seedLedger(dir, [{ usd: 13 }]);
179
+ const r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n.band), ...seat });
151
180
  assert.equal(r.notified, true);
152
- assert.equal(r.band, 90);
153
- assert.deepEqual(notices, [90], "exactly one notice surfaced");
154
- assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [50, 75, 90]);
181
+ assert.equal(r.band, 125);
182
+ assert.deepEqual(notices, [125], "exactly one notice surfaced");
183
+ assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [80, 100, 125]);
155
184
 
156
185
  // A later check at the same level does nothing.
157
- const r2 = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n.band) });
186
+ const r2 = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n.band), ...seat });
158
187
  assert.equal(r2.notified, false);
159
- assert.deepEqual(notices, [90]);
188
+ assert.deepEqual(notices, [125]);
160
189
  } finally { await rm(dir); }
161
190
  });
162
191
 
163
- test("maybeNotify reaching 100% surfaces the essential-only notice once", async () => {
192
+ test("maybeNotify past 150% surfaces the refuse notice once", async () => {
164
193
  const dir = await makeLedgerDir();
165
194
  try {
166
195
  const notices = [];
167
- seedLedger(dir, [{ usd: 11 }]); // 110%
168
- const r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n) });
196
+ const seat = { seatEnvelope: { seatBudgetCents: null, supervisorMemberId: "M-SUP" } };
197
+ seedLedger(dir, [{ usd: 16 }]); // 160%
198
+ const r = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n), ...seat });
169
199
  assert.equal(r.notified, true);
170
- assert.equal(r.band, 100);
171
- assert.match(r.message, /essential-only/i);
172
- assert.equal(r.status.essentialOnly, true);
200
+ assert.equal(r.band, 150);
201
+ assert.match(r.message, /REFUSING new sessions that are not a direct human reply/);
202
+ assert.equal(r.status.posture.refuseNonHumanSpawn, true);
173
203
  // All bands recorded; a repeat is a no-op.
174
- assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [50, 75, 90, 100]);
175
- const r2 = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n) });
204
+ assert.deepEqual(readNotified({ ledgerDir: dir, now: clk }).notified_bands, [80, 100, 125, 150]);
205
+ const r2 = maybeNotify({ ledgerDir: dir, now: clk, capUSD: 10, notify: (n) => notices.push(n), ...seat });
176
206
  assert.equal(r2.notified, false);
177
207
  assert.equal(notices.length, 1);
178
208
  } finally { await rm(dir); }
179
209
  });
180
210
 
181
- test("BANDS is the canonical 50/75/90/100 ladder", () => {
182
- assert.deepEqual(BANDS, [50, 75, 90, 100]);
211
+ test("a seat with NO supervisor edge is never told its own budget band", async () => {
212
+ const dir = await makeLedgerDir();
213
+ try {
214
+ const notices = [];
215
+ const logged = [];
216
+ seedLedger(dir, [{ usd: 9 }]); // 90% → band 80
217
+ const r = maybeNotify({
218
+ ledgerDir: dir, now: clk, capUSD: 10,
219
+ notify: (n) => notices.push(n),
220
+ log: (level, msg) => logged.push(`${level}:${msg}`),
221
+ });
222
+ assert.equal(r.band, 80);
223
+ assert.equal(r.recipient, null);
224
+ assert.equal(notices.length, 0, "no notice may be delivered to the owner");
225
+ assert.ok(logged.some((l) => l.startsWith("error:") && /NO supervisor edge/.test(l)),
226
+ "the missing edge must be LOUD, not silent");
227
+ } finally { await rm(dir); }
228
+ });
229
+
230
+ test("BANDS is the canonical 80/100/125/150 governance ladder", () => {
231
+ assert.deepEqual(BANDS, [80, 100, 125, 150]);
183
232
  });
184
233
 
185
234
  // ---------------------------------------------------------------------------
@@ -204,7 +253,7 @@ test("dailyStatus reads the cap from config/recovery.yaml when present", async (
204
253
  assert.equal(s.capUSD, 40, "cap should come from recovery.yaml, not the 25 default");
205
254
  assert.equal(s.spentUSD, 20);
206
255
  assert.equal(s.pct, 50, "20/40 = 50%");
207
- assert.equal(s.band, 50);
256
+ assert.equal(s.band, 0, "50% of the ceiling degrades nothing");
208
257
  } finally { await rm(dir); }
209
258
  });
210
259
 
@@ -218,8 +267,8 @@ test("recovery.yaml cap takes precedence over DAILY_SPEND_CAP_USD env", async ()
218
267
  seedLedger(join(dir, "state", "cost-tracking"), [{ usd: 10 }]);
219
268
  const s = dailyStatus({ agentRoot: dir, now: clk });
220
269
  assert.equal(s.capUSD, 10, "yaml cap must win over the env cap");
221
- assert.equal(s.band, 100, "10/10 spent under the yaml cap is 100% (essential-only)");
222
- assert.equal(s.essentialOnly, true);
270
+ assert.equal(s.band, 100, "10/10 spent under the yaml cap is the 100% degrade rung");
271
+ assert.equal(s.mode, "degraded");
223
272
  } finally {
224
273
  if (prevEnv === undefined) delete process.env.DAILY_SPEND_CAP_USD;
225
274
  else process.env.DAILY_SPEND_CAP_USD = prevEnv;
package/lib/cadences.mjs CHANGED
@@ -48,6 +48,11 @@ export const STANDARD_CADENCES = [
48
48
  // job is actually due, so the consumer spawns a session to action it.
49
49
  // nightly-cost-reconcile: three-way spend reconcile vs the Admin Cost API
50
50
  // (fails open with no fetchImpl); flags drift > 10%. Inline, no prompt.
51
+ // nightly-backup: the DR job. Inline (tar + optional upload, no LLM). 03:10 —
52
+ // quiet hours, and ahead of nightly-cost-reconcile so the archive captures the
53
+ // day's ledger before anything rewrites it. See lib/backup/policy.mjs for what
54
+ // is archived, where, and what is never archived at all.
55
+ { id: "nightly-backup", scope: "standard", mode: "inline", calendar: { hour: 3, minute: 10 } },
51
56
  { id: "nightly-cost-reconcile", scope: "standard", mode: "inline", calendar: { hour: 3, minute: 30 } },
52
57
  // fleet-cost-digest: build the daily fleet LLM-spend digest. Inline, no prompt.
53
58
  { id: "fleet-cost-digest", scope: "standard", mode: "inline", calendar: { hour: 7, minute: 0 } },
@@ -113,6 +118,34 @@ export const ALTITUDE_CADENCES = {
113
118
  ],
114
119
  };
115
120
 
121
+ /**
122
+ * THE HUMAN LANE. Cadences whose job is getting a human's message to the agent
123
+ * and the agent's answer back — the work Invariant #1 says never goes dark.
124
+ *
125
+ * This exists because the budget ladder's top rung ("refuse anything that is not
126
+ * a direct human reply") was implemented as `obligation.kind !== "REACT"`, and
127
+ * every cadence the consumer evaluates is a SCHEDULE. At band 150 that blocked
128
+ * 19 of 26 obligations including `inbox-processor` and `messaging-inbound` — the
129
+ * cadence that PULLS Cohort DMs and @mentions into the inbox pipeline at all. It
130
+ * only looked harmless because both guards default to `decision:"inline"` and
131
+ * return before the gate; set `INBOX_CADENCE_ESCALATE=1` or
132
+ * `MESSAGING_INBOUND_ESCALATE=1` (the documented cadence-only mode for operators
133
+ * running without the reactive daemon) and the seat stopped answering its inbox
134
+ * at 150 %, which is the exact failure the rung was written to avoid.
135
+ *
136
+ * Membership test: does a human wait on the other end of this cadence?
137
+ * inbox-processor — drains the inbox a human wrote into.
138
+ * messaging-inbound — the only puller of org DMs/@mentions.
139
+ * dynamic-jobs — DELIBERATELY NOT here: those are the agent's own
140
+ * reminders to itself, not somebody waiting on a reply.
141
+ */
142
+ export const HUMAN_LANE_CADENCES = Object.freeze(new Set(["inbox-processor", "messaging-inbound"]));
143
+
144
+ /** Is this cadence part of the human-reply lane? See {@link HUMAN_LANE_CADENCES}. */
145
+ export function isHumanLaneCadence(id) {
146
+ return HUMAN_LANE_CADENCES.has(String(id || ""));
147
+ }
148
+
116
149
  /** Cadence keywords that map to a recurring schedule (event-driven is excluded). */
117
150
  export const RECURRING = new Set(["daily", "weekly", "monthly", "quarterly", "continuous"]);
118
151
 
@@ -50,6 +50,10 @@ import { BaseAdapter, CHANNEL_COUNTERS } from "../base-adapter.mjs";
50
50
  import { randomUUID } from "node:crypto";
51
51
  import { loadOrgConfig, configFromAgent } from "../../org/client.mjs";
52
52
  import { emailSend, emailMarkRead, emailInbox, emailMessage } from "../../org/ui-parity.mjs";
53
+ import { RepeatSuppressor } from "../repeat-suppressor.mjs";
54
+ // One classifier for every surface that reports an email.* failure (this poll
55
+ // loop, `maestro doctor`, `maestro setup`) — see lib/org/email-remedy.mjs.
56
+ import { remedyFor } from "../../org/email-remedy.mjs";
53
57
 
54
58
  export const ORGMAIL_CAPABILITIES = [
55
59
  CHANNEL_CAPABILITIES.SEND,
@@ -60,6 +64,17 @@ export const ORGMAIL_CAPABILITIES = [
60
64
  /** Default poll cadence — the messaging-inbound rhythm. */
61
65
  export const DEFAULT_POLL_MS = 45_000;
62
66
 
67
+ /**
68
+ * Poll cadence while a TERMINAL condition stands (no mailbox assigned, key not
69
+ * paired, key lacks the email scope). These need a human act in Cohort; the
70
+ * 45s rhythm cannot fix them and only produces log noise and pointless load, so
71
+ * the loop drops to 10 minutes and self-heals the moment the condition clears.
72
+ */
73
+ export const DEGRADED_POLL_MS = 10 * 60_000;
74
+
75
+ /** Condition key for the standing-inbox-failure suppressor. */
76
+ const INBOX_CONDITION = "email.inbox";
77
+
63
78
  export class OrgMailAdapter extends BaseAdapter {
64
79
  /**
65
80
  * @param {object} opts
@@ -88,6 +103,13 @@ export class OrgMailAdapter extends BaseAdapter {
88
103
  this._pollMs = resolvePollMs(opts.pollMs, gate, process.env);
89
104
  this._timer = null;
90
105
  this._polling = false;
106
+ // Standing-condition tracker: a mailbox that does not exist is one FACT,
107
+ // not one fact per 45 seconds. See lib/channels/repeat-suppressor.mjs.
108
+ this._suppressor = opts.suppressor || new RepeatSuppressor({ restateMs: opts.restateMs });
109
+ // Set while a terminal condition stands, so healthCheck and the daemon can
110
+ // report WHY the lane is down without re-probing.
111
+ this._standingFault = null;
112
+ this._degraded = false;
91
113
  }
92
114
 
93
115
  /** Resolve {base, token, orgId} from config/org.yaml (the enrollment SoT). */
@@ -122,10 +144,31 @@ export class OrgMailAdapter extends BaseAdapter {
122
144
  }
123
145
 
124
146
  await tick(); // first sweep immediately so start() sees a live adapter
125
- this._timer = this._setInterval(tick, this._pollMs);
147
+ this._startTimer(tick);
148
+ }
149
+
150
+ /**
151
+ * (Re)arm the poll timer at the cadence the CURRENT state deserves: the
152
+ * configured rhythm normally, DEGRADED_POLL_MS while a terminal condition
153
+ * stands. Called on every transition so recovery is immediate — the first
154
+ * successful sweep after an admin assigns the mailbox restores 45s polling.
155
+ */
156
+ _startTimer(tick) {
157
+ if (this._timer) this._clearInterval(this._timer);
158
+ const ms = this._degraded ? DEGRADED_POLL_MS : this._pollMs;
159
+ this._tick = tick;
160
+ this._timer = this._setInterval(tick, ms);
126
161
  if (this._timer && typeof this._timer.unref === "function") this._timer.unref();
127
162
  }
128
163
 
164
+ /** Enter/leave degraded cadence, re-arming the timer only on a transition. */
165
+ _setDegraded(next, reason) {
166
+ if (this._degraded === next) return;
167
+ this._degraded = next;
168
+ this._standingFault = next ? reason : null;
169
+ if (this._tick) this._startTimer(this._tick);
170
+ }
171
+
129
172
  /**
130
173
  * One inbox sweep: list unread summaries, fetch each full DTO (newest LAST
131
174
  * so downstream sees chronological order), hand off, then markRead ONLY on a
@@ -137,9 +180,30 @@ export class OrgMailAdapter extends BaseAdapter {
137
180
  if (!o.base || !o.token) return; // not enrolled — dormant, never throws
138
181
  const inbox = await emailInbox({ unreadOnly: true, limit: 50 }, o);
139
182
  if (!inbox.ok) {
140
- this.log("warn", `email.inbox error: ${inbox.error?.code || "?"} ${inbox.error?.message || ""}`);
183
+ // A standing failure is ONE fact, not one fact per tick. Before this, a
184
+ // seat with no mailbox wrote the same NOT_FOUND line 478 times in a day —
185
+ // true every time, useful once, and loud enough to bury everything else.
186
+ // Now: full detail on the first occurrence, a re-statement with the
187
+ // suppressed count once an hour, an explicit RESOLVED line on recovery,
188
+ // and a 10-minute poll while the condition needs a human to clear it.
189
+ const code = inbox.error?.code || "?";
190
+ const seen = this._suppressor.observe(INBOX_CONDITION, { code });
191
+ if (seen.emit) {
192
+ this.log(
193
+ seen.terminal ? "error" : "warn",
194
+ `email.inbox ${code}: ${inbox.error?.message || "no detail"}` +
195
+ (seen.terminal ? ` — ${remedyFor(code)}` : "") +
196
+ seen.suffix
197
+ );
198
+ }
199
+ this._setDegraded(seen.terminal, seen.terminal ? { code, message: inbox.error?.message || "", since: new Date().toISOString() } : null);
141
200
  return;
142
201
  }
202
+ const recovered = this._suppressor.clear(INBOX_CONDITION);
203
+ if (recovered.wasActive) {
204
+ this.log("info", `email.inbox ${recovered.message} — the workspace mailbox lane is live again`);
205
+ }
206
+ this._setDegraded(false, null);
143
207
  const summaries = Array.isArray(inbox.result?.messages) ? inbox.result.messages : [];
144
208
  // Server returns newest-first (keyset createdAt desc); we deliver newest-LAST.
145
209
  const ordered = [...summaries].reverse();
@@ -289,7 +353,28 @@ export class OrgMailAdapter extends BaseAdapter {
289
353
  const addr = r.result?.mailbox?.address || "";
290
354
  return { ok: true, detail: `orgmail mailbox ${addr || "reachable"}` };
291
355
  }
292
- return { ok: false, detail: `orgmail: ${r.error?.code || "?"} ${r.error?.message || "unreachable"}` };
356
+ // Suppressing the POLL log must not suppress the health ANSWER: the whole
357
+ // point of throttling repeats is that the standing condition stays legible
358
+ // somewhere. It lives here, and in `standingFault()`, so anything that asks
359
+ // gets the full story including the remedy.
360
+ const code = r.error?.code || "?";
361
+ const remedy = remedyFor(code);
362
+ return {
363
+ ok: false,
364
+ detail:
365
+ `orgmail: ${code} ${r.error?.message || "unreachable"}` +
366
+ (remedy ? ` — ${remedy}` : "") +
367
+ (this._degraded ? " (poll is degraded to 10m until this clears)" : ""),
368
+ };
369
+ }
370
+
371
+ /**
372
+ * The standing terminal condition, if any — `{code, message, since}`. Read by
373
+ * the daemon/doctor so "this lane is down and here is why" survives the log
374
+ * throttle. Null when the lane is healthy.
375
+ */
376
+ standingFault() {
377
+ return this._standingFault;
293
378
  }
294
379
 
295
380
  async stop() {
@@ -20,6 +20,7 @@ import {
20
20
  resolvePollMs,
21
21
  htmlToText,
22
22
  DEFAULT_POLL_MS,
23
+ DEGRADED_POLL_MS,
23
24
  ORGMAIL_CAPABILITIES,
24
25
  } from "./adapter.mjs";
25
26
  import { getPlatform, isDaemonManagedPlatform } from "../contract.mjs";
@@ -309,3 +310,139 @@ test("healthCheck: missing enrollment is an honest unhealthy, not a throw", asyn
309
310
  assert.equal(h.ok, false);
310
311
  assert.match(h.detail, /enrollment missing/);
311
312
  });
313
+
314
+ // ── standing-fault throttle (the 478-lines-a-day incident) ─────────────────
315
+ //
316
+ // A member with no mailbox made every 45s poll write the same NOT_FOUND line:
317
+ // 478 of them on 2026-08-11, 164 the day before. The lane was dead and the
318
+ // only signal was noise. These tests pin the fix: full detail once, silence in
319
+ // between, a degraded poll while it needs a human, and an explicit recovery.
320
+
321
+ /** A fake hq whose email.inbox fails with `code` until `heal()` is called. */
322
+ function failingInbox(code, message = "no active mailbox is assigned to this agent") {
323
+ const state = { healed: false };
324
+ const fn = async (url) => {
325
+ const u = String(url);
326
+ const respond = (payload, status = 200, ok = true) => ({ ok, status, json: async () => payload, headers: { get: () => undefined } });
327
+ if (u.endsWith("/api/v1/email.inbox")) {
328
+ if (state.healed) {
329
+ return respond({ ok: true, result: { mailbox: { address: "astra@agents.example" }, messages: [], nextCursor: null } });
330
+ }
331
+ return respond({ ok: false, error: { code, message } }, 404, false);
332
+ }
333
+ return respond({ ok: true, result: {} });
334
+ };
335
+ fn.heal = () => { state.healed = true; };
336
+ return fn;
337
+ }
338
+
339
+ /** Build an adapter that captures its log lines and lets tests drive ticks. */
340
+ function makeLoggingAdapter(fetchImpl, over = {}) {
341
+ const logs = [];
342
+ const intervals = [];
343
+ const a = new OrgMailAdapter({
344
+ agentRoot: "/tmp/orgmail-test",
345
+ orgConfig: ORG_CFG,
346
+ config: { orgmail: { enabled: true, poll_seconds: 45, mark_read: true } },
347
+ fetchImpl,
348
+ supervise: false,
349
+ sendGate: false,
350
+ setInterval: (fn, ms) => { intervals.push({ fn, ms }); return { unref() {} }; },
351
+ clearInterval: () => {},
352
+ log: (level, msg) => logs.push({ level, msg }),
353
+ ...over,
354
+ });
355
+ return { a, logs, intervals };
356
+ }
357
+
358
+ test("standing NOT_FOUND: the first poll logs it IN FULL with the remedy", async () => {
359
+ const { a, logs } = makeLoggingAdapter(failingInbox("NOT_FOUND"));
360
+ await a.start({ onInbound: () => {} });
361
+ await settle();
362
+ const line = logs.find((l) => /email\.inbox NOT_FOUND/.test(l.msg));
363
+ assert.ok(line, "the first occurrence must be logged");
364
+ assert.equal(line.level, "error", "a dead lane is not a warning");
365
+ assert.match(line.msg, /workspace ADMIN/, "an ACTOR");
366
+ assert.match(line.msg, /Settings → Email → Mailboxes/, "and an ACT");
367
+ await a.stop();
368
+ });
369
+
370
+ test("standing NOT_FOUND: 40 further polls add ZERO lines (the incident, prevented)", async () => {
371
+ const { a, logs, intervals } = makeLoggingAdapter(failingInbox("NOT_FOUND"));
372
+ await a.start({ onInbound: () => {} });
373
+ await settle();
374
+ const afterFirst = logs.length;
375
+ const tick = intervals[intervals.length - 1].fn;
376
+ for (let i = 0; i < 40; i += 1) { await tick(); await settle(); }
377
+ assert.equal(logs.length, afterFirst, `expected no repeats, got ${logs.length - afterFirst}`);
378
+ await a.stop();
379
+ });
380
+
381
+ test("standing NOT_FOUND degrades the poll to 10m — no point hammering a human-gated fault", async () => {
382
+ const { a, intervals } = makeLoggingAdapter(failingInbox("NOT_FOUND"));
383
+ await a.start({ onInbound: () => {} });
384
+ await settle();
385
+ assert.equal(intervals[intervals.length - 1].ms, DEGRADED_POLL_MS);
386
+ await a.stop();
387
+ });
388
+
389
+ test("recovery: the mailbox appearing logs RESOLVED and restores the normal cadence", async () => {
390
+ const fetchImpl = failingInbox("NOT_FOUND");
391
+ const { a, logs, intervals } = makeLoggingAdapter(fetchImpl);
392
+ await a.start({ onInbound: () => {} });
393
+ await settle();
394
+ const tick = intervals[intervals.length - 1].fn;
395
+ for (let i = 0; i < 5; i += 1) { await tick(); await settle(); }
396
+
397
+ fetchImpl.heal(); // an admin assigns the mailbox
398
+ await tick();
399
+ await settle();
400
+
401
+ const resolved = logs.find((l) => /RESOLVED/.test(l.msg));
402
+ assert.ok(resolved, "recovery must be stated explicitly, not inferred from silence");
403
+ assert.match(resolved.msg, /6 failed attempt/);
404
+ assert.match(resolved.msg, /live again/);
405
+ assert.equal(intervals[intervals.length - 1].ms, 45_000, "normal cadence restored");
406
+ assert.equal(a.standingFault(), null);
407
+ await a.stop();
408
+ });
409
+
410
+ test("a NON-terminal error keeps the normal cadence and stays a warn", async () => {
411
+ const { a, logs, intervals } = makeLoggingAdapter(failingInbox("INTERNAL", "boom"));
412
+ await a.start({ onInbound: () => {} });
413
+ await settle();
414
+ const line = logs.find((l) => /email\.inbox INTERNAL/.test(l.msg));
415
+ assert.equal(line.level, "warn", "a transient server error is not a dead lane");
416
+ assert.equal(intervals[intervals.length - 1].ms, 45_000);
417
+ assert.equal(a.standingFault(), null);
418
+ await a.stop();
419
+ });
420
+
421
+ test("throttling the LOG never throttles the ANSWER: healthCheck still states the fault + remedy", async () => {
422
+ const { a, intervals } = makeLoggingAdapter(failingInbox("NOT_FOUND"));
423
+ await a.start({ onInbound: () => {} });
424
+ await settle();
425
+ const tick = intervals[intervals.length - 1].fn;
426
+ for (let i = 0; i < 20; i += 1) { await tick(); await settle(); }
427
+ const h = await a.healthCheck();
428
+ assert.equal(h.ok, false);
429
+ assert.match(h.detail, /NOT_FOUND/);
430
+ assert.match(h.detail, /workspace ADMIN/);
431
+ assert.match(h.detail, /degraded to 10m/);
432
+ // And the machine-readable form, for doctor / state files.
433
+ const fault = a.standingFault();
434
+ assert.equal(fault.code, "NOT_FOUND");
435
+ assert.match(fault.since, /^\d{4}-/);
436
+ await a.stop();
437
+ });
438
+
439
+ test("FORBIDDEN_SCOPE gets the KEY remedy, not the mailbox one", async () => {
440
+ const { a, logs } = makeLoggingAdapter(failingInbox("FORBIDDEN_SCOPE", "not paired"));
441
+ await a.start({ onInbound: () => {} });
442
+ await settle();
443
+ const line = logs.find((l) => /email\.inbox FORBIDDEN_SCOPE/.test(l.msg));
444
+ assert.equal(line.level, "error");
445
+ assert.match(line.msg, /Settings → API keys/);
446
+ assert.ok(!/Settings → Email → Mailboxes/.test(line.msg), "wrong remedy for this cause");
447
+ await a.stop();
448
+ });