@cohortapp/agent-sdk 2.5.1 → 2.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +305 -89
- package/bin/maestro.test.mjs +357 -48
- package/docs/runbooks/backup-restore.md +65 -33
- package/framework-features.json +4 -4
- package/lib/backup/policy.mjs +710 -0
- package/lib/backup/policy.test.mjs +305 -0
- package/lib/budget-escalate.mjs +133 -0
- package/lib/budget-escalate.test.mjs +232 -0
- package/lib/budget-guard.envelope.test.mjs +476 -0
- package/lib/budget-guard.mjs +853 -75
- package/lib/budget-guard.test.mjs +91 -42
- package/lib/cadences.mjs +33 -0
- package/lib/channels/orgmail/adapter.mjs +88 -3
- package/lib/channels/orgmail/adapter.test.mjs +137 -0
- package/lib/channels/repeat-suppressor.mjs +198 -0
- package/lib/channels/repeat-suppressor.test.mjs +134 -0
- package/lib/comms/receipts.mjs +297 -0
- package/lib/cost/ledger-row.mjs +333 -0
- package/lib/cost/ledger-row.test.mjs +183 -0
- package/lib/execution/drive.mjs +28 -1
- package/lib/execution/effects.mjs +191 -12
- package/lib/execution/effects.test.mjs +50 -11
- package/lib/goals/admission.mjs +13 -1
- package/lib/goals/admission.test.mjs +26 -1
- package/lib/goals/loop.mjs +13 -0
- package/lib/kpi-sensors.test.mjs +3 -0
- package/lib/mandate/cache.mjs +13 -5
- package/lib/mandate/derive.mjs +146 -21
- package/lib/mandate/derive.test.mjs +50 -6
- package/lib/mandate/model.mjs +32 -4
- package/lib/mandate/refresh.test.mjs +16 -2
- package/lib/mcp/server.test.mjs +12 -3
- package/lib/model-router/economics.mjs +107 -76
- package/lib/model-router/economics.test.mjs +64 -46
- package/lib/model-router/integration-coverage.test.mjs +39 -37
- package/lib/model-router/ledger.mjs +75 -22
- package/lib/model-router/ledger.test.mjs +35 -2
- package/lib/org/client.mjs +14 -0
- package/lib/org/cost-sync.mjs +16 -2
- package/lib/org/doctor.mjs +62 -1
- package/lib/org/doctor.test.mjs +36 -3
- package/lib/org/email-remedy.mjs +49 -0
- package/lib/org/engagement-ledger.mjs +376 -0
- package/lib/org/engagement-ledger.test.mjs +112 -0
- package/lib/org/engagement.mjs +1056 -0
- package/lib/org/engagement.test.mjs +739 -0
- package/lib/org/messaging.mjs +230 -3
- package/lib/org/messaging.test.mjs +110 -1
- package/lib/org/param-contract.mjs +56 -2
- package/lib/org/param-contract.test.mjs +26 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +5 -0
- package/lib/org/protocol.test.mjs +7 -1
- package/lib/org/tool-surface.mjs +506 -10
- package/lib/org/tool-surface.test.mjs +191 -7
- package/lib/org/ui-parity.mjs +333 -6
- package/lib/org/ui-parity.test.mjs +96 -3
- package/lib/org/work-ledger.mjs +241 -0
- package/lib/org/work-ledger.test.mjs +237 -0
- package/lib/plan/adoption-e2e.test.mjs +366 -0
- package/lib/plan/budget-enforcement.test.mjs +400 -0
- package/lib/plan/budget-runtime.mjs +215 -0
- package/lib/plan/compile.mjs +201 -5
- package/lib/plan/compile.test.mjs +19 -5
- package/lib/plan/emit.mjs +8 -0
- package/lib/plan/emit.test.mjs +18 -0
- package/lib/resource-governor.mjs +58 -12
- package/lib/resource-governor.test.mjs +41 -1
- package/lib/security/audit-engine.mjs +45 -8
- package/lib/security/audit-engine.test.mjs +35 -0
- package/lib/setup/enroll-from-cohort.mjs +14 -1
- package/lib/setup/sections/mandate.mjs +48 -7
- package/lib/setup/sections/mandate.test.mjs +17 -2
- package/lib/setup/sections/orgmail.mjs +10 -2
- package/lib/setup/state.mjs +83 -2
- package/lib/telemetry/collect.mjs +360 -20
- package/lib/telemetry/collect.test.mjs +266 -0
- package/package.json +1 -1
- package/scripts/cost/track-claude-usage.mjs +207 -48
- package/scripts/cost/track-claude-usage.test.mjs +148 -0
- package/scripts/daemon/agent-daemon.mjs +315 -17
- package/scripts/daemon/assurance-e2e.test.mjs +421 -0
- package/scripts/daemon/assurance.mjs +944 -0
- package/scripts/daemon/assurance.test.mjs +668 -0
- package/scripts/daemon/cadence-consumer-governance.test.mjs +56 -0
- package/scripts/daemon/cadence-consumer.mjs +147 -9
- package/scripts/daemon/cadence-consumer.test.mjs +6 -0
- package/scripts/daemon/cadence-handlers.mjs +158 -0
- package/scripts/daemon/cadence-handlers.test.mjs +64 -0
- package/scripts/daemon/classifier.test.mjs +18 -9
- package/scripts/daemon/deliver.mjs +314 -0
- package/scripts/daemon/dispatcher-governance.test.mjs +10 -0
- package/scripts/daemon/dispatcher.mjs +64 -6
- package/scripts/daemon/responder-cost.test.mjs +68 -0
- package/scripts/daemon/responder.mjs +351 -298
- package/scripts/local-triggers/generate-plists.test.mjs +7 -4
- package/scripts/maintenance/backup-run.mjs +415 -0
- package/scripts/maintenance/backup-to-cloud.sh +16 -116
- package/scripts/org/send-orgmail.mjs +16 -0
- package/scripts/record-receipt.sh +63 -0
- package/scripts/restore-from-backup.sh +14 -3
- package/scripts/restore-from-backup.test.mjs +8 -5
- package/scripts/send-email-threaded.py +47 -0
- package/scripts/send-sms.sh +4 -0
- package/scripts/send-whatsapp.sh +4 -0
- package/scripts/setup/init-backup.mjs +93 -38
- package/scripts/slack-send.sh +12 -0
|
@@ -0,0 +1,944 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* assurance.mjs — the guarantee that an ask is always answered by something.
|
|
3
|
+
*
|
|
4
|
+
* THE COMPLAINT THIS EXISTS TO FIX
|
|
5
|
+
*
|
|
6
|
+
* "The typing indicator activates, so I know she received it… if the work takes
|
|
7
|
+
* long or errors out, she either never replies, or replies 15-20-30 minutes
|
|
8
|
+
* later… right now it dies silently and I never hear from her again unless I
|
|
9
|
+
* chase her."
|
|
10
|
+
*
|
|
11
|
+
* Measured, that was: acknowledgements delivered 34% of the time (the other 66%
|
|
12
|
+
* died on a 60s `claude --print` timeout and told the human nothing); sessions
|
|
13
|
+
* with a median duration of 14.7 minutes and a p90 of 39; and three separate
|
|
14
|
+
* ways for a session to end without a single word reaching the requester —
|
|
15
|
+
* non-zero exit (telemetry only), SIGTERM at the 45-minute cap (telemetry only),
|
|
16
|
+
* and, worst of all, exit 0 having said nothing (marked processed forever and
|
|
17
|
+
* emitted as `sent`).
|
|
18
|
+
*
|
|
19
|
+
* THE MODEL
|
|
20
|
+
*
|
|
21
|
+
* An inbound ask that will take real work opens an OBLIGATION: a durable
|
|
22
|
+
* on-disk debt saying "a human is waiting to hear from me about this." The
|
|
23
|
+
* obligation is discharged only by evidence that something reached them — a
|
|
24
|
+
* delivery RECEIPT (lib/comms/receipts), not an exit code. Until it is
|
|
25
|
+
* discharged it is swept, and every sweep that finds an overdue debt SPEAKS.
|
|
26
|
+
*
|
|
27
|
+
* open → ack immediately (template, no model, cannot time out)
|
|
28
|
+
* overdue → interim progress update, capped, never spam
|
|
29
|
+
* failed → say what happened and what happens next; retry once if the
|
|
30
|
+
* cause was transient; when out of retries, say so and escalate
|
|
31
|
+
* finished → verify a receipt exists; if the session said nothing, say it
|
|
32
|
+
* orphaned → a daemon that restarts finds the debt and speaks to it
|
|
33
|
+
*
|
|
34
|
+
* WHY THE LEDGER IS ON DISK AND THE SWEEP IS NOT A TIMER
|
|
35
|
+
*
|
|
36
|
+
* Every in-process mechanism here dies with the process, and the process dying
|
|
37
|
+
* is one of the failure modes. An obligation file outlives the daemon; the
|
|
38
|
+
* sweep re-derives what is owed from disk on every tick, including the first
|
|
39
|
+
* tick after a restart. `daemonPid` on the record is what makes an interrupted
|
|
40
|
+
* obligation self-evident.
|
|
41
|
+
*
|
|
42
|
+
* FAIL-OPEN, NEVER SILENT. Nothing in this module throws into the hot path. But
|
|
43
|
+
* the fail-open direction is chosen deliberately in each case: when we cannot
|
|
44
|
+
* tell whether the human was spoken to, we assume they were not. The worst case
|
|
45
|
+
* is one redundant message; the bug being fixed is silence.
|
|
46
|
+
*
|
|
47
|
+
* @module scripts/daemon/assurance
|
|
48
|
+
*/
|
|
49
|
+
|
|
50
|
+
"use strict";
|
|
51
|
+
|
|
52
|
+
import { mkdirSync, readdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "fs";
|
|
53
|
+
import { join } from "path";
|
|
54
|
+
import { deliver, deliverWithRetry, replyTargetOf, canDeliverTo } from "./deliver.mjs";
|
|
55
|
+
import { spokeFor } from "../../lib/comms/receipts.mjs";
|
|
56
|
+
import { resultTextFromStdout } from "./session-outcomes.mjs";
|
|
57
|
+
|
|
58
|
+
const AGENT_REPO_DIR = process.env.AGENT_DIR || join(new URL(".", import.meta.url).pathname, "../..");
|
|
59
|
+
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
// Tuning — every threshold is env-overridable so an operator can tighten or
|
|
62
|
+
// loosen the promise without a code change.
|
|
63
|
+
// ---------------------------------------------------------------------------
|
|
64
|
+
|
|
65
|
+
const num = (v, d) => { const n = parseInt(v, 10); return Number.isFinite(n) ? n : d; };
|
|
66
|
+
|
|
67
|
+
/** How long after opening we expect the ack to already be out. The sweep
|
|
68
|
+
* compensates anything still un-acked past this — the fix for "the holding
|
|
69
|
+
* message failed and nothing ever retried it". */
|
|
70
|
+
export const ACK_GRACE_MS = num(process.env.ASSURANCE_ACK_GRACE_MS, 45_000);
|
|
71
|
+
|
|
72
|
+
/** Work that outruns this gets an interim update rather than silence. Set just
|
|
73
|
+
* under the measured p50 session (14.7 min) so the common case is covered,
|
|
74
|
+
* and well above the quick path so a fast answer never triggers one. */
|
|
75
|
+
export const PROGRESS_AFTER_MS = num(process.env.ASSURANCE_PROGRESS_AFTER_MS, 5 * 60_000);
|
|
76
|
+
|
|
77
|
+
/** Gap between interim updates. */
|
|
78
|
+
export const PROGRESS_EVERY_MS = num(process.env.ASSURANCE_PROGRESS_EVERY_MS, 10 * 60_000);
|
|
79
|
+
|
|
80
|
+
/** Hard cap on interim updates per obligation. "Do not spam" is a requirement,
|
|
81
|
+
* not a nicety: two updates over 45 minutes is attentive, six is noise. */
|
|
82
|
+
export const PROGRESS_MAX = num(process.env.ASSURANCE_PROGRESS_MAX, 2);
|
|
83
|
+
|
|
84
|
+
/** An obligation still open this long after admission has outlived every
|
|
85
|
+
* session timeout in the dispatcher (45 min for opus/inbox). Past this the
|
|
86
|
+
* work is gone, whatever the logs say, and the human is told. */
|
|
87
|
+
export const STALE_AFTER_MS = num(process.env.ASSURANCE_STALE_AFTER_MS, 50 * 60_000);
|
|
88
|
+
|
|
89
|
+
/** Automatic retries of the WORK (not of a send) before we stop and escalate.
|
|
90
|
+
* Deliberately small: each attempt costs 15-45 minutes of a human's patience,
|
|
91
|
+
* so an unbounded silent retry loop is itself a defect. */
|
|
92
|
+
export const RETRY_MAX = num(process.env.ASSURANCE_RETRY_MAX, 1);
|
|
93
|
+
|
|
94
|
+
/** Retained after discharge so an audit can see what was promised and when. */
|
|
95
|
+
const CLOSED_RETENTION_MS = num(process.env.ASSURANCE_RETENTION_MS, 3 * 24 * 60 * 60 * 1000);
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* How many times the sweep will re-attempt an acknowledgement that would not
|
|
99
|
+
* send, before it concludes the channel is unreachable rather than flaky.
|
|
100
|
+
*
|
|
101
|
+
* This bound is not a nicety. Without it the sweep's ack branch `continue`d on
|
|
102
|
+
* every tick, so an item on a service transport cannot reach — telegram,
|
|
103
|
+
* whatsapp, voice, calendar all route through the daemon and none of them
|
|
104
|
+
* through `deliver` — was retried once a minute forever: 1,440 impossible sends
|
|
105
|
+
* a day, an obligation that could never reach any other branch, and a human who
|
|
106
|
+
* was never told a thing. Five attempts spans four minutes of genuine flakiness;
|
|
107
|
+
* past that the honest answer is "I cannot reach this person" and the debt
|
|
108
|
+
* belongs in needs-attention where an operator will see it.
|
|
109
|
+
*/
|
|
110
|
+
export const ACK_MAX_ATTEMPTS = num(process.env.ASSURANCE_ACK_MAX_ATTEMPTS, 5);
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Minimum age before the sweep will call an obligation "interrupted" on the
|
|
114
|
+
* strength of a pid mismatch.
|
|
115
|
+
*
|
|
116
|
+
* A mismatch alone proves nothing. Two daemons sharing one AGENT_DIR — a manual
|
|
117
|
+
* run alongside the launchd one, which is exactly what an operator does while
|
|
118
|
+
* debugging — each see the other's fresh, healthy, actively-running obligations
|
|
119
|
+
* as foreign, and with no guard would tell those requesters "my session was
|
|
120
|
+
* interrupted" one second after it started, while it runs to completion behind
|
|
121
|
+
* the apology. Liveness is checked first (a pid we can signal is a daemon that
|
|
122
|
+
* is still working); this window is the backstop for a pid that has been
|
|
123
|
+
* recycled onto an unrelated process.
|
|
124
|
+
*/
|
|
125
|
+
export const INTERRUPT_GRACE_MS = num(process.env.ASSURANCE_INTERRUPT_GRACE_MS, 3 * 60_000);
|
|
126
|
+
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
// Ledger
|
|
129
|
+
// ---------------------------------------------------------------------------
|
|
130
|
+
|
|
131
|
+
function obligationDir() {
|
|
132
|
+
const dir = join(AGENT_REPO_DIR, "state", "obligations");
|
|
133
|
+
mkdirSync(dir, { recursive: true });
|
|
134
|
+
return dir;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Filesystem-safe form of an arbitrary item key. */
|
|
138
|
+
export function sanitiseKey(key) {
|
|
139
|
+
return String(key || "unknown").replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 180);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* The obligation key. Same precedence as the daemon's per-item lock and
|
|
144
|
+
* in-flight admission (raw_ref, then id) so all three name the same thing and a
|
|
145
|
+
* cross-reference is possible during an incident.
|
|
146
|
+
*/
|
|
147
|
+
export function obligationKey(item) {
|
|
148
|
+
if (!item) return null;
|
|
149
|
+
return item.raw_ref || item.id || item.message_id || null;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function pathFor(key) {
|
|
153
|
+
return join(obligationDir(), `${sanitiseKey(key)}.json`);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function writeRecord(rec) {
|
|
157
|
+
try {
|
|
158
|
+
const p = pathFor(rec.key);
|
|
159
|
+
const tmp = `${p}.tmp`;
|
|
160
|
+
writeFileSync(tmp, JSON.stringify(rec, null, 2));
|
|
161
|
+
renameSync(tmp, p); // atomic — a torn record is an unreadable debt
|
|
162
|
+
return true;
|
|
163
|
+
} catch (err) {
|
|
164
|
+
console.warn(`[assurance] could not persist obligation ${rec && rec.key}: ${err.message}`);
|
|
165
|
+
return false;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** Read one obligation. Returns null when absent or corrupt. */
|
|
170
|
+
export function readObligation(key) {
|
|
171
|
+
try { return JSON.parse(readFileSync(pathFor(key), "utf-8")); }
|
|
172
|
+
catch { return null; }
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Every obligation on disk, open and closed. */
|
|
176
|
+
export function listObligations() {
|
|
177
|
+
const out = [];
|
|
178
|
+
let files = [];
|
|
179
|
+
try { files = readdirSync(obligationDir()).filter((f) => f.endsWith(".json")); }
|
|
180
|
+
catch { return out; }
|
|
181
|
+
for (const f of files) {
|
|
182
|
+
try { out.push(JSON.parse(readFileSync(join(obligationDir(), f), "utf-8"))); }
|
|
183
|
+
catch { /* a corrupt record must not hide the rest */ }
|
|
184
|
+
}
|
|
185
|
+
return out;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** Only what is still owed. */
|
|
189
|
+
export function openObligations() {
|
|
190
|
+
return listObligations().filter((r) => r && r.state === "open");
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* The minimal snapshot of an item needed to SPEAK to its human later — from a
|
|
195
|
+
* sweep tick, possibly in a different process, long after the original item
|
|
196
|
+
* object is gone. Content is truncated: this file is a debt record, not a
|
|
197
|
+
* message archive.
|
|
198
|
+
*/
|
|
199
|
+
function itemSnapshot(item) {
|
|
200
|
+
return {
|
|
201
|
+
id: item.id || null,
|
|
202
|
+
raw_ref: item.raw_ref || null,
|
|
203
|
+
message_id: item.message_id || null,
|
|
204
|
+
service: item.service || null,
|
|
205
|
+
channel: item.channel || null,
|
|
206
|
+
channel_id: item.channel_id || null,
|
|
207
|
+
thread_id: item.thread_id || null,
|
|
208
|
+
sender: item.sender || null,
|
|
209
|
+
sender_email: item.sender_email || null,
|
|
210
|
+
subject: item.subject || null,
|
|
211
|
+
content: typeof item.content === "string" ? item.content.slice(0, 400) : null,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// ---------------------------------------------------------------------------
|
|
216
|
+
// The judgement: does this ask need an acknowledgement at all?
|
|
217
|
+
// ---------------------------------------------------------------------------
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* "A simple ask should still get a direct answer without a pointless 'I'll look
|
|
221
|
+
* into it' first — getting that distinction right is the whole feature."
|
|
222
|
+
*
|
|
223
|
+
* The distinction is NOT a classifier enum. It is a fact about control flow:
|
|
224
|
+
* an ack is owed exactly when the answer will NOT arrive in this turn — i.e.
|
|
225
|
+
* when a session is being spawned. If the quick path answered, the human
|
|
226
|
+
* already has their answer and an ack would be noise; if a session is spawning,
|
|
227
|
+
* the human is about to wait minutes and an ack is the whole point.
|
|
228
|
+
*
|
|
229
|
+
* The previous gate asked the classifier instead (`respond || draft ||
|
|
230
|
+
* research`), which silently excluded `queue` — the action the HEURISTIC
|
|
231
|
+
* FALLBACK emits when the LLM classifier itself fails. So exactly when the
|
|
232
|
+
* system was degraded, a directed DM got a 15-minute session and no ack at all.
|
|
233
|
+
*
|
|
234
|
+
* @param {object} o
|
|
235
|
+
* @param {boolean} o.willSpawnSession is a session being dispatched for this item
|
|
236
|
+
* @param {object} [o.item] the inbox item
|
|
237
|
+
* @param {string} [o.source] "inbox" | "backlog"
|
|
238
|
+
* @returns {{ack:boolean, reason:string}}
|
|
239
|
+
*/
|
|
240
|
+
export function shouldAcknowledge(o = {}) {
|
|
241
|
+
if (!o.willSpawnSession) return { ack: false, reason: "answered-in-turn" };
|
|
242
|
+
if (o.source && o.source !== "inbox") return { ack: false, reason: "no-waiting-human" };
|
|
243
|
+
const item = o.item || {};
|
|
244
|
+
if (!item.service) return { ack: false, reason: "no-service" };
|
|
245
|
+
if (!replyTargetOf(item)) return { ack: false, reason: "no-reply-target" };
|
|
246
|
+
// A service this agent has no transport for. Saying "I'm on it" into a
|
|
247
|
+
// channel we cannot write to is not an acknowledgement, it is an exception
|
|
248
|
+
// with a nicer name — and the sweep would then retry that impossible send
|
|
249
|
+
// once a minute forever. The DEBT is still opened (see openAndAcknowledge):
|
|
250
|
+
// an ask we cannot answer in public is exactly the one an operator must be
|
|
251
|
+
// told about, and that is what needs-attention is for.
|
|
252
|
+
if (!canDeliverTo(item)) return { ack: false, reason: "no-transport" };
|
|
253
|
+
// Nobody is waiting on work the agent gave itself.
|
|
254
|
+
if (item.self_originated === true) return { ack: false, reason: "self-originated" };
|
|
255
|
+
return { ack: true, reason: "session-dispatched" };
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// ---------------------------------------------------------------------------
|
|
259
|
+
// Composition — deterministic strings. No model, therefore no timeout.
|
|
260
|
+
// ---------------------------------------------------------------------------
|
|
261
|
+
|
|
262
|
+
/** Deterministic variant pick, so repeated acks in a thread are not identical
|
|
263
|
+
* boilerplate but are still perfectly reproducible in a test. */
|
|
264
|
+
function variant(seed, n) {
|
|
265
|
+
let h = 0;
|
|
266
|
+
const s = String(seed || "");
|
|
267
|
+
for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) >>> 0;
|
|
268
|
+
return h % n;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Trim the classifier's summary into a clause that reads naturally mid-sentence. */
|
|
272
|
+
function topicClause(classResult, item) {
|
|
273
|
+
const raw = (classResult && classResult.summary) || (item && item.subject) || "";
|
|
274
|
+
const t = String(raw).trim().replace(/\s+/g, " ").replace(/[.!?]+$/, "");
|
|
275
|
+
if (!t || t.length > 120) return null;
|
|
276
|
+
return t.charAt(0).toLowerCase() + t.slice(1);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** Rough, honest expectation. Derived from the measured distribution of real
|
|
280
|
+
* sessions, not invented: sonnet work clusters near a minute, opus work near
|
|
281
|
+
* fifteen. Phrased as a range because it IS a range. */
|
|
282
|
+
export function expectedWindow(classResult) {
|
|
283
|
+
const model = classResult && classResult.model;
|
|
284
|
+
const priority = classResult && classResult.priority;
|
|
285
|
+
if (model === "opus" || priority === "critical") return "usually within 10-20 minutes";
|
|
286
|
+
return "usually within a few minutes";
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* The acknowledgement. Two or three sentences, warm, specific where it can be,
|
|
291
|
+
* and — critically — it commits to coming back "either way". That clause is a
|
|
292
|
+
* promise the rest of this module now actually keeps: failure notices, interim
|
|
293
|
+
* updates and the stale sweep all exist so it is never a lie.
|
|
294
|
+
*/
|
|
295
|
+
export function composeAck(item, classResult = {}) {
|
|
296
|
+
const topic = topicClause(classResult, item);
|
|
297
|
+
const v = variant(obligationKey(item) || (item && item.sender), 3);
|
|
298
|
+
const openers = [
|
|
299
|
+
"No problem — let me look into this and sort it out.",
|
|
300
|
+
"Got it — I'm on this now.",
|
|
301
|
+
"Understood — let me dig into this.",
|
|
302
|
+
];
|
|
303
|
+
const parts = [openers[v]];
|
|
304
|
+
if (topic) parts.push(`Picking up: ${topic}.`);
|
|
305
|
+
parts.push(`It needs a bit of work, so I'll come back to you here with an answer — ${expectedWindow(classResult)}, and I'll message you either way.`);
|
|
306
|
+
return parts.join(" ");
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* The topic, set off in parentheses.
|
|
311
|
+
*
|
|
312
|
+
* It is NOT interpolated as the object of a preposition, because the classifier's
|
|
313
|
+
* summary is as often a verb phrase ("fix the issues and push them") as a noun
|
|
314
|
+
* phrase ("July spend reconciliation") — and "Still working on fix the issues"
|
|
315
|
+
* is the kind of sentence that tells a reader they are talking to a machine.
|
|
316
|
+
* Parentheses read correctly for both.
|
|
317
|
+
*/
|
|
318
|
+
function topicAside(rec) {
|
|
319
|
+
return rec && rec.summary ? ` (${rec.summary})` : "";
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** An interim update. Says how long it has been and that the work is live. */
|
|
323
|
+
export function composeProgress(rec, o = {}) {
|
|
324
|
+
const now = Number.isFinite(o.now) ? o.now : Date.now();
|
|
325
|
+
const mins = Math.max(1, Math.round((now - (rec.openedAt || now)) / 60_000));
|
|
326
|
+
const topic = topicAside(rec);
|
|
327
|
+
if ((rec.progressSent || 0) === 0) {
|
|
328
|
+
return `Still working on this${topic} — ${mins} minutes in, and it's taking longer than I expected. Nothing's stuck; I'll come back as soon as I have something worth sending.`;
|
|
329
|
+
}
|
|
330
|
+
return `Quick check-in: still on this${topic}, ${mins} minutes in. I haven't forgotten it — I'll follow up the moment it's done, or tell you if I can't finish it.`;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* The failure notice. Says WHAT HAPPENED and WHAT HAPPENS NEXT, in that order,
|
|
335
|
+
* because those are the two things the human is missing when a session dies.
|
|
336
|
+
* Never blames the human, never hides behind "an error occurred", and never
|
|
337
|
+
* ends without a next step.
|
|
338
|
+
*/
|
|
339
|
+
export function composeFailure(rec, o = {}) {
|
|
340
|
+
const f = o.failure || {};
|
|
341
|
+
const topic = topicAside(rec);
|
|
342
|
+
const cause = f.human || "the working session ended unexpectedly";
|
|
343
|
+
if (o.willRetry) {
|
|
344
|
+
return `I hit a problem with this${topic} — ${cause}. I'm retrying it now (attempt ${(rec.attempts || 1) + 1} of ${RETRY_MAX + 1}); if it fails again I'll come straight back to you rather than leave you waiting.`;
|
|
345
|
+
}
|
|
346
|
+
return `I couldn't get this done${topic} — ${cause}. I've stopped retrying so I'm not silently burning time on it, and I've flagged it so it isn't lost. Do you want me to try a narrower version of this, hand it to someone else, or leave it with you?`;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* The one the old code could not even conceive of: the session exited ZERO and
|
|
351
|
+
* said nothing. Observed live — a 241-second, $2.36 session whose own result
|
|
352
|
+
* text ended "Nothing was sent.", marked processed forever and emitted as
|
|
353
|
+
* `sent`. This message is what the human gets instead of that silence.
|
|
354
|
+
*/
|
|
355
|
+
export function composeSilentSuccess(rec, o = {}) {
|
|
356
|
+
const topic = topicAside(rec);
|
|
357
|
+
const tail = (o.finalText || "").trim();
|
|
358
|
+
const head = `I finished working on this${topic} but didn't get a reply out to you — that's my fault, not yours.`;
|
|
359
|
+
if (tail) {
|
|
360
|
+
const excerpt = tail.length > 900 ? `${tail.slice(0, 900).trimEnd()}…` : tail;
|
|
361
|
+
return `${head} Here's where I got to:\n\n${excerpt}\n\nIf that doesn't answer it, say so and I'll take another run at it.`;
|
|
362
|
+
}
|
|
363
|
+
return `${head} I don't have a clean result to show you, so I'd rather say that than pretend otherwise. Want me to run it again?`;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** A session interrupted by the daemon itself dying/restarting. */
|
|
367
|
+
export function composeInterrupted(rec) {
|
|
368
|
+
const topic = topicAside(rec);
|
|
369
|
+
return `Heads up — my working session on this${topic} was interrupted before it finished (my end restarted). I've put it back in the queue and I'm picking it up again now; I'll come back to you with the answer.`;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// ---------------------------------------------------------------------------
|
|
373
|
+
// Failure classification — what happened, and is it worth trying again?
|
|
374
|
+
// ---------------------------------------------------------------------------
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Map a terminal session state onto a plain-English cause and a retry verdict.
|
|
378
|
+
*
|
|
379
|
+
* The distinction matters because the two wrong answers are both bad: retrying
|
|
380
|
+
* a permanent fault burns another 45 minutes of the human's patience for the
|
|
381
|
+
* same outcome, and refusing to retry a transient one throws away work that
|
|
382
|
+
* would have succeeded.
|
|
383
|
+
*
|
|
384
|
+
* @param {object} o
|
|
385
|
+
* @param {number|null} [o.code] process exit code
|
|
386
|
+
* @param {string} [o.error] error text (spawn error, stderr tail)
|
|
387
|
+
* @param {string} [o.stage]
|
|
388
|
+
* @returns {{transient:boolean, label:string, human:string}}
|
|
389
|
+
*/
|
|
390
|
+
export function classifyFailure(o = {}) {
|
|
391
|
+
const err = String(o.error || "");
|
|
392
|
+
const code = o.code;
|
|
393
|
+
|
|
394
|
+
if (/rate.?limit|429|overloaded|529/i.test(err)) {
|
|
395
|
+
return { transient: true, label: "rate_limited", human: "the model API was rate-limiting me" };
|
|
396
|
+
}
|
|
397
|
+
if (/ENOENT|EACCES|command not found|spawn error/i.test(err)) {
|
|
398
|
+
// The binary or its permissions are wrong. Another attempt changes nothing.
|
|
399
|
+
return { transient: false, label: "spawn_failed", human: "I couldn't start my working session at all (a setup problem on my machine)" };
|
|
400
|
+
}
|
|
401
|
+
if (/budget|spend cap|daily budget|essential-only/i.test(err)) {
|
|
402
|
+
return { transient: false, label: "budget_refused", human: "I've hit my spend cap for today, so I can't run the work that would answer this" };
|
|
403
|
+
}
|
|
404
|
+
if (/out of memory|ENOMEM|low free memory/i.test(err)) {
|
|
405
|
+
return { transient: true, label: "resource_starved", human: "my machine ran out of memory partway through" };
|
|
406
|
+
}
|
|
407
|
+
if (code === 143 || /SIGTERM|timed out/i.test(err)) {
|
|
408
|
+
return { transient: true, label: "timeout", human: "the work ran past its time limit and got cut off" };
|
|
409
|
+
}
|
|
410
|
+
if (code === 137 || /SIGKILL/i.test(err)) {
|
|
411
|
+
return { transient: true, label: "killed", human: "the working session was killed before it finished" };
|
|
412
|
+
}
|
|
413
|
+
if (code === 0) {
|
|
414
|
+
return { transient: false, label: "silent_success", human: "the work finished but produced nothing I could send you" };
|
|
415
|
+
}
|
|
416
|
+
return { transient: true, label: `exit_${code == null ? "unknown" : code}`, human: "the working session ended with an error" };
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// ---------------------------------------------------------------------------
|
|
420
|
+
// Lifecycle
|
|
421
|
+
// ---------------------------------------------------------------------------
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Open a debt and acknowledge it — in that order.
|
|
425
|
+
*
|
|
426
|
+
* The record is written BEFORE the send, so a crash between the two leaves an
|
|
427
|
+
* un-acked obligation the sweep will notice and compensate. Written after, a
|
|
428
|
+
* crash would leave a human waiting on a debt that nothing knows exists.
|
|
429
|
+
*
|
|
430
|
+
* @param {object} a
|
|
431
|
+
* @param {object} a.item
|
|
432
|
+
* @param {object} [a.classResult]
|
|
433
|
+
* @param {string} [a.service]
|
|
434
|
+
* @param {string} [a.traceId]
|
|
435
|
+
* @param {number} [a.now]
|
|
436
|
+
* @param {object} [a.deps] {ackSender} — `(item, classResult) => {sent, holdingText,
|
|
437
|
+
* error}`, i.e. exactly responder.sendHoldingMessage's contract, so the
|
|
438
|
+
* daemon's existing injected-fake seam keeps working unchanged. Defaults
|
|
439
|
+
* to composing the text here and handing it to transport.
|
|
440
|
+
* @param {boolean} [a.ack=true] open the debt but say nothing. Used when there
|
|
441
|
+
* is no transport to the requester: the debt is REAL — a session is
|
|
442
|
+
* about to run and may die — but the compensating action is an operator
|
|
443
|
+
* escalation, not a message into a channel that does not exist. Opening
|
|
444
|
+
* it anyway is what turned "one failed session silently deletes the ask"
|
|
445
|
+
* into a bounded retry with a durable trace.
|
|
446
|
+
* @returns {Promise<{key:string|null, opened:boolean, acked:boolean, ackText:string|null, error?:string}>}
|
|
447
|
+
*/
|
|
448
|
+
export async function openAndAcknowledge(a = {}) {
|
|
449
|
+
const item = a.item || {};
|
|
450
|
+
const key = obligationKey(item);
|
|
451
|
+
if (!key) return { key: null, opened: false, acked: false, ackText: null, error: "item has no stable key" };
|
|
452
|
+
const now = Number.isFinite(a.now) ? a.now : Date.now();
|
|
453
|
+
const classResult = a.classResult || {};
|
|
454
|
+
const wantAck = a.ack !== false;
|
|
455
|
+
|
|
456
|
+
const existing = readObligation(key);
|
|
457
|
+
if (existing && existing.state === "open" && existing.acknowledged) {
|
|
458
|
+
// Already acknowledged (a re-delivery of the same item). Acknowledge ONCE.
|
|
459
|
+
return { key, opened: false, acked: true, ackText: null, reason: "already-acknowledged" };
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
const rec = existing && existing.state === "open" ? existing : {
|
|
463
|
+
key,
|
|
464
|
+
itemId: item.id || null,
|
|
465
|
+
raw_ref: item.raw_ref || null,
|
|
466
|
+
service: a.service || item.service || null,
|
|
467
|
+
channel: replyTargetOf(item),
|
|
468
|
+
sender: item.sender || null,
|
|
469
|
+
summary: topicClause(classResult, item),
|
|
470
|
+
priority: classResult.priority || null,
|
|
471
|
+
model: classResult.model || null,
|
|
472
|
+
openedAt: now,
|
|
473
|
+
// The clock the STALE check runs against. It is distinct from openedAt
|
|
474
|
+
// because a retry restarts the work: measuring "has this outrun every
|
|
475
|
+
// session timeout" from the original arrival declares a running retry dead
|
|
476
|
+
// and tells the human the opposite of what it was told five minutes ago.
|
|
477
|
+
lastAttemptAt: now,
|
|
478
|
+
acknowledged: false,
|
|
479
|
+
ackAt: null,
|
|
480
|
+
ackAttempts: 0,
|
|
481
|
+
deliverable: canDeliverTo(item),
|
|
482
|
+
progressSent: 0,
|
|
483
|
+
lastProgressAt: null,
|
|
484
|
+
attempts: 0,
|
|
485
|
+
lastError: null,
|
|
486
|
+
sessionId: null,
|
|
487
|
+
traceId: a.traceId || item.trace_id || null,
|
|
488
|
+
daemonPid: process.pid,
|
|
489
|
+
state: "open",
|
|
490
|
+
item: itemSnapshot(item),
|
|
491
|
+
};
|
|
492
|
+
rec.daemonPid = process.pid;
|
|
493
|
+
if (rec.deliverable === undefined) rec.deliverable = canDeliverTo(item);
|
|
494
|
+
writeRecord(rec);
|
|
495
|
+
|
|
496
|
+
if (!wantAck || !rec.deliverable) {
|
|
497
|
+
// The debt is on the books and the sweep will not try to speak into a
|
|
498
|
+
// channel that does not exist (see sweepObligations branch (b)). What it
|
|
499
|
+
// WILL do is escalate if the work then fails — which is the whole point of
|
|
500
|
+
// opening a record we cannot acknowledge.
|
|
501
|
+
return { key, opened: true, acked: false, ackText: null, reason: rec.deliverable ? "ack-suppressed" : "no-transport" };
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
const ackSender = (a.deps && a.deps.ackSender)
|
|
505
|
+
|| (async (i, cr) => {
|
|
506
|
+
const t = composeAck(i, cr);
|
|
507
|
+
const r = await deliverWithRetry(i, t, { kind: "ack", ...(a.deps && a.deps.sleep ? { sleep: a.deps.sleep } : {}) });
|
|
508
|
+
return { sent: r.sent, holdingText: t, error: r.error };
|
|
509
|
+
});
|
|
510
|
+
const res = await ackSender(item, classResult);
|
|
511
|
+
const text = (res && res.holdingText) || composeAck(item, classResult);
|
|
512
|
+
|
|
513
|
+
if (res && res.sent) {
|
|
514
|
+
rec.acknowledged = true;
|
|
515
|
+
rec.ackAt = Number.isFinite(a.now) ? a.now : Date.now();
|
|
516
|
+
writeRecord(rec);
|
|
517
|
+
return { key, opened: true, acked: true, ackText: text };
|
|
518
|
+
}
|
|
519
|
+
// NOT fatal, and NOT forgotten: the debt stays open and un-acked, and the
|
|
520
|
+
// sweep retries it within ACK_GRACE_MS. This is precisely the case the old
|
|
521
|
+
// code logged and dropped.
|
|
522
|
+
rec.lastError = (res && res.error) || "ack send failed";
|
|
523
|
+
rec.ackAttempts = (rec.ackAttempts || 0) + 1;
|
|
524
|
+
writeRecord(rec);
|
|
525
|
+
console.warn(`[assurance] ack not delivered for ${key} (${rec.lastError}) — obligation left open for sweep`);
|
|
526
|
+
return { key, opened: true, acked: false, ackText: text, error: rec.lastError };
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/**
|
|
530
|
+
* Attach the dispatched session id, so an incident can join debt ↔ session log.
|
|
531
|
+
*
|
|
532
|
+
* Also restarts the staleness clock: this is a fresh attempt at the work, and
|
|
533
|
+
* the question the STALE branch asks — "has this outrun every session timeout
|
|
534
|
+
* there is?" — is about the RUNNING session, not about how long ago the human
|
|
535
|
+
* first asked.
|
|
536
|
+
*/
|
|
537
|
+
export function noteSession(key, sessionId, o = {}) {
|
|
538
|
+
const rec = readObligation(key);
|
|
539
|
+
if (!rec) return false;
|
|
540
|
+
rec.sessionId = sessionId || null;
|
|
541
|
+
rec.attempts = (rec.attempts || 0) + 1;
|
|
542
|
+
rec.lastAttemptAt = Number.isFinite(o.now) ? o.now : Date.now();
|
|
543
|
+
rec.daemonPid = process.pid;
|
|
544
|
+
return writeRecord(rec);
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* Is the process that opened this obligation still running?
|
|
549
|
+
*
|
|
550
|
+
* `kill(pid, 0)` signals nothing and throws ESRCH only when no such process
|
|
551
|
+
* exists, so it is the cheapest honest liveness test available. EPERM means the
|
|
552
|
+
* pid exists but belongs to another user — still alive, still not ours to
|
|
553
|
+
* declare dead. Anything unexpected is treated as ALIVE, because the cost of a
|
|
554
|
+
* false "dead" is telling a human their work was interrupted while it runs.
|
|
555
|
+
*/
|
|
556
|
+
export function isPidAlive(pid) {
|
|
557
|
+
if (!Number.isFinite(pid) || pid <= 0) return false;
|
|
558
|
+
try { process.kill(pid, 0); return true; }
|
|
559
|
+
catch (err) { return err && err.code === "EPERM"; }
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* How many OPEN debts share a room with this one.
|
|
564
|
+
*
|
|
565
|
+
* The receipt ledger's fallback rung — "an unattributed message reached this
|
|
566
|
+
* room, so assume it was the answer" — is only safe when the room holds exactly
|
|
567
|
+
* one unanswered ask. With two, the message answered one of them and guessing
|
|
568
|
+
* which is how an unanswered ask gets closed as answered.
|
|
569
|
+
*/
|
|
570
|
+
function siblingsInRoom(rec, open) {
|
|
571
|
+
const ch = String(rec.channel || "").trim().toLowerCase();
|
|
572
|
+
if (!ch) return 1;
|
|
573
|
+
return open.filter((r) => String(r.channel || "").trim().toLowerCase() === ch).length;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* Ask "was this debt's human actually spoken to?", through whichever seam the
|
|
578
|
+
* caller injected.
|
|
579
|
+
*
|
|
580
|
+
* The real implementation is `spokeFor`, which returns `{heard, basis}` — the
|
|
581
|
+
* basis matters, because "a receipt carrying this obligation's key" and "an
|
|
582
|
+
* unattributed message in a quiet room" are different strengths of evidence and
|
|
583
|
+
* an incident needs to know which one closed a debt. A test that only wants to
|
|
584
|
+
* pin the answer may inject a plain boolean; normalising here keeps that seam
|
|
585
|
+
* honest without every caller having to fabricate a basis.
|
|
586
|
+
*/
|
|
587
|
+
function askHeard(impl, query) {
|
|
588
|
+
const r = impl(query);
|
|
589
|
+
if (typeof r === "boolean") return { heard: r, basis: "injected" };
|
|
590
|
+
return r && typeof r === "object" ? r : { heard: false, basis: "unknown" };
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
/**
|
|
594
|
+
* Discharge the debt: the human has their answer.
|
|
595
|
+
* @param {string} key
|
|
596
|
+
* @param {object} [o] {outcome, now}
|
|
597
|
+
*/
|
|
598
|
+
export function closeObligation(key, o = {}) {
|
|
599
|
+
const rec = readObligation(key);
|
|
600
|
+
if (!rec) return false;
|
|
601
|
+
rec.state = o.outcome || "answered";
|
|
602
|
+
rec.closedAt = Number.isFinite(o.now) ? o.now : Date.now();
|
|
603
|
+
if (o.note) rec.closeNote = o.note;
|
|
604
|
+
return writeRecord(rec);
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/**
|
|
608
|
+
* A session ended. Decide what the human hears, and say it.
|
|
609
|
+
*
|
|
610
|
+
* This is the single place that turns every terminal session state into a
|
|
611
|
+
* message. The four cases it covers are the four ways the old code went silent:
|
|
612
|
+
*
|
|
613
|
+
* ok:true + a receipt exists → the session spoke. Close the debt quietly;
|
|
614
|
+
* do NOT add a message (that would be spam).
|
|
615
|
+
* ok:true + no receipt → silent success. Say so, with the result text.
|
|
616
|
+
* ok:false + transient + tries → say what broke and that a retry is running.
|
|
617
|
+
* ok:false + terminal → say it failed, stop, escalate durably.
|
|
618
|
+
*
|
|
619
|
+
* @param {object} a
|
|
620
|
+
* @param {string} a.key
|
|
621
|
+
* @param {boolean} a.ok
|
|
622
|
+
* @param {number|null} [a.code]
|
|
623
|
+
* @param {string} [a.stdout] the session's raw stdout (for the result text)
|
|
624
|
+
* @param {string} [a.error]
|
|
625
|
+
* @param {number} [a.now]
|
|
626
|
+
* @param {object} [a.deps] {deliverImpl, spokeSinceImpl}
|
|
627
|
+
* @returns {Promise<{spoke:boolean, text:string|null, verdict:string, willRetry:boolean}>}
|
|
628
|
+
*/
|
|
629
|
+
export async function settleSession(a = {}) {
|
|
630
|
+
const rec = readObligation(a.key);
|
|
631
|
+
if (!rec) return { spoke: false, text: null, verdict: "no-obligation", willRetry: false };
|
|
632
|
+
const now = Number.isFinite(a.now) ? a.now : Date.now();
|
|
633
|
+
if (a.sessionId && rec.sessionId !== a.sessionId) { rec.sessionId = a.sessionId; writeRecord(rec); }
|
|
634
|
+
const item = rec.item || {};
|
|
635
|
+
const send = (a.deps && a.deps.deliverImpl) || deliver;
|
|
636
|
+
|
|
637
|
+
// Did THIS session's work reach THIS person? Not "did anything land in that
|
|
638
|
+
// room" — a room is shared, and a channel-only check let one session's answer
|
|
639
|
+
// discharge every other debt open in the same DM, closing unanswered asks as
|
|
640
|
+
// answered with nobody told. `spokeFor` resolves the receipt against the
|
|
641
|
+
// obligation's own key and session id; only an unambiguous room falls back.
|
|
642
|
+
// Our own courtesy messages are excluded throughout: an ack is not an answer.
|
|
643
|
+
const heardBy = (a.deps && (a.deps.spokeForImpl || a.deps.spokeSinceImpl)) || spokeFor;
|
|
644
|
+
const verdictHeard = askHeard(heardBy, {
|
|
645
|
+
obligation: rec,
|
|
646
|
+
now,
|
|
647
|
+
siblingOpenCount: siblingsInRoom(rec, openObligations()),
|
|
648
|
+
excludeKinds: ["ack", "progress", "failure"],
|
|
649
|
+
agentRoot: AGENT_REPO_DIR,
|
|
650
|
+
// Legacy shape, for a seam injected as the older channel-scoped predicate.
|
|
651
|
+
channel: rec.channel,
|
|
652
|
+
sinceMs: rec.openedAt || now - 3_600_000,
|
|
653
|
+
});
|
|
654
|
+
const heard = !!(verdictHeard && verdictHeard.heard);
|
|
655
|
+
|
|
656
|
+
if (a.ok && heard) {
|
|
657
|
+
closeObligation(a.key, { outcome: "answered", now, note: `receipt:${verdictHeard.basis}` });
|
|
658
|
+
return { spoke: false, text: null, verdict: "answered-by-session", willRetry: false, basis: verdictHeard.basis };
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
if (a.ok && !heard) {
|
|
662
|
+
const finalText = safeResultText(a.stdout);
|
|
663
|
+
const text = composeSilentSuccess(rec, { finalText });
|
|
664
|
+
const res = await send(item, text, { kind: "reply", idempotencySuffix: `rescue-${rec.attempts || 0}` });
|
|
665
|
+
closeObligation(a.key, {
|
|
666
|
+
outcome: res && res.sent ? "answered" : "undeliverable",
|
|
667
|
+
now,
|
|
668
|
+
note: res && res.sent ? "session exited 0 without speaking; daemon delivered its result text" : `silent-success fallback undeliverable: ${res && res.error}`,
|
|
669
|
+
});
|
|
670
|
+
return { spoke: !!(res && res.sent), text, verdict: "silent-success", willRetry: false };
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// ── failure ────────────────────────────────────────────────────────────
|
|
674
|
+
const failure = classifyFailure({ code: a.code, error: a.error });
|
|
675
|
+
const attempts = rec.attempts || 1;
|
|
676
|
+
const willRetry = failure.transient && attempts <= RETRY_MAX;
|
|
677
|
+
|
|
678
|
+
rec.lastError = failure.label;
|
|
679
|
+
rec.lastFailureAt = now;
|
|
680
|
+
writeRecord(rec);
|
|
681
|
+
|
|
682
|
+
const text = composeFailure(rec, { failure, willRetry });
|
|
683
|
+
const res = await send(item, text, { kind: "failure", idempotencySuffix: `failure-${attempts}` });
|
|
684
|
+
|
|
685
|
+
if (willRetry) {
|
|
686
|
+
// The debt stays OPEN and the inbox item stays un-`.processed`, so the next
|
|
687
|
+
// poll re-delivers it. The difference from before is that the human now
|
|
688
|
+
// knows a retry is happening instead of watching a typing dot stop.
|
|
689
|
+
//
|
|
690
|
+
// Restart the staleness clock HERE, not only at the next noteSession. The
|
|
691
|
+
// retry is announced at this instant ("I'm retrying it now"), and until the
|
|
692
|
+
// next poll picks the item up there is a window in which the sweep would
|
|
693
|
+
// otherwise measure age from the original arrival, cross STALE_AFTER_MS and
|
|
694
|
+
// tell the same human "I've stopped retrying" — a flat contradiction of a
|
|
695
|
+
// message they received minutes earlier, while the retry runs.
|
|
696
|
+
rec.lastAttemptAt = now;
|
|
697
|
+
writeRecord(rec);
|
|
698
|
+
return { spoke: !!(res && res.sent), text, verdict: `retrying:${failure.label}`, willRetry: true };
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
escalate(rec, { failure, now, told: !!(res && res.sent) });
|
|
702
|
+
closeObligation(a.key, { outcome: "failed", now, note: failure.label });
|
|
703
|
+
return { spoke: !!(res && res.sent), text, verdict: `failed:${failure.label}`, willRetry: false };
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
function safeResultText(stdout) {
|
|
707
|
+
try { return resultTextFromStdout(stdout || "") || ""; }
|
|
708
|
+
catch { return ""; }
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
/**
|
|
712
|
+
* Durable "this was not delivered and needs a person" record.
|
|
713
|
+
*
|
|
714
|
+
* Honest scope note: this does NOT raise an hq `Escalation` row. The escalation
|
|
715
|
+
* effect in lib/execution/effects.mjs writes a row and speaks to nobody
|
|
716
|
+
* (journalled `spoke:false` on all 9 observed escalations), which is the same
|
|
717
|
+
* silence in a different table. What a waiting human actually needs is to be
|
|
718
|
+
* TOLD — that already happened above — plus a trace an operator can sweep. This
|
|
719
|
+
* file is that trace.
|
|
720
|
+
*/
|
|
721
|
+
export function escalate(rec, o = {}) {
|
|
722
|
+
try {
|
|
723
|
+
const dir = join(obligationDir(), "needs-attention");
|
|
724
|
+
mkdirSync(dir, { recursive: true });
|
|
725
|
+
const p = join(dir, `${sanitiseKey(rec.key)}.json`);
|
|
726
|
+
writeFileSync(p, JSON.stringify({
|
|
727
|
+
key: rec.key,
|
|
728
|
+
sender: rec.sender,
|
|
729
|
+
service: rec.service,
|
|
730
|
+
channel: rec.channel,
|
|
731
|
+
summary: rec.summary,
|
|
732
|
+
openedAt: rec.openedAt ? new Date(rec.openedAt).toISOString() : null,
|
|
733
|
+
failedAt: new Date(Number.isFinite(o.now) ? o.now : Date.now()).toISOString(),
|
|
734
|
+
cause: o.failure ? o.failure.label : (rec.lastError || "unknown"),
|
|
735
|
+
requesterWasTold: !!o.told,
|
|
736
|
+
attempts: rec.attempts || 0,
|
|
737
|
+
sessionId: rec.sessionId || null,
|
|
738
|
+
traceId: rec.traceId || null,
|
|
739
|
+
itemPreview: rec.item ? rec.item.content : null,
|
|
740
|
+
}, null, 2));
|
|
741
|
+
console.error(`[assurance] ESCALATED ${rec.key} — ${o.failure ? o.failure.label : rec.lastError}; requester told: ${!!o.told}`);
|
|
742
|
+
return true;
|
|
743
|
+
} catch (err) {
|
|
744
|
+
console.error(`[assurance] could not write escalation for ${rec && rec.key}: ${err.message}`);
|
|
745
|
+
return false;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
// ---------------------------------------------------------------------------
|
|
750
|
+
// The sweep — the safety net under every path above
|
|
751
|
+
// ---------------------------------------------------------------------------
|
|
752
|
+
|
|
753
|
+
/**
|
|
754
|
+
* Walk the open debts and speak to whichever are overdue.
|
|
755
|
+
*
|
|
756
|
+
* This is what makes the guarantee hold across process death, unhandled
|
|
757
|
+
* rejections, a dispatcher that never calls back, and an ack whose send failed.
|
|
758
|
+
* It is intentionally the only mechanism with no in-memory state: everything it
|
|
759
|
+
* needs is on disk, so the first tick after a restart is as effective as the
|
|
760
|
+
* hundredth tick of a healthy process.
|
|
761
|
+
*
|
|
762
|
+
* @param {object} [a]
|
|
763
|
+
* @param {number} [a.now]
|
|
764
|
+
* @param {object} [a.deps] {deliverImpl, spokeSinceImpl}
|
|
765
|
+
* @returns {Promise<{swept:number, acked:number, progressed:number, staled:number, closed:number}>}
|
|
766
|
+
*/
|
|
767
|
+
export async function sweepObligations(a = {}) {
|
|
768
|
+
const now = Number.isFinite(a.now) ? a.now : Date.now();
|
|
769
|
+
const send = (a.deps && a.deps.deliverImpl) || deliver;
|
|
770
|
+
const heardBy = (a.deps && (a.deps.spokeForImpl || a.deps.spokeSinceImpl)) || spokeFor;
|
|
771
|
+
const alive = (a.deps && a.deps.isPidAliveImpl) || isPidAlive;
|
|
772
|
+
const stats = { swept: 0, acked: 0, progressed: 0, staled: 0, closed: 0, interrupted: 0, unreachable: 0 };
|
|
773
|
+
|
|
774
|
+
const open = openObligations();
|
|
775
|
+
for (const rec of open) {
|
|
776
|
+
stats.swept++;
|
|
777
|
+
// Two clocks, deliberately. `age` is how long the HUMAN has been waiting and
|
|
778
|
+
// is what progress messages talk about. `workAge` is how long the current
|
|
779
|
+
// attempt has been running and is what the stale check judges — a retry
|
|
780
|
+
// resets it, because declaring a running retry dead is worse than waiting.
|
|
781
|
+
const age = now - (rec.openedAt || now);
|
|
782
|
+
const workAge = now - (rec.lastAttemptAt || rec.openedAt || now);
|
|
783
|
+
const item = rec.item || {};
|
|
784
|
+
|
|
785
|
+
// (a) The session already answered — discharge without saying anything.
|
|
786
|
+
// Attributed, not room-wide: see spokeFor. An unattributed message in a room
|
|
787
|
+
// holding two open debts closes neither.
|
|
788
|
+
const v = askHeard(heardBy, {
|
|
789
|
+
obligation: rec,
|
|
790
|
+
now,
|
|
791
|
+
siblingOpenCount: siblingsInRoom(rec, open),
|
|
792
|
+
excludeKinds: ["ack", "progress", "failure"],
|
|
793
|
+
agentRoot: AGENT_REPO_DIR,
|
|
794
|
+
channel: rec.channel,
|
|
795
|
+
sinceMs: rec.openedAt || now,
|
|
796
|
+
});
|
|
797
|
+
if (v && v.heard) {
|
|
798
|
+
closeObligation(rec.key, { outcome: "answered", now, note: `receipt:${v.basis}` });
|
|
799
|
+
stats.closed++;
|
|
800
|
+
continue;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
// (b) No transport to this person at all. Every branch below this one ends
|
|
804
|
+
// in a send, so without this the obligation is unserviceable AND immortal:
|
|
805
|
+
// the ack branch's `continue` meant the stale branch was never reached, and
|
|
806
|
+
// an impossible send was retried every 60 seconds for as long as the daemon
|
|
807
|
+
// lived. Escalate ONCE, close the debt, and let an operator see it.
|
|
808
|
+
if (rec.deliverable === false || (rec.ackAttempts || 0) >= ACK_MAX_ATTEMPTS) {
|
|
809
|
+
const failure = {
|
|
810
|
+
transient: false,
|
|
811
|
+
label: rec.deliverable === false ? "no_transport" : "ack_undeliverable",
|
|
812
|
+
human: "I have no way to reply to you on this channel",
|
|
813
|
+
};
|
|
814
|
+
escalate(rec, { failure, now, told: false });
|
|
815
|
+
closeObligation(rec.key, { outcome: "undeliverable", now, note: failure.label });
|
|
816
|
+
stats.unreachable++;
|
|
817
|
+
continue;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
// (c) Past every session timeout there is. The work is gone whatever the
|
|
821
|
+
// logs claim; stop pretending and tell them.
|
|
822
|
+
//
|
|
823
|
+
// This is checked BEFORE the ack retry, not after. Ordered the other way,
|
|
824
|
+
// an obligation whose ack keeps failing never advances past the ack branch's
|
|
825
|
+
// `continue`, so the promise that the sweep is a backstop is void for
|
|
826
|
+
// exactly the obligations that most need one.
|
|
827
|
+
if (workAge >= STALE_AFTER_MS) {
|
|
828
|
+
const failure = { transient: false, label: "no_outcome", human: "the work never came back with a result and has now outrun its time limit" };
|
|
829
|
+
const text = composeFailure(rec, { failure, willRetry: false });
|
|
830
|
+
const res = await send(item, text, { kind: "failure", idempotencySuffix: "failure-stale" });
|
|
831
|
+
escalate(rec, { failure, now, told: !!(res && res.sent) });
|
|
832
|
+
closeObligation(rec.key, { outcome: "failed", now, note: "stale-no-outcome" });
|
|
833
|
+
stats.staled++;
|
|
834
|
+
continue;
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
// (d) An acknowledgement that never made it out. Compensate it — this is
|
|
838
|
+
// the 66%-of-the-time case that used to be logged and dropped. Bounded by
|
|
839
|
+
// ACK_MAX_ATTEMPTS above, so a channel that will never accept a message
|
|
840
|
+
// becomes an escalation rather than a permanent retry loop.
|
|
841
|
+
if (!rec.acknowledged && age >= ACK_GRACE_MS) {
|
|
842
|
+
const text = composeAck(item, { summary: rec.summary, model: rec.model, priority: rec.priority });
|
|
843
|
+
const res = await send(item, text, { kind: "ack", idempotencySuffix: "ack" });
|
|
844
|
+
if (res && res.sent) {
|
|
845
|
+
rec.acknowledged = true;
|
|
846
|
+
rec.ackAt = now;
|
|
847
|
+
writeRecord(rec);
|
|
848
|
+
stats.acked++;
|
|
849
|
+
} else {
|
|
850
|
+
// A permanent refusal (no transport, policy block) is worth five ticks
|
|
851
|
+
// of nobody's time; count it out at once.
|
|
852
|
+
rec.ackAttempts = (rec.ackAttempts || 0) + (res && res.permanent ? ACK_MAX_ATTEMPTS : 1);
|
|
853
|
+
rec.lastError = (res && res.error) || "ack send failed";
|
|
854
|
+
writeRecord(rec);
|
|
855
|
+
}
|
|
856
|
+
continue; // one message per obligation per tick, always
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
// (e) Interrupted: the debt was opened by a daemon that is no longer this
|
|
860
|
+
// process, and nothing has closed it. The work died with that process.
|
|
861
|
+
//
|
|
862
|
+
// A pid MISMATCH alone does not establish that. Two daemons on one AGENT_DIR
|
|
863
|
+
// — a manual run next to the launchd one, which is what an operator does
|
|
864
|
+
// while debugging — each see the other's healthy, actively-running debts as
|
|
865
|
+
// foreign. Unguarded, both tell those requesters "my session was
|
|
866
|
+
// interrupted" seconds after the ask arrived, while the work runs fine
|
|
867
|
+
// behind the apology. So: the owning process must be provably GONE, and the
|
|
868
|
+
// debt must have sat still long enough that a live handover would have shown
|
|
869
|
+
// up by now.
|
|
870
|
+
if (
|
|
871
|
+
rec.daemonPid && rec.daemonPid !== process.pid && !rec.interruptedNotifiedAt
|
|
872
|
+
&& workAge >= INTERRUPT_GRACE_MS && !alive(rec.daemonPid)
|
|
873
|
+
) {
|
|
874
|
+
const text = composeInterrupted(rec);
|
|
875
|
+
const res = await send(item, text, { kind: "progress", idempotencySuffix: `interrupted-${rec.attempts || 0}` });
|
|
876
|
+
rec.interruptedNotifiedAt = now;
|
|
877
|
+
rec.daemonPid = process.pid;
|
|
878
|
+
writeRecord(rec);
|
|
879
|
+
if (res && res.sent) stats.interrupted++;
|
|
880
|
+
continue;
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
// (f) Long work gets progress, not silence — capped.
|
|
884
|
+
const sinceUpdate = now - (rec.lastProgressAt || rec.ackAt || rec.openedAt || now);
|
|
885
|
+
const due = (rec.progressSent || 0) === 0
|
|
886
|
+
? age >= PROGRESS_AFTER_MS
|
|
887
|
+
: sinceUpdate >= PROGRESS_EVERY_MS;
|
|
888
|
+
if (due && (rec.progressSent || 0) < PROGRESS_MAX) {
|
|
889
|
+
const text = composeProgress(rec, { now });
|
|
890
|
+
const res = await send(item, text, { kind: "progress", idempotencySuffix: `progress-${(rec.progressSent || 0) + 1}` });
|
|
891
|
+
if (res && res.sent) {
|
|
892
|
+
rec.progressSent = (rec.progressSent || 0) + 1;
|
|
893
|
+
rec.lastProgressAt = now;
|
|
894
|
+
writeRecord(rec);
|
|
895
|
+
stats.progressed++;
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
pruneObligations({ now });
|
|
901
|
+
return stats;
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
/** Delete discharged debts past retention. Open debts are NEVER pruned — an
|
|
905
|
+
* obligation may only leave the ledger by being discharged. */
|
|
906
|
+
export function pruneObligations(o = {}) {
|
|
907
|
+
const now = Number.isFinite(o.now) ? o.now : Date.now();
|
|
908
|
+
let removed = 0;
|
|
909
|
+
for (const rec of listObligations()) {
|
|
910
|
+
if (!rec || rec.state === "open") continue;
|
|
911
|
+
if (now - (rec.closedAt || 0) < CLOSED_RETENTION_MS) continue;
|
|
912
|
+
try { unlinkSync(pathFor(rec.key)); removed++; } catch { /* best-effort */ }
|
|
913
|
+
}
|
|
914
|
+
return removed;
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
/** Test seam: wipe the ledger. */
|
|
918
|
+
export function _resetObligations() {
|
|
919
|
+
for (const rec of listObligations()) {
|
|
920
|
+
try { unlinkSync(pathFor(rec.key)); } catch { /* */ }
|
|
921
|
+
}
|
|
922
|
+
try {
|
|
923
|
+
const dir = join(obligationDir(), "needs-attention");
|
|
924
|
+
for (const f of readdirSync(dir)) { try { unlinkSync(join(dir, f)); } catch { /* */ } }
|
|
925
|
+
} catch { /* */ }
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
export default {
|
|
929
|
+
shouldAcknowledge,
|
|
930
|
+
composeAck,
|
|
931
|
+
composeProgress,
|
|
932
|
+
composeFailure,
|
|
933
|
+
composeSilentSuccess,
|
|
934
|
+
composeInterrupted,
|
|
935
|
+
classifyFailure,
|
|
936
|
+
openAndAcknowledge,
|
|
937
|
+
noteSession,
|
|
938
|
+
settleSession,
|
|
939
|
+
sweepObligations,
|
|
940
|
+
closeObligation,
|
|
941
|
+
openObligations,
|
|
942
|
+
obligationKey,
|
|
943
|
+
isPidAlive,
|
|
944
|
+
};
|