@cohortapp/agent-sdk 2.18.36 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +3 -0
- package/docs/guides/billing-console-keys.md +16 -3
- package/docs/runbooks/cadence-requeue-storm-recovery.md +186 -0
- package/lib/assurance/first-reply.mjs +47 -2
- package/lib/assurance/notice-plane.mjs +218 -0
- package/lib/assurance/notice-voice.mjs +35 -2
- package/lib/assurance/operator-plane-store.mjs +164 -0
- package/lib/assurance/operator-plane.mjs +689 -0
- package/lib/budget-escalate.mjs +42 -4
- package/lib/budget-guard.mjs +24 -1
- package/lib/cadence-bus.mjs +359 -46
- package/lib/cli/attention.mjs +300 -0
- package/lib/cli/seat-auth.mjs +86 -6
- package/lib/comms/machine-status.mjs +377 -0
- package/lib/comms/send-gate.mjs +309 -2
- package/lib/org/awareness.mjs +15 -2
- package/lib/org/clause-match.mjs +470 -0
- package/lib/org/client.mjs +27 -1
- package/lib/org/escalation-chain.mjs +134 -0
- package/lib/org/first-run.mjs +263 -0
- package/lib/org/genesis-contract.mjs +271 -0
- package/lib/org/governance-runtime.mjs +409 -0
- package/lib/org/governance-store.mjs +238 -0
- package/lib/org/governance.mjs +326 -0
- package/lib/org/mesh.mjs +135 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +6 -0
- package/lib/org/tool-surface.mjs +28 -0
- package/lib/setup/enroll-from-cohort.mjs +11 -0
- package/lib/telemetry/alerts.mjs +12 -0
- package/lib/telemetry/collect.mjs +237 -1
- package/package.json +1 -1
- package/scripts/ci/check-durable-write-seam.mjs +3 -3
- package/scripts/daemon/agent-daemon.mjs +41 -2
- package/scripts/daemon/assurance.mjs +852 -49
- package/scripts/daemon/cadence-consumer.mjs +182 -9
- package/scripts/daemon/context-compiler.mjs +25 -0
- package/scripts/daemon/deliver.mjs +172 -2
- package/scripts/fleet/closing-sweep.mjs +266 -0
|
@@ -98,6 +98,30 @@ import {
|
|
|
98
98
|
noticeRejection,
|
|
99
99
|
fallbackNotice,
|
|
100
100
|
} from "../../lib/assurance/notice-voice.mjs";
|
|
101
|
+
// WHERE A NOTICE GOES — and the answer is never "to a person". This is the
|
|
102
|
+
// decision the wording iterations kept skipping; `notice-plane.mjs` carries the
|
|
103
|
+
// argument and the table, and the table's default for an unregistered notice is
|
|
104
|
+
// the operator plane, so a new notice kind is silent to people until someone
|
|
105
|
+
// deliberately says otherwise under that header.
|
|
106
|
+
import { noticePlane, mayReachHuman, NOTICE_PLANE } from "../../lib/assurance/notice-plane.mjs";
|
|
107
|
+
// THE OPERATOR PLANE'S READER OWNS THE SHAPES THIS FILE WRITES. The retention
|
|
108
|
+
// constant, the counter names, the journal row kinds and the session-log path are
|
|
109
|
+
// all read back by `lib/cli/attention.mjs` and by the presence beat, so they are
|
|
110
|
+
// declared once, in the pure module, and imported here rather than restated —
|
|
111
|
+
// `docs`/CLAUDE.md: a literal and the thing that checks it move together.
|
|
112
|
+
import {
|
|
113
|
+
OPERATOR_PLANE_RETENTION_MS,
|
|
114
|
+
OPERATOR_PLANE_COUNTERS,
|
|
115
|
+
JOURNAL_KIND,
|
|
116
|
+
sessionLogRelPath, outcomeIsOwed } from "../../lib/assurance/operator-plane.mjs";
|
|
117
|
+
// The SHAPE of a machine narrating itself, as ONE shared vocabulary with TWO
|
|
118
|
+
// thresholds. This file asks the inbound question ("is this peer's message a
|
|
119
|
+
// notice?") and needs only the predicate, because it has already established the
|
|
120
|
+
// sender is a seat; `deliver.mjs` asks the outbound one ("may this body leave?")
|
|
121
|
+
// and needs the apparatus subject too, because it does not know who wrote it.
|
|
122
|
+
// Both read the same lists, so a phrase added for one is live for both. The
|
|
123
|
+
// difference in thresholds is argued at `hasStatusPredicate`.
|
|
124
|
+
import { hasStatusPredicate } from "../../lib/comms/machine-status.mjs";
|
|
101
125
|
import { isRetiredFirstReply, firstReplyShape, REACTIONS, statesNoReplyOwed } from "../../lib/assurance/first-reply.mjs";
|
|
102
126
|
// The unit of acknowledgement is the CONVERSATION, not the message: four
|
|
103
127
|
// messages in a row earn ONE first response that was written with all four in
|
|
@@ -192,8 +216,13 @@ export const STALE_AFTER_MS = num(process.env.ASSURANCE_STALE_AFTER_MS, 50 * 60_
|
|
|
192
216
|
* so an unbounded silent retry loop is itself a defect. */
|
|
193
217
|
export const RETRY_MAX = num(process.env.ASSURANCE_RETRY_MAX, 1);
|
|
194
218
|
|
|
195
|
-
/** Retained after discharge so an audit can see what was promised and when.
|
|
196
|
-
|
|
219
|
+
/** Retained after discharge so an audit can see what was promised and when.
|
|
220
|
+
*
|
|
221
|
+
* The default comes from `lib/assurance/operator-plane.mjs` rather than being
|
|
222
|
+
* restated here: the reader's `orphaned` and `stale` verdicts are statements
|
|
223
|
+
* ABOUT this window ("the row outlived its record"), so the window and the thing
|
|
224
|
+
* that reasons about it must not be two literals that agree by comment. */
|
|
225
|
+
const CLOSED_RETENTION_MS = num(process.env.ASSURANCE_RETENTION_MS, OPERATOR_PLANE_RETENTION_MS);
|
|
197
226
|
|
|
198
227
|
/**
|
|
199
228
|
* How many times the sweep will re-attempt an acknowledgement that would not
|
|
@@ -507,7 +536,13 @@ export function looksLikePeerOutcomeNotice(item) {
|
|
|
507
536
|
if (REQUEST_MARKER.test(content)) return false;
|
|
508
537
|
if (IMPERATIVE_OPENER.test(content)) return false;
|
|
509
538
|
if (!/\b(i|my|i'?ve|i'?m)\b/i.test(content)) return false;
|
|
510
|
-
|
|
539
|
+
// ONE VOCABULARY, SHARED WITH THE OUTBOUND REFUSAL. This predicate list used
|
|
540
|
+
// to be a literal here, duplicating the one the outbound screen needs. The two
|
|
541
|
+
// now read `lib/comms/machine-status.mjs`, so the shape this seat REFUSES TO
|
|
542
|
+
// SEND and the shape it REFUSES TO ACK are the same shape by construction —
|
|
543
|
+
// the `narratorPatterns` convention: emitter and recogniser are edited
|
|
544
|
+
// together or not at all. A phrase added for one is live for both.
|
|
545
|
+
return hasStatusPredicate(content);
|
|
511
546
|
}
|
|
512
547
|
|
|
513
548
|
/**
|
|
@@ -596,10 +631,18 @@ export function narratorPatterns() {
|
|
|
596
631
|
// rest — if this gate cannot recognise it, a peer's refusal notice arrives
|
|
597
632
|
// as an ordinary ask, earns an ack and spawns a session behind it, which is
|
|
598
633
|
// the loop that produced 142 of 188 notices on 2026-09-30.
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
634
|
+
// BOTH CAUSE SHAPES, because the cause clause is now a MAPPED phrase and an
|
|
635
|
+
// unmapped code yields no clause at all. A pattern derived only from the
|
|
636
|
+
// clause-bearing shape has a mandatory `\(…\)` in it and silently stops
|
|
637
|
+
// matching every refusal whose code this tree does not know — which is the
|
|
638
|
+
// half the self-echo gate most needs, since an unknown code is what an older
|
|
639
|
+
// or newer peer emits.
|
|
640
|
+
...[true, false].flatMap((permanent) =>
|
|
641
|
+
[undefined, CAUSE].flatMap((finalText) =>
|
|
642
|
+
[CAUSE, ""].map((causePhrase) =>
|
|
643
|
+
composeRefusedSend({}, { why: { code: CAUSE, permanent }, finalText, causePhrase })),
|
|
644
|
+
),
|
|
645
|
+
),
|
|
603
646
|
];
|
|
604
647
|
// GLOBAL, not first-occurrence. `String.prototype.replace` with a string
|
|
605
648
|
// pattern substitutes once, so a template carrying the sentinel twice — a
|
|
@@ -1247,6 +1290,27 @@ export async function generateFailureNotice(rec, o = {}, opts = {}) {
|
|
|
1247
1290
|
|
|
1248
1291
|
const run = typeof opts.runImpl === "function" ? opts.runImpl : runAckModel;
|
|
1249
1292
|
const timeoutMs = Number.isFinite(opts.timeoutMs) ? opts.timeoutMs : NOTICE_GEN_TIMEOUT_MS;
|
|
1293
|
+
// NOBODY READS THIS, SO DO NOT PAY A MODEL TO WRITE IT.
|
|
1294
|
+
//
|
|
1295
|
+
// Every notice kind is operator-plane (`lib/assurance/notice-plane.mjs`), so
|
|
1296
|
+
// the destination of this text is a log line and a journal row — and a model
|
|
1297
|
+
// spawn to make a log line read nicely is a cold `claude --print` per dead
|
|
1298
|
+
// obligation per sweep, bounded at NOTICE_GEN_TIMEOUT_MS, for prose no person
|
|
1299
|
+
// will see. The deterministic sentence is the right artifact for the operator
|
|
1300
|
+
// plane and always was.
|
|
1301
|
+
//
|
|
1302
|
+
// `degraded` is FALSE: nothing is broken, this is the design. The counter
|
|
1303
|
+
// records the real reason so the `notice.generation_failed` rates stay
|
|
1304
|
+
// meaningful rather than being swamped by a path that was never going to run.
|
|
1305
|
+
//
|
|
1306
|
+
// The test seam still overrides, so `notice-emission.test.mjs` can keep
|
|
1307
|
+
// exercising the generator's guards (`claimsSuccess`, `inventsFacts`, the
|
|
1308
|
+
// register) against a stub — those guards remain the defence for any future
|
|
1309
|
+
// caller that is genuinely human-facing.
|
|
1310
|
+
if (typeof opts.runImpl !== "function" && !mayReachHuman(kind)) {
|
|
1311
|
+
counters.bump("notice.generation_skipped", { notice: kind, reason: "operator-plane" });
|
|
1312
|
+
return { text: degradedText(), generated: false, degraded: false, cause, plane: NOTICE_PLANE.OPERATOR };
|
|
1313
|
+
}
|
|
1250
1314
|
if (typeof opts.runImpl !== "function" && !noticeGenEnabled()) {
|
|
1251
1315
|
counters.bump("notice.generation_skipped", { notice: kind, reason: "disabled" });
|
|
1252
1316
|
return { text: degradedText(), generated: false, degraded: true, cause };
|
|
@@ -1272,7 +1336,7 @@ export async function generateFailureNotice(rec, o = {}, opts = {}) {
|
|
|
1272
1336
|
const verdict = noticeRejection(raw, { source: noticeSource(promptArgs) });
|
|
1273
1337
|
if (verdict.ok) return { text: verdict.text, generated: true, degraded: false, cause };
|
|
1274
1338
|
counters.bump("notice.generation_failed", { cause: `unusable:${verdict.reason}`, notice: kind });
|
|
1275
|
-
console.warn(`[assurance] ${kind} notice generation produced nothing usable [${verdict.reason}] —
|
|
1339
|
+
console.warn(`[assurance] ${kind} notice generation produced nothing usable [${verdict.reason}] — falling back to the plain sentence (a dead ask is never left unrecorded)`);
|
|
1276
1340
|
return { text: degradedText(), generated: false, degraded: true, cause, rejected: verdict.reason };
|
|
1277
1341
|
} catch (err) {
|
|
1278
1342
|
const msg = String((err && err.message) || "");
|
|
@@ -1280,7 +1344,7 @@ export async function generateFailureNotice(rec, o = {}, opts = {}) {
|
|
|
1280
1344
|
: /spawn error|ENOENT|EACCES/i.test(msg) ? "spawn_failed"
|
|
1281
1345
|
: "error";
|
|
1282
1346
|
counters.bump("notice.generation_failed", { cause: why, notice: kind });
|
|
1283
|
-
console.warn(`[assurance] ${kind} notice generation failed (${msg}) —
|
|
1347
|
+
console.warn(`[assurance] ${kind} notice generation failed (${msg}) — falling back to the plain sentence [cause:${why}]`);
|
|
1284
1348
|
return { text: degradedText(), generated: false, degraded: true, cause };
|
|
1285
1349
|
}
|
|
1286
1350
|
}
|
|
@@ -1314,6 +1378,174 @@ export function composeSilentSuccess(rec, o = {}) {
|
|
|
1314
1378
|
return `${head} I don't have a clean result to show you, though. Want me to run it again?`;
|
|
1315
1379
|
}
|
|
1316
1380
|
|
|
1381
|
+
// ── THE EXCERPT SHAPES ABOVE ARE RECOGNITION-ONLY NOW ────────────────────────
|
|
1382
|
+
//
|
|
1383
|
+
// `composeSilentSuccess` and `composeRefusedSend` still build their
|
|
1384
|
+
// "<head> Here's where I got to:\n\n<result>" shapes, because `narratorPatterns`
|
|
1385
|
+
// derives the self-echo templates from them and a seat that has not upgraded
|
|
1386
|
+
// still posts exactly those bytes; a receiving seat has to keep recognising them
|
|
1387
|
+
// as a notice and not dispatch them as an ask. THIS seat never sends either one:
|
|
1388
|
+
// `settleSession` holds the result text in the operator plane
|
|
1389
|
+
// (`holdResultForOwner`), and `deliver()` refuses any body that opens with
|
|
1390
|
+
// either head (`first-reply.opensAsRescueNotice`).
|
|
1391
|
+
|
|
1392
|
+
/** How much of a held result the needs-attention row carries for the owner. */
|
|
1393
|
+
export const HELD_RESULT_PREVIEW_LIMIT = 600;
|
|
1394
|
+
|
|
1395
|
+
/**
|
|
1396
|
+
* THE LATE-REPLY FALLBACK, WITH A RESULT: HELD FOR THE OWNER, NEVER POSTED.
|
|
1397
|
+
*
|
|
1398
|
+
* ── WHY THE RESULT TEXT IS NOT A REPLY ───────────────────────────────────────
|
|
1399
|
+
*
|
|
1400
|
+
* The rescue assumed that a session which exited 0 without a delivery receipt
|
|
1401
|
+
* had an answer stuck on the way out, and that its end-of-turn text WAS that
|
|
1402
|
+
* answer. The end-of-turn text is the session's report to the process that
|
|
1403
|
+
* spawned it. Sessions reply through their tools; what they print at the end is
|
|
1404
|
+
* an account of what they did — "Queued as FU-123 in state/queues/…, logged to
|
|
1405
|
+
* logs/audit/…, I'll check for a reply from a concurrent session" — written for
|
|
1406
|
+
* the operator. Measured over the window in which this rescue posted (29 posts,
|
|
1407
|
+
* five seats, two team channels, 2026-10-05 → 10-13): 23 of the 29 were a
|
|
1408
|
+
* session saying it had deliberately not replied, 25 carried the machine's own
|
|
1409
|
+
* paths, ids or session talk, and 4 read like an answer. The rescue was
|
|
1410
|
+
* publishing operator notes, under the seat's name, and overriding the
|
|
1411
|
+
* session's own decision not to speak in the process.
|
|
1412
|
+
*
|
|
1413
|
+
* `statesNoReplyOwed` was the attempt to tell the two apart by wording, and it
|
|
1414
|
+
* cannot: "sent no reply" is also what a stuck answer says. A body cannot be
|
|
1415
|
+
* judged by which function composed it, and a session's notes cannot be judged
|
|
1416
|
+
* by regex. So the rule is structural: a message to a person is composed by
|
|
1417
|
+
* something that addressed it to them. The end-of-turn text never was.
|
|
1418
|
+
*
|
|
1419
|
+
* ── WHAT HAPPENS INSTEAD, AND WHO SEES IT ────────────────────────────────────
|
|
1420
|
+
*
|
|
1421
|
+
* THE ROW `escalate` writes `needs-attention/<key>.json` on THIS seat —
|
|
1422
|
+
* the seat that wrote the text, which is the only one that could
|
|
1423
|
+
* clean up after a post (message deletion is author-only) — with
|
|
1424
|
+
* the cause, the length and a bounded preview of the held text.
|
|
1425
|
+
* The obligation closes `undeliverable`, which is OWED, so
|
|
1426
|
+
* `maestro attention list` shows it and the presence beat counts
|
|
1427
|
+
* it. `maestro attention show <key>` prints the preview and the
|
|
1428
|
+
* session-log pointer to the full text.
|
|
1429
|
+
* THE JOURNAL one operator line per (obligation, notice), once, via
|
|
1430
|
+
* `sayOutcomeNotice` → `recordOperatorNotice`.
|
|
1431
|
+
* THE COUNTER `assurance.result_held_for_owner`.
|
|
1432
|
+
*
|
|
1433
|
+
* Nothing goes to the conversation and nothing goes to any other seat. The
|
|
1434
|
+
* reply-shaped minority loses its automatic post; it is in the owner's queue
|
|
1435
|
+
* with its text, which is where a decision about a reply nobody composed
|
|
1436
|
+
* belongs.
|
|
1437
|
+
*
|
|
1438
|
+
* @param {object} o
|
|
1439
|
+
* @param {object} o.rec the obligation record
|
|
1440
|
+
* @param {object} o.a the settleSession arguments (key, deps)
|
|
1441
|
+
* @param {object} o.item the item snapshot
|
|
1442
|
+
* @param {number} o.now
|
|
1443
|
+
* @param {function} o.send the delivery impl (only reached if a notice were ever human-plane)
|
|
1444
|
+
* @param {string} o.notice NOTICE.SILENT_SUCCESS | NOTICE.SEND_REFUSED
|
|
1445
|
+
* @param {string} o.verdict the verdict string the caller returns
|
|
1446
|
+
* @param {string} o.finalText the session's end-of-turn text, non-empty
|
|
1447
|
+
* @param {string} o.cause the operator-facing cause label
|
|
1448
|
+
* @param {object} [o.noSend] the no-send verdict, for the refused branch
|
|
1449
|
+
* @returns {Promise<object>} the settleSession result
|
|
1450
|
+
*/
|
|
1451
|
+
async function holdResultForOwner(o) {
|
|
1452
|
+
const { rec, a, item, now, send, notice, verdict, finalText, cause } = o;
|
|
1453
|
+
const label = `${cause}; result text held for the owner, not posted`;
|
|
1454
|
+
rec.lastError = label;
|
|
1455
|
+
const res = await sayOutcomeNotice({
|
|
1456
|
+
rec,
|
|
1457
|
+
notice,
|
|
1458
|
+
// An OPERATOR sentence, deliberately not either composer: those apologise to
|
|
1459
|
+
// a person, and this line is for the owner reading the journal.
|
|
1460
|
+
text: `${cause}. The session's result text (${finalText.length} chars) is held for the owner on this seat and was not posted.`,
|
|
1461
|
+
item,
|
|
1462
|
+
kind: "notice",
|
|
1463
|
+
idempotencySuffix: `${notice === NOTICE.SEND_REFUSED ? "refused" : "rescue"}-${rec.attempts || 0}`,
|
|
1464
|
+
now,
|
|
1465
|
+
send,
|
|
1466
|
+
roomNoticeImpl: a.deps && a.deps.roomNoticeImpl,
|
|
1467
|
+
});
|
|
1468
|
+
escalate(rec, { failure: { label }, now, told: false, heldResult: finalText });
|
|
1469
|
+
closeObligation(a.key, {
|
|
1470
|
+
outcome: "undeliverable",
|
|
1471
|
+
now,
|
|
1472
|
+
note: `${label} (${finalText.length} chars in the operator plane); no conversation told`,
|
|
1473
|
+
});
|
|
1474
|
+
counters.bump("assurance.result_held_for_owner", { service: rec.service, notice });
|
|
1475
|
+
return {
|
|
1476
|
+
spoke: false,
|
|
1477
|
+
text: null,
|
|
1478
|
+
verdict,
|
|
1479
|
+
willRetry: false,
|
|
1480
|
+
plane: res && res.plane ? res.plane : NOTICE_PLANE.OPERATOR,
|
|
1481
|
+
held: true,
|
|
1482
|
+
...(o.noSend ? { noSend: o.noSend } : {}),
|
|
1483
|
+
};
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
/**
|
|
1487
|
+
* A machine send-block code → the one clause a PERSON can do something with.
|
|
1488
|
+
*
|
|
1489
|
+
* ── WHY A TABLE AND NOT THE RAW REASON ──────────────────────────────────────
|
|
1490
|
+
*
|
|
1491
|
+
* The previous version interpolated `why.reason` — the no-send receipt's own
|
|
1492
|
+
* `frame.error` string — straight into a body a human reads. Two things are
|
|
1493
|
+
* wrong with that, and the second one only became visible once this branch
|
|
1494
|
+
* landed:
|
|
1495
|
+
*
|
|
1496
|
+
* 1. It is machine noise. "FORBIDDEN_SCOPE: blocked by send-gate" is the
|
|
1497
|
+
* exact register this subsystem keeps being told to stop writing in.
|
|
1498
|
+
* 2. THE GATES' OWN REFUSAL MESSAGES ARE NOW AMONG THOSE REASONS. `deliver`
|
|
1499
|
+
* records `code:"OPERATOR_PLANE", error:'machine-status kind "failure"
|
|
1500
|
+
* may not be delivered to a conversation — route it to the operator
|
|
1501
|
+
* plane'`, and `noteNoSend` writes that verbatim into the receipt this
|
|
1502
|
+
* composer reads. So the fix for broken machine messages could compose a
|
|
1503
|
+
* broken machine message that quoted the gate which stopped it.
|
|
1504
|
+
*
|
|
1505
|
+
* The raw string is not lost and an operator never has to guess: it is in the
|
|
1506
|
+
* no-send receipt, in the `needs-attention` row and in the notice journal, which
|
|
1507
|
+
* is where it is read. The person gets what it means for them.
|
|
1508
|
+
*
|
|
1509
|
+
* THE DEFAULT IS NO CLAUSE AT ALL, not a fallback to the raw text. That is the
|
|
1510
|
+
* structural half: an unmapped or newly-invented code cannot reach a person as
|
|
1511
|
+
* machine prose, because this function has no path that returns `why.reason`.
|
|
1512
|
+
* Add a row here when a new code starts appearing in receipts.
|
|
1513
|
+
*
|
|
1514
|
+
* @type {Readonly<Record<string,string>>}
|
|
1515
|
+
*/
|
|
1516
|
+
export const SEND_BLOCK_PHRASES = Object.freeze({
|
|
1517
|
+
FORBIDDEN_SCOPE: "I don't have permission to post there",
|
|
1518
|
+
HOOK_BLOCKED: "the wording got stopped by our own outbound rules",
|
|
1519
|
+
BANNED_PHRASE: "the wording got stopped by our own outbound rules",
|
|
1520
|
+
FORBIDDEN: "I don't have permission to post there",
|
|
1521
|
+
403: "I don't have permission to post there",
|
|
1522
|
+
NOT_A_MEMBER: "I'm not in that conversation",
|
|
1523
|
+
CHANNEL_NOT_FOUND: "that conversation wasn't reachable from here",
|
|
1524
|
+
404: "that conversation wasn't reachable from here",
|
|
1525
|
+
UNAUTHORIZED: "my access to that workspace wasn't accepted",
|
|
1526
|
+
401: "my access to that workspace wasn't accepted",
|
|
1527
|
+
RATE_LIMITED: "the channel was rate-limiting me",
|
|
1528
|
+
429: "the channel was rate-limiting me",
|
|
1529
|
+
NETWORK: "the network dropped on the way out",
|
|
1530
|
+
ETIMEDOUT: "the network dropped on the way out",
|
|
1531
|
+
ECONNRESET: "the network dropped on the way out",
|
|
1532
|
+
});
|
|
1533
|
+
|
|
1534
|
+
/**
|
|
1535
|
+
* The human-facing cause clause for a blocked send, or "" when there isn't one.
|
|
1536
|
+
*
|
|
1537
|
+
* @param {object} why a `noSendFor` verdict (`{code, reason, permanent, …}`)
|
|
1538
|
+
* @returns {string} a phrase, never a machine string, possibly empty
|
|
1539
|
+
*/
|
|
1540
|
+
export function humanSendBlockPhrase(why = {}) {
|
|
1541
|
+
const code = why && why.code != null ? String(why.code).trim() : "";
|
|
1542
|
+
if (!code) return "";
|
|
1543
|
+
const key = code.toUpperCase();
|
|
1544
|
+
if (Object.prototype.hasOwnProperty.call(SEND_BLOCK_PHRASES, key)) return SEND_BLOCK_PHRASES[key];
|
|
1545
|
+
if (Object.prototype.hasOwnProperty.call(SEND_BLOCK_PHRASES, code)) return SEND_BLOCK_PHRASES[code];
|
|
1546
|
+
return "";
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1317
1549
|
/**
|
|
1318
1550
|
* The case `composeSilentSuccess` was apologising for by mistake: a send that
|
|
1319
1551
|
* WAS attempted and was STOPPED, with a recorded cause.
|
|
@@ -1347,11 +1579,14 @@ export function composeSilentSuccess(rec, o = {}) {
|
|
|
1347
1579
|
*/
|
|
1348
1580
|
export function composeRefusedSend(rec, o = {}) {
|
|
1349
1581
|
const why = o.why || {};
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
//
|
|
1353
|
-
//
|
|
1354
|
-
|
|
1582
|
+
// `o.causePhrase` is a PATTERN SEAM, not an escape hatch for a caller. The
|
|
1583
|
+
// cause slot is a MAPPED phrase now, and `narratorPatterns` has to derive a
|
|
1584
|
+
// template carrying its regex sentinel in that slot — otherwise the self-echo
|
|
1585
|
+
// gate stops recognising this seat's own refusal notices, a peer's refusal
|
|
1586
|
+
// arrives as an ordinary ask, and the 142-of-188 loop restarts. Nothing on the
|
|
1587
|
+
// delivery path passes it; `assurance-no-send-gate.test.mjs` pins that no raw
|
|
1588
|
+
// receipt string can reach the body through `why`.
|
|
1589
|
+
const cause = typeof o.causePhrase === "string" ? o.causePhrase : humanSendBlockPhrase(why);
|
|
1355
1590
|
const head = `Couldn't get this out to you — the send was blocked${cause ? ` (${cause})` : ""}.`;
|
|
1356
1591
|
// Never promise a retry this path does not perform. The silent-success branch
|
|
1357
1592
|
// has no retry loop — that lives in the failure branch — so a permanent cause
|
|
@@ -1582,7 +1817,13 @@ export const NOTICE = Object.freeze({
|
|
|
1582
1817
|
FAILURE_RETRYING: "failure:retrying",
|
|
1583
1818
|
FAILURE_FINAL: "failure:final",
|
|
1584
1819
|
INTERRUPTED: "interrupted",
|
|
1820
|
+
// The two answer-or-nothing ids. Both of these have a HUMAN shape (the one
|
|
1821
|
+
// that carries the session's result text, delivered as a `reply`) and an
|
|
1822
|
+
// OPERATOR shape (the same composer with no result text, which is a template
|
|
1823
|
+
// about this machine's own run). Only the operator shape reaches here, and
|
|
1824
|
+
// `notice-plane.NOTICE_PLANES` declares it operator like all the rest.
|
|
1585
1825
|
SILENT_SUCCESS: "silent-success",
|
|
1826
|
+
SEND_REFUSED: "send-refused",
|
|
1586
1827
|
});
|
|
1587
1828
|
|
|
1588
1829
|
/**
|
|
@@ -1650,9 +1891,186 @@ export function markNoticeSaid(rec, notice, now) {
|
|
|
1650
1891
|
* @param {function} [o.roomNoticeImpl] test seam for the room-scoped dedupe
|
|
1651
1892
|
* @returns {Promise<{sent:boolean, reason:string}>}
|
|
1652
1893
|
*/
|
|
1894
|
+
/**
|
|
1895
|
+
* THE OPERATOR SINK — where a machine's status about itself actually goes.
|
|
1896
|
+
*
|
|
1897
|
+
* This replaces the send, it does not accompany it. The three things an operator
|
|
1898
|
+
* needs are all written here or beside it:
|
|
1899
|
+
*
|
|
1900
|
+
* THE ROW `escalate` writes `needs-attention/<key>.json` on every one of
|
|
1901
|
+
* these failures, carrying the key, the sender, the cause, the
|
|
1902
|
+
* trace id and the session id — strictly more than the DM ever
|
|
1903
|
+
* said. That is unchanged and is written by the caller.
|
|
1904
|
+
* THE JOURNAL this function appends one line per (obligation, notice) to
|
|
1905
|
+
* `state/obligations/notice-journal.jsonl`, so the sequence of
|
|
1906
|
+
* what a seat decided about a dead ask is readable in order
|
|
1907
|
+
* rather than reconstructed from a row that only holds the last
|
|
1908
|
+
* state. Append-only, one line, no prose a model wrote.
|
|
1909
|
+
* THE COUNTER `assurance.notice_to_operator`, broken out by notice and
|
|
1910
|
+
* reason, so the volume is a number on a health surface instead
|
|
1911
|
+
* of a thing a person notices in their DMs.
|
|
1912
|
+
*
|
|
1913
|
+
* ONCE-ONLY, DURABLY, AND IT IS THE LATCH THAT ALREADY EXISTED.
|
|
1914
|
+
* `noticeAlreadySaid` / `markNoticeSaid` are keyed on
|
|
1915
|
+
* (channel, obligationKey, notice) and INHERITED across a retry that reopens the
|
|
1916
|
+
* same key (`openAndAcknowledge`), so one dead ask yields one journal line
|
|
1917
|
+
* however many sweeps, retries and daemon restarts touch it. Checking it here
|
|
1918
|
+
* rather than only on the human path is what makes "the same thing twice" a
|
|
1919
|
+
* property of the sink too — the owner received the same sentence twice eleven
|
|
1920
|
+
* minutes apart, and a sink that appends twice would have reproduced that defect
|
|
1921
|
+
* in the log it replaced the DM with.
|
|
1922
|
+
*
|
|
1923
|
+
* THE TEXT IS KEPT, deliberately. It is the deterministic sentence (no model is
|
|
1924
|
+
* spawned for an operator line — see `generateFailureNotice`), and it is the
|
|
1925
|
+
* most readable one-line statement of what happened that this code can produce.
|
|
1926
|
+
* An operator reading the journal wants it. A person in a conversation does not,
|
|
1927
|
+
* which is the entire distinction this module draws.
|
|
1928
|
+
*
|
|
1929
|
+
* Never throws: a sink that can fail the sweep is worse than a sink that loses a
|
|
1930
|
+
* line, and the escalation row is written independently by the caller.
|
|
1931
|
+
*
|
|
1932
|
+
* @param {object} o {rec, notice, text, now, reason}
|
|
1933
|
+
* @returns {{sent:false, reason:string, plane:string, journalled:boolean}}
|
|
1934
|
+
*/
|
|
1935
|
+
/**
|
|
1936
|
+
* Has this notice already been JOURNALLED for this attempt?
|
|
1937
|
+
*
|
|
1938
|
+
* ── THE LATCH THE JOURNAL NEEDS IS NOT THE LATCH THE PERSON NEEDED ──────────
|
|
1939
|
+
*
|
|
1940
|
+
* `noticeAlreadySaid` is keyed `(notice, channel)` and inherited across the retry
|
|
1941
|
+
* that reopens the same key. That is exactly right for a MESSAGE: one dead ask,
|
|
1942
|
+
* one sentence, however many sweeps touch it. Applied to the journal it caps the
|
|
1943
|
+
* file at two lines per debt ever (`failure:retrying`, then `failure:final`) — so
|
|
1944
|
+
* the one file whose whole purpose is chronology could not record a sequence, and
|
|
1945
|
+
* "this ask has failed 31 times in 6 hours" was unsayable by construction.
|
|
1946
|
+
*
|
|
1947
|
+
* Nobody is being told anything twice any more, so the journal's duplicate
|
|
1948
|
+
* problem is a different one: the SAME EVENT reported twice (a second settle of
|
|
1949
|
+
* the same session, a second sweep in the same tick). The discriminator for that
|
|
1950
|
+
* is the attempt — `noteSession` bumps `attemptsTotal` once per dispatched
|
|
1951
|
+
* session, and a retry is a new session. So the journal dedupes on
|
|
1952
|
+
* `(notice, channel, attempt)`: the same event stays one line, a genuine second
|
|
1953
|
+
* failure appends one.
|
|
1954
|
+
*
|
|
1955
|
+
* Stored in its own map rather than reusing `rec.notices`, because that map is the
|
|
1956
|
+
* human-path latch and widening its key would quietly re-enable a repeat notice
|
|
1957
|
+
* on every retry.
|
|
1958
|
+
*/
|
|
1959
|
+
function journalId(rec, notice, attempt) {
|
|
1960
|
+
const ch = String((rec && rec.channel) || "").trim().toLowerCase() || "unknown";
|
|
1961
|
+
return `${notice}|${ch}|${Number.isFinite(attempt) ? attempt : 0}`;
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1964
|
+
function recordOperatorNotice(o = {}) {
|
|
1965
|
+
const { rec, notice, text, now } = o;
|
|
1966
|
+
const already = noticeAlreadySaid(rec, notice);
|
|
1967
|
+
const attempt = Math.max(0, Number((rec && (rec.attemptsTotal ?? rec.attempts)) || 0));
|
|
1968
|
+
if (!rec.journalled || typeof rec.journalled !== "object") rec.journalled = {};
|
|
1969
|
+
const jid = journalId(rec, notice, attempt);
|
|
1970
|
+
const journalledAlready = Number.isFinite(rec.journalled[jid]);
|
|
1971
|
+
const stamp = Number.isFinite(now) ? now : Date.now();
|
|
1972
|
+
if (!already || !journalledAlready) {
|
|
1973
|
+
// THE SINK PERSISTS ITS OWN LATCH. `markNoticeSaid` mutates `rec` and
|
|
1974
|
+
// documents that the caller writes it — which was true of the failure branch
|
|
1975
|
+
// (`writeRecord` immediately after `sayOutcomeNotice`) and NOT of the paths
|
|
1976
|
+
// that go straight to `closeObligation`, because `closeObligation` re-reads
|
|
1977
|
+
// the record from disk and the in-memory mark is lost. The once-only property
|
|
1978
|
+
// is this function's guarantee, so this function makes it durable rather than
|
|
1979
|
+
// depending on what each caller happens to do next.
|
|
1980
|
+
if (!already) markNoticeSaid(rec, notice, now);
|
|
1981
|
+
if (!journalledAlready) rec.journalled[jid] = stamp;
|
|
1982
|
+
try { writeRecord(rec); } catch { /* best-effort: a lost latch is one extra row, not a lost debt */ }
|
|
1983
|
+
}
|
|
1984
|
+
counters.bump(OPERATOR_PLANE_COUNTERS.notice, {
|
|
1985
|
+
notice,
|
|
1986
|
+
reason: o.reason || "declared-operator",
|
|
1987
|
+
repeat: already ? "suppressed" : "first",
|
|
1988
|
+
});
|
|
1989
|
+
if (journalledAlready) {
|
|
1990
|
+
return { sent: false, reason: "operator-plane-already-recorded", plane: NOTICE_PLANE.OPERATOR, journalled: false };
|
|
1991
|
+
}
|
|
1992
|
+
let journalled = false;
|
|
1993
|
+
try {
|
|
1994
|
+
const line = JSON.stringify({
|
|
1995
|
+
at: new Date(stamp).toISOString(),
|
|
1996
|
+
// `kind` separates a notice from the terminal `resolution` row added below.
|
|
1997
|
+
// Absent on every line written before this change, and the reader treats a
|
|
1998
|
+
// line without one as a notice, which is the only kind there was.
|
|
1999
|
+
kind: JOURNAL_KIND.NOTICE,
|
|
2000
|
+
notice,
|
|
2001
|
+
key: rec && rec.key,
|
|
2002
|
+
service: rec && rec.service,
|
|
2003
|
+
channel: rec && rec.channel,
|
|
2004
|
+
sender: (rec && rec.sender) || null,
|
|
2005
|
+
cause: (rec && rec.lastError) || null,
|
|
2006
|
+
attempts: (rec && rec.attempts) || 0,
|
|
2007
|
+
// ── WHAT THE JOURNAL WAS MISSING TO STAND ON ITS OWN ──────────────────
|
|
2008
|
+
//
|
|
2009
|
+
// `attemptsTotal` survives the retry that reopens this key, so the journal
|
|
2010
|
+
// can show a sequence; `attempt` is which one THIS line is about, and is
|
|
2011
|
+
// the key the duplicate latch above uses. `summary` and `itemPreview` are
|
|
2012
|
+
// the ask: they lived only on the needs-attention row and on the obligation
|
|
2013
|
+
// record, and the record is pruned at three days — which is why 72 of 78
|
|
2014
|
+
// rows on this machine could no longer say what had failed. `exitCode` and
|
|
2015
|
+
// `sessionLog` are the only two facts in the whole plane that distinguish a
|
|
2016
|
+
// timeout from a crash from a clean exit that never spoke, against a cause
|
|
2017
|
+
// field that reads the single label `no_outcome` on 94% of the population.
|
|
2018
|
+
attemptsTotal: Math.max(0, Number((rec && rec.attemptsTotal) || 0)),
|
|
2019
|
+
attempt,
|
|
2020
|
+
repeat: already === true,
|
|
2021
|
+
summary: (rec && rec.summary) || null,
|
|
2022
|
+
itemPreview: rec && rec.item && typeof rec.item.content === "string" ? rec.item.content.slice(0, 300) : null,
|
|
2023
|
+
exitCode: rec && Number.isFinite(rec.exitCode) ? rec.exitCode : null,
|
|
2024
|
+
sessionId: (rec && rec.sessionId) || null,
|
|
2025
|
+
sessionLog: sessionLogRelPath(rec && rec.sessionId),
|
|
2026
|
+
traceId: (rec && rec.traceId) || null,
|
|
2027
|
+
// The plane, on every row, so a reader of the journal never has to wonder
|
|
2028
|
+
// whether this one also went to a person. None of them do.
|
|
2029
|
+
plane: NOTICE_PLANE.OPERATOR,
|
|
2030
|
+
requesterTold: false,
|
|
2031
|
+
statusLine: typeof text === "string" ? text.slice(0, 400) : null,
|
|
2032
|
+
});
|
|
2033
|
+
appendFileSync(join(obligationDir(), "notice-journal.jsonl"), `${line}\n`);
|
|
2034
|
+
journalled = true;
|
|
2035
|
+
} catch (err) {
|
|
2036
|
+
console.error(`[assurance] could not journal ${notice} for ${rec && rec.key}: ${err && err.message}`);
|
|
2037
|
+
}
|
|
2038
|
+
console.log(`[assurance] ${notice} for ${rec && rec.key} → operator plane (${o.reason || "declared-operator"}); no conversation was told. Cause: ${(rec && rec.lastError) || "unknown"}`);
|
|
2039
|
+
return { sent: false, reason: "operator-plane", plane: NOTICE_PLANE.OPERATOR, journalled };
|
|
2040
|
+
}
|
|
2041
|
+
|
|
1653
2042
|
async function sayOutcomeNotice(o) {
|
|
1654
2043
|
const { rec, notice, text, item, kind, now } = o;
|
|
1655
2044
|
|
|
2045
|
+
// ── THE PLANE DECISION, BEFORE EVERYTHING ELSE ───────────────────────────
|
|
2046
|
+
//
|
|
2047
|
+
// This is the question this subsystem never asked, and the reason the owner
|
|
2048
|
+
// received a new variant after each of three fixes. Everything below this
|
|
2049
|
+
// point — two dedupes, a document-surface test, a room claim — is about HOW
|
|
2050
|
+
// MANY copies of a message a person gets and WHAT WORDS it wears. The answer
|
|
2051
|
+
// is that they get none: status about this machine's own apparatus is not a
|
|
2052
|
+
// message to a person. The argument, the measured costs, and the narrow case
|
|
2053
|
+
// that does NOT survive are in `lib/assurance/notice-plane.mjs`. Read that
|
|
2054
|
+
// before changing this line.
|
|
2055
|
+
//
|
|
2056
|
+
// WHY HERE AND NOT ONLY AT THE TRANSPORT. `deliver()` refuses these kinds too,
|
|
2057
|
+
// and that is the backstop for a caller nobody has written yet. But a refusal
|
|
2058
|
+
// at the transport produces the DEBRIS of an attempted send: a no-send
|
|
2059
|
+
// receipt, a `{sent:false}` the caller reads as "the room heard nothing", and
|
|
2060
|
+
// an escalation row whose `requesterWasTold:false` is right by accident.
|
|
2061
|
+
// Deciding here writes the operator plane a deliberate record instead.
|
|
2062
|
+
//
|
|
2063
|
+
// WHY IT IS ABOVE THE DOCUMENT TEST. The document test is a SURFACE test, and
|
|
2064
|
+
// it was the last iteration of the old axis — withhold on one more surface. It
|
|
2065
|
+
// is kept below rather than deleted because it answers a different question
|
|
2066
|
+
// and would be the only guard still standing if anyone ever registers a
|
|
2067
|
+
// human-facing notice in `NOTICE_PLANES`. For every notice that exists today,
|
|
2068
|
+
// this block returns first.
|
|
2069
|
+
const plane = noticePlane(notice);
|
|
2070
|
+
if (!plane.human) {
|
|
2071
|
+
return recordOperatorNotice({ rec, notice, text, now, reason: plane.reason });
|
|
2072
|
+
}
|
|
2073
|
+
|
|
1656
2074
|
// ── NO OUTCOME NOTICE REACHES A DOCUMENT, AT THE CHOKEPOINT ───────────────
|
|
1657
2075
|
//
|
|
1658
2076
|
// 2.18.30 put this test in `settleSession`, which covered the failed,
|
|
@@ -1835,8 +2253,28 @@ export async function openAndAcknowledge(a = {}) {
|
|
|
1835
2253
|
// Which outcome notices have already been delivered for this (channel,
|
|
1836
2254
|
// key). Also inherited, for the same reason.
|
|
1837
2255
|
notices: {},
|
|
2256
|
+
// Which (notice, channel, attempt) triples have already reached the operator
|
|
2257
|
+
// JOURNAL. Separate from `notices` because that map is the human-path latch
|
|
2258
|
+
// and is deliberately coarser — see `journalId`. Inherited below for the same
|
|
2259
|
+
// reason `notices` is: without it a retry re-journals the whole lifecycle.
|
|
2260
|
+
journalled: {},
|
|
1838
2261
|
attempts: 0,
|
|
2262
|
+
// ── THE ATTEMPT COUNT THAT SURVIVES A RETRY ───────────────────────────────
|
|
2263
|
+
//
|
|
2264
|
+
// `attempts` is reset by construction: a retry CLOSES this record and the next
|
|
2265
|
+
// poll opens a fresh one under the same key, so `attempts` counts the sessions
|
|
2266
|
+
// of the CURRENT record only. On this machine it read 1 on all 78
|
|
2267
|
+
// needs-attention rows and on 21 of 23 records, while `assurance.retrying` had
|
|
2268
|
+
// been bumped 198 times — the plane's retry history was a flat lie, and the
|
|
2269
|
+
// journal copied the same field and inherited it. `attemptsTotal` is inherited
|
|
2270
|
+
// across the reopen, so "this ask has been attempted four times" is sayable.
|
|
2271
|
+
attemptsTotal: 0,
|
|
1839
2272
|
lastError: null,
|
|
2273
|
+
// Which exit code the last session ended on, when one is known. `cause` is a
|
|
2274
|
+
// four-value enum of which 94% of the observed population is `no_outcome`;
|
|
2275
|
+
// this is one of the two fields that can tell those apart (the other is the
|
|
2276
|
+
// session log path, derived from `sessionId`).
|
|
2277
|
+
exitCode: null,
|
|
1840
2278
|
sessionId: null,
|
|
1841
2279
|
traceId: a.traceId || item.trace_id || null,
|
|
1842
2280
|
daemonPid: process.pid,
|
|
@@ -1852,6 +2290,19 @@ export async function openAndAcknowledge(a = {}) {
|
|
|
1852
2290
|
rec.interimSaid = existing.interimSaid === true;
|
|
1853
2291
|
rec.interimAt = existing.interimAt || null;
|
|
1854
2292
|
rec.notices = (existing.notices && typeof existing.notices === "object") ? { ...existing.notices } : {};
|
|
2293
|
+
rec.journalled = (existing.journalled && typeof existing.journalled === "object") ? { ...existing.journalled } : {};
|
|
2294
|
+
// THE COUNT, NOT JUST THE LATCHES. The previous pass inherited `interimSaid`,
|
|
2295
|
+
// `interimAt` and `notices` — everything needed to stay QUIET across a retry —
|
|
2296
|
+
// and nothing needed to say how many times the work had been tried. So the
|
|
2297
|
+
// plane could not answer "was it retried?", which is one of the two questions
|
|
2298
|
+
// an operator opens it with. Carried across with the latches, in the same block,
|
|
2299
|
+
// so the next author who adds an inherited field sees the count beside them.
|
|
2300
|
+
rec.attemptsTotal = Math.max(
|
|
2301
|
+
0,
|
|
2302
|
+
Number(existing.attemptsTotal || 0),
|
|
2303
|
+
Number(existing.attempts || 0),
|
|
2304
|
+
);
|
|
2305
|
+
rec.exitCode = Number.isFinite(existing.exitCode) ? existing.exitCode : null;
|
|
1855
2306
|
}
|
|
1856
2307
|
rec.tier = tier;
|
|
1857
2308
|
rec.daemonPid = process.pid;
|
|
@@ -1981,6 +2432,10 @@ export function noteSession(key, sessionId, o = {}) {
|
|
|
1981
2432
|
if (!rec) return false;
|
|
1982
2433
|
rec.sessionId = sessionId || null;
|
|
1983
2434
|
rec.attempts = (rec.attempts || 0) + 1;
|
|
2435
|
+
// The count that survives the retry's reopen. Bumped in lockstep here — the one
|
|
2436
|
+
// place an attempt is counted at all — so the two cannot disagree about this
|
|
2437
|
+
// record while differing about its history, which is the whole point.
|
|
2438
|
+
rec.attemptsTotal = Math.max(Number(rec.attemptsTotal || 0), 0) + 1;
|
|
1984
2439
|
rec.lastAttemptAt = Number.isFinite(o.now) ? o.now : Date.now();
|
|
1985
2440
|
rec.daemonPid = process.pid;
|
|
1986
2441
|
return writeRecord(rec);
|
|
@@ -2043,6 +2498,9 @@ export function closeObligation(key, o = {}) {
|
|
|
2043
2498
|
rec.state = o.outcome || "answered";
|
|
2044
2499
|
rec.closedAt = Number.isFinite(o.now) ? o.now : Date.now();
|
|
2045
2500
|
if (o.note) rec.closeNote = o.note;
|
|
2501
|
+
// THE PLANE LEARNS HOW THIS ENDED. Before `writeRecord`, so the one write
|
|
2502
|
+
// covers both the record and the resolution latch.
|
|
2503
|
+
resolveOperatorPlane(rec);
|
|
2046
2504
|
const wrote = writeRecord(rec);
|
|
2047
2505
|
// Every close path in this module funnels through here — answered, silent
|
|
2048
2506
|
// success, failed, undeliverable, stale — so this is the one place the
|
|
@@ -2060,7 +2518,8 @@ export function closeObligation(key, o = {}) {
|
|
|
2060
2518
|
*
|
|
2061
2519
|
* ok:true + a receipt exists → the session spoke. Close the debt quietly;
|
|
2062
2520
|
* do NOT add a message (that would be spam).
|
|
2063
|
-
* ok:true + no receipt → silent success.
|
|
2521
|
+
* ok:true + no receipt → silent success. Hold the result text for the
|
|
2522
|
+
* owner in the operator plane; post nothing.
|
|
2064
2523
|
* ok:false + transient + tries → say what broke and that a retry is running.
|
|
2065
2524
|
* ok:false + terminal → say it failed, stop, escalate durably.
|
|
2066
2525
|
*
|
|
@@ -2079,6 +2538,19 @@ export async function settleSession(a = {}) {
|
|
|
2079
2538
|
if (!rec) return { spoke: false, text: null, verdict: "no-obligation", willRetry: false };
|
|
2080
2539
|
const now = Number.isFinite(a.now) ? a.now : Date.now();
|
|
2081
2540
|
if (a.sessionId && rec.sessionId !== a.sessionId) { rec.sessionId = a.sessionId; writeRecord(rec); }
|
|
2541
|
+
// THE EXIT CODE, ON THE RECORD, BEFORE ANY BRANCH READS IT. This is the only
|
|
2542
|
+
// discriminating fact this call site is handed: `classifyFailure` collapses every
|
|
2543
|
+
// terminal state into a four-value label, and 94% of the plane's population reads
|
|
2544
|
+
// the single one of those that says least (`no_outcome` — "the session produced
|
|
2545
|
+
// nothing"). An exit code separates a timeout from a crash from a kill. Stamped
|
|
2546
|
+
// for every settle, not only the failures, so a close that later resolves a row
|
|
2547
|
+
// carries it too.
|
|
2548
|
+
//
|
|
2549
|
+
// WHAT IS STILL MISSING AND IS NOT INVENTED HERE: a stderr tail. The dispatcher's
|
|
2550
|
+
// `onClose` passes `{ok, code, sessionId, stdout}` and no stderr, so there is
|
|
2551
|
+
// nothing to record; the session log path (derived from `sessionId`) is the
|
|
2552
|
+
// pointer at it instead. Capturing stderr is a dispatcher change, not a plane one.
|
|
2553
|
+
if (Number.isFinite(a.code) && rec.exitCode !== a.code) { rec.exitCode = Number(a.code); writeRecord(rec); }
|
|
2082
2554
|
const item = rec.item || {};
|
|
2083
2555
|
const send = (a.deps && a.deps.deliverImpl) || deliver;
|
|
2084
2556
|
|
|
@@ -2133,10 +2605,11 @@ export async function settleSession(a = {}) {
|
|
|
2133
2605
|
// unchanged — including for every seat that has not upgraded,
|
|
2134
2606
|
// which writes no rows at all.
|
|
2135
2607
|
//
|
|
2136
|
-
// FAILS OPEN TO THE
|
|
2137
|
-
// corrupt ledger line: all of them land on
|
|
2138
|
-
//
|
|
2139
|
-
//
|
|
2608
|
+
// FAILS OPEN TO THE OWNER, NOT TO THE ROOM. A null verdict, a thrown
|
|
2609
|
+
// reader, a corrupt ledger line: all of them land on the silent-success
|
|
2610
|
+
// path below, which holds whatever the session produced in the operator
|
|
2611
|
+
// plane (`holdResultForOwner`) or records that it produced nothing. Never a
|
|
2612
|
+
// silently dropped debt, and never the session's notes in a conversation.
|
|
2140
2613
|
const noSeen = (a.deps && a.deps.noSendForImpl) || noSendFor;
|
|
2141
2614
|
let why = null;
|
|
2142
2615
|
try {
|
|
@@ -2157,21 +2630,50 @@ export async function settleSession(a = {}) {
|
|
|
2157
2630
|
// shared room when nobody asked, and this is not narration — it is the
|
|
2158
2631
|
// one line that says a message this seat owed was stopped, plus its cause.
|
|
2159
2632
|
// Withholding it is how a blocked send lane stays invisible for days.
|
|
2633
|
+
//
|
|
2634
|
+
// ── TOLD TO THE OWNER, NOT THE ROOM ────────────────────────────────────
|
|
2635
|
+
//
|
|
2636
|
+
// That paragraph is true of the FAULT, not of the sentence. The fault has
|
|
2637
|
+
// to reach somebody; nothing says that somebody is the person in the
|
|
2638
|
+
// conversation. Without result text, all a room would get is "Couldn't get
|
|
2639
|
+
// this out to you — the send was blocked (…). That won't clear on its
|
|
2640
|
+
// own": a template about this machine's own plumbing, unactionable by the
|
|
2641
|
+
// reasoning in `notice-plane.mjs`. WITH result text it is no better — the
|
|
2642
|
+
// text is the session's end-of-turn report, not a reply (see
|
|
2643
|
+
// `holdResultForOwner`). Both shapes go to the operator plane, which gets
|
|
2644
|
+
// strictly more than the sentence ever did, including the raw code.
|
|
2160
2645
|
const finalText = safeResultText(a.stdout);
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2646
|
+
if (!finalText) {
|
|
2647
|
+
const res = await sayOutcomeNotice({
|
|
2648
|
+
rec, notice: NOTICE.SEND_REFUSED, text: composeRefusedSend(rec, { why }),
|
|
2649
|
+
item, kind: "notice", idempotencySuffix: `refused-${rec.attempts || 0}`, now, send,
|
|
2650
|
+
roomNoticeImpl: a.deps && a.deps.roomNoticeImpl,
|
|
2651
|
+
});
|
|
2652
|
+
rec.lastError = `send refused: ${why.code || "no code"}${why.reason ? ` — ${String(why.reason).slice(0, 200)}` : ""}`;
|
|
2653
|
+
escalate(rec, { failure: { label: rec.lastError }, now, told: false });
|
|
2654
|
+
closeObligation(a.key, {
|
|
2655
|
+
outcome: "undeliverable",
|
|
2656
|
+
now,
|
|
2657
|
+
note: `send refused (${why.code || "no code"}${why.surface ? ` on ${why.surface}` : ""}) and the session produced no result text; recorded in the operator plane, no conversation told`,
|
|
2658
|
+
});
|
|
2659
|
+
counters.bump("assurance.no_send_refused", { service: rec.service, code: why.code || "none", plane: "operator" });
|
|
2660
|
+
return { spoke: false, text: null, verdict: "send-refused", willRetry: false, noSend: why, plane: res.plane };
|
|
2661
|
+
}
|
|
2662
|
+
// WITH result text it is held too, never posted. The session tried to
|
|
2663
|
+
// send, and something — very often the outbound gate itself — refused
|
|
2664
|
+
// it. Re-posting the session's end-of-turn text under a "the send was
|
|
2665
|
+
// blocked" head is how a gate refusal got routed around: the notes that
|
|
2666
|
+
// follow the head are the session's account of its own work, frequently
|
|
2667
|
+
// carrying the very content the gate stopped. See `holdResultForOwner`.
|
|
2668
|
+
counters.bump("assurance.no_send_refused", { service: rec.service, code: why.code || "none", plane: "operator" });
|
|
2669
|
+
return holdResultForOwner({
|
|
2670
|
+
rec, a, item, now, send,
|
|
2671
|
+
notice: NOTICE.SEND_REFUSED,
|
|
2672
|
+
verdict: "send-refused",
|
|
2673
|
+
finalText,
|
|
2674
|
+
cause: `send refused: ${why.code || "no code"}${why.surface ? ` on ${why.surface}` : ""}${why.reason ? ` — ${String(why.reason).slice(0, 200)}` : ""}`,
|
|
2675
|
+
noSend: why,
|
|
2172
2676
|
});
|
|
2173
|
-
counters.bump("assurance.no_send_refused", { service: rec.service, code: why.code || "none" });
|
|
2174
|
-
return { spoke: !!(res && res.sent), text, verdict: "send-refused", willRetry: false, noSend: why };
|
|
2175
2677
|
}
|
|
2176
2678
|
|
|
2177
2679
|
// A CLEAN SILENT EXIT OUTSIDE A DM CLOSES QUIETLY, LIKE A DOC COMMENT.
|
|
@@ -2225,22 +2727,64 @@ export async function settleSession(a = {}) {
|
|
|
2225
2727
|
return { spoke: false, text: null, verdict: "silent-success", willRetry: false };
|
|
2226
2728
|
}
|
|
2227
2729
|
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2730
|
+
// ── NO RESULT TEXT IS NOT AN ANSWER, SO IT IS NOT A REPLY ───────────────
|
|
2731
|
+
//
|
|
2732
|
+
// `composeSilentSuccess` has two shapes and NEITHER is a reply. The one WITH
|
|
2733
|
+
// the session's result text was treated as one until 2026-10-13 and is now
|
|
2734
|
+
// held for the owner below (`holdResultForOwner`). WITHOUT it, the whole
|
|
2735
|
+
// message is:
|
|
2736
|
+
//
|
|
2737
|
+
// Finished this but didn't get a reply out to you — my fault. I don't
|
|
2738
|
+
// have a clean result to show you, though. Want me to run it again?
|
|
2739
|
+
//
|
|
2740
|
+
// which is "my session finished, you got nothing, shall I try again" — a
|
|
2741
|
+
// template about this machine's own run with no work product in it, i.e.
|
|
2742
|
+
// precisely the category the owner complained about twice. This repo's own
|
|
2743
|
+
// canonical recogniser agrees: `first-reply.isAgentStatusNotice` returns
|
|
2744
|
+
// true for it, and `STATUS_NOTICE_PATTERNS` lists its head as a frozen
|
|
2745
|
+
// literal of the status-notice family.
|
|
2746
|
+
//
|
|
2747
|
+
// WHY IT SURVIVED THE FIRST PASS OF THIS FIX, which is worth recording
|
|
2748
|
+
// because it is the same shape as every earlier miss: it is delivered with
|
|
2749
|
+
// `kind:"reply"`, so the origin gate (keyed on `kind`) never sees it, and it
|
|
2750
|
+
// names no apparatus noun, so the shape gate does not either. Both gates are
|
|
2751
|
+
// about machine STATUS and this is machine status wearing a reply's kind.
|
|
2752
|
+
// The first pass then PINNED it as a reply that must go out, on the reading
|
|
2753
|
+
// that `composeSilentSuccess` "carries the answer" — true of one of its two
|
|
2754
|
+
// shapes. The split is by what the body carries, never by which function
|
|
2755
|
+
// composed it.
|
|
2756
|
+
if (!finalText) {
|
|
2757
|
+
const res = await sayOutcomeNotice({
|
|
2758
|
+
rec, notice: NOTICE.SILENT_SUCCESS, text: composeSilentSuccess(rec, {}),
|
|
2759
|
+
item, kind: "notice", idempotencySuffix: `rescue-${rec.attempts || 0}`, now, send,
|
|
2760
|
+
roomNoticeImpl: a.deps && a.deps.roomNoticeImpl,
|
|
2761
|
+
});
|
|
2762
|
+
rec.lastError = rec.lastError || "session exited 0, sent nothing, produced no result text";
|
|
2763
|
+
escalate(rec, { failure: { label: "session exited 0 with no result text and nothing sent" }, now, told: false });
|
|
2764
|
+
closeObligation(a.key, {
|
|
2765
|
+
outcome: "undeliverable",
|
|
2766
|
+
now,
|
|
2767
|
+
note: "session exited 0 without speaking and produced no result text; recorded in the operator plane, no conversation told",
|
|
2768
|
+
});
|
|
2769
|
+
counters.bump("assurance.silent_success_to_operator", { service: rec.service });
|
|
2770
|
+
return { spoke: false, text: null, verdict: "silent-success", willRetry: false, plane: res.plane };
|
|
2771
|
+
}
|
|
2772
|
+
|
|
2773
|
+
// ── WITH RESULT TEXT IT IS STILL NOT A REPLY ─────────────────────────────
|
|
2774
|
+
//
|
|
2775
|
+
// This branch used to post `composeSilentSuccess(rec, {finalText})` as a
|
|
2776
|
+
// `kind:"reply"`, on the reading that the result text IS the answer. It is
|
|
2777
|
+
// the session's end-of-turn report to this daemon, and posting it put this
|
|
2778
|
+
// machine's file paths, queue ids and session talk into team channels under
|
|
2779
|
+
// the seat's name. The text is held for the owner; nothing is sent. See
|
|
2780
|
+
// `holdResultForOwner` for the measurement and the design.
|
|
2781
|
+
return holdResultForOwner({
|
|
2782
|
+
rec, a, item, now, send,
|
|
2783
|
+
notice: NOTICE.SILENT_SUCCESS,
|
|
2784
|
+
verdict: "silent-success",
|
|
2785
|
+
finalText,
|
|
2786
|
+
cause: "session exited 0 without sending a reply",
|
|
2242
2787
|
});
|
|
2243
|
-
return { spoke: !!(res && res.sent), text, verdict: "silent-success", willRetry: false };
|
|
2244
2788
|
}
|
|
2245
2789
|
|
|
2246
2790
|
// ── failure ────────────────────────────────────────────────────────────
|
|
@@ -2333,6 +2877,103 @@ function safeResultText(stdout) {
|
|
|
2333
2877
|
* TOLD — that already happened above — plus a trace an operator can sweep. This
|
|
2334
2878
|
* file is that trace.
|
|
2335
2879
|
*/
|
|
2880
|
+
/**
|
|
2881
|
+
* THE TERMINAL FACT. Tell the operator plane how a debt it escalated ended.
|
|
2882
|
+
*
|
|
2883
|
+
* ── WHY THE PLANE COULD NOT ANSWER "DID IT EVENTUALLY WORK?" ─────────────────
|
|
2884
|
+
*
|
|
2885
|
+
* `escalate` is the only writer of a needs-attention row and it runs only on
|
|
2886
|
+
* failure. `recordOperatorNotice` is never called on a success path. So nothing
|
|
2887
|
+
* in the plane had a `status`, a `resolved` or a `closedAt` — the eventual outcome
|
|
2888
|
+
* existed ONLY in the obligation record's `state`/`closeNote`, which is deleted at
|
|
2889
|
+
* CLOSED_RETENTION_MS. Measured on this machine: 78 rows, none resolved, and 72
|
|
2890
|
+
* (92%) with no record left to join to. An operator opening the plane in week six
|
|
2891
|
+
* saw 78 rows of which 72 could not be worked and 0 could be closed.
|
|
2892
|
+
*
|
|
2893
|
+
* SCOPED TO WHAT THE PLANE ALREADY KNOWS ABOUT, deliberately. `closeObligation`
|
|
2894
|
+
* is on every discharge path including the common happy one, and appending a
|
|
2895
|
+
* journal row for every answered ask would turn the operator journal into a
|
|
2896
|
+
* firehose of successes — which is the volume failure this subsystem has already
|
|
2897
|
+
* made twice. So this fires only when the plane has previously said something
|
|
2898
|
+
* about this key: a needs-attention row exists, or a notice was journalled.
|
|
2899
|
+
*
|
|
2900
|
+
* IDEMPOTENT. `rec.planeResolvedAt` latches it, because `closeObligation` can be
|
|
2901
|
+
* reached twice for one debt (a settle racing the sweep) and two resolution rows
|
|
2902
|
+
* would make the reader's `closedAt` depend on which one it read last.
|
|
2903
|
+
*
|
|
2904
|
+
* Never throws: a close that fails because of its own bookkeeping is strictly
|
|
2905
|
+
* worse than a plane that misses one terminal row.
|
|
2906
|
+
*
|
|
2907
|
+
* @param {object} rec the obligation record, already carrying its terminal state;
|
|
2908
|
+
* MUTATED (`planeResolvedAt`), caller persists
|
|
2909
|
+
* @returns {boolean} whether anything was written
|
|
2910
|
+
*/
|
|
2911
|
+
export function resolveOperatorPlane(rec) {
|
|
2912
|
+
if (!rec || !rec.key) return false;
|
|
2913
|
+
if (Number.isFinite(rec.planeResolvedAt)) return false;
|
|
2914
|
+
const at = Number.isFinite(rec.closedAt) ? rec.closedAt : Date.now();
|
|
2915
|
+
let rowPath = null;
|
|
2916
|
+
let row = null;
|
|
2917
|
+
try {
|
|
2918
|
+
rowPath = join(obligationDir(), "needs-attention", `${sanitiseKey(rec.key)}.json`);
|
|
2919
|
+
row = JSON.parse(readFileSync(rowPath, "utf-8"));
|
|
2920
|
+
} catch { row = null; }
|
|
2921
|
+
const journalled = !!(rec.journalled && typeof rec.journalled === "object" && Object.keys(rec.journalled).length)
|
|
2922
|
+
|| !!(rec.notices && typeof rec.notices === "object" && Object.keys(rec.notices).length);
|
|
2923
|
+
if (!row && !journalled) return false;
|
|
2924
|
+
|
|
2925
|
+
const outcome = String(rec.state || "closed");
|
|
2926
|
+
rec.planeResolvedAt = at;
|
|
2927
|
+
let wrote = false;
|
|
2928
|
+
|
|
2929
|
+
// THE ROW, REWRITTEN IN PLACE. It is the surface an operator lands on, so it
|
|
2930
|
+
// must be able to say "done" without a join — that join is the thing that is
|
|
2931
|
+
// already dead for 92% of the population.
|
|
2932
|
+
if (row && rowPath) {
|
|
2933
|
+
try {
|
|
2934
|
+
writeFileSync(rowPath, JSON.stringify({
|
|
2935
|
+
...row,
|
|
2936
|
+
outcome,
|
|
2937
|
+
resolvedAt: new Date(at).toISOString(),
|
|
2938
|
+
closeNote: rec.closeNote || row.closeNote || null,
|
|
2939
|
+
attemptsTotal: Math.max(0, Number(rec.attemptsTotal || rec.attempts || row.attemptsTotal || 0)),
|
|
2940
|
+
exitCode: Number.isFinite(rec.exitCode) ? rec.exitCode : (Number.isFinite(row.exitCode) ? row.exitCode : null),
|
|
2941
|
+
}, null, 2));
|
|
2942
|
+
wrote = true;
|
|
2943
|
+
} catch (err) {
|
|
2944
|
+
console.error(`[assurance] could not resolve the needs-attention row for ${rec.key}: ${err && err.message}`);
|
|
2945
|
+
}
|
|
2946
|
+
}
|
|
2947
|
+
|
|
2948
|
+
// THE JOURNAL, APPENDED TO. The row holds only the last state; the journal is
|
|
2949
|
+
// the ordered record, and a sequence that never terminates is a sequence a
|
|
2950
|
+
// reader has to guess the end of.
|
|
2951
|
+
try {
|
|
2952
|
+
appendFileSync(join(obligationDir(), "notice-journal.jsonl"), `${JSON.stringify({
|
|
2953
|
+
at: new Date(at).toISOString(),
|
|
2954
|
+
kind: JOURNAL_KIND.RESOLUTION,
|
|
2955
|
+
notice: null,
|
|
2956
|
+
key: rec.key,
|
|
2957
|
+
service: rec.service || null,
|
|
2958
|
+
channel: rec.channel || null,
|
|
2959
|
+
sender: rec.sender || null,
|
|
2960
|
+
summary: rec.summary || null,
|
|
2961
|
+
outcome,
|
|
2962
|
+
closeNote: rec.closeNote || null,
|
|
2963
|
+
attemptsTotal: Math.max(0, Number(rec.attemptsTotal || rec.attempts || 0)),
|
|
2964
|
+
sessionId: rec.sessionId || null,
|
|
2965
|
+
traceId: rec.traceId || null,
|
|
2966
|
+
plane: NOTICE_PLANE.OPERATOR,
|
|
2967
|
+
requesterTold: false,
|
|
2968
|
+
})}\n`);
|
|
2969
|
+
wrote = true;
|
|
2970
|
+
} catch (err) {
|
|
2971
|
+
console.error(`[assurance] could not journal the resolution of ${rec.key}: ${err && err.message}`);
|
|
2972
|
+
}
|
|
2973
|
+
counters.bump(OPERATOR_PLANE_COUNTERS.resolved, { outcome, service: rec.service || "unknown" });
|
|
2974
|
+
return wrote;
|
|
2975
|
+
}
|
|
2976
|
+
|
|
2336
2977
|
export function escalate(rec, o = {}) {
|
|
2337
2978
|
try {
|
|
2338
2979
|
const dir = join(obligationDir(), "needs-attention");
|
|
@@ -2354,9 +2995,50 @@ export function escalate(rec, o = {}) {
|
|
|
2354
2995
|
// than leaving them to infer it from `cause`.
|
|
2355
2996
|
configFault: rec.configFault || null,
|
|
2356
2997
|
attempts: rec.attempts || 0,
|
|
2998
|
+
// ── THE THREE FIELDS A READER NEEDED AND THE ROW DID NOT HAVE ──────────
|
|
2999
|
+
//
|
|
3000
|
+
// `attemptsTotal` because `attempts` read 1 on all 78 rows on this machine
|
|
3001
|
+
// while `assurance.retrying` had been bumped 198 times: a retry CLOSES the
|
|
3002
|
+
// record and the next poll opens a fresh one, which inherited the notice
|
|
3003
|
+
// latch but not the attempt count, so the row's own retry history was a lie.
|
|
3004
|
+
// `exitCode` and `sessionLog` because `cause` is a four-value enum of which
|
|
3005
|
+
// 94% is `no_outcome` — "the session produced nothing" — and nothing in the
|
|
3006
|
+
// plane distinguished a timeout from a crash from a kill from a clean exit
|
|
3007
|
+
// that never spoke.
|
|
3008
|
+
attemptsTotal: Math.max(0, Number(rec.attemptsTotal || rec.attempts || 0)),
|
|
3009
|
+
exitCode: Number.isFinite(rec.exitCode) ? rec.exitCode : null,
|
|
2357
3010
|
sessionId: rec.sessionId || null,
|
|
3011
|
+
sessionLog: sessionLogRelPath(rec.sessionId),
|
|
2358
3012
|
traceId: rec.traceId || null,
|
|
2359
3013
|
itemPreview: rec.item ? rec.item.content : null,
|
|
3014
|
+
// ── A HELD RESULT, FOR THE OWNER ───────────────────────────────────────
|
|
3015
|
+
//
|
|
3016
|
+
// Present only when the late-reply fallback held a session's end-of-turn
|
|
3017
|
+
// text instead of posting it (`holdResultForOwner`). The length is exact;
|
|
3018
|
+
// the preview is bounded and whitespace-collapsed, and the full text is at
|
|
3019
|
+
// `sessionLog`. It stays on this seat: the presence beat lifts counts off
|
|
3020
|
+
// the box, never row text.
|
|
3021
|
+
...(typeof o.heldResult === "string" && o.heldResult.trim()
|
|
3022
|
+
? {
|
|
3023
|
+
heldResultChars: o.heldResult.length,
|
|
3024
|
+
heldResultPreview: (() => {
|
|
3025
|
+
const flat = o.heldResult.replace(/\s+/g, " ").trim();
|
|
3026
|
+
return flat.length > HELD_RESULT_PREVIEW_LIMIT ? `${flat.slice(0, HELD_RESULT_PREVIEW_LIMIT).trimEnd()}…` : flat;
|
|
3027
|
+
})(),
|
|
3028
|
+
}
|
|
3029
|
+
: {}),
|
|
3030
|
+
// ── THE TERMINAL FACT, DECLARED AS ABSENT ─────────────────────────────
|
|
3031
|
+
//
|
|
3032
|
+
// `escalate` is the only writer of a row and it runs only on failure, so
|
|
3033
|
+
// before this change there was no `outcome`, `resolved` or `closedAt` field
|
|
3034
|
+
// anywhere in the plane and every row was permanently "unknown" — which is
|
|
3035
|
+
// how 78 of them accumulated with none ever marked done. `closeObligation`
|
|
3036
|
+
// now stamps these (see `resolveOperatorPlane`); writing them as null here
|
|
3037
|
+
// is what makes "not yet resolved" a stated fact rather than a missing key
|
|
3038
|
+
// the reader has to guess the meaning of.
|
|
3039
|
+
outcome: null,
|
|
3040
|
+
resolvedAt: null,
|
|
3041
|
+
closeNote: null,
|
|
2360
3042
|
}, null, 2));
|
|
2361
3043
|
console.error(`[assurance] ESCALATED ${rec.key} — ${o.failure ? o.failure.label : rec.lastError}; requester told: ${!!o.told}`);
|
|
2362
3044
|
return true;
|
|
@@ -2971,9 +3653,120 @@ async function _sweepObligations(a = {}) {
|
|
|
2971
3653
|
}
|
|
2972
3654
|
|
|
2973
3655
|
pruneObligations({ now });
|
|
3656
|
+
pruneNoticeJournal({ now });
|
|
3657
|
+
pruneNeedsAttention({ now });
|
|
2974
3658
|
return stats;
|
|
2975
3659
|
}
|
|
2976
3660
|
|
|
3661
|
+
/**
|
|
3662
|
+
* Trim RESOLVED needs-attention rows past the retention window.
|
|
3663
|
+
*
|
|
3664
|
+
* ── RETENTION WAS INVERTED, AND THAT IS WHY 92% OF THE PLANE IS DEAD ─────────
|
|
3665
|
+
*
|
|
3666
|
+
* The journal — the only ordered record, the thing this subsystem defends as what
|
|
3667
|
+
* replaced the DM — is trimmed at CLOSED_RETENTION_MS. The rows, which hold no
|
|
3668
|
+
* sequence and are overwritten per key, were NEVER pruned in production: the only
|
|
3669
|
+
* code that cleared one was the `_resetObligations` test seam. So on this machine
|
|
3670
|
+
* there were 78 rows spanning five weeks and 23 obligation records, and 72 of the
|
|
3671
|
+
* rows (92%) had outlived the record they must join to — unworkable by
|
|
3672
|
+
* construction, and indistinguishable from live incidents to anything reading the
|
|
3673
|
+
* directory.
|
|
3674
|
+
*
|
|
3675
|
+
* ONLY RESOLVED ROWS GO, and this is the conservative half of the fix. A row whose
|
|
3676
|
+
* debt never reached a terminal state is the ONE artefact that says a person asked
|
|
3677
|
+
* and got nothing; deleting it to tidy a directory would destroy the only evidence
|
|
3678
|
+
* of the failure this plane exists to hold. Those are kept forever and the reader
|
|
3679
|
+
* labels them `unresolvable` with the reason, which is the honest rendering of a
|
|
3680
|
+
* row that cannot be joined. Pruning them is a policy question about how long an
|
|
3681
|
+
* unacknowledged dead ask is worth keeping, and nobody has answered it.
|
|
3682
|
+
*
|
|
3683
|
+
* Never throws, for the same reason as the other two prunes.
|
|
3684
|
+
*
|
|
3685
|
+
* @param {object} [o] `{now}`
|
|
3686
|
+
* @returns {number} rows removed
|
|
3687
|
+
*/
|
|
3688
|
+
export function pruneNeedsAttention(o = {}) {
|
|
3689
|
+
const now = Number.isFinite(o.now) ? o.now : Date.now();
|
|
3690
|
+
const dir = join(obligationDir(), "needs-attention");
|
|
3691
|
+
let names = [];
|
|
3692
|
+
try { names = readdirSync(dir); } catch { return 0; }
|
|
3693
|
+
let removed = 0;
|
|
3694
|
+
for (const name of names) {
|
|
3695
|
+
if (!name.endsWith(".json")) continue;
|
|
3696
|
+
let row = null;
|
|
3697
|
+
try { row = JSON.parse(readFileSync(join(dir, name), "utf-8")); } catch { continue; }
|
|
3698
|
+
if (!row || typeof row !== "object") continue;
|
|
3699
|
+
const outcome = row.outcome == null ? "" : String(row.outcome).trim();
|
|
3700
|
+
if (!outcome) continue; // unresolved: the evidence stays
|
|
3701
|
+
// AND AN OWED ASK IS EVIDENCE TOO. "failed"/"undeliverable" mean the machine
|
|
3702
|
+
// gave up and the person never got an answer — only "answered" settles. The
|
|
3703
|
+
// first version of this guard checked only that an outcome EXISTED, which
|
|
3704
|
+
// meant every escalated dead ask became prunable at 3 days. Measured: a dead
|
|
3705
|
+
// ask written through the real path was deleted by `pruneNeedsAttention({now:
|
|
3706
|
+
// now + 4d})`. Before this prune existed those rows survived indefinitely,
|
|
3707
|
+
// which is the only reason the 78 unanswered asks were discoverable at all.
|
|
3708
|
+
if (outcomeIsOwed(outcome)) continue;
|
|
3709
|
+
const at = Date.parse(row.resolvedAt || row.failedAt || "");
|
|
3710
|
+
if (!Number.isFinite(at) || now - at < CLOSED_RETENTION_MS) continue;
|
|
3711
|
+
try { unlinkSync(join(dir, name)); removed++; } catch { /* best-effort */ }
|
|
3712
|
+
}
|
|
3713
|
+
if (removed) counters.bump(OPERATOR_PLANE_COUNTERS.pruned, { removed: String(removed) });
|
|
3714
|
+
return removed;
|
|
3715
|
+
}
|
|
3716
|
+
|
|
3717
|
+
/**
|
|
3718
|
+
* Trim the operator notice journal to the retention window.
|
|
3719
|
+
*
|
|
3720
|
+
* WHY THIS EXISTS AND WHY IT IS A REWRITE RATHER THAN A DELETE. The journal
|
|
3721
|
+
* replaced a message, so it had better not have the defect the message had: it
|
|
3722
|
+
* is append-only, one line per dead ask, deliberately NOT coalesced (an operator
|
|
3723
|
+
* who sees one row of twenty has been told nineteen lies by omission). That
|
|
3724
|
+
* makes it the only file in this subsystem that grows without bound, on every
|
|
3725
|
+
* seat, forever — and `pruneReceipts` and `pruneObligations` both exist because
|
|
3726
|
+
* an untrimmed log is the next thing someone deletes wholesale. A log nobody
|
|
3727
|
+
* reads and nobody trims gets truncated by a stranger.
|
|
3728
|
+
*
|
|
3729
|
+
* Retention is the OBLIGATION window (`CLOSED_RETENTION_MS`), not the receipts'
|
|
3730
|
+
* seven days: a journal line is about a debt, and once the debt itself has left
|
|
3731
|
+
* the ledger the line has nothing to join to.
|
|
3732
|
+
*
|
|
3733
|
+
* Rewrites in place, atomically, and never throws. A journal that can fail the
|
|
3734
|
+
* sweep is worse than a journal that keeps a few stale lines.
|
|
3735
|
+
*
|
|
3736
|
+
* @param {object} [o] `{now}`
|
|
3737
|
+
* @returns {number} lines dropped
|
|
3738
|
+
*/
|
|
3739
|
+
export function pruneNoticeJournal(o = {}) {
|
|
3740
|
+
const now = Number.isFinite(o.now) ? o.now : Date.now();
|
|
3741
|
+
const p = join(obligationDir(), "notice-journal.jsonl");
|
|
3742
|
+
let raw;
|
|
3743
|
+
try { raw = readFileSync(p, "utf-8"); }
|
|
3744
|
+
catch { return 0; } // absent is the normal case on a seat that has had no failures
|
|
3745
|
+
const lines = raw.split("\n").filter((l) => l.trim());
|
|
3746
|
+
const keep = [];
|
|
3747
|
+
let dropped = 0;
|
|
3748
|
+
for (const line of lines) {
|
|
3749
|
+
let at = NaN;
|
|
3750
|
+
try { at = Date.parse(JSON.parse(line).at); } catch { at = NaN; }
|
|
3751
|
+
// A line whose stamp cannot be read is dropped, not kept forever: it is a
|
|
3752
|
+
// corrupt row in a file whose only purpose is chronology.
|
|
3753
|
+
if (!Number.isFinite(at)) { dropped++; continue; }
|
|
3754
|
+
if (now - at >= CLOSED_RETENTION_MS) { dropped++; continue; }
|
|
3755
|
+
keep.push(line);
|
|
3756
|
+
}
|
|
3757
|
+
if (!dropped) return 0;
|
|
3758
|
+
try {
|
|
3759
|
+
if (!keep.length) { unlinkSync(p); return dropped; }
|
|
3760
|
+
const tmp = `${p}.tmp`;
|
|
3761
|
+
writeFileSync(tmp, `${keep.join("\n")}\n`);
|
|
3762
|
+
renameSync(tmp, p);
|
|
3763
|
+
} catch (err) {
|
|
3764
|
+
console.error(`[assurance] could not prune the notice journal: ${err && err.message}`);
|
|
3765
|
+
return 0;
|
|
3766
|
+
}
|
|
3767
|
+
return dropped;
|
|
3768
|
+
}
|
|
3769
|
+
|
|
2977
3770
|
/** Delete discharged debts past retention. Open debts are NEVER pruned — an
|
|
2978
3771
|
* obligation may only leave the ledger by being discharged. */
|
|
2979
3772
|
export function pruneObligations(o = {}) {
|
|
@@ -2995,16 +3788,21 @@ export function pruneObligations(o = {}) {
|
|
|
2995
3788
|
return removed;
|
|
2996
3789
|
}
|
|
2997
3790
|
|
|
2998
|
-
/** Test seam: wipe the ledger — obligations, escalations
|
|
3791
|
+
/** Test seam: wipe the ledger — obligations, escalations, the room budget AND
|
|
3792
|
+
* the operator notice journal.
|
|
2999
3793
|
* The room budget lives in the same directory and is part of the same state:
|
|
3000
3794
|
* a reset that leaves it behind makes the next test's first interim depend on
|
|
3001
|
-
* which tests ran before it.
|
|
3795
|
+
* which tests ran before it. The notice journal is append-only and keyed by
|
|
3796
|
+
* obligation, and every test in a file typically settles the SAME item, so a
|
|
3797
|
+
* reset that leaves it behind makes "exactly one row" depend on which tests ran
|
|
3798
|
+
* first — the same hazard, in the file that replaced the message. */
|
|
3002
3799
|
export function _resetObligations() {
|
|
3003
3800
|
_sweepInFlight = false; // a test that threw mid-sweep must not disable the next one
|
|
3004
3801
|
for (const rec of listObligations()) {
|
|
3005
3802
|
try { unlinkSync(pathFor(rec.key)); } catch { /* */ }
|
|
3006
3803
|
}
|
|
3007
3804
|
try { unlinkSync(roomBudgetPath(AGENT_REPO_DIR)); } catch { /* absent is the normal case */ }
|
|
3805
|
+
try { unlinkSync(join(obligationDir(), "notice-journal.jsonl")); } catch { /* absent is the normal case */ }
|
|
3008
3806
|
try {
|
|
3009
3807
|
const dir = join(obligationDir(), "needs-attention");
|
|
3010
3808
|
for (const f of readdirSync(dir)) { try { unlinkSync(join(dir, f)); } catch { /* */ } }
|
|
@@ -3012,6 +3810,9 @@ export function _resetObligations() {
|
|
|
3012
3810
|
}
|
|
3013
3811
|
|
|
3014
3812
|
export default {
|
|
3813
|
+
pruneNoticeJournal,
|
|
3814
|
+
pruneNeedsAttention,
|
|
3815
|
+
resolveOperatorPlane,
|
|
3015
3816
|
shouldAcknowledge,
|
|
3016
3817
|
senderLooksLikeAgentSeat,
|
|
3017
3818
|
looksLikePeerAckShape,
|
|
@@ -3025,6 +3826,8 @@ export default {
|
|
|
3025
3826
|
_buildAckUserPromptForTest: buildAckUserPrompt,
|
|
3026
3827
|
composeSilentSuccess,
|
|
3027
3828
|
composeRefusedSend,
|
|
3829
|
+
SEND_BLOCK_PHRASES,
|
|
3830
|
+
humanSendBlockPhrase,
|
|
3028
3831
|
composeInterrupted,
|
|
3029
3832
|
looksLikePeerOutcomeNotice,
|
|
3030
3833
|
classifyFailure,
|