@cohortapp/agent-sdk 2.18.37 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +3 -0
- package/docs/guides/billing-console-keys.md +16 -3
- package/lib/assurance/first-reply.mjs +31 -2
- package/lib/assurance/operator-plane-store.mjs +164 -0
- package/lib/assurance/operator-plane.mjs +689 -0
- package/lib/budget-guard.mjs +24 -1
- package/lib/cli/attention.mjs +300 -0
- package/lib/cli/seat-auth.mjs +86 -6
- package/lib/comms/send-gate.mjs +309 -2
- package/lib/org/awareness.mjs +15 -2
- package/lib/org/clause-match.mjs +470 -0
- package/lib/org/client.mjs +27 -1
- package/lib/org/escalation-chain.mjs +134 -0
- package/lib/org/first-run.mjs +263 -0
- package/lib/org/genesis-contract.mjs +271 -0
- package/lib/org/governance-runtime.mjs +409 -0
- package/lib/org/governance-store.mjs +238 -0
- package/lib/org/governance.mjs +326 -0
- package/lib/org/mesh.mjs +135 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +6 -0
- package/lib/org/tool-surface.mjs +28 -0
- package/lib/setup/enroll-from-cohort.mjs +11 -0
- package/lib/telemetry/alerts.mjs +12 -0
- package/lib/telemetry/collect.mjs +237 -1
- package/package.json +1 -1
- package/scripts/daemon/agent-daemon.mjs +41 -2
- package/scripts/daemon/assurance.mjs +482 -55
- package/scripts/daemon/cadence-consumer.mjs +17 -0
- package/scripts/daemon/context-compiler.mjs +25 -0
- package/scripts/daemon/deliver.mjs +71 -5
- package/scripts/fleet/closing-sweep.mjs +266 -0
|
@@ -104,6 +104,16 @@ import {
|
|
|
104
104
|
// the operator plane, so a new notice kind is silent to people until someone
|
|
105
105
|
// deliberately says otherwise under that header.
|
|
106
106
|
import { noticePlane, mayReachHuman, NOTICE_PLANE } from "../../lib/assurance/notice-plane.mjs";
|
|
107
|
+
// THE OPERATOR PLANE'S READER OWNS THE SHAPES THIS FILE WRITES. The retention
|
|
108
|
+
// constant, the counter names, the journal row kinds and the session-log path are
|
|
109
|
+
// all read back by `lib/cli/attention.mjs` and by the presence beat, so they are
|
|
110
|
+
// declared once, in the pure module, and imported here rather than restated —
|
|
111
|
+
// `docs`/CLAUDE.md: a literal and the thing that checks it move together.
|
|
112
|
+
import {
|
|
113
|
+
OPERATOR_PLANE_RETENTION_MS,
|
|
114
|
+
OPERATOR_PLANE_COUNTERS,
|
|
115
|
+
JOURNAL_KIND,
|
|
116
|
+
sessionLogRelPath, outcomeIsOwed } from "../../lib/assurance/operator-plane.mjs";
|
|
107
117
|
// The SHAPE of a machine narrating itself, as ONE shared vocabulary with TWO
|
|
108
118
|
// thresholds. This file asks the inbound question ("is this peer's message a
|
|
109
119
|
// notice?") and needs only the predicate, because it has already established the
|
|
@@ -206,8 +216,13 @@ export const STALE_AFTER_MS = num(process.env.ASSURANCE_STALE_AFTER_MS, 50 * 60_
|
|
|
206
216
|
* so an unbounded silent retry loop is itself a defect. */
|
|
207
217
|
export const RETRY_MAX = num(process.env.ASSURANCE_RETRY_MAX, 1);
|
|
208
218
|
|
|
209
|
-
/** Retained after discharge so an audit can see what was promised and when.
|
|
210
|
-
|
|
219
|
+
/** Retained after discharge so an audit can see what was promised and when.
|
|
220
|
+
*
|
|
221
|
+
* The default comes from `lib/assurance/operator-plane.mjs` rather than being
|
|
222
|
+
* restated here: the reader's `orphaned` and `stale` verdicts are statements
|
|
223
|
+
* ABOUT this window ("the row outlived its record"), so the window and the thing
|
|
224
|
+
* that reasons about it must not be two literals that agree by comment. */
|
|
225
|
+
const CLOSED_RETENTION_MS = num(process.env.ASSURANCE_RETENTION_MS, OPERATOR_PLANE_RETENTION_MS);
|
|
211
226
|
|
|
212
227
|
/**
|
|
213
228
|
* How many times the sweep will re-attempt an acknowledgement that would not
|
|
@@ -1363,6 +1378,111 @@ export function composeSilentSuccess(rec, o = {}) {
|
|
|
1363
1378
|
return `${head} I don't have a clean result to show you, though. Want me to run it again?`;
|
|
1364
1379
|
}
|
|
1365
1380
|
|
|
1381
|
+
// ── THE EXCERPT SHAPES ABOVE ARE RECOGNITION-ONLY NOW ────────────────────────
|
|
1382
|
+
//
|
|
1383
|
+
// `composeSilentSuccess` and `composeRefusedSend` still build their
|
|
1384
|
+
// "<head> Here's where I got to:\n\n<result>" shapes, because `narratorPatterns`
|
|
1385
|
+
// derives the self-echo templates from them and a seat that has not upgraded
|
|
1386
|
+
// still posts exactly those bytes; a receiving seat has to keep recognising them
|
|
1387
|
+
// as a notice and not dispatch them as an ask. THIS seat never sends either one:
|
|
1388
|
+
// `settleSession` holds the result text in the operator plane
|
|
1389
|
+
// (`holdResultForOwner`), and `deliver()` refuses any body that opens with
|
|
1390
|
+
// either head (`first-reply.opensAsRescueNotice`).
|
|
1391
|
+
|
|
1392
|
+
/** How much of a held result the needs-attention row carries for the owner. */
|
|
1393
|
+
export const HELD_RESULT_PREVIEW_LIMIT = 600;
|
|
1394
|
+
|
|
1395
|
+
/**
|
|
1396
|
+
* THE LATE-REPLY FALLBACK, WITH A RESULT: HELD FOR THE OWNER, NEVER POSTED.
|
|
1397
|
+
*
|
|
1398
|
+
* ── WHY THE RESULT TEXT IS NOT A REPLY ───────────────────────────────────────
|
|
1399
|
+
*
|
|
1400
|
+
* The rescue assumed that a session which exited 0 without a delivery receipt
|
|
1401
|
+
* had an answer stuck on the way out, and that its end-of-turn text WAS that
|
|
1402
|
+
* answer. The end-of-turn text is the session's report to the process that
|
|
1403
|
+
* spawned it. Sessions reply through their tools; what they print at the end is
|
|
1404
|
+
* an account of what they did — "Queued as FU-123 in state/queues/…, logged to
|
|
1405
|
+
* logs/audit/…, I'll check for a reply from a concurrent session" — written for
|
|
1406
|
+
* the operator. Measured over the window in which this rescue posted (29 posts,
|
|
1407
|
+
* five seats, two team channels, 2026-10-05 → 10-13): 23 of the 29 were a
|
|
1408
|
+
* session saying it had deliberately not replied, 25 carried the machine's own
|
|
1409
|
+
* paths, ids or session talk, and 4 read like an answer. The rescue was
|
|
1410
|
+
* publishing operator notes, under the seat's name, and overriding the
|
|
1411
|
+
* session's own decision not to speak in the process.
|
|
1412
|
+
*
|
|
1413
|
+
* `statesNoReplyOwed` was the attempt to tell the two apart by wording, and it
|
|
1414
|
+
* cannot: "sent no reply" is also what a stuck answer says. A body cannot be
|
|
1415
|
+
* judged by which function composed it, and a session's notes cannot be judged
|
|
1416
|
+
* by regex. So the rule is structural: a message to a person is composed by
|
|
1417
|
+
* something that addressed it to them. The end-of-turn text never was.
|
|
1418
|
+
*
|
|
1419
|
+
* ── WHAT HAPPENS INSTEAD, AND WHO SEES IT ────────────────────────────────────
|
|
1420
|
+
*
|
|
1421
|
+
* THE ROW `escalate` writes `needs-attention/<key>.json` on THIS seat —
|
|
1422
|
+
* the seat that wrote the text, which is the only one that could
|
|
1423
|
+
* clean up after a post (message deletion is author-only) — with
|
|
1424
|
+
* the cause, the length and a bounded preview of the held text.
|
|
1425
|
+
* The obligation closes `undeliverable`, which is OWED, so
|
|
1426
|
+
* `maestro attention list` shows it and the presence beat counts
|
|
1427
|
+
* it. `maestro attention show <key>` prints the preview and the
|
|
1428
|
+
* session-log pointer to the full text.
|
|
1429
|
+
* THE JOURNAL one operator line per (obligation, notice), once, via
|
|
1430
|
+
* `sayOutcomeNotice` → `recordOperatorNotice`.
|
|
1431
|
+
* THE COUNTER `assurance.result_held_for_owner`.
|
|
1432
|
+
*
|
|
1433
|
+
* Nothing goes to the conversation and nothing goes to any other seat. The
|
|
1434
|
+
* reply-shaped minority loses its automatic post; it is in the owner's queue
|
|
1435
|
+
* with its text, which is where a decision about a reply nobody composed
|
|
1436
|
+
* belongs.
|
|
1437
|
+
*
|
|
1438
|
+
* @param {object} o
|
|
1439
|
+
* @param {object} o.rec the obligation record
|
|
1440
|
+
* @param {object} o.a the settleSession arguments (key, deps)
|
|
1441
|
+
* @param {object} o.item the item snapshot
|
|
1442
|
+
* @param {number} o.now
|
|
1443
|
+
* @param {function} o.send the delivery impl (only reached if a notice were ever human-plane)
|
|
1444
|
+
* @param {string} o.notice NOTICE.SILENT_SUCCESS | NOTICE.SEND_REFUSED
|
|
1445
|
+
* @param {string} o.verdict the verdict string the caller returns
|
|
1446
|
+
* @param {string} o.finalText the session's end-of-turn text, non-empty
|
|
1447
|
+
* @param {string} o.cause the operator-facing cause label
|
|
1448
|
+
* @param {object} [o.noSend] the no-send verdict, for the refused branch
|
|
1449
|
+
* @returns {Promise<object>} the settleSession result
|
|
1450
|
+
*/
|
|
1451
|
+
async function holdResultForOwner(o) {
|
|
1452
|
+
const { rec, a, item, now, send, notice, verdict, finalText, cause } = o;
|
|
1453
|
+
const label = `${cause}; result text held for the owner, not posted`;
|
|
1454
|
+
rec.lastError = label;
|
|
1455
|
+
const res = await sayOutcomeNotice({
|
|
1456
|
+
rec,
|
|
1457
|
+
notice,
|
|
1458
|
+
// An OPERATOR sentence, deliberately not either composer: those apologise to
|
|
1459
|
+
// a person, and this line is for the owner reading the journal.
|
|
1460
|
+
text: `${cause}. The session's result text (${finalText.length} chars) is held for the owner on this seat and was not posted.`,
|
|
1461
|
+
item,
|
|
1462
|
+
kind: "notice",
|
|
1463
|
+
idempotencySuffix: `${notice === NOTICE.SEND_REFUSED ? "refused" : "rescue"}-${rec.attempts || 0}`,
|
|
1464
|
+
now,
|
|
1465
|
+
send,
|
|
1466
|
+
roomNoticeImpl: a.deps && a.deps.roomNoticeImpl,
|
|
1467
|
+
});
|
|
1468
|
+
escalate(rec, { failure: { label }, now, told: false, heldResult: finalText });
|
|
1469
|
+
closeObligation(a.key, {
|
|
1470
|
+
outcome: "undeliverable",
|
|
1471
|
+
now,
|
|
1472
|
+
note: `${label} (${finalText.length} chars in the operator plane); no conversation told`,
|
|
1473
|
+
});
|
|
1474
|
+
counters.bump("assurance.result_held_for_owner", { service: rec.service, notice });
|
|
1475
|
+
return {
|
|
1476
|
+
spoke: false,
|
|
1477
|
+
text: null,
|
|
1478
|
+
verdict,
|
|
1479
|
+
willRetry: false,
|
|
1480
|
+
plane: res && res.plane ? res.plane : NOTICE_PLANE.OPERATOR,
|
|
1481
|
+
held: true,
|
|
1482
|
+
...(o.noSend ? { noSend: o.noSend } : {}),
|
|
1483
|
+
};
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1366
1486
|
/**
|
|
1367
1487
|
* A machine send-block code → the one clause a PERSON can do something with.
|
|
1368
1488
|
*
|
|
@@ -1376,7 +1496,7 @@ export function composeSilentSuccess(rec, o = {}) {
|
|
|
1376
1496
|
* 1. It is machine noise. "FORBIDDEN_SCOPE: blocked by send-gate" is the
|
|
1377
1497
|
* exact register this subsystem keeps being told to stop writing in.
|
|
1378
1498
|
* 2. THE GATES' OWN REFUSAL MESSAGES ARE NOW AMONG THOSE REASONS. `deliver`
|
|
1379
|
-
* records `code:"
|
|
1499
|
+
* records `code:"OPERATOR_PLANE", error:'machine-status kind "failure"
|
|
1380
1500
|
* may not be delivered to a conversation — route it to the operator
|
|
1381
1501
|
* plane'`, and `noteNoSend` writes that verbatim into the receipt this
|
|
1382
1502
|
* composer reads. So the fix for broken machine messages could compose a
|
|
@@ -1812,11 +1932,44 @@ export function markNoticeSaid(rec, notice, now) {
|
|
|
1812
1932
|
* @param {object} o {rec, notice, text, now, reason}
|
|
1813
1933
|
* @returns {{sent:false, reason:string, plane:string, journalled:boolean}}
|
|
1814
1934
|
*/
|
|
1935
|
+
/**
|
|
1936
|
+
* Has this notice already been JOURNALLED for this attempt?
|
|
1937
|
+
*
|
|
1938
|
+
* ── THE LATCH THE JOURNAL NEEDS IS NOT THE LATCH THE PERSON NEEDED ──────────
|
|
1939
|
+
*
|
|
1940
|
+
* `noticeAlreadySaid` is keyed `(notice, channel)` and inherited across the retry
|
|
1941
|
+
* that reopens the same key. That is exactly right for a MESSAGE: one dead ask,
|
|
1942
|
+
* one sentence, however many sweeps touch it. Applied to the journal it caps the
|
|
1943
|
+
* file at two lines per debt ever (`failure:retrying`, then `failure:final`) — so
|
|
1944
|
+
* the one file whose whole purpose is chronology could not record a sequence, and
|
|
1945
|
+
* "this ask has failed 31 times in 6 hours" was unsayable by construction.
|
|
1946
|
+
*
|
|
1947
|
+
* Nobody is being told anything twice any more, so the journal's duplicate
|
|
1948
|
+
* problem is a different one: the SAME EVENT reported twice (a second settle of
|
|
1949
|
+
* the same session, a second sweep in the same tick). The discriminator for that
|
|
1950
|
+
* is the attempt — `noteSession` bumps `attemptsTotal` once per dispatched
|
|
1951
|
+
* session, and a retry is a new session. So the journal dedupes on
|
|
1952
|
+
* `(notice, channel, attempt)`: the same event stays one line, a genuine second
|
|
1953
|
+
* failure appends one.
|
|
1954
|
+
*
|
|
1955
|
+
* Stored in its own map rather than reusing `rec.notices`, because that map is the
|
|
1956
|
+
* human-path latch and widening its key would quietly re-enable a repeat notice
|
|
1957
|
+
* on every retry.
|
|
1958
|
+
*/
|
|
1959
|
+
function journalId(rec, notice, attempt) {
|
|
1960
|
+
const ch = String((rec && rec.channel) || "").trim().toLowerCase() || "unknown";
|
|
1961
|
+
return `${notice}|${ch}|${Number.isFinite(attempt) ? attempt : 0}`;
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1815
1964
|
function recordOperatorNotice(o = {}) {
|
|
1816
1965
|
const { rec, notice, text, now } = o;
|
|
1817
1966
|
const already = noticeAlreadySaid(rec, notice);
|
|
1818
|
-
|
|
1819
|
-
|
|
1967
|
+
const attempt = Math.max(0, Number((rec && (rec.attemptsTotal ?? rec.attempts)) || 0));
|
|
1968
|
+
if (!rec.journalled || typeof rec.journalled !== "object") rec.journalled = {};
|
|
1969
|
+
const jid = journalId(rec, notice, attempt);
|
|
1970
|
+
const journalledAlready = Number.isFinite(rec.journalled[jid]);
|
|
1971
|
+
const stamp = Number.isFinite(now) ? now : Date.now();
|
|
1972
|
+
if (!already || !journalledAlready) {
|
|
1820
1973
|
// THE SINK PERSISTS ITS OWN LATCH. `markNoticeSaid` mutates `rec` and
|
|
1821
1974
|
// documents that the caller writes it — which was true of the failure branch
|
|
1822
1975
|
// (`writeRecord` immediately after `sayOutcomeNotice`) and NOT of the paths
|
|
@@ -1824,20 +1977,26 @@ function recordOperatorNotice(o = {}) {
|
|
|
1824
1977
|
// the record from disk and the in-memory mark is lost. The once-only property
|
|
1825
1978
|
// is this function's guarantee, so this function makes it durable rather than
|
|
1826
1979
|
// depending on what each caller happens to do next.
|
|
1980
|
+
if (!already) markNoticeSaid(rec, notice, now);
|
|
1981
|
+
if (!journalledAlready) rec.journalled[jid] = stamp;
|
|
1827
1982
|
try { writeRecord(rec); } catch { /* best-effort: a lost latch is one extra row, not a lost debt */ }
|
|
1828
1983
|
}
|
|
1829
|
-
counters.bump(
|
|
1984
|
+
counters.bump(OPERATOR_PLANE_COUNTERS.notice, {
|
|
1830
1985
|
notice,
|
|
1831
1986
|
reason: o.reason || "declared-operator",
|
|
1832
1987
|
repeat: already ? "suppressed" : "first",
|
|
1833
1988
|
});
|
|
1834
|
-
if (
|
|
1989
|
+
if (journalledAlready) {
|
|
1835
1990
|
return { sent: false, reason: "operator-plane-already-recorded", plane: NOTICE_PLANE.OPERATOR, journalled: false };
|
|
1836
1991
|
}
|
|
1837
1992
|
let journalled = false;
|
|
1838
1993
|
try {
|
|
1839
1994
|
const line = JSON.stringify({
|
|
1840
|
-
at: new Date(
|
|
1995
|
+
at: new Date(stamp).toISOString(),
|
|
1996
|
+
// `kind` separates a notice from the terminal `resolution` row added below.
|
|
1997
|
+
// Absent on every line written before this change, and the reader treats a
|
|
1998
|
+
// line without one as a notice, which is the only kind there was.
|
|
1999
|
+
kind: JOURNAL_KIND.NOTICE,
|
|
1841
2000
|
notice,
|
|
1842
2001
|
key: rec && rec.key,
|
|
1843
2002
|
service: rec && rec.service,
|
|
@@ -1845,7 +2004,25 @@ function recordOperatorNotice(o = {}) {
|
|
|
1845
2004
|
sender: (rec && rec.sender) || null,
|
|
1846
2005
|
cause: (rec && rec.lastError) || null,
|
|
1847
2006
|
attempts: (rec && rec.attempts) || 0,
|
|
2007
|
+
// ── WHAT THE JOURNAL WAS MISSING TO STAND ON ITS OWN ──────────────────
|
|
2008
|
+
//
|
|
2009
|
+
// `attemptsTotal` survives the retry that reopens this key, so the journal
|
|
2010
|
+
// can show a sequence; `attempt` is which one THIS line is about, and is
|
|
2011
|
+
// the key the duplicate latch above uses. `summary` and `itemPreview` are
|
|
2012
|
+
// the ask: they lived only on the needs-attention row and on the obligation
|
|
2013
|
+
// record, and the record is pruned at three days — which is why 72 of 78
|
|
2014
|
+
// rows on this machine could no longer say what had failed. `exitCode` and
|
|
2015
|
+
// `sessionLog` are the only two facts in the whole plane that distinguish a
|
|
2016
|
+
// timeout from a crash from a clean exit that never spoke, against a cause
|
|
2017
|
+
// field that reads the single label `no_outcome` on 94% of the population.
|
|
2018
|
+
attemptsTotal: Math.max(0, Number((rec && rec.attemptsTotal) || 0)),
|
|
2019
|
+
attempt,
|
|
2020
|
+
repeat: already === true,
|
|
2021
|
+
summary: (rec && rec.summary) || null,
|
|
2022
|
+
itemPreview: rec && rec.item && typeof rec.item.content === "string" ? rec.item.content.slice(0, 300) : null,
|
|
2023
|
+
exitCode: rec && Number.isFinite(rec.exitCode) ? rec.exitCode : null,
|
|
1848
2024
|
sessionId: (rec && rec.sessionId) || null,
|
|
2025
|
+
sessionLog: sessionLogRelPath(rec && rec.sessionId),
|
|
1849
2026
|
traceId: (rec && rec.traceId) || null,
|
|
1850
2027
|
// The plane, on every row, so a reader of the journal never has to wonder
|
|
1851
2028
|
// whether this one also went to a person. None of them do.
|
|
@@ -2076,8 +2253,28 @@ export async function openAndAcknowledge(a = {}) {
|
|
|
2076
2253
|
// Which outcome notices have already been delivered for this (channel,
|
|
2077
2254
|
// key). Also inherited, for the same reason.
|
|
2078
2255
|
notices: {},
|
|
2256
|
+
// Which (notice, channel, attempt) triples have already reached the operator
|
|
2257
|
+
// JOURNAL. Separate from `notices` because that map is the human-path latch
|
|
2258
|
+
// and is deliberately coarser — see `journalId`. Inherited below for the same
|
|
2259
|
+
// reason `notices` is: without it a retry re-journals the whole lifecycle.
|
|
2260
|
+
journalled: {},
|
|
2079
2261
|
attempts: 0,
|
|
2262
|
+
// ── THE ATTEMPT COUNT THAT SURVIVES A RETRY ───────────────────────────────
|
|
2263
|
+
//
|
|
2264
|
+
// `attempts` is reset by construction: a retry CLOSES this record and the next
|
|
2265
|
+
// poll opens a fresh one under the same key, so `attempts` counts the sessions
|
|
2266
|
+
// of the CURRENT record only. On this machine it read 1 on all 78
|
|
2267
|
+
// needs-attention rows and on 21 of 23 records, while `assurance.retrying` had
|
|
2268
|
+
// been bumped 198 times — the plane's retry history was a flat lie, and the
|
|
2269
|
+
// journal copied the same field and inherited it. `attemptsTotal` is inherited
|
|
2270
|
+
// across the reopen, so "this ask has been attempted four times" is sayable.
|
|
2271
|
+
attemptsTotal: 0,
|
|
2080
2272
|
lastError: null,
|
|
2273
|
+
// Which exit code the last session ended on, when one is known. `cause` is a
|
|
2274
|
+
// four-value enum of which 94% of the observed population is `no_outcome`;
|
|
2275
|
+
// this is one of the two fields that can tell those apart (the other is the
|
|
2276
|
+
// session log path, derived from `sessionId`).
|
|
2277
|
+
exitCode: null,
|
|
2081
2278
|
sessionId: null,
|
|
2082
2279
|
traceId: a.traceId || item.trace_id || null,
|
|
2083
2280
|
daemonPid: process.pid,
|
|
@@ -2093,6 +2290,19 @@ export async function openAndAcknowledge(a = {}) {
|
|
|
2093
2290
|
rec.interimSaid = existing.interimSaid === true;
|
|
2094
2291
|
rec.interimAt = existing.interimAt || null;
|
|
2095
2292
|
rec.notices = (existing.notices && typeof existing.notices === "object") ? { ...existing.notices } : {};
|
|
2293
|
+
rec.journalled = (existing.journalled && typeof existing.journalled === "object") ? { ...existing.journalled } : {};
|
|
2294
|
+
// THE COUNT, NOT JUST THE LATCHES. The previous pass inherited `interimSaid`,
|
|
2295
|
+
// `interimAt` and `notices` — everything needed to stay QUIET across a retry —
|
|
2296
|
+
// and nothing needed to say how many times the work had been tried. So the
|
|
2297
|
+
// plane could not answer "was it retried?", which is one of the two questions
|
|
2298
|
+
// an operator opens it with. Carried across with the latches, in the same block,
|
|
2299
|
+
// so the next author who adds an inherited field sees the count beside them.
|
|
2300
|
+
rec.attemptsTotal = Math.max(
|
|
2301
|
+
0,
|
|
2302
|
+
Number(existing.attemptsTotal || 0),
|
|
2303
|
+
Number(existing.attempts || 0),
|
|
2304
|
+
);
|
|
2305
|
+
rec.exitCode = Number.isFinite(existing.exitCode) ? existing.exitCode : null;
|
|
2096
2306
|
}
|
|
2097
2307
|
rec.tier = tier;
|
|
2098
2308
|
rec.daemonPid = process.pid;
|
|
@@ -2222,6 +2432,10 @@ export function noteSession(key, sessionId, o = {}) {
|
|
|
2222
2432
|
if (!rec) return false;
|
|
2223
2433
|
rec.sessionId = sessionId || null;
|
|
2224
2434
|
rec.attempts = (rec.attempts || 0) + 1;
|
|
2435
|
+
// The count that survives the retry's reopen. Bumped in lockstep here — the one
|
|
2436
|
+
// place an attempt is counted at all — so the two cannot disagree about this
|
|
2437
|
+
// record while differing about its history, which is the whole point.
|
|
2438
|
+
rec.attemptsTotal = Math.max(Number(rec.attemptsTotal || 0), 0) + 1;
|
|
2225
2439
|
rec.lastAttemptAt = Number.isFinite(o.now) ? o.now : Date.now();
|
|
2226
2440
|
rec.daemonPid = process.pid;
|
|
2227
2441
|
return writeRecord(rec);
|
|
@@ -2284,6 +2498,9 @@ export function closeObligation(key, o = {}) {
|
|
|
2284
2498
|
rec.state = o.outcome || "answered";
|
|
2285
2499
|
rec.closedAt = Number.isFinite(o.now) ? o.now : Date.now();
|
|
2286
2500
|
if (o.note) rec.closeNote = o.note;
|
|
2501
|
+
// THE PLANE LEARNS HOW THIS ENDED. Before `writeRecord`, so the one write
|
|
2502
|
+
// covers both the record and the resolution latch.
|
|
2503
|
+
resolveOperatorPlane(rec);
|
|
2287
2504
|
const wrote = writeRecord(rec);
|
|
2288
2505
|
// Every close path in this module funnels through here — answered, silent
|
|
2289
2506
|
// success, failed, undeliverable, stale — so this is the one place the
|
|
@@ -2301,7 +2518,8 @@ export function closeObligation(key, o = {}) {
|
|
|
2301
2518
|
*
|
|
2302
2519
|
* ok:true + a receipt exists → the session spoke. Close the debt quietly;
|
|
2303
2520
|
* do NOT add a message (that would be spam).
|
|
2304
|
-
* ok:true + no receipt → silent success.
|
|
2521
|
+
* ok:true + no receipt → silent success. Hold the result text for the
|
|
2522
|
+
* owner in the operator plane; post nothing.
|
|
2305
2523
|
* ok:false + transient + tries → say what broke and that a retry is running.
|
|
2306
2524
|
* ok:false + terminal → say it failed, stop, escalate durably.
|
|
2307
2525
|
*
|
|
@@ -2320,6 +2538,19 @@ export async function settleSession(a = {}) {
|
|
|
2320
2538
|
if (!rec) return { spoke: false, text: null, verdict: "no-obligation", willRetry: false };
|
|
2321
2539
|
const now = Number.isFinite(a.now) ? a.now : Date.now();
|
|
2322
2540
|
if (a.sessionId && rec.sessionId !== a.sessionId) { rec.sessionId = a.sessionId; writeRecord(rec); }
|
|
2541
|
+
// THE EXIT CODE, ON THE RECORD, BEFORE ANY BRANCH READS IT. This is the only
|
|
2542
|
+
// discriminating fact this call site is handed: `classifyFailure` collapses every
|
|
2543
|
+
// terminal state into a four-value label, and 94% of the plane's population reads
|
|
2544
|
+
// the single one of those that says least (`no_outcome` — "the session produced
|
|
2545
|
+
// nothing"). An exit code separates a timeout from a crash from a kill. Stamped
|
|
2546
|
+
// for every settle, not only the failures, so a close that later resolves a row
|
|
2547
|
+
// carries it too.
|
|
2548
|
+
//
|
|
2549
|
+
// WHAT IS STILL MISSING AND IS NOT INVENTED HERE: a stderr tail. The dispatcher's
|
|
2550
|
+
// `onClose` passes `{ok, code, sessionId, stdout}` and no stderr, so there is
|
|
2551
|
+
// nothing to record; the session log path (derived from `sessionId`) is the
|
|
2552
|
+
// pointer at it instead. Capturing stderr is a dispatcher change, not a plane one.
|
|
2553
|
+
if (Number.isFinite(a.code) && rec.exitCode !== a.code) { rec.exitCode = Number(a.code); writeRecord(rec); }
|
|
2323
2554
|
const item = rec.item || {};
|
|
2324
2555
|
const send = (a.deps && a.deps.deliverImpl) || deliver;
|
|
2325
2556
|
|
|
@@ -2374,10 +2605,11 @@ export async function settleSession(a = {}) {
|
|
|
2374
2605
|
// unchanged — including for every seat that has not upgraded,
|
|
2375
2606
|
// which writes no rows at all.
|
|
2376
2607
|
//
|
|
2377
|
-
// FAILS OPEN TO THE
|
|
2378
|
-
// corrupt ledger line: all of them land on
|
|
2379
|
-
//
|
|
2380
|
-
//
|
|
2608
|
+
// FAILS OPEN TO THE OWNER, NOT TO THE ROOM. A null verdict, a thrown
|
|
2609
|
+
// reader, a corrupt ledger line: all of them land on the silent-success
|
|
2610
|
+
// path below, which holds whatever the session produced in the operator
|
|
2611
|
+
// plane (`holdResultForOwner`) or records that it produced nothing. Never a
|
|
2612
|
+
// silently dropped debt, and never the session's notes in a conversation.
|
|
2381
2613
|
const noSeen = (a.deps && a.deps.noSendForImpl) || noSendFor;
|
|
2382
2614
|
let why = null;
|
|
2383
2615
|
try {
|
|
@@ -2399,19 +2631,17 @@ export async function settleSession(a = {}) {
|
|
|
2399
2631
|
// one line that says a message this seat owed was stopped, plus its cause.
|
|
2400
2632
|
// Withholding it is how a blocked send lane stays invisible for days.
|
|
2401
2633
|
//
|
|
2402
|
-
// ──
|
|
2634
|
+
// ── TOLD TO THE OWNER, NOT THE ROOM ────────────────────────────────────
|
|
2403
2635
|
//
|
|
2404
2636
|
// That paragraph is true of the FAULT, not of the sentence. The fault has
|
|
2405
2637
|
// to reach somebody; nothing says that somebody is the person in the
|
|
2406
|
-
// conversation.
|
|
2407
|
-
//
|
|
2408
|
-
//
|
|
2409
|
-
//
|
|
2410
|
-
//
|
|
2411
|
-
// `
|
|
2412
|
-
//
|
|
2413
|
-
// operator sink gets strictly more than the sentence did, including the
|
|
2414
|
-
// raw code that no longer goes to a person.
|
|
2638
|
+
// conversation. Without result text, all a room would get is "Couldn't get
|
|
2639
|
+
// this out to you — the send was blocked (…). That won't clear on its
|
|
2640
|
+
// own": a template about this machine's own plumbing, unactionable by the
|
|
2641
|
+
// reasoning in `notice-plane.mjs`. WITH result text it is no better — the
|
|
2642
|
+
// text is the session's end-of-turn report, not a reply (see
|
|
2643
|
+
// `holdResultForOwner`). Both shapes go to the operator plane, which gets
|
|
2644
|
+
// strictly more than the sentence ever did, including the raw code.
|
|
2415
2645
|
const finalText = safeResultText(a.stdout);
|
|
2416
2646
|
if (!finalText) {
|
|
2417
2647
|
const res = await sayOutcomeNotice({
|
|
@@ -2429,20 +2659,21 @@ export async function settleSession(a = {}) {
|
|
|
2429
2659
|
counters.bump("assurance.no_send_refused", { service: rec.service, code: why.code || "none", plane: "operator" });
|
|
2430
2660
|
return { spoke: false, text: null, verdict: "send-refused", willRetry: false, noSend: why, plane: res.plane };
|
|
2431
2661
|
}
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2662
|
+
// WITH result text it is held too, never posted. The session tried to
|
|
2663
|
+
// send, and something — very often the outbound gate itself — refused
|
|
2664
|
+
// it. Re-posting the session's end-of-turn text under a "the send was
|
|
2665
|
+
// blocked" head is how a gate refusal got routed around: the notes that
|
|
2666
|
+
// follow the head are the session's account of its own work, frequently
|
|
2667
|
+
// carrying the very content the gate stopped. See `holdResultForOwner`.
|
|
2668
|
+
counters.bump("assurance.no_send_refused", { service: rec.service, code: why.code || "none", plane: "operator" });
|
|
2669
|
+
return holdResultForOwner({
|
|
2670
|
+
rec, a, item, now, send,
|
|
2671
|
+
notice: NOTICE.SEND_REFUSED,
|
|
2672
|
+
verdict: "send-refused",
|
|
2673
|
+
finalText,
|
|
2674
|
+
cause: `send refused: ${why.code || "no code"}${why.surface ? ` on ${why.surface}` : ""}${why.reason ? ` — ${String(why.reason).slice(0, 200)}` : ""}`,
|
|
2675
|
+
noSend: why,
|
|
2438
2676
|
});
|
|
2439
|
-
closeObligation(a.key, {
|
|
2440
|
-
outcome: res && res.sent ? "answered" : "undeliverable",
|
|
2441
|
-
now,
|
|
2442
|
-
note: `send refused (${why.code || "no code"}${why.surface ? ` on ${why.surface}` : ""}); ${res && res.sent ? "cause reported to the room" : `cause also undeliverable: ${res && res.error}`}`,
|
|
2443
|
-
});
|
|
2444
|
-
counters.bump("assurance.no_send_refused", { service: rec.service, code: why.code || "none", plane: "human" });
|
|
2445
|
-
return { spoke: !!(res && res.sent), text, verdict: "send-refused", willRetry: false, noSend: why };
|
|
2446
2677
|
}
|
|
2447
2678
|
|
|
2448
2679
|
// A CLEAN SILENT EXIT OUTSIDE A DM CLOSES QUIETLY, LIKE A DOC COMMENT.
|
|
@@ -2498,10 +2729,10 @@ export async function settleSession(a = {}) {
|
|
|
2498
2729
|
|
|
2499
2730
|
// ── NO RESULT TEXT IS NOT AN ANSWER, SO IT IS NOT A REPLY ───────────────
|
|
2500
2731
|
//
|
|
2501
|
-
// `composeSilentSuccess` has two shapes and
|
|
2502
|
-
//
|
|
2503
|
-
//
|
|
2504
|
-
//
|
|
2732
|
+
// `composeSilentSuccess` has two shapes and NEITHER is a reply. The one WITH
|
|
2733
|
+
// the session's result text was treated as one until 2026-10-13 and is now
|
|
2734
|
+
// held for the owner below (`holdResultForOwner`). WITHOUT it, the whole
|
|
2735
|
+
// message is:
|
|
2505
2736
|
//
|
|
2506
2737
|
// Finished this but didn't get a reply out to you — my fault. I don't
|
|
2507
2738
|
// have a clean result to show you, though. Want me to run it again?
|
|
@@ -2539,22 +2770,21 @@ export async function settleSession(a = {}) {
|
|
|
2539
2770
|
return { spoke: false, text: null, verdict: "silent-success", willRetry: false, plane: res.plane };
|
|
2540
2771
|
}
|
|
2541
2772
|
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2773
|
+
// ── WITH RESULT TEXT IT IS STILL NOT A REPLY ─────────────────────────────
|
|
2774
|
+
//
|
|
2775
|
+
// This branch used to post `composeSilentSuccess(rec, {finalText})` as a
|
|
2776
|
+
// `kind:"reply"`, on the reading that the result text IS the answer. It is
|
|
2777
|
+
// the session's end-of-turn report to this daemon, and posting it put this
|
|
2778
|
+
// machine's file paths, queue ids and session talk into team channels under
|
|
2779
|
+
// the seat's name. The text is held for the owner; nothing is sent. See
|
|
2780
|
+
// `holdResultForOwner` for the measurement and the design.
|
|
2781
|
+
return holdResultForOwner({
|
|
2782
|
+
rec, a, item, now, send,
|
|
2783
|
+
notice: NOTICE.SILENT_SUCCESS,
|
|
2784
|
+
verdict: "silent-success",
|
|
2785
|
+
finalText,
|
|
2786
|
+
cause: "session exited 0 without sending a reply",
|
|
2556
2787
|
});
|
|
2557
|
-
return { spoke: !!(res && res.sent), text, verdict: "silent-success", willRetry: false };
|
|
2558
2788
|
}
|
|
2559
2789
|
|
|
2560
2790
|
// ── failure ────────────────────────────────────────────────────────────
|
|
@@ -2647,6 +2877,103 @@ function safeResultText(stdout) {
|
|
|
2647
2877
|
* TOLD — that already happened above — plus a trace an operator can sweep. This
|
|
2648
2878
|
* file is that trace.
|
|
2649
2879
|
*/
|
|
2880
|
+
/**
|
|
2881
|
+
* THE TERMINAL FACT. Tell the operator plane how a debt it escalated ended.
|
|
2882
|
+
*
|
|
2883
|
+
* ── WHY THE PLANE COULD NOT ANSWER "DID IT EVENTUALLY WORK?" ─────────────────
|
|
2884
|
+
*
|
|
2885
|
+
* `escalate` is the only writer of a needs-attention row and it runs only on
|
|
2886
|
+
* failure. `recordOperatorNotice` is never called on a success path. So nothing
|
|
2887
|
+
* in the plane had a `status`, a `resolved` or a `closedAt` — the eventual outcome
|
|
2888
|
+
* existed ONLY in the obligation record's `state`/`closeNote`, which is deleted at
|
|
2889
|
+
* CLOSED_RETENTION_MS. Measured on this machine: 78 rows, none resolved, and 72
|
|
2890
|
+
* (92%) with no record left to join to. An operator opening the plane in week six
|
|
2891
|
+
* saw 78 rows of which 72 could not be worked and 0 could be closed.
|
|
2892
|
+
*
|
|
2893
|
+
* SCOPED TO WHAT THE PLANE ALREADY KNOWS ABOUT, deliberately. `closeObligation`
|
|
2894
|
+
* is on every discharge path including the common happy one, and appending a
|
|
2895
|
+
* journal row for every answered ask would turn the operator journal into a
|
|
2896
|
+
* firehose of successes — which is the volume failure this subsystem has already
|
|
2897
|
+
* made twice. So this fires only when the plane has previously said something
|
|
2898
|
+
* about this key: a needs-attention row exists, or a notice was journalled.
|
|
2899
|
+
*
|
|
2900
|
+
* IDEMPOTENT. `rec.planeResolvedAt` latches it, because `closeObligation` can be
|
|
2901
|
+
* reached twice for one debt (a settle racing the sweep) and two resolution rows
|
|
2902
|
+
* would make the reader's `closedAt` depend on which one it read last.
|
|
2903
|
+
*
|
|
2904
|
+
* Never throws: a close that fails because of its own bookkeeping is strictly
|
|
2905
|
+
* worse than a plane that misses one terminal row.
|
|
2906
|
+
*
|
|
2907
|
+
* @param {object} rec the obligation record, already carrying its terminal state;
|
|
2908
|
+
* MUTATED (`planeResolvedAt`), caller persists
|
|
2909
|
+
* @returns {boolean} whether anything was written
|
|
2910
|
+
*/
|
|
2911
|
+
export function resolveOperatorPlane(rec) {
|
|
2912
|
+
if (!rec || !rec.key) return false;
|
|
2913
|
+
if (Number.isFinite(rec.planeResolvedAt)) return false;
|
|
2914
|
+
const at = Number.isFinite(rec.closedAt) ? rec.closedAt : Date.now();
|
|
2915
|
+
let rowPath = null;
|
|
2916
|
+
let row = null;
|
|
2917
|
+
try {
|
|
2918
|
+
rowPath = join(obligationDir(), "needs-attention", `${sanitiseKey(rec.key)}.json`);
|
|
2919
|
+
row = JSON.parse(readFileSync(rowPath, "utf-8"));
|
|
2920
|
+
} catch { row = null; }
|
|
2921
|
+
const journalled = !!(rec.journalled && typeof rec.journalled === "object" && Object.keys(rec.journalled).length)
|
|
2922
|
+
|| !!(rec.notices && typeof rec.notices === "object" && Object.keys(rec.notices).length);
|
|
2923
|
+
if (!row && !journalled) return false;
|
|
2924
|
+
|
|
2925
|
+
const outcome = String(rec.state || "closed");
|
|
2926
|
+
rec.planeResolvedAt = at;
|
|
2927
|
+
let wrote = false;
|
|
2928
|
+
|
|
2929
|
+
// THE ROW, REWRITTEN IN PLACE. It is the surface an operator lands on, so it
|
|
2930
|
+
// must be able to say "done" without a join — that join is the thing that is
|
|
2931
|
+
// already dead for 92% of the population.
|
|
2932
|
+
if (row && rowPath) {
|
|
2933
|
+
try {
|
|
2934
|
+
writeFileSync(rowPath, JSON.stringify({
|
|
2935
|
+
...row,
|
|
2936
|
+
outcome,
|
|
2937
|
+
resolvedAt: new Date(at).toISOString(),
|
|
2938
|
+
closeNote: rec.closeNote || row.closeNote || null,
|
|
2939
|
+
attemptsTotal: Math.max(0, Number(rec.attemptsTotal || rec.attempts || row.attemptsTotal || 0)),
|
|
2940
|
+
exitCode: Number.isFinite(rec.exitCode) ? rec.exitCode : (Number.isFinite(row.exitCode) ? row.exitCode : null),
|
|
2941
|
+
}, null, 2));
|
|
2942
|
+
wrote = true;
|
|
2943
|
+
} catch (err) {
|
|
2944
|
+
console.error(`[assurance] could not resolve the needs-attention row for ${rec.key}: ${err && err.message}`);
|
|
2945
|
+
}
|
|
2946
|
+
}
|
|
2947
|
+
|
|
2948
|
+
// THE JOURNAL, APPENDED TO. The row holds only the last state; the journal is
|
|
2949
|
+
// the ordered record, and a sequence that never terminates is a sequence a
|
|
2950
|
+
// reader has to guess the end of.
|
|
2951
|
+
try {
|
|
2952
|
+
appendFileSync(join(obligationDir(), "notice-journal.jsonl"), `${JSON.stringify({
|
|
2953
|
+
at: new Date(at).toISOString(),
|
|
2954
|
+
kind: JOURNAL_KIND.RESOLUTION,
|
|
2955
|
+
notice: null,
|
|
2956
|
+
key: rec.key,
|
|
2957
|
+
service: rec.service || null,
|
|
2958
|
+
channel: rec.channel || null,
|
|
2959
|
+
sender: rec.sender || null,
|
|
2960
|
+
summary: rec.summary || null,
|
|
2961
|
+
outcome,
|
|
2962
|
+
closeNote: rec.closeNote || null,
|
|
2963
|
+
attemptsTotal: Math.max(0, Number(rec.attemptsTotal || rec.attempts || 0)),
|
|
2964
|
+
sessionId: rec.sessionId || null,
|
|
2965
|
+
traceId: rec.traceId || null,
|
|
2966
|
+
plane: NOTICE_PLANE.OPERATOR,
|
|
2967
|
+
requesterTold: false,
|
|
2968
|
+
})}\n`);
|
|
2969
|
+
wrote = true;
|
|
2970
|
+
} catch (err) {
|
|
2971
|
+
console.error(`[assurance] could not journal the resolution of ${rec.key}: ${err && err.message}`);
|
|
2972
|
+
}
|
|
2973
|
+
counters.bump(OPERATOR_PLANE_COUNTERS.resolved, { outcome, service: rec.service || "unknown" });
|
|
2974
|
+
return wrote;
|
|
2975
|
+
}
|
|
2976
|
+
|
|
2650
2977
|
export function escalate(rec, o = {}) {
|
|
2651
2978
|
try {
|
|
2652
2979
|
const dir = join(obligationDir(), "needs-attention");
|
|
@@ -2668,9 +2995,50 @@ export function escalate(rec, o = {}) {
|
|
|
2668
2995
|
// than leaving them to infer it from `cause`.
|
|
2669
2996
|
configFault: rec.configFault || null,
|
|
2670
2997
|
attempts: rec.attempts || 0,
|
|
2998
|
+
// ── THE THREE FIELDS A READER NEEDED AND THE ROW DID NOT HAVE ──────────
|
|
2999
|
+
//
|
|
3000
|
+
// `attemptsTotal` because `attempts` read 1 on all 78 rows on this machine
|
|
3001
|
+
// while `assurance.retrying` had been bumped 198 times: a retry CLOSES the
|
|
3002
|
+
// record and the next poll opens a fresh one, which inherited the notice
|
|
3003
|
+
// latch but not the attempt count, so the row's own retry history was a lie.
|
|
3004
|
+
// `exitCode` and `sessionLog` because `cause` is a four-value enum of which
|
|
3005
|
+
// 94% is `no_outcome` — "the session produced nothing" — and nothing in the
|
|
3006
|
+
// plane distinguished a timeout from a crash from a kill from a clean exit
|
|
3007
|
+
// that never spoke.
|
|
3008
|
+
attemptsTotal: Math.max(0, Number(rec.attemptsTotal || rec.attempts || 0)),
|
|
3009
|
+
exitCode: Number.isFinite(rec.exitCode) ? rec.exitCode : null,
|
|
2671
3010
|
sessionId: rec.sessionId || null,
|
|
3011
|
+
sessionLog: sessionLogRelPath(rec.sessionId),
|
|
2672
3012
|
traceId: rec.traceId || null,
|
|
2673
3013
|
itemPreview: rec.item ? rec.item.content : null,
|
|
3014
|
+
// ── A HELD RESULT, FOR THE OWNER ───────────────────────────────────────
|
|
3015
|
+
//
|
|
3016
|
+
// Present only when the late-reply fallback held a session's end-of-turn
|
|
3017
|
+
// text instead of posting it (`holdResultForOwner`). The length is exact;
|
|
3018
|
+
// the preview is bounded and whitespace-collapsed, and the full text is at
|
|
3019
|
+
// `sessionLog`. It stays on this seat: the presence beat lifts counts off
|
|
3020
|
+
// the box, never row text.
|
|
3021
|
+
...(typeof o.heldResult === "string" && o.heldResult.trim()
|
|
3022
|
+
? {
|
|
3023
|
+
heldResultChars: o.heldResult.length,
|
|
3024
|
+
heldResultPreview: (() => {
|
|
3025
|
+
const flat = o.heldResult.replace(/\s+/g, " ").trim();
|
|
3026
|
+
return flat.length > HELD_RESULT_PREVIEW_LIMIT ? `${flat.slice(0, HELD_RESULT_PREVIEW_LIMIT).trimEnd()}…` : flat;
|
|
3027
|
+
})(),
|
|
3028
|
+
}
|
|
3029
|
+
: {}),
|
|
3030
|
+
// ── THE TERMINAL FACT, DECLARED AS ABSENT ─────────────────────────────
|
|
3031
|
+
//
|
|
3032
|
+
// `escalate` is the only writer of a row and it runs only on failure, so
|
|
3033
|
+
// before this change there was no `outcome`, `resolved` or `closedAt` field
|
|
3034
|
+
// anywhere in the plane and every row was permanently "unknown" — which is
|
|
3035
|
+
// how 78 of them accumulated with none ever marked done. `closeObligation`
|
|
3036
|
+
// now stamps these (see `resolveOperatorPlane`); writing them as null here
|
|
3037
|
+
// is what makes "not yet resolved" a stated fact rather than a missing key
|
|
3038
|
+
// the reader has to guess the meaning of.
|
|
3039
|
+
outcome: null,
|
|
3040
|
+
resolvedAt: null,
|
|
3041
|
+
closeNote: null,
|
|
2674
3042
|
}, null, 2));
|
|
2675
3043
|
console.error(`[assurance] ESCALATED ${rec.key} — ${o.failure ? o.failure.label : rec.lastError}; requester told: ${!!o.told}`);
|
|
2676
3044
|
return true;
|
|
@@ -3286,9 +3654,66 @@ async function _sweepObligations(a = {}) {
|
|
|
3286
3654
|
|
|
3287
3655
|
pruneObligations({ now });
|
|
3288
3656
|
pruneNoticeJournal({ now });
|
|
3657
|
+
pruneNeedsAttention({ now });
|
|
3289
3658
|
return stats;
|
|
3290
3659
|
}
|
|
3291
3660
|
|
|
3661
|
+
/**
|
|
3662
|
+
* Trim RESOLVED needs-attention rows past the retention window.
|
|
3663
|
+
*
|
|
3664
|
+
* ── RETENTION WAS INVERTED, AND THAT IS WHY 92% OF THE PLANE IS DEAD ─────────
|
|
3665
|
+
*
|
|
3666
|
+
* The journal — the only ordered record, the thing this subsystem defends as what
|
|
3667
|
+
* replaced the DM — is trimmed at CLOSED_RETENTION_MS. The rows, which hold no
|
|
3668
|
+
* sequence and are overwritten per key, were NEVER pruned in production: the only
|
|
3669
|
+
* code that cleared one was the `_resetObligations` test seam. So on this machine
|
|
3670
|
+
* there were 78 rows spanning five weeks and 23 obligation records, and 72 of the
|
|
3671
|
+
* rows (92%) had outlived the record they must join to — unworkable by
|
|
3672
|
+
* construction, and indistinguishable from live incidents to anything reading the
|
|
3673
|
+
* directory.
|
|
3674
|
+
*
|
|
3675
|
+
* ONLY RESOLVED ROWS GO, and this is the conservative half of the fix. A row whose
|
|
3676
|
+
* debt never reached a terminal state is the ONE artefact that says a person asked
|
|
3677
|
+
* and got nothing; deleting it to tidy a directory would destroy the only evidence
|
|
3678
|
+
* of the failure this plane exists to hold. Those are kept forever and the reader
|
|
3679
|
+
* labels them `unresolvable` with the reason, which is the honest rendering of a
|
|
3680
|
+
* row that cannot be joined. Pruning them is a policy question about how long an
|
|
3681
|
+
* unacknowledged dead ask is worth keeping, and nobody has answered it.
|
|
3682
|
+
*
|
|
3683
|
+
* Never throws, for the same reason as the other two prunes.
|
|
3684
|
+
*
|
|
3685
|
+
* @param {object} [o] `{now}`
|
|
3686
|
+
* @returns {number} rows removed
|
|
3687
|
+
*/
|
|
3688
|
+
export function pruneNeedsAttention(o = {}) {
|
|
3689
|
+
const now = Number.isFinite(o.now) ? o.now : Date.now();
|
|
3690
|
+
const dir = join(obligationDir(), "needs-attention");
|
|
3691
|
+
let names = [];
|
|
3692
|
+
try { names = readdirSync(dir); } catch { return 0; }
|
|
3693
|
+
let removed = 0;
|
|
3694
|
+
for (const name of names) {
|
|
3695
|
+
if (!name.endsWith(".json")) continue;
|
|
3696
|
+
let row = null;
|
|
3697
|
+
try { row = JSON.parse(readFileSync(join(dir, name), "utf-8")); } catch { continue; }
|
|
3698
|
+
if (!row || typeof row !== "object") continue;
|
|
3699
|
+
const outcome = row.outcome == null ? "" : String(row.outcome).trim();
|
|
3700
|
+
if (!outcome) continue; // unresolved: the evidence stays
|
|
3701
|
+
// AND AN OWED ASK IS EVIDENCE TOO. "failed"/"undeliverable" mean the machine
|
|
3702
|
+
// gave up and the person never got an answer — only "answered" settles. The
|
|
3703
|
+
// first version of this guard checked only that an outcome EXISTED, which
|
|
3704
|
+
// meant every escalated dead ask became prunable at 3 days. Measured: a dead
|
|
3705
|
+
// ask written through the real path was deleted by `pruneNeedsAttention({now:
|
|
3706
|
+
// now + 4d})`. Before this prune existed those rows survived indefinitely,
|
|
3707
|
+
// which is the only reason the 78 unanswered asks were discoverable at all.
|
|
3708
|
+
if (outcomeIsOwed(outcome)) continue;
|
|
3709
|
+
const at = Date.parse(row.resolvedAt || row.failedAt || "");
|
|
3710
|
+
if (!Number.isFinite(at) || now - at < CLOSED_RETENTION_MS) continue;
|
|
3711
|
+
try { unlinkSync(join(dir, name)); removed++; } catch { /* best-effort */ }
|
|
3712
|
+
}
|
|
3713
|
+
if (removed) counters.bump(OPERATOR_PLANE_COUNTERS.pruned, { removed: String(removed) });
|
|
3714
|
+
return removed;
|
|
3715
|
+
}
|
|
3716
|
+
|
|
3292
3717
|
/**
|
|
3293
3718
|
* Trim the operator notice journal to the retention window.
|
|
3294
3719
|
*
|
|
@@ -3386,6 +3811,8 @@ export function _resetObligations() {
|
|
|
3386
3811
|
|
|
3387
3812
|
export default {
|
|
3388
3813
|
pruneNoticeJournal,
|
|
3814
|
+
pruneNeedsAttention,
|
|
3815
|
+
resolveOperatorPlane,
|
|
3389
3816
|
shouldAcknowledge,
|
|
3390
3817
|
senderLooksLikeAgentSeat,
|
|
3391
3818
|
looksLikePeerAckShape,
|