@cohortapp/agent-sdk 2.18.36 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +3 -0
- package/docs/guides/billing-console-keys.md +16 -3
- package/docs/runbooks/cadence-requeue-storm-recovery.md +186 -0
- package/lib/assurance/first-reply.mjs +47 -2
- package/lib/assurance/notice-plane.mjs +218 -0
- package/lib/assurance/notice-voice.mjs +35 -2
- package/lib/assurance/operator-plane-store.mjs +164 -0
- package/lib/assurance/operator-plane.mjs +689 -0
- package/lib/budget-escalate.mjs +42 -4
- package/lib/budget-guard.mjs +24 -1
- package/lib/cadence-bus.mjs +359 -46
- package/lib/cli/attention.mjs +300 -0
- package/lib/cli/seat-auth.mjs +86 -6
- package/lib/comms/machine-status.mjs +377 -0
- package/lib/comms/send-gate.mjs +309 -2
- package/lib/org/awareness.mjs +15 -2
- package/lib/org/clause-match.mjs +470 -0
- package/lib/org/client.mjs +27 -1
- package/lib/org/escalation-chain.mjs +134 -0
- package/lib/org/first-run.mjs +263 -0
- package/lib/org/genesis-contract.mjs +271 -0
- package/lib/org/governance-runtime.mjs +409 -0
- package/lib/org/governance-store.mjs +238 -0
- package/lib/org/governance.mjs +326 -0
- package/lib/org/mesh.mjs +135 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +6 -0
- package/lib/org/tool-surface.mjs +28 -0
- package/lib/setup/enroll-from-cohort.mjs +11 -0
- package/lib/telemetry/alerts.mjs +12 -0
- package/lib/telemetry/collect.mjs +237 -1
- package/package.json +1 -1
- package/scripts/ci/check-durable-write-seam.mjs +3 -3
- package/scripts/daemon/agent-daemon.mjs +41 -2
- package/scripts/daemon/assurance.mjs +852 -49
- package/scripts/daemon/cadence-consumer.mjs +182 -9
- package/scripts/daemon/context-compiler.mjs +25 -0
- package/scripts/daemon/deliver.mjs +172 -2
- package/scripts/fleet/closing-sweep.mjs +266 -0
|
@@ -43,6 +43,9 @@
|
|
|
43
43
|
* pollMs drain interval (default 2_000).
|
|
44
44
|
* heartbeatMs health.json refresh interval (default 15_000).
|
|
45
45
|
* recoveryMs stale-claim sweep interval (default 5 * 60_000).
|
|
46
|
+
* deferHoldMs how long a tick deferred by an upstream gate is held off
|
|
47
|
+
* the claim path before the gate is re-tested
|
|
48
|
+
* (default 30_000). Never affects attempt accounting.
|
|
46
49
|
* budgetEscalateMs how often to check whether a budget-band notice is due
|
|
47
50
|
* for the SUPERVISOR (default 5 * 60_000; 0 disables).
|
|
48
51
|
* spawnSession injected spawner for tests; defaults to a real
|
|
@@ -75,6 +78,8 @@ import {
|
|
|
75
78
|
completeTick,
|
|
76
79
|
failTick,
|
|
77
80
|
requeueTick,
|
|
81
|
+
MAX_DEFER_HOLD_MS,
|
|
82
|
+
DEDUPE_LANE_HANDOFF_TIMEOUT,
|
|
78
83
|
enqueueTick,
|
|
79
84
|
recoverStaleClaims,
|
|
80
85
|
sweepRetention,
|
|
@@ -105,6 +110,8 @@ import { classifyCadenceFailure, isPermanent, PERMANENT_MAX_ATTEMPTS } from "../
|
|
|
105
110
|
import { bump as bumpCounter } from "../../lib/diagnostics/counters.mjs";
|
|
106
111
|
import { obligationAllowedUnderPosture } from "../../lib/plan/compile.mjs";
|
|
107
112
|
import { isHumanLaneCadence } from "../../lib/cadences.mjs";
|
|
113
|
+
import { readPause as readGovernancePause, loadGovernance } from "../../lib/org/governance-store.mjs";
|
|
114
|
+
import { teamModePosture } from "../../lib/org/governance.mjs";
|
|
108
115
|
import { sessionPermissionArgs } from "../../lib/session-permissions.mjs";
|
|
109
116
|
import { renderTemplate, buildContext } from "../../lib/render.mjs";
|
|
110
117
|
import { withParallelism } from "../../lib/prompts/parallelism.mjs";
|
|
@@ -156,6 +163,12 @@ const MAX_CONCURRENT_SUB_SESSIONS = 1;
|
|
|
156
163
|
// the burn. 2 retries with exponential back-off is the right balance.
|
|
157
164
|
const DEFAULT_MAX_ATTEMPTS = 2;
|
|
158
165
|
const BACKOFF_SCHEDULE_MS = [0, 30_000, 120_000]; // 1st retry +30s, 2nd retry +2m
|
|
166
|
+
// How long a tick deferred by an upstream gate is held off the claim path
|
|
167
|
+
// before the gate is re-tested (W19). Attempt-count-neutral: see deferTick.
|
|
168
|
+
// 30 s matches the first non-zero backoff rung, so a gate that clears quickly
|
|
169
|
+
// costs at most one wasted interval and a gate that stays shut costs 2 polls'
|
|
170
|
+
// worth of work per minute instead of 30.
|
|
171
|
+
const DEFER_HOLD_MS = 30_000;
|
|
159
172
|
|
|
160
173
|
// Circuit breaker — when 3 same-cadence failures land in a row, stop
|
|
161
174
|
// spawning that cadence for 30 minutes. Prevents launchd-rate runaway.
|
|
@@ -703,6 +716,9 @@ export function startConsumer(opts = {}) {
|
|
|
703
716
|
const backoffSchedule = opts.backoffSchedule || BACKOFF_SCHEDULE_MS;
|
|
704
717
|
const circuitThreshold = opts.circuitThreshold ?? CIRCUIT_OPEN_THRESHOLD;
|
|
705
718
|
const circuitDurationMs = opts.circuitDurationMs ?? CIRCUIT_OPEN_DURATION_MS;
|
|
719
|
+
// Overridable like `backoffSchedule`, so a test can drive the deferral path
|
|
720
|
+
// hard without waiting out a real hold.
|
|
721
|
+
const deferHoldMs = Number.isFinite(opts.deferHoldMs) ? opts.deferHoldMs : DEFER_HOLD_MS;
|
|
706
722
|
const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
707
723
|
// WS4 governance modules (injectable for hermetic tests).
|
|
708
724
|
const governor = opts.governor || resourceGovernor;
|
|
@@ -854,6 +870,21 @@ export function startConsumer(opts = {}) {
|
|
|
854
870
|
* Returns { admit:boolean, reason }.
|
|
855
871
|
*/
|
|
856
872
|
function governanceGate(cadence) {
|
|
873
|
+
// Genesis governance (WS6): a spend-cap / directive pause, or a team that
|
|
874
|
+
// has the seat switched OFF, stops PROACTIVE cadences. The human lane
|
|
875
|
+
// (inbox-processor, messaging-inbound) is never held — a paused seat still
|
|
876
|
+
// answers when addressed. A seat whose server sends no governance fields
|
|
877
|
+
// has no pause file and a null teamMode, so nothing here changes for it.
|
|
878
|
+
if (!(cadence && isHumanLaneCadence(cadence))) {
|
|
879
|
+
try {
|
|
880
|
+
const pause = readGovernancePause(agentRoot);
|
|
881
|
+
if (pause) return { admit: false, reason: `governance-paused:${pause.source}` };
|
|
882
|
+
const g = loadGovernance(agentRoot);
|
|
883
|
+
if (g.present && g.governance && !teamModePosture(g.governance.teamMode).proactive) {
|
|
884
|
+
return { admit: false, reason: "team-mode-off" };
|
|
885
|
+
}
|
|
886
|
+
} catch { /* a governance read fault admits: a governance bug must not wedge cadences (same invariant as below) */ }
|
|
887
|
+
}
|
|
857
888
|
try {
|
|
858
889
|
const rb = rateGuard.checkRateLimit(RATE_PROVIDER, { agentRoot });
|
|
859
890
|
if (!rb.allowed) return { admit: false, reason: "rate-limited" };
|
|
@@ -976,6 +1007,41 @@ export function startConsumer(opts = {}) {
|
|
|
976
1007
|
// cadence are the duplicate this guards against.
|
|
977
1008
|
const inFlightCadences = new Set();
|
|
978
1009
|
|
|
1010
|
+
// PER-CADENCE DEFERRAL HOLDS (W19 rework) — cadence → epoch-ms deadline.
|
|
1011
|
+
//
|
|
1012
|
+
// WHY NOT PER TICK. The first cut stamped the hold on the deferred TICK only.
|
|
1013
|
+
// That makes the requeue rate linear in queue depth: with `depth` pending
|
|
1014
|
+
// copies, `depth` independent deadlines expire inside one hold window and the
|
|
1015
|
+
// drain claims and re-defers every one of them. MEASURED on this branch over
|
|
1016
|
+
// 15 min of 2 s polls behind a gate that never opens (30 s hold): depth 1 →
|
|
1017
|
+
// 8.0/s to 0.03/s, but depth 87 (the live inbox) → 8.0/s to only 2.9/s, and
|
|
1018
|
+
// depth 240 and 1000 → 8.0/s BEFORE AND AFTER. The incident's own queue was
|
|
1019
|
+
// 15,674 deep, so the per-tick hold alone would have delivered nothing.
|
|
1020
|
+
//
|
|
1021
|
+
// A gate is shut for a CADENCE, not for a file. One hold therefore covers
|
|
1022
|
+
// every pending tick of that cadence, and the rate stops depending on how
|
|
1023
|
+
// many there are: 120 deferrals/hour at any depth.
|
|
1024
|
+
//
|
|
1025
|
+
// In memory, deliberately. The durable half is already on disk (each
|
|
1026
|
+
// deferred tick carries `metadata.notBefore`), so a restart loses only the
|
|
1027
|
+
// grouping — the first poll after a restart re-establishes one hold per shut
|
|
1028
|
+
// cadence at a cost of one deferral each.
|
|
1029
|
+
const cadenceHoldUntil = new Map();
|
|
1030
|
+
|
|
1031
|
+
/** Set (never shorten) the hold on a cadence. */
|
|
1032
|
+
function holdCadence(cadence, until) {
|
|
1033
|
+
if (!cadence || !Number.isFinite(until)) return;
|
|
1034
|
+
const prior = cadenceHoldUntil.get(cadence);
|
|
1035
|
+
if (!Number.isFinite(prior) || until > prior) cadenceHoldUntil.set(cadence, until);
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
/** Drop holds that have expired, so the map stays the size of the shut set. */
|
|
1039
|
+
function pruneCadenceHolds(now) {
|
|
1040
|
+
for (const [cadence, until] of cadenceHoldUntil) {
|
|
1041
|
+
if (!(until > now)) cadenceHoldUntil.delete(cadence);
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
|
|
979
1045
|
// Injectable clock so tests can drive the stale window deterministically.
|
|
980
1046
|
const nowMs = typeof opts.now === "function" ? opts.now : Date.now;
|
|
981
1047
|
|
|
@@ -990,19 +1056,57 @@ export function startConsumer(opts = {}) {
|
|
|
990
1056
|
|
|
991
1057
|
/**
|
|
992
1058
|
* Re-queue a tick that cannot run YET (an upstream gate, never a per-event
|
|
993
|
-
* failure) to the TAIL under a new id (audit F8)
|
|
1059
|
+
* failure) to the TAIL under a new id (audit F8), HELD until it is worth
|
|
1060
|
+
* re-testing the gate. Attempts untouched.
|
|
1061
|
+
*
|
|
994
1062
|
* `count:false` (the circuit/backoff skip) re-keys without touching the
|
|
995
1063
|
* retry/deferral counters or last_decision — a held-back cadence was never
|
|
996
1064
|
* attempted, and the skip is already logged and counted as skipped_*; a
|
|
997
1065
|
* 30 s poll inside a 60 min circuit window must not read as 120 retries.
|
|
1066
|
+
*
|
|
1067
|
+
* THE HOT LOOP THIS CLOSES (W19, measured on this seat 2026-10-01).
|
|
1068
|
+
* Re-keying alone left the tick immediately claimable, so every poll
|
|
1069
|
+
* re-claimed it, re-tested the shut gate and wrote it back: 1,186,383
|
|
1070
|
+
* deferrals in 68 h (8/s, the 16-per-drain batch cap over a 2 s poll), one
|
|
1071
|
+
* event at 40,150 deferrals, 443 MB/day of audit log. The incident note
|
|
1072
|
+
* blamed the CONCURRENCY call site; the log says otherwise — the live
|
|
1073
|
+
* reasons were `cadence-in-flight:sub-session` (35,043/day),
|
|
1074
|
+
* `cadence-in-flight:handoff` (12,035), `skipped_circuit_open` (14,307) and
|
|
1075
|
+
* the governance gate, with concurrency not in the top twenty. Every one of
|
|
1076
|
+
* them lands HERE, so the hold belongs here and not at any one call site —
|
|
1077
|
+
* fixing only the concurrency path would have changed nothing measurable.
|
|
1078
|
+
*
|
|
1079
|
+
* ATTEMPT-COUNT-NEUTRAL, BY CONSTRUCTION AND BY TEST. The hold is a
|
|
1080
|
+
* timestamp on the requeued copy; `attempts` is written only by `failTick`
|
|
1081
|
+
* (audit M5), which is not called on this path. A gate closure therefore
|
|
1082
|
+
* cannot spend retry budget or move a tick toward dlq/, however long it
|
|
1083
|
+
* stays shut. That is the expensive mistake to make here: reusing the retry
|
|
1084
|
+
* machinery for the delay would dead-letter a perfectly healthy cadence for
|
|
1085
|
+
* the crime of being deferred often, turning a delay into a lost task.
|
|
1086
|
+
* `defer-hold.test.mjs` pins it.
|
|
1087
|
+
*
|
|
1088
|
+
* @param {number} [opts.holdMs] override the hold (e.g. a gate that knows
|
|
1089
|
+
* its own `retry_at`). 0 means "claimable now".
|
|
998
1090
|
*/
|
|
999
|
-
function deferTick(event, reason, extra = {}, { count = true } = {}) {
|
|
1000
|
-
|
|
1091
|
+
function deferTick(event, reason, extra = {}, { count = true, holdMs = deferHoldMs } = {}) {
|
|
1092
|
+
// Clamped here as well as in requeueTick: the number that reaches the
|
|
1093
|
+
// per-cadence map must be the same one that reaches the tick's own stamp,
|
|
1094
|
+
// or claimNextTick's two holds would disagree about when this is due.
|
|
1095
|
+
const hold = Number.isFinite(holdMs) ? Math.min(Math.max(holdMs, 0), MAX_DEFER_HOLD_MS) : 0;
|
|
1096
|
+
const r = requeueTick(agentRoot, event, { defer: true, now: new Date(nowMs()), holdMs: hold });
|
|
1097
|
+
// THE DEPTH-INDEPENDENT HALF. Hold the CADENCE, not just this tick — see
|
|
1098
|
+
// `cadenceHoldUntil`. Every pending copy of this cadence is now skipped
|
|
1099
|
+
// without being claimed until the hold expires, so a shut gate costs one
|
|
1100
|
+
// deferral per hold window however deep the queue is.
|
|
1101
|
+
if (hold > 0) holdCadence(event.cadence, nowMs() + hold);
|
|
1001
1102
|
if (count) {
|
|
1002
|
-
stats.retries
|
|
1103
|
+
// NOT stats.retries. A deferral is not a retry, and conflating them made
|
|
1104
|
+
// the health file read `retries: 1186419` against 36 real retries — an
|
|
1105
|
+
// operator seeing a million retries reasonably concludes the DLQ is
|
|
1106
|
+
// about to fill. `stats.deferred` is the honest counter.
|
|
1003
1107
|
stats.deferred += 1;
|
|
1004
1108
|
stats.last_decision = "deferred";
|
|
1005
|
-
log({ level: "info", stage: "escalate_deferred", reason, id: event.id, requeued_as: r && r.id, cadence: event.cadence, ...extra });
|
|
1109
|
+
log({ level: "info", stage: "escalate_deferred", reason, id: event.id, requeued_as: r && r.id, cadence: event.cadence, hold_ms: hold, ...extra });
|
|
1006
1110
|
}
|
|
1007
1111
|
return { ok: false, decision: "deferred", reason };
|
|
1008
1112
|
}
|
|
@@ -1124,9 +1228,28 @@ export function startConsumer(opts = {}) {
|
|
|
1124
1228
|
const r = enqueueTick({
|
|
1125
1229
|
cadence: h.cadence,
|
|
1126
1230
|
source: "handoff-timeout",
|
|
1231
|
+
// THE MAJORITY LANE OF THE LIVE DEPTH (W19 rework). This sweep has
|
|
1232
|
+
// no time gate of any kind: every expired handoff enqueues, and on
|
|
1233
|
+
// 2026-10-01 that was 47 of the live seat's 88 pending ticks, all
|
|
1234
|
+
// of them `meeting-action-capture` — 47 copies of "the session
|
|
1235
|
+
// never acked it, run it again", which cannot each do distinct
|
|
1236
|
+
// work. Its own lane, so a pending periodic tick never suppresses
|
|
1237
|
+
// it and it never suppresses one.
|
|
1238
|
+
dedupeLane: DEDUPE_LANE_HANDOFF_TIMEOUT,
|
|
1127
1239
|
agentRoot,
|
|
1128
1240
|
metadata: { ...meta, handoffTimedOut: true, handoffTickId: h.tickId, reason: `handoff to session timed out (${h.deadlineAt || "no deadline"})` },
|
|
1129
1241
|
});
|
|
1242
|
+
// A SUPPRESSED DUPLICATE IS A SUCCESS, AND GETTING THIS WRONG MAKES A
|
|
1243
|
+
// SECOND HOT LOOP. The work is already pending, so this handoff is
|
|
1244
|
+
// done and must be retired — if it threw instead, the handoff would
|
|
1245
|
+
// stay open, `cadenceInFlight` would report "handoff" forever, and
|
|
1246
|
+
// this sweep (which runs on EVERY poll, ahead of the drain) would
|
|
1247
|
+
// re-enqueue, re-suppress and write two log rows every 2 s.
|
|
1248
|
+
if (r && r.skipped === "pending-duplicate") {
|
|
1249
|
+
log({ level: "info", stage: "handoff_timed_out_duplicate", id: h.tickId, cadence: h.cadence, pending_id: r.pendingId || null, note: "a handoff-timeout tick for this cadence is already pending; handoff retired" });
|
|
1250
|
+
stats.handoff_timeouts += 1;
|
|
1251
|
+
return true;
|
|
1252
|
+
}
|
|
1130
1253
|
// enqueueTick swallows an unwritable inbox (fallbackOnly, audit row
|
|
1131
1254
|
// only) — that is NOT back on the bus: keep the handoff open.
|
|
1132
1255
|
if (!r || !r.path) throw new Error(`tick not enqueued (${r && r.fallbackOnly ? "inbox unwritable" : "no path"})`);
|
|
@@ -1312,9 +1435,31 @@ export function startConsumer(opts = {}) {
|
|
|
1312
1435
|
}
|
|
1313
1436
|
}
|
|
1314
1437
|
|
|
1438
|
+
/**
|
|
1439
|
+
* How many cadences are held right now, and until when.
|
|
1440
|
+
*
|
|
1441
|
+
* `held` makes a deferral hold OBSERVABLE. Without it an operator cannot tell
|
|
1442
|
+
* "this tick is waiting 30 s for a shut gate" from "this tick is stuck",
|
|
1443
|
+
* because claimNextTick skips a held tick silently and nothing else in the
|
|
1444
|
+
* health file mentions it. Reported in both the heartbeat and getStats so the
|
|
1445
|
+
* two never disagree.
|
|
1446
|
+
*/
|
|
1447
|
+
function heldSnapshot() {
|
|
1448
|
+
const now = nowMs();
|
|
1449
|
+
let held = 0;
|
|
1450
|
+
let furthest = 0;
|
|
1451
|
+
for (const until of cadenceHoldUntil.values()) {
|
|
1452
|
+
if (until > now) {
|
|
1453
|
+
held += 1;
|
|
1454
|
+
if (until > furthest) furthest = until;
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
return { held, held_until: furthest ? new Date(furthest).toISOString() : null };
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1315
1460
|
function heartbeat() {
|
|
1316
1461
|
writeHealth(agentRoot, {
|
|
1317
|
-
stats: { ...stats, depth: busDepth(agentRoot) },
|
|
1462
|
+
stats: { ...stats, depth: busDepth(agentRoot), ...heldSnapshot() },
|
|
1318
1463
|
active_subsessions: activeSubSessions,
|
|
1319
1464
|
stopping,
|
|
1320
1465
|
});
|
|
@@ -1338,7 +1483,28 @@ export function startConsumer(opts = {}) {
|
|
|
1338
1483
|
// Put the event back — to the TAIL (F8). Attempt accounting is
|
|
1339
1484
|
// single-sourced in failTick (audit M5), so we never touch it here: a
|
|
1340
1485
|
// held-back-by-circuit re-queue is not a failed attempt, nor a retry.
|
|
1341
|
-
|
|
1486
|
+
// This gate knows its own deadline, so hold to exactly that rather than
|
|
1487
|
+
// the generic interval: a 30 min open circuit should cost ONE requeue,
|
|
1488
|
+
// not sixty. Floored at the generic hold so a retry_at already in the
|
|
1489
|
+
// past (clock skew, a just-expired window) cannot re-open the hot loop.
|
|
1490
|
+
deferTick(
|
|
1491
|
+
event,
|
|
1492
|
+
gate.reason,
|
|
1493
|
+
{ retry_at: new Date(gate.retry_at).toISOString() },
|
|
1494
|
+
// Hold to this gate's own deadline, so a 30 min open circuit costs ONE
|
|
1495
|
+
// requeue not sixty. FLOORED at the generic hold so a retry_at already
|
|
1496
|
+
// in the past cannot re-open the hot loop, and CEILINGED at
|
|
1497
|
+
// MAX_DEFER_HOLD_MS so a retry_at from a provider Retry-After, a
|
|
1498
|
+
// forward clock jump or a corrupted number cannot park the tick out of
|
|
1499
|
+
// sight for days (claimNextTick skips a held tick silently).
|
|
1500
|
+
{
|
|
1501
|
+
count: false,
|
|
1502
|
+
holdMs: Math.min(
|
|
1503
|
+
MAX_DEFER_HOLD_MS,
|
|
1504
|
+
Math.max(deferHoldMs, gate.retry_at - nowMs()),
|
|
1505
|
+
),
|
|
1506
|
+
},
|
|
1507
|
+
);
|
|
1342
1508
|
return { ok: false, decision: gate.reason };
|
|
1343
1509
|
}
|
|
1344
1510
|
|
|
@@ -1710,6 +1876,7 @@ export function startConsumer(opts = {}) {
|
|
|
1710
1876
|
|
|
1711
1877
|
let processed = 0;
|
|
1712
1878
|
let spawnedThisTick = 0;
|
|
1879
|
+
pruneCadenceHolds(nowMs());
|
|
1713
1880
|
// Head-of-line (F8): a deferred tick is re-keyed to the tail, so the loop
|
|
1714
1881
|
// KEEPS SCANNING past it to the next cadence instead of breaking. This set
|
|
1715
1882
|
// (keyed by the tick's original id, which a re-key preserves) is how the
|
|
@@ -1718,7 +1885,10 @@ export function startConsumer(opts = {}) {
|
|
|
1718
1885
|
// holding only un-runnable ticks costs one visit per tick, not sixteen.
|
|
1719
1886
|
const seenThisTick = new Set();
|
|
1720
1887
|
while (!stopping) {
|
|
1721
|
-
|
|
1888
|
+
// Due-aware (W19): a tick still inside its deferral hold is skipped
|
|
1889
|
+
// without being claimed, so a shut gate costs a cached read rather than
|
|
1890
|
+
// a claim + requeue + audit row.
|
|
1891
|
+
const claim = claimNextTick(agentRoot, { now: nowMs(), heldCadences: cadenceHoldUntil });
|
|
1722
1892
|
if (!claim) break;
|
|
1723
1893
|
const event = claim.event;
|
|
1724
1894
|
const key = (event.metadata && event.metadata.originalId) || event.id;
|
|
@@ -1876,7 +2046,7 @@ export function startConsumer(opts = {}) {
|
|
|
1876
2046
|
}
|
|
1877
2047
|
|
|
1878
2048
|
function getStats() {
|
|
1879
|
-
return { ...stats, depth: busDepth(agentRoot), agent_root: agentRoot };
|
|
2049
|
+
return { ...stats, depth: busDepth(agentRoot), ...heldSnapshot(), agent_root: agentRoot };
|
|
1880
2050
|
}
|
|
1881
2051
|
|
|
1882
2052
|
return {
|
|
@@ -1891,6 +2061,9 @@ export function startConsumer(opts = {}) {
|
|
|
1891
2061
|
_clearInFlight: (id) => inFlightClaimIds.delete(id),
|
|
1892
2062
|
// Front door: the open handoffs this consumer has written for the session.
|
|
1893
2063
|
_handoffs: () => listHandoffs(agentRoot),
|
|
2064
|
+
// Write the health file on demand, so a test can read `held` without
|
|
2065
|
+
// waiting on the heartbeat timer.
|
|
2066
|
+
_heartbeat: heartbeat,
|
|
1894
2067
|
};
|
|
1895
2068
|
}
|
|
1896
2069
|
|
|
@@ -20,6 +20,8 @@ import { isPrivateConversation, historyDirNames } from "../../lib/context/histor
|
|
|
20
20
|
import { stripDisclosureInstructionsAndWarn } from "../../lib/identity/disclosure-instructions.mjs";
|
|
21
21
|
import { fitSections } from "../../lib/context/budget.mjs";
|
|
22
22
|
import { currentWorkBlock, audienceForItem } from "../../lib/session/current-work.mjs";
|
|
23
|
+
import { loadGovernance, readPause as readGovernancePause } from "../../lib/org/governance-store.mjs";
|
|
24
|
+
import { governancePromptBlock } from "../../lib/org/governance.mjs";
|
|
23
25
|
|
|
24
26
|
const AGENT_REPO_DIR = process.env.__TEST_AGENT_DIR || process.env.AGENT_DIR || join(new URL(".", import.meta.url).pathname, "../..");
|
|
25
27
|
const SESSION_DIR = process.env.__TEST_SESSION_DIR || join(AGENT_REPO_DIR, "state", "sessions");
|
|
@@ -462,6 +464,19 @@ function loadCachedOrgCfg() {
|
|
|
462
464
|
}
|
|
463
465
|
|
|
464
466
|
/** For tests: reset the cached org config so a fresh config/org.yaml is re-read. */
|
|
467
|
+
/** The decision-rights block for this seat, or "" (old server / unreadable). */
|
|
468
|
+
function loadDecisionRights() {
|
|
469
|
+
try {
|
|
470
|
+
const g = loadGovernance(AGENT_REPO_DIR);
|
|
471
|
+
if (!g.present) return "";
|
|
472
|
+
return governancePromptBlock(g.governance, { pause: readGovernancePause(AGENT_REPO_DIR) });
|
|
473
|
+
} catch {
|
|
474
|
+
// The block is advisory; the tool gate is what enforces. Omitting it on a
|
|
475
|
+
// read fault leaves the context exactly as it was before WS6.
|
|
476
|
+
return "";
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
|
|
465
480
|
export function _resetOrgCfgCache() {
|
|
466
481
|
cachedOrgCfg = undefined;
|
|
467
482
|
}
|
|
@@ -610,6 +625,16 @@ export async function compileContext(item, classResult, options = {}) {
|
|
|
610
625
|
sections.push("");
|
|
611
626
|
}
|
|
612
627
|
|
|
628
|
+
// Genesis decision rights (WS6). The tool gate enforces only the clauses it
|
|
629
|
+
// can match clearly; every clause still reaches the model here. Absent for
|
|
630
|
+
// a seat whose server sends no governance fields — the block is omitted.
|
|
631
|
+
const rightsText = loadDecisionRights();
|
|
632
|
+
if (rightsText) {
|
|
633
|
+
sections.push("## Decision Rights");
|
|
634
|
+
sections.push(rightsText);
|
|
635
|
+
sections.push("");
|
|
636
|
+
}
|
|
637
|
+
|
|
613
638
|
if (orgMemoryText) {
|
|
614
639
|
sections.push("## Org Shared Memory");
|
|
615
640
|
sections.push(orgMemoryText);
|
|
@@ -28,14 +28,21 @@
|
|
|
28
28
|
import { execFileSync } from "child_process";
|
|
29
29
|
import { mkdirSync, writeFileSync } from "fs";
|
|
30
30
|
import { join } from "path";
|
|
31
|
-
import { screenOutbound } from "../../lib/comms/send-gate.mjs";
|
|
31
|
+
import { screenOutbound, machineLeakVerdict } from "../../lib/comms/send-gate.mjs";
|
|
32
32
|
import { getHookBus } from "../../lib/hooks/bus.mjs";
|
|
33
33
|
import { recordNoSend, recordOutbound } from "../../lib/comms/receipts.mjs";
|
|
34
34
|
// Pure data (a frozen table, no imports of its own) — the surface vocabulary the
|
|
35
35
|
// inbound projection stamps into `raw_ref`. Imported rather than re-listed so a
|
|
36
36
|
// new surface cannot be added upstream without this file's switch noticing.
|
|
37
37
|
import { ROOM_SURFACES, SURFACE_NAMES } from "../../lib/org/inbound/surfaces.mjs";
|
|
38
|
-
import { isAgentStatusNotice, isRetiredFirstReply } from "../../lib/assurance/first-reply.mjs";
|
|
38
|
+
import { isAgentStatusNotice, isRetiredFirstReply, opensAsRescueNotice } from "../../lib/assurance/first-reply.mjs";
|
|
39
|
+
// THE TWO HALVES OF THE MACHINE-STATUS REFUSAL. `isMachineStatusDeliveryKind`
|
|
40
|
+
// is the ORIGIN test (what composed this) and `machineStatusNarration` is the
|
|
41
|
+
// SHAPE test (what it reads like). Both are pure data + pure predicates, imported
|
|
42
|
+
// rather than restated, so the decision in `notice-plane.mjs` and the enforcement
|
|
43
|
+
// here cannot drift. See the refusal block in `deliver` below.
|
|
44
|
+
import { isMachineStatusDeliveryKind } from "../../lib/assurance/notice-plane.mjs";
|
|
45
|
+
import { machineStatusNarration } from "../../lib/comms/machine-status.mjs";
|
|
39
46
|
|
|
40
47
|
const AGENT_REPO_DIR = process.env.AGENT_DIR || join(new URL(".", import.meta.url).pathname, "../..");
|
|
41
48
|
|
|
@@ -142,6 +149,43 @@ const CHANNEL_SURFACES = ROOM_SURFACES;
|
|
|
142
149
|
* otherwise healthy deployment, which is a strictly worse failure than one
|
|
143
150
|
* extra attempt.
|
|
144
151
|
*/
|
|
152
|
+
/**
|
|
153
|
+
* The code the FOUR MACHINE-STATUS refusals below carry — and why it is not
|
|
154
|
+
* `FORBIDDEN_SCOPE` any more.
|
|
155
|
+
*
|
|
156
|
+
* All three used to be coded `FORBIDDEN_SCOPE`, which is hq's code for a 403.
|
|
157
|
+
* `isScopeFault()` in `lib/daemon/reply-debt.mjs` returns true for exactly that
|
|
158
|
+
* string, and that predicate drives both `configFault:"missing-scope"` on a
|
|
159
|
+
* needs-attention row (`agent-daemon.mjs`) and the `reply.scope_refused` counter
|
|
160
|
+
* behind the `seat_missing_scope` CRITICAL alert, whose own detail text reads
|
|
161
|
+
* "Grant the scope; the seat cannot."
|
|
162
|
+
*
|
|
163
|
+
* So "I correctly refused to post my own status line" was about to become
|
|
164
|
+
* indistinguishable from "this seat is missing a scope and a human must grant
|
|
165
|
+
* one" — a critical alert, on a working seat, with an instruction nobody can
|
|
166
|
+
* carry out. Latent only because no seat had shipped the routing change yet; it
|
|
167
|
+
* lands the first time one does.
|
|
168
|
+
*
|
|
169
|
+
* Deliberately NOT added to PERMANENT_RPC_CODES below: that list is about error
|
|
170
|
+
* frames hq returns, and these refusals are this daemon's own and already carry
|
|
171
|
+
* `permanent:true` explicitly.
|
|
172
|
+
*
|
|
173
|
+
* `lib/assurance/operator-plane.test.mjs` pins that `isScopeFault` is false for
|
|
174
|
+
* this code, because the whole value of the distinction is that one predicate
|
|
175
|
+
* stops seeing the other thing.
|
|
176
|
+
*/
|
|
177
|
+
export const OPERATOR_PLANE_REFUSAL = "OPERATOR_PLANE";
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* The code a CONTENT refusal carries: a body this daemon will not send because
|
|
181
|
+
* of what it says (send-gate's MACHINE_LEAK_RULES), not because of who it may
|
|
182
|
+
* talk to. Distinct from `FORBIDDEN_SCOPE` for the reason given above for
|
|
183
|
+
* OPERATOR_PLANE_REFUSAL: `isScopeFault` reads that string as "a human must
|
|
184
|
+
* grant this seat a scope" and raises the `seat_missing_scope` CRITICAL alert,
|
|
185
|
+
* and the remedy for a leak is a rewrite, not a grant.
|
|
186
|
+
*/
|
|
187
|
+
export const CONTENT_POLICY_REFUSAL = "CONTENT_POLICY";
|
|
188
|
+
|
|
145
189
|
const PERMANENT_RPC_CODES = Object.freeze([
|
|
146
190
|
"NOT_FOUND",
|
|
147
191
|
"BAD_REQUEST",
|
|
@@ -945,6 +989,33 @@ export async function deliverReaction(item, emoji, o = {}) {
|
|
|
945
989
|
* @param {object} frame the `{sent:false, …}` result being returned
|
|
946
990
|
* @returns {object} `frame`
|
|
947
991
|
*/
|
|
992
|
+
/**
|
|
993
|
+
* Is this body one of this daemon's OWN status templates, carrying no answer?
|
|
994
|
+
*
|
|
995
|
+
* One paragraph (so the composers' excerpt paragraph is absent) AND an exact
|
|
996
|
+
* whole-body match against `assurance.narratorPatterns`, which is derived from
|
|
997
|
+
* the composers rather than written out — so rewording a template keeps it
|
|
998
|
+
* recognised here, and a real reply that merely opens with a familiar phrase is
|
|
999
|
+
* not matched, because the match spans the whole body.
|
|
1000
|
+
*
|
|
1001
|
+
* FAILS OPEN. A dynamic import that throws, or an assurance module mid-edit, must
|
|
1002
|
+
* never be able to hold up a real reply.
|
|
1003
|
+
*
|
|
1004
|
+
* @param {unknown} text
|
|
1005
|
+
* @returns {Promise<string|null>} a short excerpt of the matched body, or null
|
|
1006
|
+
*/
|
|
1007
|
+
async function answerlessOwnTemplate(text) {
|
|
1008
|
+
const raw = String(text == null ? "" : text).trim();
|
|
1009
|
+
if (!raw) return null;
|
|
1010
|
+
const paragraphs = raw.split(/\n\s*\n/).map((p) => p.trim()).filter(Boolean);
|
|
1011
|
+
if (paragraphs.length !== 1) return null;
|
|
1012
|
+
try {
|
|
1013
|
+
const { isOwnNarratorText } = await import("./assurance.mjs");
|
|
1014
|
+
if (typeof isOwnNarratorText !== "function") return null;
|
|
1015
|
+
return isOwnNarratorText(raw) ? raw.slice(0, 90) : null;
|
|
1016
|
+
} catch { return null; }
|
|
1017
|
+
}
|
|
1018
|
+
|
|
948
1019
|
function noteNoSend(item, o, frame) {
|
|
949
1020
|
recordNoSend({
|
|
950
1021
|
decision: "refused",
|
|
@@ -1017,6 +1088,90 @@ export async function deliver(item, text, o = {}) {
|
|
|
1017
1088
|
try { console.error(`[deliver] refused: retired placeholder reply in outbound body ${JSON.stringify(String(text).slice(0, 120))} — this is a stale composer, not a message (permanent, not retrying)`); } catch { /* */ }
|
|
1018
1089
|
return noteNoSend(item, o, { sent: false, via: null, permanent: true, channel: item && item.channel_id, code: "FORBIDDEN_SCOPE", error: "FORBIDDEN_SCOPE: retired placeholder reply — the first line must be generated from the actual thread, or nothing" });
|
|
1019
1090
|
}
|
|
1091
|
+
// ── MACHINE STATUS NEVER REACHES A PERSON ────────────────────────────────
|
|
1092
|
+
//
|
|
1093
|
+
// The register above is a list of sentences that HAVE shipped. It cannot
|
|
1094
|
+
// refuse the next one, and the history says the next one is what gets sent:
|
|
1095
|
+
// three fixes, three registrations, three new variants, and on 2026-10-01 the
|
|
1096
|
+
// owner received the current variant twice, the second time eleven minutes
|
|
1097
|
+
// after complaining about the first. `lib/assurance/notice-plane.mjs` carries
|
|
1098
|
+
// the decision and the argument; this is where it is enforced on bytes.
|
|
1099
|
+
//
|
|
1100
|
+
// TWO TESTS, BECAUSE THEY FAIL DIFFERENTLY.
|
|
1101
|
+
//
|
|
1102
|
+
// ORIGIN — `kind` is "failure" or "notice", i.e. the caller has declared
|
|
1103
|
+
// that a machine composed this about its own apparatus. No
|
|
1104
|
+
// sentence can get around this one, which is the property every
|
|
1105
|
+
// previous fix lacked. `sayOutcomeNotice` no longer reaches this
|
|
1106
|
+
// function at all (it routes to the operator sink), so a refusal
|
|
1107
|
+
// here means a NEW caller has been wired to the human transport —
|
|
1108
|
+
// exactly the regression this is here to catch.
|
|
1109
|
+
// SHAPE — an apparatus subject plus a status predicate, in a body carrying
|
|
1110
|
+
// nothing else. This is what catches a sentence the codebase has
|
|
1111
|
+
// never contained, including one declared as a `reply` by an author
|
|
1112
|
+
// who routed around the origin test. Narrow by construction: a real
|
|
1113
|
+
// answer that reports the WORK failing has no apparatus subject,
|
|
1114
|
+
// and anything carrying a link, a fence, a list or real length is
|
|
1115
|
+
// exonerated outright. See `lib/comms/machine-status.mjs`.
|
|
1116
|
+
//
|
|
1117
|
+
// Permanent, like the other two refusals here: retrying re-runs a
|
|
1118
|
+
// deterministic screen and gets the same answer. The obligation is not lost —
|
|
1119
|
+
// `noteNoSend` writes the receipt and the assurance sweep's `escalate` has
|
|
1120
|
+
// already written the `needs-attention` row an operator sweeps.
|
|
1121
|
+
if (isMachineStatusDeliveryKind(kind)) {
|
|
1122
|
+
try { console.error(`[deliver] refused: machine-status delivery kind ${JSON.stringify(kind)} — status about this machine's own work goes to the operator plane (needs-attention, counters, logs), never to a conversation (permanent, not retrying)`); } catch { /* */ }
|
|
1123
|
+
return noteNoSend(item, o, { sent: false, via: null, permanent: true, channel: item && item.channel_id, code: OPERATOR_PLANE_REFUSAL, error: `${OPERATOR_PLANE_REFUSAL}: machine-status kind "${kind}" may not be delivered to a conversation — route it to the operator plane` });
|
|
1124
|
+
}
|
|
1125
|
+
const narration = machineStatusNarration(text);
|
|
1126
|
+
if (narration) {
|
|
1127
|
+
try { console.error(`[deliver] refused: outbound body narrates this machine's own apparatus (${JSON.stringify(narration)}) — a person is owed the answer or nothing, not a status line (permanent, not retrying)`); } catch { /* */ }
|
|
1128
|
+
return noteNoSend(item, o, { sent: false, via: null, permanent: true, channel: item && item.channel_id, code: OPERATOR_PLANE_REFUSAL, error: `${OPERATOR_PLANE_REFUSAL}: machine-status narration in outbound body (${narration}) — this belongs in the operator plane, not in a conversation` });
|
|
1129
|
+
}
|
|
1130
|
+
// ── THE THIRD SHAPE: ONE OF THIS DAEMON'S OWN TEMPLATES, WITH NO ANSWER ───
|
|
1131
|
+
//
|
|
1132
|
+
// Found by an adversarial pass on 2026-10-02, which got one through both gates
|
|
1133
|
+
// above. `composeSilentSuccess` with no result text —
|
|
1134
|
+
//
|
|
1135
|
+
// Finished this but didn't get a reply out to you — my fault. I don't have a
|
|
1136
|
+
// clean result to show you, though. Want me to run it again?
|
|
1137
|
+
//
|
|
1138
|
+
// — is machine status wearing a reply's `kind`: the ORIGIN test never sees it,
|
|
1139
|
+
// and the SHAPE test finds no apparatus noun in it. `composeRefusedSend` with
|
|
1140
|
+
// no result text is the same. Both are now routed to the operator plane at the
|
|
1141
|
+
// emitter; this is the backstop, because the emitter decision is one `if` and
|
|
1142
|
+
// this subsystem's guards have been accidentally undone three times.
|
|
1143
|
+
//
|
|
1144
|
+
// EXACT, DERIVED, AND IT CANNOT EAT AN ANSWER. `isOwnNarratorText` matches a
|
|
1145
|
+
// body against patterns DERIVED FROM THE COMPOSERS THEMSELVES (so a reworded
|
|
1146
|
+
// template is still recognised, which a literal register could not do), anchored
|
|
1147
|
+
// `^…$` over the whole body. The one-paragraph condition is what separates the
|
|
1148
|
+
// two shapes of those composers: both of them put the session's result text in
|
|
1149
|
+
// a SECOND paragraph, so a body with no second paragraph is the template alone.
|
|
1150
|
+
// A real reply that happens to open with a familiar phrase cannot match, because
|
|
1151
|
+
// the match is the entire body, not a prefix.
|
|
1152
|
+
//
|
|
1153
|
+
// Lazily imported: `assurance.mjs` imports this module, and the patterns are
|
|
1154
|
+
// derived from composers that live there. Fails OPEN — an import error must
|
|
1155
|
+
// never hold up a real reply.
|
|
1156
|
+
const answerless = await answerlessOwnTemplate(text);
|
|
1157
|
+
if (answerless) {
|
|
1158
|
+
try { console.error(`[deliver] refused: outbound body is this daemon's own status template with no answer in it — ${JSON.stringify(answerless)} (permanent, not retrying)`); } catch { /* */ }
|
|
1159
|
+
return noteNoSend(item, o, { sent: false, via: null, permanent: true, channel: item && item.channel_id, code: OPERATOR_PLANE_REFUSAL, error: `${OPERATOR_PLANE_REFUSAL}: machine-status template with no work product in it — route it to the operator plane, not to a conversation` });
|
|
1160
|
+
}
|
|
1161
|
+
// ── THE FOURTH SHAPE: A RESCUE WRAPPER, WITH A SESSION'S NOTES INSIDE IT ──
|
|
1162
|
+
//
|
|
1163
|
+
// The gate above stops the two rescue composers when they carry NOTHING. With
|
|
1164
|
+
// a second paragraph they used to be treated as replies, and the second
|
|
1165
|
+
// paragraph is the session's end-of-turn text — its report to the process that
|
|
1166
|
+
// spawned it, with this machine's file paths, queue ids and session talk in it.
|
|
1167
|
+
// That body went to team channels under the seat's name from five seats.
|
|
1168
|
+
// `settleSession` now holds that text in the operator plane and sends neither
|
|
1169
|
+
// wrapper to anyone; this refuses the wrapper here too, whatever follows the
|
|
1170
|
+
// head, so the emitter decision is not the only thing standing.
|
|
1171
|
+
if (opensAsRescueNotice(text)) {
|
|
1172
|
+
try { console.error(`[deliver] refused: outbound body opens as the assurance rescue wrapper — a session's end-of-turn text is held for the owner in the operator plane, never posted (permanent, not retrying)`); } catch { /* */ }
|
|
1173
|
+
return noteNoSend(item, o, { sent: false, via: null, permanent: true, channel: item && item.channel_id, code: OPERATOR_PLANE_REFUSAL, error: `${OPERATOR_PLANE_REFUSAL}: the assurance rescue wrapper may not reach a conversation — the session's text is held for the owner (maestro attention)` });
|
|
1174
|
+
}
|
|
1020
1175
|
const leak = scaffoldMarkerLeak(text);
|
|
1021
1176
|
if (leak) {
|
|
1022
1177
|
const surface = (item && item.channel_id) ? "room" : "unknown";
|
|
@@ -1033,6 +1188,21 @@ export async function deliver(item, text, o = {}) {
|
|
|
1033
1188
|
try { console.error(`[deliver] refused: machine-local path ${JSON.stringify(pathLeak)} in outbound body (permanent, not retrying)`); } catch { /* */ }
|
|
1034
1189
|
return noteNoSend(item, o, { sent: false, via: null, permanent: true, channel: item && item.channel_id, surface, code: "FORBIDDEN_SCOPE", error: `FORBIDDEN_SCOPE: machine-local path ${pathLeak} in outbound body — no one off this machine can open it; a reply must not carry a local path` });
|
|
1035
1190
|
}
|
|
1191
|
+
// ── MACHINE INTERNALS, ON EVERY LANE ────────────────────────────────────────
|
|
1192
|
+
//
|
|
1193
|
+
// send-gate's MACHINE_LEAK_RULES (agent-state paths, queue and inbox ids, the
|
|
1194
|
+
// agent's own sessions) run inside `screenOutbound`, which the cohort and
|
|
1195
|
+
// slack lanes call — but the gmail lane below shells out to a script that runs
|
|
1196
|
+
// only the Layer-0 validator, so a rule that lives only in the lanes is a rule
|
|
1197
|
+
// one lane routes around. Applied here, once, before any lane. Pure, so it
|
|
1198
|
+
// cannot fail open; the refusal names the rule and the span, and nothing is
|
|
1199
|
+
// rewritten — the caller escalates, the sender restates.
|
|
1200
|
+
const machineLeak = machineLeakVerdict(text, { agentRoot: AGENT_REPO_DIR });
|
|
1201
|
+
if (machineLeak) {
|
|
1202
|
+
const surface = (item && item.channel_id) ? "room" : "unknown";
|
|
1203
|
+
try { console.error(`[deliver] refused: ${machineLeak.reason} (permanent, not retrying)`); } catch { /* */ }
|
|
1204
|
+
return noteNoSend(item, o, { sent: false, via: null, permanent: true, channel: item && item.channel_id, surface, code: CONTENT_POLICY_REFUSAL, error: `${CONTENT_POLICY_REFUSAL}: ${machineLeak.reason}` });
|
|
1205
|
+
}
|
|
1036
1206
|
// `permanent` says retrying changes nothing. A caller that cannot tell the
|
|
1037
1207
|
// difference between "hq blipped" and "there is no transport for telegram"
|
|
1038
1208
|
// retries both at the same cadence, which is how an impossible send becomes an
|