@cohortapp/agent-sdk 2.18.15 → 2.18.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/org/inbound/hydrate.mjs +35 -1
- package/lib/org/inbound/project.mjs +3 -0
- package/lib/session/frontdoor.mjs +87 -8
- package/lib/session/handoffs.mjs +57 -0
- package/lib/session/inbox-claims.mjs +47 -2
- package/lib/session/revive.mjs +302 -5
- package/lib/telemetry/collect.mjs +224 -0
- package/package.json +1 -1
- package/scripts/daemon/agent-daemon.mjs +242 -9
- package/scripts/daemon/assurance.mjs +56 -1
- package/scripts/fleet/rollout.mjs +48 -4
- package/scripts/hooks/pre-write-yaml-validate.mjs +63 -2
- package/scripts/local-triggers/autoupdate.sh +194 -9
|
@@ -160,8 +160,10 @@ import { processOne } from "../../lib/execution/pipeline.mjs";
|
|
|
160
160
|
// of spawning `claude --print`; not live → today's dispatch, unchanged. The
|
|
161
161
|
// gate is pure (lib/session/frontdoor.mjs); the assurance sweep reopens a
|
|
162
162
|
// session claim that was neither replied nor done within 20 min.
|
|
163
|
-
import { shouldReviveFrontDoor, reviveCommand, sessionJobLabelOnDisk } from "../../lib/session/revive.mjs";
|
|
164
|
-
import {
|
|
163
|
+
import { shouldReviveFrontDoor, reviveCommand, sessionJobLabelOnDisk, confirmRevived, isTerminalReviveFailure, detectUnrequestedRestart } from "../../lib/session/revive.mjs";
|
|
164
|
+
import { budgetSpentNote } from "../../lib/telemetry/collect.mjs";
|
|
165
|
+
import { checkLock as checkProcessLock } from "../../lib/singleton.js";
|
|
166
|
+
import { makeFrontDoorGate, sessionLiveFromHeartbeat, readFrontDoorState, DEFAULT_STALE_MS } from "../../lib/session/frontdoor.mjs";
|
|
165
167
|
import { CADENCE_REGISTRY } from "./cadence-handlers.mjs";
|
|
166
168
|
import { sweepSessionInboxForDaemon } from "../../lib/session/inbox-claims.mjs";
|
|
167
169
|
import { defaultEffects, scheduleToQueue } from "../../lib/execution/effects.mjs";
|
|
@@ -294,33 +296,264 @@ const _frontDoorGate = makeFrontDoorGate({ agentRoot: AGENT_REPO_DIR });
|
|
|
294
296
|
// we want rather than a stale count carried across it.
|
|
295
297
|
/** How often the daemon asks whether the front door has gone quiet. */
|
|
296
298
|
const REVIVE_CHECK_INTERVAL_MS = 60 * 1000;
|
|
297
|
-
|
|
298
|
-
|
|
299
|
+
// `notLive` is the count of CONSECUTIVE not-live reads — the actor-side
|
|
300
|
+
// hysteresis (Ravi). Takeover is harmless and flips at one read; a kickstart
|
|
301
|
+
// drops a working session, so the FIRST one waits for two consecutive not-live
|
|
302
|
+
// reads. A live read resets it. Per-process and deliberately not persisted: a
|
|
303
|
+
// daemon restart is itself a change of circumstances.
|
|
304
|
+
// `halted` latches when an assert comes back TERMINAL (phantom pid / crash
|
|
305
|
+
// loop): the seat needs a person, and a blind second kickstart would bury the
|
|
306
|
+
// first failure's log under the loop. One attempt on that road, then hand up.
|
|
307
|
+
//
|
|
308
|
+
// `confirmedIdentity` carries the last poll's {pid, startTime} ACROSS the 60s
|
|
309
|
+
// poll cadence, and `kickstartThisInterval` records whether THIS daemon
|
|
310
|
+
// kickstarted the job since that identity was recorded. A within-window
|
|
311
|
+
// continuity check cannot see a crash loop whose period exceeds the settle
|
|
312
|
+
// window — the same identity appears twice inside it. Comparing the confirmed
|
|
313
|
+
// identity against the next poll's observation, with the kickstart flag,
|
|
314
|
+
// catches an UNREQUESTED restart at ANY period, with no window to tune.
|
|
315
|
+
const _revive = { attempts: 0, lastAt: null, notLive: 0, halted: false, confirmedIdentity: null, kickstartThisInterval: false };
|
|
316
|
+
/** TEST-ONLY: clear the per-process revive state between cases. */
|
|
317
|
+
export function _resetRevive() {
|
|
318
|
+
_revive.attempts = 0; _revive.lastAt = null; _revive.notLive = 0; _revive.halted = false;
|
|
319
|
+
_revive.confirmedIdentity = null; _revive.kickstartThisInterval = false;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** Gap between the two settle-window reads that prove the new session did not
|
|
323
|
+
* come up and immediately crash. Short — the door was already dark for the
|
|
324
|
+
* full grace before we kickstarted, so this only has to outlast a boot wobble. */
|
|
325
|
+
const CONFIRM_SETTLE_MS = 5000;
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* One observation of the session job: its launchctl pid, whether that pid is a
|
|
329
|
+
* GENUINELY LIVE process (not a phantom launchctl lists but `ps` says is dead —
|
|
330
|
+
* a silent crash loop), the process start-time that fingerprints the instance,
|
|
331
|
+
* and the COMM (the binary running under the pid). A relaunch changes the
|
|
332
|
+
* start-time even if the pid is reused, so comparing start-times across the
|
|
333
|
+
* settle window is how a crash loop is told apart from one continuous process;
|
|
334
|
+
* the comm is the third, complementary check — a live pid running the WRONG
|
|
335
|
+
* binary is not the process we kickstarted.
|
|
336
|
+
*
|
|
337
|
+
* ABSOLUTE `/bin/ps`, never bare `ps`: `ps` is aliased to `pnpm start` on this
|
|
338
|
+
* fleet, so a bare `ps` is poison. `kill -0` (a builtin/alias-immune existence
|
|
339
|
+
* signal) decides liveness; `/bin/ps` reads the immutable start-time and comm.
|
|
340
|
+
*/
|
|
341
|
+
async function readSessionSample(label) {
|
|
342
|
+
let pid = null, alive = false, startTime = null, comm = null;
|
|
343
|
+
try {
|
|
344
|
+
const { stdout } = await execFileAsync("launchctl", ["list", label]);
|
|
345
|
+
const m = /"PID"\s*=\s*(\d+)/.exec(String(stdout || ""));
|
|
346
|
+
if (m) pid = Number(m[1]);
|
|
347
|
+
} catch { /* no pid — the kickstart produced no running process */ }
|
|
348
|
+
if (Number.isInteger(pid) && pid > 0) {
|
|
349
|
+
try { process.kill(pid, 0); alive = true; } catch { alive = false; } // signal 0 = existence check
|
|
350
|
+
try {
|
|
351
|
+
// One /bin/ps call for both the start-time and the comm — absolute path so
|
|
352
|
+
// the `ps`→`pnpm start` alias cannot poison it.
|
|
353
|
+
const { stdout } = await execFileAsync("/bin/ps", ["-o", "lstart=,comm=", "-p", String(pid)]);
|
|
354
|
+
const line = String(stdout || "").trim();
|
|
355
|
+
// `lstart` is a fixed-width 24-char date ("Wed Sep 25 12:00:00 2026")
|
|
356
|
+
// followed by the comm; split on the last field boundary the date ends at.
|
|
357
|
+
const m = /^(.{24})\s+(.*)$/.exec(line);
|
|
358
|
+
if (m) { startTime = m[1].trim() || null; comm = m[2].trim() || null; }
|
|
359
|
+
else { startTime = line || null; } // unexpected shape — keep what we have, comm unknown
|
|
360
|
+
} catch { startTime = null; comm = null; } // ps gone or pid vanished — degrade; pid continuity still carries
|
|
361
|
+
}
|
|
362
|
+
return { pid, alive, startTime, comm };
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** The binary the session job should be running — matched by suffix against the
|
|
366
|
+
* observed comm. Bare "node" so any install prefix (/opt/homebrew/bin/node,
|
|
367
|
+
* /usr/local/bin/node) matches; a live pid running anything else (a `ps`-alias
|
|
368
|
+
* stray `pnpm`, a wrong process) fails the comm-check and names `wrong-comm`. */
|
|
369
|
+
const EXPECTED_SESSION_COMM = "node";
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* After a kickstart, ASSERT the door came back before recording success — a
|
|
373
|
+
* restart helper's exit code is not a revive (Jacob's drain-then-restart-daemon.sh
|
|
374
|
+
* exited 0 and left the process stopped with NO pid). A ONE-SHOT read is not
|
|
375
|
+
* enough: it passes a phantom pid and a come-up-then-crash loop whose local beat
|
|
376
|
+
* file always looks fresh. So take TWO reads across a settle window (proving the
|
|
377
|
+
* process is one continuous instance, not a crash loop) and pair them with the
|
|
378
|
+
* session's OWN beat freshness — for a session that beat IS the after-signal a
|
|
379
|
+
* wedge kills, because the heartbeat rides tool use, so a session that stops
|
|
380
|
+
* working stops beating (unlike a daemon, which a crash loop keeps re-stamping).
|
|
381
|
+
*/
|
|
382
|
+
async function defaultConfirmAfter({ label, now, staleMs, sleep }) {
|
|
383
|
+
const _sleep = sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
384
|
+
const s1 = await readSessionSample(label);
|
|
385
|
+
await _sleep(CONFIRM_SETTLE_MS);
|
|
386
|
+
const s2 = await readSessionSample(label);
|
|
387
|
+
// serverBeatFresh for the SESSION: is the heartbeat it should now be writing
|
|
388
|
+
// fresh? undefined (couldn't read) does not block — continuity carries it.
|
|
389
|
+
let serverBeatFresh;
|
|
390
|
+
try {
|
|
391
|
+
const st = readFrontDoorState(AGENT_REPO_DIR, { now, staleMs });
|
|
392
|
+
if (st && st.liveness && Number.isFinite(st.liveness.ageMs)) {
|
|
393
|
+
serverBeatFresh = st.liveness.ageMs < (staleMs || DEFAULT_STALE_MS);
|
|
394
|
+
}
|
|
395
|
+
} catch { /* still stale — a fresh session has not beaten yet; leave undefined */ }
|
|
396
|
+
return { samples: [s1, s2], serverBeatFresh };
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// A failed revive has to reach the FLEET, not just this log: a kickstart that
|
|
400
|
+
// returned with no live pid is a failure to escalate — the seat needs a person.
|
|
401
|
+
// collect.mjs reads this into `machine.reviveNote`, so the presence beat carries
|
|
402
|
+
// it. Best-effort (the console line already fired); cleared on a confirmed revive.
|
|
403
|
+
const REVIVE_NOTE_PATH = join(AGENT_REPO_DIR, "state", "telemetry", "revive-note.json");
|
|
404
|
+
function writeReviveNote({ reason, target, detail, at }) {
|
|
405
|
+
try {
|
|
406
|
+
mkdirSync(join(AGENT_REPO_DIR, "state", "telemetry"), { recursive: true });
|
|
407
|
+
const tmp = `${REVIVE_NOTE_PATH}.${process.pid}.tmp`;
|
|
408
|
+
writeFileSync(tmp, JSON.stringify({ reason, target: target || null, detail: detail || "", at }));
|
|
409
|
+
renameSync(tmp, REVIVE_NOTE_PATH);
|
|
410
|
+
} catch { /* the beat surface is best-effort; the log line already fired */ }
|
|
411
|
+
}
|
|
412
|
+
function clearReviveNote() { try { unlinkSync(REVIVE_NOTE_PATH); } catch { /* nothing to clear */ } }
|
|
413
|
+
|
|
414
|
+
export async function reviveFrontDoorIfShut(state, deps = {}) {
|
|
299
415
|
const now = deps.now ? deps.now() : Date.now();
|
|
416
|
+
// TEST-ONLY seam: seed the per-process ladder state so a case can start on a
|
|
417
|
+
// given rung (e.g. budget already spent) without threading a dozen polls.
|
|
418
|
+
if (deps.seedRevive && typeof deps.seedRevive === "object") Object.assign(_revive, deps.seedRevive);
|
|
419
|
+
// A latched terminal failure (a prior kickstart came up and crash-looped, or
|
|
420
|
+
// produced a phantom pid) means the seat is a person's problem now — do not
|
|
421
|
+
// fire another blind kickstart on top of a loop that would bury the evidence.
|
|
422
|
+
if (_revive.halted) return { revive: false, reason: "halted-escalated" };
|
|
300
423
|
const live = sessionLiveFromHeartbeat(state && state.heartbeat, { now });
|
|
424
|
+
const label = deps.label || sessionJobLabelOnDisk(readdirSync, homedir());
|
|
425
|
+
|
|
426
|
+
// ── CROSS-POLL CRASH-LOOP CATCH (unbounded, no threshold) ─────────────────
|
|
427
|
+
//
|
|
428
|
+
// The within-window continuity check inside confirmRevived cannot see a crash
|
|
429
|
+
// loop whose period EXCEEDS the settle window: it shows the same identity
|
|
430
|
+
// twice inside a few seconds and false-confirms. Between two 60s polls,
|
|
431
|
+
// though, the loop has restarted — so if we hold a previously CONFIRMED
|
|
432
|
+
// identity and this poll observes a DIFFERENT one with no kickstart issued in
|
|
433
|
+
// the interval, that is an unrequested restart at ANY period. HALT and note it
|
|
434
|
+
// rather than blindly re-confirm/re-kickstart a loop forever.
|
|
435
|
+
//
|
|
436
|
+
// Armed ONLY while the door is still in trouble (not reading live). A door
|
|
437
|
+
// that is healthily answering again has recovered — a later identity change is
|
|
438
|
+
// ordinary churn (an SDK upgrade, a `session restart`, a supervisor resume
|
|
439
|
+
// rotation), none of which this daemon should police as a crash loop. So a
|
|
440
|
+
// healthy live read DISARMS the check; the loop this hunts is precisely one
|
|
441
|
+
// whose relaunched instance never beats a full grace, so it stays not-live.
|
|
442
|
+
if (label && _revive.confirmedIdentity && state && state.sessionLive !== true) {
|
|
443
|
+
const readSample = deps.readSessionSample || readSessionSample;
|
|
444
|
+
let curr = null;
|
|
445
|
+
try { curr = await readSample(label); } catch { curr = null; }
|
|
446
|
+
const drift = detectUnrequestedRestart({
|
|
447
|
+
prev: _revive.confirmedIdentity,
|
|
448
|
+
curr,
|
|
449
|
+
kickstartIssuedSince: _revive.kickstartThisInterval,
|
|
450
|
+
});
|
|
451
|
+
if (drift.restarted) {
|
|
452
|
+
_revive.halted = true;
|
|
453
|
+
const detail = `crash-loop-cross-poll: pid ${_revive.confirmedIdentity.pid}(${_revive.confirmedIdentity.startTime}) → pid ${curr && curr.pid}(${curr && curr.startTime}) with no kickstart since`;
|
|
454
|
+
console.error(`[daemon] ${label} identity changed across the poll cadence with no kickstart issued (${detail}) — a crash loop whose period outran the settle window; halting and escalating (this seat needs a person)`);
|
|
455
|
+
writeReviveNote({ reason: "session-revive-failed", target: label, detail, at: new Date(now).toISOString() });
|
|
456
|
+
return { revive: false, reason: "crash-loop-cross-poll", confirmed: { ok: false, reason: "crash-loop-cross-poll" } };
|
|
457
|
+
}
|
|
458
|
+
} else if (state && state.sessionLive === true && _revive.confirmedIdentity) {
|
|
459
|
+
// Recovered and answering — disarm; a healthy session's later identity
|
|
460
|
+
// changes are churn, not the loop.
|
|
461
|
+
_revive.confirmedIdentity = null;
|
|
462
|
+
_revive.kickstartThisInterval = false;
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
// Track the consecutive not-live streak BEFORE the decision; a live read
|
|
466
|
+
// resets it so a session that beats between two quiet reads never accrues.
|
|
467
|
+
const notLiveNow = !(state && state.sessionLive === true);
|
|
468
|
+
_revive.notLive = notLiveNow ? _revive.notLive + 1 : 0;
|
|
469
|
+
const reviveAfterMs = Number(process.env.MAESTRO_SESSION_STALE_S || 0) * 1000 || undefined;
|
|
301
470
|
const verdict = shouldReviveFrontDoor({
|
|
302
471
|
frontDoor: state && state.frontDoor,
|
|
303
472
|
sessionLive: state && state.sessionLive,
|
|
304
473
|
silentMs: live.ageMs,
|
|
474
|
+
notLiveReads: _revive.notLive,
|
|
305
475
|
attempts: _revive.attempts,
|
|
306
476
|
lastAttemptAt: _revive.lastAt,
|
|
307
477
|
now,
|
|
308
|
-
reviveAfterMs
|
|
478
|
+
reviveAfterMs,
|
|
309
479
|
});
|
|
310
|
-
if (!verdict.revive)
|
|
311
|
-
|
|
480
|
+
if (!verdict.revive) {
|
|
481
|
+
// A ladder that spent its whole budget and STOPPED is a failure that does
|
|
482
|
+
// not name itself — the seat goes quiet and a quiet system looks healthy.
|
|
483
|
+
// Turn that silence into a beat note through the same reviveNote channel.
|
|
484
|
+
if (verdict.reason === "budget-spent") {
|
|
485
|
+
const note = budgetSpentNote({ verdict: verdict.reason, target: label || null, attempts: _revive.attempts, at: new Date(now).toISOString() });
|
|
486
|
+
if (note) writeReviveNote(note);
|
|
487
|
+
}
|
|
488
|
+
return verdict;
|
|
489
|
+
}
|
|
312
490
|
if (!label) return { revive: false, reason: "no-session-job" };
|
|
491
|
+
|
|
492
|
+
// Supervisor coordination via the resume loop's OWN lock, not a soft note
|
|
493
|
+
// (Jacob). The supervisor holds the `session` singleton for its whole life
|
|
494
|
+
// and writes the heartbeat as a healthy resume makes progress. We only reach
|
|
495
|
+
// here after the door has been dark past the FULL grace AND confirmed over
|
|
496
|
+
// two reads 60s apart — so a resume gap is already excluded by time, and a
|
|
497
|
+
// live holder here is a supervisor PARKED in its launch probe (the 3.5-day /
|
|
498
|
+
// 15-day case this module exists for), for which the hard kickstart is
|
|
499
|
+
// exactly right. Consulting the lock tells a parked supervisor (override,
|
|
500
|
+
// logged) apart from a dead one (a clean restart), and never blocks the
|
|
501
|
+
// parked case — the failure that a hard "skip if held" would reintroduce.
|
|
502
|
+
const supervisor = (deps.checkSessionLock || (() => checkProcessLock("session")))();
|
|
503
|
+
if (supervisor && supervisor.running) {
|
|
504
|
+
console.error(`[daemon] ${label} front door dark past grace while a supervisor (pid ${supervisor.pid}) still holds the session lock — parked in its launch probe; hard-kickstarting over it`);
|
|
505
|
+
}
|
|
506
|
+
|
|
313
507
|
_revive.attempts += 1;
|
|
314
508
|
_revive.lastAt = now;
|
|
509
|
+
// This daemon is now the one that asked for the restart, so a new identity on
|
|
510
|
+
// the NEXT poll is expected, not the crash loop the cross-poll check hunts.
|
|
511
|
+
_revive.kickstartThisInterval = true;
|
|
315
512
|
const cmd = reviveCommand(label, typeof process.getuid === "function" ? process.getuid() : 0);
|
|
316
513
|
console.error(`[daemon] front door shut — ${verdict.reason}; restarting ${label}`);
|
|
317
514
|
try {
|
|
318
515
|
await (deps.execFile || execFileAsync)(cmd.file, cmd.args);
|
|
319
|
-
console.error(`[daemon] ${label} restarted; a fresh session clears whatever the old one was sitting on`);
|
|
320
516
|
} catch (err) {
|
|
321
517
|
console.error(`[daemon] could not restart ${label}: ${err && err.message ? err.message : err} — this seat needs a person`);
|
|
518
|
+
return { ...verdict, confirmed: { ok: false, reason: "kickstart-threw" } };
|
|
519
|
+
}
|
|
520
|
+
// The kickstart returned. Now prove it actually came back — across a settle
|
|
521
|
+
// window, not a one-shot read that a phantom pid or a come-up-then-crash
|
|
522
|
+
// would pass.
|
|
523
|
+
const staleMs = reviveAfterMs;
|
|
524
|
+
const after = deps.confirmAfter
|
|
525
|
+
? await deps.confirmAfter({ label, now, staleMs })
|
|
526
|
+
: await defaultConfirmAfter({ label, now, staleMs });
|
|
527
|
+
const confirmed = confirmRevived({ ...after, staleMs, expectedComm: EXPECTED_SESSION_COMM });
|
|
528
|
+
if (confirmed.ok) {
|
|
529
|
+
console.error(`[daemon] ${label} restarted and answering (${confirmed.reason}); a fresh session clears whatever the old one was sitting on`);
|
|
530
|
+
clearReviveNote();
|
|
531
|
+
// Record the identity this revive confirmed so the NEXT poll can tell one
|
|
532
|
+
// continuous instance from a crash loop whose period outran the settle
|
|
533
|
+
// window. Reset the kickstart flag: the interval it justified is over.
|
|
534
|
+
const alive = Array.isArray(after && after.samples) ? after.samples.find((s) => s && Number.isInteger(s.pid) && s.pid > 0 && s.alive === true) : null;
|
|
535
|
+
_revive.confirmedIdentity = alive ? { pid: alive.pid, startTime: alive.startTime } : null;
|
|
536
|
+
_revive.kickstartThisInterval = false;
|
|
537
|
+
} else if (isTerminalReviveFailure(confirmed)) {
|
|
538
|
+
// Phantom pid or a restart boundary inside the settle window: the kickstart
|
|
539
|
+
// produced no living, continuous process. STOP — one attempt, then hand up
|
|
540
|
+
// with what we saw, rather than kickstart again into a loop that buries it.
|
|
541
|
+
_revive.halted = true;
|
|
542
|
+
const evidence = describeSamples(after && after.samples);
|
|
543
|
+
console.error(`[daemon] ${label} kickstart did NOT revive it (${confirmed.reason}): ${evidence} — a helper's exit code is not a revive; this seat needs a person (failure to escalate)`);
|
|
544
|
+
writeReviveNote({ reason: "session-revive-failed", target: label, detail: `${confirmed.reason}: ${evidence}`, at: new Date(now).toISOString() });
|
|
545
|
+
} else {
|
|
546
|
+
console.error(`[daemon] ${label} restarted; awaiting its first beat (${confirmed.reason}) — the next check confirms or re-attempts within budget`);
|
|
322
547
|
}
|
|
323
|
-
return verdict;
|
|
548
|
+
return { ...verdict, confirmed };
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/** One-line evidence of what the settle-window reads actually saw, for the log
|
|
552
|
+
* and the beat note — so a failed escalation carries the fingerprint, not just
|
|
553
|
+
* a verdict. */
|
|
554
|
+
function describeSamples(samples) {
|
|
555
|
+
if (!Array.isArray(samples) || !samples.length) return "no reads";
|
|
556
|
+
return samples.map((s) => `pid=${s && s.pid != null ? s.pid : "none"}${s && s.alive ? "(alive)" : "(dead)"}`).join(" then ");
|
|
324
557
|
}
|
|
325
558
|
|
|
326
559
|
/**
|
|
@@ -78,7 +78,8 @@ import * as counters from "../../lib/diagnostics/counters.mjs";
|
|
|
78
78
|
// the only composer of interim copy left in the system, and carries the two
|
|
79
79
|
// copy rules (GENERIC_OPENER, FORWARD_PROMISE) every interim must satisfy.
|
|
80
80
|
import { replyTier } from "../../lib/assurance/tier.mjs";
|
|
81
|
-
import { isRollCallItem } from "../../lib/org/inbound/broadcast.mjs";
|
|
81
|
+
import { isRollCallItem, BROADCAST_REASON } from "../../lib/org/inbound/broadcast.mjs";
|
|
82
|
+
import { isMembershipReason } from "../../lib/org/inbound/directedness.mjs";
|
|
82
83
|
import {
|
|
83
84
|
claimRoomInterim,
|
|
84
85
|
claimRoomNotice,
|
|
@@ -397,6 +398,16 @@ export function itemSnapshot(item) {
|
|
|
397
398
|
// reads belongs in this snapshot.
|
|
398
399
|
scope_id: item.scope_id || null,
|
|
399
400
|
thread_id: item.thread_id || null,
|
|
401
|
+
// WHY THIS ADDRESS PROVENANCE IS A DEBT FIELD, not decoration. The sweep and
|
|
402
|
+
// the silent-success path SPEAK from `rec.item`, long after the live item is
|
|
403
|
+
// gone, and one decision they make is whether a rescued result belongs in
|
|
404
|
+
// this room at all: a membership/broadcast-proved surface (a doc comment
|
|
405
|
+
// shared to a channel, a room this seat merely belongs to) is ambient, and
|
|
406
|
+
// dumping a finished session's result text into it is narration nobody is
|
|
407
|
+
// waiting on 1:1 — the exact leak doc-comment surfaces close silently to
|
|
408
|
+
// avoid. Without `direct_reason` in the snapshot the DM/ambient distinction
|
|
409
|
+
// is unrecoverable at sweep time; with it, the guard fails open to a DM.
|
|
410
|
+
direct_reason: item.direct_reason || null,
|
|
400
411
|
sender: item.sender || null,
|
|
401
412
|
sender_email: item.sender_email || null,
|
|
402
413
|
subject: item.subject || null,
|
|
@@ -1280,6 +1291,32 @@ export function composeSilentSuccess(rec, o = {}) {
|
|
|
1280
1291
|
return `${head} I don't have a clean result to show you, though. Want me to run it again?`;
|
|
1281
1292
|
}
|
|
1282
1293
|
|
|
1294
|
+
/**
|
|
1295
|
+
* Is this obligation's surface AMBIENT rather than a 1:1 the requester waits on?
|
|
1296
|
+
*
|
|
1297
|
+
* The signal is `direct_reason` — WHY the inbound join decided this event was
|
|
1298
|
+
* this seat's — carried into the snapshot by `itemSnapshot`. A membership or
|
|
1299
|
+
* broadcast reason (`channel`/`participant`/`shared`, or `collective`) is
|
|
1300
|
+
* access-shaped: the seat was addressed because it can SEE the thing, not
|
|
1301
|
+
* because anything named it, so any number of seats in the room hold the same
|
|
1302
|
+
* event. An identity-shaped reason (a DM, `owner`, `named`, an assignee) is one
|
|
1303
|
+
* person waiting on one reply.
|
|
1304
|
+
*
|
|
1305
|
+
* FAILS OPEN TO "NOT AMBIENT" (i.e. narrate). An item whose provenance was
|
|
1306
|
+
* never stamped — the historical default, and every pre-existing silent-success
|
|
1307
|
+
* test — is treated as a waiting DM, because the cost of narrating into a real
|
|
1308
|
+
* DM is nothing and the cost of silencing one is the very defect this module
|
|
1309
|
+
* exists to fix.
|
|
1310
|
+
*
|
|
1311
|
+
* @param {object} rec the obligation record; its `item.direct_reason` is read
|
|
1312
|
+
* @returns {boolean}
|
|
1313
|
+
*/
|
|
1314
|
+
function isAmbientSurface(rec) {
|
|
1315
|
+
const reason = String((rec && rec.item && rec.item.direct_reason) || "");
|
|
1316
|
+
if (!reason) return false; // unknown provenance → treat as a DM, narrate
|
|
1317
|
+
return isMembershipReason(reason) || reason === BROADCAST_REASON;
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1283
1320
|
/** A session interrupted by the daemon itself dying/restarting. */
|
|
1284
1321
|
export function composeInterrupted() {
|
|
1285
1322
|
return `Heads up — my session on this was interrupted before it finished (my end restarted). I've picked it back up; I'll come back with the answer.`;
|
|
@@ -1843,6 +1880,24 @@ export async function settleSession(a = {}) {
|
|
|
1843
1880
|
}
|
|
1844
1881
|
|
|
1845
1882
|
if (a.ok && !heard) {
|
|
1883
|
+
// A CLEAN SILENT EXIT OUTSIDE A DM CLOSES QUIETLY, LIKE A DOC COMMENT.
|
|
1884
|
+
// The rescue message exists to hand a waiting requester the result their
|
|
1885
|
+
// session produced but never sent. On an ambient/membership-proved surface
|
|
1886
|
+
// there is no such requester — the seat was addressed because it can see
|
|
1887
|
+
// the room, not because anyone named it — so the same message is result
|
|
1888
|
+
// text NARRATED into a shared room, the leak doc-comment surfaces already
|
|
1889
|
+
// close silently to avoid. The debt is still discharged (silent is not
|
|
1890
|
+
// un-closed); only the narration is withheld. `direct_reason` reaches here
|
|
1891
|
+
// through `itemSnapshot`, and the guard fails open to narrating.
|
|
1892
|
+
if (isAmbientSurface(rec)) {
|
|
1893
|
+
closeObligation(a.key, {
|
|
1894
|
+
outcome: "answered",
|
|
1895
|
+
now,
|
|
1896
|
+
note: "session exited 0 without speaking; ambient surface closed without narration",
|
|
1897
|
+
});
|
|
1898
|
+
counters.bump("assurance.silent_success_ambient", { service: rec.service });
|
|
1899
|
+
return { spoke: false, text: null, verdict: "silent-success", willRetry: false };
|
|
1900
|
+
}
|
|
1846
1901
|
const finalText = safeResultText(a.stdout);
|
|
1847
1902
|
const text = composeSilentSuccess(rec, { finalText });
|
|
1848
1903
|
const res = await send(item, text, { kind: "reply", idempotencySuffix: `rescue-${rec.attempts || 0}` });
|
|
@@ -545,6 +545,15 @@ export function seatVersion(entry) {
|
|
|
545
545
|
* cannot see, and it would be wrong on whichever side of the hq deploy it was
|
|
546
546
|
* not written for.
|
|
547
547
|
*
|
|
548
|
+
* Each row also carries `wedged` (bool) and `wedgeReason` (string|null),
|
|
549
|
+
* derived from the same `machine` block {@link seatStuck} reads: a seat is
|
|
550
|
+
* wedged when the beat itself reports `machine.wedge.reason` (published by
|
|
551
|
+
* collect.mjs's sessionWedge — beating, but not reading its inbox/handoffs),
|
|
552
|
+
* OR when `machine.frontDoor === "session"` and `machine.sessionLive === false`
|
|
553
|
+
* (the front door is shut outright). The version stays on the row either way —
|
|
554
|
+
* wedged is a verdict {@link classifySeat} applies, not a fact this function
|
|
555
|
+
* withholds.
|
|
556
|
+
*
|
|
548
557
|
* @param {object[]} members
|
|
549
558
|
* @param {{seatWindowMs?:number, now?:number}} [o]
|
|
550
559
|
* @returns {{seats:object[], nonSeats:object[], readCarriesVersionField:boolean}}
|
|
@@ -563,6 +572,10 @@ export function toSeats(members, o = {}) {
|
|
|
563
572
|
}
|
|
564
573
|
const p = m.presence && typeof m.presence === "object" ? m.presence : null;
|
|
565
574
|
const beatMs = p && Number.isFinite(p.lastBeatMs) ? p.lastBeatMs : null;
|
|
575
|
+
// Same accessor seatStuck uses: the beat's machine block, wherever it hangs.
|
|
576
|
+
const mach = (m.status && typeof m.status === "object" && m.status.machine) || m.machine || {};
|
|
577
|
+
const wedge = mach.wedge && typeof mach.wedge === "object" ? mach.wedge : null;
|
|
578
|
+
const doorDown = mach.frontDoor === "session" && mach.sessionLive === false;
|
|
566
579
|
const row = {
|
|
567
580
|
id: String(m.id || ""),
|
|
568
581
|
slug: String(m.slug || ""),
|
|
@@ -572,6 +585,8 @@ export function toSeats(members, o = {}) {
|
|
|
572
585
|
lastBeatMs: beatMs,
|
|
573
586
|
...seatVersions(m),
|
|
574
587
|
stuck: seatStuck(m, o.now),
|
|
588
|
+
wedged: !!((wedge && wedge.reason) || doorDown),
|
|
589
|
+
wedgeReason: (wedge && typeof wedge.reason === "string" && wedge.reason) || (doorDown ? "front-door-not-live" : null),
|
|
575
590
|
};
|
|
576
591
|
if (beatMs !== null && beatMs <= windowMs) seats.push(row);
|
|
577
592
|
else nonSeats.push(row);
|
|
@@ -582,14 +597,18 @@ export function toSeats(members, o = {}) {
|
|
|
582
597
|
|
|
583
598
|
/**
|
|
584
599
|
* Verdict for one seat against the target version. Pure.
|
|
600
|
+
* wedged — beating, but its front door is not answering (a shut
|
|
601
|
+
* session or an overdue handoff); outranks current/stale —
|
|
602
|
+
* the version stays on the row for display, unread here
|
|
585
603
|
* current — reported version equals the target
|
|
586
604
|
* stale — reported a different (older or newer) version
|
|
587
605
|
* unverifiable — beating, but the read carries no version at all
|
|
588
606
|
* @param {object} seat
|
|
589
607
|
* @param {string} target
|
|
590
|
-
* @returns {"current"|"stale"|"unverifiable"}
|
|
608
|
+
* @returns {"wedged"|"current"|"stale"|"unverifiable"}
|
|
591
609
|
*/
|
|
592
610
|
export function classifySeat(seat, target) {
|
|
611
|
+
if (seat && seat.wedged) return "wedged";
|
|
593
612
|
const v = seat && seat.version;
|
|
594
613
|
if (!v) return "unverifiable";
|
|
595
614
|
return compareVersions(v, target) === 0 ? "current" : "stale";
|
|
@@ -604,7 +623,7 @@ export function classifySeat(seat, target) {
|
|
|
604
623
|
*/
|
|
605
624
|
export function summarise(fleet, target) {
|
|
606
625
|
const rows = (fleet.seats || []).map((s) => ({ ...s, verdict: classifySeat(s, target) }));
|
|
607
|
-
const counts = { current: 0, stale: 0, unverifiable: 0 };
|
|
626
|
+
const counts = { current: 0, stale: 0, unverifiable: 0, wedged: 0 };
|
|
608
627
|
for (const r of rows) counts[r.verdict] += 1;
|
|
609
628
|
// Behind AND proved it cannot arrive — a strictly stronger statement than
|
|
610
629
|
// `stale`, which clears itself on the next hourly run.
|
|
@@ -623,6 +642,7 @@ export function summarise(fleet, target) {
|
|
|
623
642
|
// AND no seat is known to be unable to move.
|
|
624
643
|
const stuck = rows.filter((r) => r.stuck && r.stuck.verdict === "stuck");
|
|
625
644
|
const stale = rows.filter((r) => r.verdict === "stale");
|
|
645
|
+
const wedged = rows.filter((r) => r.verdict === "wedged");
|
|
626
646
|
const staleSlugs = new Set(stale.map((r) => r.slug));
|
|
627
647
|
return {
|
|
628
648
|
target,
|
|
@@ -636,8 +656,10 @@ export function summarise(fleet, target) {
|
|
|
636
656
|
stuckBehind: stuck.filter((r) => staleSlugs.has(r.slug)),
|
|
637
657
|
/** Stuck against some other version while current on this one. Still a finding. */
|
|
638
658
|
stuckElsewhere: stuck.filter((r) => !staleSlugs.has(r.slug)),
|
|
659
|
+
/** Beating, but its front door is not answering — a 4th verdict beside stale. */
|
|
660
|
+
wedged,
|
|
639
661
|
readCarriesVersionField: fleet.readCarriesVersionField === true,
|
|
640
|
-
done: rows.length > 0 && counts.current === rows.length && stuck.length === 0,
|
|
662
|
+
done: rows.length > 0 && counts.current === rows.length && stuck.length === 0 && wedged.length === 0,
|
|
641
663
|
};
|
|
642
664
|
}
|
|
643
665
|
|
|
@@ -793,10 +815,18 @@ export function stuckClause(row, target) {
|
|
|
793
815
|
* exit code: `behind` and `unverifiable` are different problems with different
|
|
794
816
|
* fixes, and only the first is answered by waiting.
|
|
795
817
|
*
|
|
818
|
+
* `wedged` is a fourth problem with a fourth fix: the seat is beating (so it is
|
|
819
|
+
* not `unverifiable`, and waiting for its next beat proves nothing) but its
|
|
820
|
+
* front door is not answering — a shut session or an overdue handoff. It is
|
|
821
|
+
* checked after `stuck` (stuck is the longer-standing, already-acted-on
|
|
822
|
+
* signal) and before `empty-fleet`/`unverifiable`/`behind`: reporting a wedged
|
|
823
|
+
* seat as merely "behind" would invite waiting it out, and waiting does
|
|
824
|
+
* nothing for a seat that is not reading.
|
|
825
|
+
*
|
|
796
826
|
* Pure.
|
|
797
827
|
*
|
|
798
828
|
* @param {ReturnType<typeof summarise>|null|undefined} summary
|
|
799
|
-
* @returns {{code:number, reason:"verified"|"stuck"|"behind"|"unverifiable"|"empty-fleet"|"no-fleet-read", note:string|null}}
|
|
829
|
+
* @returns {{code:number, reason:"verified"|"stuck"|"wedged"|"behind"|"unverifiable"|"empty-fleet"|"no-fleet-read", note:string|null}}
|
|
800
830
|
*/
|
|
801
831
|
export function propagationOutcome(summary) {
|
|
802
832
|
if (!summary || !Array.isArray(summary.rows)) {
|
|
@@ -829,6 +859,20 @@ export function propagationOutcome(summary) {
|
|
|
829
859
|
renderVerdict(summary),
|
|
830
860
|
};
|
|
831
861
|
}
|
|
862
|
+
// WEDGED comes after stuck (stuck is longer-standing and still the one a
|
|
863
|
+
// person must act on first) but before empty-fleet/unverifiable/behind: a
|
|
864
|
+
// seat that is beating but not reading is not a seat anyone can wait out
|
|
865
|
+
// either, and it must not be reported as merely "behind" — waiting changes
|
|
866
|
+
// nothing when the front door itself is what is not answering.
|
|
867
|
+
if (summary.wedged && summary.wedged.length) {
|
|
868
|
+
return {
|
|
869
|
+
code: 3,
|
|
870
|
+
reason: "wedged",
|
|
871
|
+
note:
|
|
872
|
+
`${summary.wedged.length} seat(s) are beating but their front door is not answering (wedged) — the ` +
|
|
873
|
+
`daemon revive or a person must clear them; a version cannot be verified on a seat that is not reading.`,
|
|
874
|
+
};
|
|
875
|
+
}
|
|
832
876
|
if (!summary.rows.length) {
|
|
833
877
|
return {
|
|
834
878
|
code: 3,
|
|
@@ -17,7 +17,14 @@
|
|
|
17
17
|
* old_string replaced by new_string) and REFUSES — exit 2, a permissionDecision:"deny" JSON on
|
|
18
18
|
* stdout and the reason on stderr — when:
|
|
19
19
|
* - an Edit's old_string is absent from the file, or matches more than once without
|
|
20
|
-
* replace_all (the edit would land somewhere the writer did not look at);
|
|
20
|
+
* replace_all (the edit would land somewhere the writer did not look at); when that repeated
|
|
21
|
+
* match is an `id:` line, the refusal names the real cause — the file already carries a
|
|
22
|
+
* duplicate id and is unwritable via id-anchored edits until the collision is resolved;
|
|
23
|
+
* - the would-be top-level `items:` list (or a top-level sequence) would carry two rows sharing
|
|
24
|
+
* the same `id` value that was not already colliding on disk. A duplicate id slips past js-yaml
|
|
25
|
+
* (the rows are distinct list entries, not duplicate map keys) but makes every id-anchored edit
|
|
26
|
+
* ambiguous — the "collision then silently unwritable all day" failure — so it is refused loudly
|
|
27
|
+
* as a WHOLE FILE BLOCK at the write boundary;
|
|
21
28
|
* - the result does not parse as YAML (js-yaml, which also rejects duplicate keys);
|
|
22
29
|
* - a provenance-keyed value NEW in this write ends on a dangling connector. The delta is
|
|
23
30
|
* judged, not the whole file, so a pre-existing truncated row never blocks an unrelated
|
|
@@ -108,6 +115,39 @@ export function guardedRowCount(doc) {
|
|
|
108
115
|
return null;
|
|
109
116
|
}
|
|
110
117
|
|
|
118
|
+
/** The top-level list the id-uniqueness guard protects: `items:` or a top-level sequence, else null. */
|
|
119
|
+
function guardedList(doc) {
|
|
120
|
+
if (Array.isArray(doc)) return doc;
|
|
121
|
+
if (doc && typeof doc === "object" && Array.isArray(doc.items)) return doc.items;
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* First `id` value that appears on two or more rows of the guarded top-level list, else null.
|
|
127
|
+
* A duplicate id makes every id-anchored edit ambiguous, so js-yaml lets it through (the rows are
|
|
128
|
+
* distinct list entries, not duplicate map keys) but the file becomes unwritable via id-anchored edits.
|
|
129
|
+
*/
|
|
130
|
+
export function duplicateItemId(doc) {
|
|
131
|
+
const list = guardedList(doc);
|
|
132
|
+
if (!list) return null;
|
|
133
|
+
const seen = new Set();
|
|
134
|
+
for (const row of list) {
|
|
135
|
+
if (!row || typeof row !== "object") continue;
|
|
136
|
+
const id = row.id;
|
|
137
|
+
if (typeof id !== "string" && typeof id !== "number") continue;
|
|
138
|
+
const key = String(id);
|
|
139
|
+
if (seen.has(key)) return key;
|
|
140
|
+
seen.add(key);
|
|
141
|
+
}
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** If `oldStr` is anchored on an `id:` line, return the id value; else null. */
|
|
146
|
+
function repeatedIdAnchor(oldStr) {
|
|
147
|
+
const m = /(?:^|\n)\s*(?:-\s+)?id:\s*(.+?)\s*(?:\n|$)/.exec(oldStr);
|
|
148
|
+
return m ? m[1].replace(/^["']|["']$/g, "") : null;
|
|
149
|
+
}
|
|
150
|
+
|
|
111
151
|
/** Count non-overlapping occurrences of `needle` in `hay`. */
|
|
112
152
|
function countOccurrences(hay, needle) {
|
|
113
153
|
if (!needle) return 0;
|
|
@@ -138,7 +178,15 @@ export function wouldBeContent(toolName, ti, current) {
|
|
|
138
178
|
if (typeof oldStr !== "string" || oldStr === "") return { refuse: "Edit with an empty old_string" };
|
|
139
179
|
const n = countOccurrences(current, oldStr);
|
|
140
180
|
if (n === 0) return { refuse: "old_string is not present in the file on disk — the edit would not land where you looked" };
|
|
141
|
-
if (n > 1 && !ti.replace_all)
|
|
181
|
+
if (n > 1 && !ti.replace_all) {
|
|
182
|
+
// A repeated `id:` line means the file already carries a duplicate id — the collision, not the
|
|
183
|
+
// anchor, is why every id-anchored edit matches twice. Name that instead of the generic advice.
|
|
184
|
+
const dupId = repeatedIdAnchor(oldStr);
|
|
185
|
+
if (dupId !== null) {
|
|
186
|
+
return { refuse: `WHOLE FILE BLOCKED — the file already contains a duplicate id '${dupId}', so this id-anchored edit matches ${n} places and cannot land. The file is unwritable via id-anchored edits until the duplicate id is resolved (give one of the colliding rows a unique id).` };
|
|
187
|
+
}
|
|
188
|
+
return { refuse: `old_string matches ${n} places — anchor it to one row or pass replace_all` };
|
|
189
|
+
}
|
|
142
190
|
return { content: ti.replace_all ? current.split(oldStr).join(newStr) : current.replace(oldStr, () => newStr) };
|
|
143
191
|
}
|
|
144
192
|
if (toolName === "MultiEdit" && Array.isArray(ti.edits)) {
|
|
@@ -200,6 +248,19 @@ export function evaluate(payload, o = {}) {
|
|
|
200
248
|
try { prev = yaml.load(current); prevParsed = true; } catch { prevParsed = false; }
|
|
201
249
|
}
|
|
202
250
|
|
|
251
|
+
// Duplicate-id guard (id-uniqueness reserved at the write boundary). A second row sharing an id
|
|
252
|
+
// slips past js-yaml — the rows are distinct list entries, not duplicate map keys — but it makes
|
|
253
|
+
// every id-anchored edit ambiguous and the file effectively unwritable. Judged on the DELTA: a
|
|
254
|
+
// brand-new collision this write introduces always blocks; a pre-existing one the write leaves in
|
|
255
|
+
// place is surfaced by the id-anchored-Edit match-count path, not penalised twice here.
|
|
256
|
+
const nextDup = duplicateItemId(next);
|
|
257
|
+
if (nextDup !== null && (!prevParsed || duplicateItemId(prev) !== nextDup)) {
|
|
258
|
+
return {
|
|
259
|
+
decision: "deny",
|
|
260
|
+
reason: `${toolName} ${fp}: WHOLE FILE BLOCKED — two rows share id '${nextDup}'. A duplicate id makes every id-anchored edit ambiguous and the file unwritable. Give the new row a unique id before writing.`,
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
|
|
203
264
|
// Provenance truncation — judged on the DELTA so pre-existing rows never block a new write.
|
|
204
265
|
const before = new Set(prevParsed ? provenanceHits(prev).map((h) => `${h.id}\u0000${h.key}\u0000${h.value}`) : []);
|
|
205
266
|
const fresh = provenanceHits(next).filter((h) => !before.has(`${h.id}\u0000${h.key}\u0000${h.value}`));
|