@agent-compose/sdk 0.8.2 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/__tests__/perf-sampler.test.d.ts +10 -0
- package/dist/agent/agent-context.d.ts +1 -1
- package/dist/agent/agent-loop.d.ts +9 -1
- package/dist/agent/desktop-open.d.ts +184 -0
- package/dist/agent/perf-sampler.d.ts +99 -0
- package/dist/agent/services-manifest.d.ts +88 -0
- package/dist/agent/services-restore.d.ts +58 -0
- package/dist/client.d.ts +164 -8
- package/dist/display.d.ts +17 -0
- package/dist/index.d.ts +14 -5
- package/dist/index.js +1393 -53
- package/dist/runtimes/_cli-agent.d.ts +359 -2
- package/dist/runtimes/claude-code.d.ts +12 -0
- package/dist/runtimes/codex.d.ts +8 -0
- package/dist/runtimes/openai-desktop.js +1329 -53
- package/dist/runtimes/session-env.test.d.ts +14 -0
- package/dist/sandbox.d.ts +1 -1
- package/dist/types/api-conversations.d.ts +309 -1
- package/dist/types/api-factory.d.ts +115 -10
- package/dist/types/api-runs.d.ts +21 -0
- package/dist/types/protocol.d.ts +44 -1
- package/dist/types/runtime.d.ts +120 -0
- package/package.json +1 -1
- package/src/agent/agent-context.ts +100 -11
- package/src/agent/agent-loop.ts +15 -3
- package/src/agent/desktop-open.ts +418 -0
- package/src/agent/perf-sampler.ts +202 -0
- package/src/agent/services-manifest.ts +356 -0
- package/src/agent/services-restore.ts +195 -0
- package/src/client.ts +328 -12
- package/src/display.ts +44 -1
- package/src/index.ts +65 -2
- package/src/runtimes/_cli-agent.ts +911 -35
- package/src/runtimes/claude-code.ts +198 -14
- package/src/runtimes/codex.ts +58 -1
- package/src/sandbox/providers/e2b.ts +29 -1
- package/src/sandbox/providers/local.ts +16 -4
- package/src/sandbox.ts +1 -1
- package/src/types/api-conversations.ts +307 -3
- package/src/types/api-factory.ts +118 -10
- package/src/types/api-runs.ts +23 -0
- package/src/types/protocol.ts +44 -1
- package/src/types/runtime.ts +122 -0
|
@@ -24,9 +24,15 @@
|
|
|
24
24
|
* values set via `agentc secrets set` are visible to the CLI.
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
|
-
import type { AgentMessage, ModelExecutionContract, RuntimeOptions, SandboxProvider, ToolCallGateResult } from "../index.js";
|
|
27
|
+
import type { AgentMessage, ModelExecutionContract, RunnerLivenessVerdict, RuntimeOptions, SandboxProvider, ToolCallGateResult, TurnExitNotify } from "../index.js";
|
|
28
28
|
import { defineRuntime } from "../types/runtime.js";
|
|
29
29
|
import { AsyncQueue } from "../agent/async-queue.js";
|
|
30
|
+
// Function-only cycle with perf-sampler.ts (it imports `shellQuote` from
|
|
31
|
+
// here) — safe: both sides are hoisted declarations used at call time.
|
|
32
|
+
import {
|
|
33
|
+
parsePerfToken, perfSamplerFunctionFragment, PERF_TOKEN_MAX_CHARS,
|
|
34
|
+
} from "../agent/perf-sampler.js";
|
|
35
|
+
import type { GuestPerfSample } from "../agent/perf-sampler.js";
|
|
30
36
|
import { formatError } from "../utils/errors.js";
|
|
31
37
|
import { ndJsonStream, type Stream } from "@agentclientprotocol/sdk";
|
|
32
38
|
import { runProcessorChain } from "../processors/runner.js";
|
|
@@ -48,6 +54,402 @@ function now(): string { return new Date().toISOString(); }
|
|
|
48
54
|
* this loop — bounds a truly dead sandbox. Exported for tests. */
|
|
49
55
|
export const TAIL_REATTACH_DELAY_MS = 2_000;
|
|
50
56
|
|
|
57
|
+
// ── Durable-truth liveness (2026-08-15 incident, turn 18dc5261) ─────────────
|
|
58
|
+
// DURABLE STATE IS THE ONLY TRUTH; STREAMS ARE A LATENCY OPTIMIZATION.
|
|
59
|
+
// The incident: the attached tail's connect-web stream wedged SILENTLY —
|
|
60
|
+
// `wait()` never settled, no error, no EOF — while the runner finished its
|
|
61
|
+
// answer into the durable file (full transcript + exit sentinel on disk at
|
|
62
|
+
// +87s). Every recovery arm lived AFTER `await handle.wait()`, so the
|
|
63
|
+
// transport was blind forever; the finished turn sat unharvested for ten
|
|
64
|
+
// minutes until the next dispatch superseded it. The pieces below make the
|
|
65
|
+
// durable plane authoritative:
|
|
66
|
+
// - the detached runner writes a HEARTBEAT file (epoch ms, every ~10s,
|
|
67
|
+
// from a subshell inside the setsid session that self-terminates with
|
|
68
|
+
// the wrapper) next to the stdout/pid/sentinel files;
|
|
69
|
+
// - an attached tail is WATCHED, never trusted: a fixed-cadence probe (one
|
|
70
|
+
// fresh short exec) reads the durable trio, and durable bytes the stream
|
|
71
|
+
// failed to deliver across a full interval prove the stream dead — kill
|
|
72
|
+
// the tail (never the runner) and catch up from the byte offset;
|
|
73
|
+
// - `probeTurnLiveness` serves the same durable reading to the executor's
|
|
74
|
+
// evidence ticker as a three-valued verdict (alive / dead /
|
|
75
|
+
// probe-failed), replacing process-grep guesswork.
|
|
76
|
+
|
|
77
|
+
/** Cadence of the in-guest heartbeat writer (seconds — it is a shell loop). */
|
|
78
|
+
export const RUNNER_HEARTBEAT_INTERVAL_SECONDS = 10;
|
|
79
|
+
/** How recently a file under the CLI's own task state dir (~/.claude/tasks)
|
|
80
|
+
* must have been touched to count as a LIVE background task in the busy
|
|
81
|
+
* sentinel (minutes — it is `find -mmin`). A running task's state/output
|
|
82
|
+
* files are written continuously, so two minutes is generous; a finished
|
|
83
|
+
* task's files go quiet and age out on their own. */
|
|
84
|
+
export const RUNNER_BUSY_TASK_FRESH_MINUTES = 2;
|
|
85
|
+
/** Cap on the task-id list the busy sentinel publishes — the ids are for
|
|
86
|
+
* server-side log attribution, not an inventory. */
|
|
87
|
+
export const RUNNER_BUSY_TASK_ID_LIMIT = 16;
|
|
88
|
+
/** The sh fragment the heartbeat subshell runs each beat to maintain the
|
|
89
|
+
* busy sentinel — the guest half of the background-work marker contract:
|
|
90
|
+
* - `<promptPath>.busy` — the COUNT of recently-active files under the
|
|
91
|
+
* CLI's task state dir (`$ac_busy` stays set in-shell for the loop's
|
|
92
|
+
* own exit decision);
|
|
93
|
+
* - `<promptPath>.tasks` — the task IDS those files belong to (first
|
|
94
|
+
* path segment under tasks/, deduped, capped).
|
|
95
|
+
* The marker's freshness stamp is the files' own mtime (each beat rewrites
|
|
96
|
+
* them; the server probe gates on `-mmin -1`). Cheap (one bounded find
|
|
97
|
+
* over a small dir), and read by the server's park path so a VM hosting
|
|
98
|
+
* live background work is never frozen mid-flight (2026-08-15 forensics:
|
|
99
|
+
* a 3-min idle park suspended a VM with three research agents mid-flight).
|
|
100
|
+
* A missing tasks dir counts 0. Exported for tests. */
|
|
101
|
+
export function runnerBusySentinelFragment(busyPath: string, tasksPath: string): string {
|
|
102
|
+
return `ac_task_files=$(find "$HOME/.claude/tasks" -type f -mmin -${RUNNER_BUSY_TASK_FRESH_MINUTES} 2>/dev/null); `
|
|
103
|
+
+ `if [ -n "$ac_task_files" ]; then ac_busy=$(printf '%s\\n' "$ac_task_files" | wc -l | tr -cd 0-9); else ac_busy=0; fi; `
|
|
104
|
+
+ `printf '%s' "$ac_busy" > ${shellQuote(busyPath)} 2>/dev/null; `
|
|
105
|
+
+ `printf '%s\\n' "$ac_task_files" | sed 's|.*/tasks/||; s|/.*||' | grep . | sort -u `
|
|
106
|
+
+ `| head -${RUNNER_BUSY_TASK_ID_LIMIT} > ${shellQuote(tasksPath)} 2>/dev/null`;
|
|
107
|
+
}
|
|
108
|
+
/** A heartbeat older than this (by the GUEST's own clock — the probe reads
|
|
109
|
+
* `date +%s%3N` in the same exec, so host clock skew is irrelevant) is
|
|
110
|
+
* stale: three missed writes. */
|
|
111
|
+
export const RUNNER_HEARTBEAT_STALE_MS = RUNNER_HEARTBEAT_INTERVAL_SECONDS * 1000 * 3;
|
|
112
|
+
/** Watchdog cadence while a tail stream is attached. Each tick is one fresh
|
|
113
|
+
* short exec; a healthy stream makes every tick a no-op. */
|
|
114
|
+
export const TAIL_WATCHDOG_INTERVAL_MS = 15_000;
|
|
115
|
+
/** Explicit ceiling on one durable-trio probe exec. A probe that can't
|
|
116
|
+
* answer is "probe-failed" — never a verdict. */
|
|
117
|
+
export const TURN_PROBE_TIMEOUT_MS = 10_000;
|
|
118
|
+
/** After this many stream deaths in ONE turn, stop re-attaching tails and
|
|
119
|
+
* finish the turn on durable-file polling alone (the GHA permanent-
|
|
120
|
+
* fallback pattern): a wire that killed three streams will kill the
|
|
121
|
+
* fourth, and the durable plane already delivers everything. */
|
|
122
|
+
export const TAIL_ABANDON_AFTER_STREAM_DEATHS = 3;
|
|
123
|
+
/** Poll cadence once streaming is abandoned — a full durable read per poll
|
|
124
|
+
* (envd HTTP, ~65ms RTT), consumed from the byte offset. Latency-tuned:
|
|
125
|
+
* this is the degraded path, correctness never depends on it being fast. */
|
|
126
|
+
export const DURABLE_POLL_INTERVAL_MS = 1_000;
|
|
127
|
+
/** The claude CLI's stderr complaint when `--resume <id>` names a thread
|
|
128
|
+
* that does not exist on this machine (the 2026-08-18 split-brain
|
|
129
|
+
* forensic). When a nonzero exit's stderr carries it, the transport
|
|
130
|
+
* surfaces the stderr as the turn's LAST error event even though a bare
|
|
131
|
+
* result error already streamed — the consumer's classifier needs the
|
|
132
|
+
* specific complaint, not the generic `error_during_execution` token. */
|
|
133
|
+
export const RESUME_TARGET_MISSING_STDERR = /no conversation found with session id/i;
|
|
134
|
+
|
|
135
|
+
/** One durable-trio reading, parsed from the probe exec's single line. */
|
|
136
|
+
export interface TurnProbeReading {
|
|
137
|
+
/** Bytes currently in the durable stdout file (-1: unreadable/absent). */
|
|
138
|
+
size: number;
|
|
139
|
+
/** Is the detached runner's process group leader alive (kill -0)? */
|
|
140
|
+
alive: boolean;
|
|
141
|
+
/** Last heartbeat value (epoch ms, guest clock; 0 = no heartbeat yet). */
|
|
142
|
+
hbMs: number;
|
|
143
|
+
/** The guest's own clock at probe time (epoch ms). */
|
|
144
|
+
nowMs: number;
|
|
145
|
+
/** Is the exit sentinel already in the durable file (turn FINISHED)? */
|
|
146
|
+
sentinelSeen: boolean;
|
|
147
|
+
/** Latest guest perf sample (the trailing `.perf` token field), when the
|
|
148
|
+
* guest wrote one AND it parsed clean. Absent = older guest image, no
|
|
149
|
+
* sample yet, or a garbled token — evidence-absent, never an error.
|
|
150
|
+
* Telemetry-only: no liveness verdict may ever read it. */
|
|
151
|
+
perf?: GuestPerfSample;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** The probe exec: one line, six fields, always exit 0 — faults surface as
|
|
155
|
+
* parse failures, not exec failures. Reads the guest's own clock in the
|
|
156
|
+
* same exec so staleness math never involves the host clock. The sixth
|
|
157
|
+
* field is the perf token (perf-sampler.ts), `-` when absent; the char
|
|
158
|
+
* whitelist (`tr -cd`) collapses any garbled write to at most one token —
|
|
159
|
+
* it can never add whitespace and desync the field positions. */
|
|
160
|
+
export function turnLivenessProbeCommand(
|
|
161
|
+
paths: { outPath: string; hbPath: string; perfPath: string; pid: number; sentinel: string },
|
|
162
|
+
): string {
|
|
163
|
+
return `sz=$(wc -c < ${shellQuote(paths.outPath)} 2>/dev/null || echo -1); `
|
|
164
|
+
+ `alive=0; kill -0 ${paths.pid} 2>/dev/null && alive=1; `
|
|
165
|
+
+ `hb=$(cat ${shellQuote(paths.hbPath)} 2>/dev/null || echo 0); `
|
|
166
|
+
+ `now=$(date +%s%3N); `
|
|
167
|
+
+ `sent=$(grep -c -F ${shellQuote(paths.sentinel)} ${shellQuote(paths.outPath)} 2>/dev/null || echo 0); `
|
|
168
|
+
+ `pf=$(head -c ${PERF_TOKEN_MAX_CHARS} ${shellQuote(paths.perfPath)} 2>/dev/null | tr -cd 'A-Za-z0-9=,.-'); `
|
|
169
|
+
+ `printf '%s %s %s %s %s %s\\n' "$sz" "$alive" "$hb" "$now" "$sent" "\${pf:--}"`;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Parse the probe's line. Null = unusable output (a wedged or faulted exec)
|
|
173
|
+
* — the caller treats that as "probe-failed", never as a verdict. The
|
|
174
|
+
* sixth field (perf token) is OPTIONAL both ways: a five-field line from
|
|
175
|
+
* an old guest parses fine, and a token that fails its own clamp simply
|
|
176
|
+
* leaves `perf` absent — perf can never fail a liveness reading. */
|
|
177
|
+
export function parseTurnProbeOutput(raw: string): TurnProbeReading | null {
|
|
178
|
+
const fields = raw.trim().split(/\s+/);
|
|
179
|
+
if (fields.length < 5) return null;
|
|
180
|
+
const size = Number.parseInt(fields[0]!, 10);
|
|
181
|
+
const alive = fields[1] === "1" ? true : fields[1] === "0" ? false : null;
|
|
182
|
+
const hbMs = Number.parseInt(fields[2]!, 10);
|
|
183
|
+
const nowMs = Number.parseInt(fields[3]!, 10);
|
|
184
|
+
const sentinelCount = Number.parseInt(fields[4]!, 10);
|
|
185
|
+
if (!Number.isFinite(size) || alive === null || !Number.isFinite(hbMs)
|
|
186
|
+
|| !Number.isFinite(nowMs) || !Number.isFinite(sentinelCount)) return null;
|
|
187
|
+
const perfTok = fields[5];
|
|
188
|
+
const perf = perfTok !== undefined && perfTok !== "-" ? parsePerfToken(perfTok) : null;
|
|
189
|
+
return {
|
|
190
|
+
size, alive, hbMs, nowMs, sentinelSeen: sentinelCount > 0,
|
|
191
|
+
...(perf ? { perf } : {}),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** The three-valued verdict from one durable reading. A FINISHED turn
|
|
196
|
+
* (sentinel durable in the file) is ALIVE — the transport's watchdog
|
|
197
|
+
* harvests it within one interval; reporting it dead is exactly the
|
|
198
|
+
* incident's ten-minute blindness. */
|
|
199
|
+
export function livenessVerdictFromReading(reading: TurnProbeReading | null): RunnerLivenessVerdict {
|
|
200
|
+
if (reading === null) return "probe-failed";
|
|
201
|
+
if (reading.alive) return "alive";
|
|
202
|
+
if (reading.sentinelSeen) return "alive";
|
|
203
|
+
if (reading.hbMs > 0 && reading.nowMs - reading.hbMs <= RUNNER_HEARTBEAT_STALE_MS) return "alive";
|
|
204
|
+
return "dead";
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// ── Mid-turn stream input (the delivery half of injectUserMessage) ──────────
|
|
208
|
+
// The durable transport's stdin, when the spec declares `streamInput`, is a
|
|
209
|
+
// FIFO fed by an in-guest subshell: the prompt line first, then any complete
|
|
210
|
+
// lines appended to the turn's durable INBOX file. A steering-capable CLI
|
|
211
|
+
// (claude -p --input-format stream-json) folds a message that arrives while
|
|
212
|
+
// the turn runs into the RUNNING turn at its next tool boundary — verified
|
|
213
|
+
// live against claude 2.1.233. The feeder checks for the CLI's terminal
|
|
214
|
+
// `result` line BEFORE each delivery (a message that races the result stays
|
|
215
|
+
// owed for the follow-up turn, never half-consumed) and stops on it; closing
|
|
216
|
+
// the FIFO is what EOFs the CLI's stdin and lets it exit. The DELIVERED
|
|
217
|
+
// counter file is the ack the server trusts: it advances only after the
|
|
218
|
+
// feeder forwarded those inbox lines into the CLI's stdin.
|
|
219
|
+
|
|
220
|
+
/** Feeder poll cadence (seconds — it is a shell `sleep`; GNU sleep accepts
|
|
221
|
+
* fractions and the durable transport only runs on the GNU guest image). */
|
|
222
|
+
export const INJECT_FEEDER_POLL_SECONDS = "0.3";
|
|
223
|
+
/** How much of the durable stdout tail the feeder greps for the terminal
|
|
224
|
+
* result line each poll — bounded so a long transcript never turns the
|
|
225
|
+
* poll into a full-file scan. Comfortably above the JSONL guard's line cap,
|
|
226
|
+
* so a capped result line still fits whole. */
|
|
227
|
+
export const INJECT_FEEDER_RESULT_TAIL_BYTES = 262_144;
|
|
228
|
+
/** One in-guest delivery-ack exec: append the line, then poll the delivered
|
|
229
|
+
* counter until it covers this append (delivered), the runner dies, or the
|
|
230
|
+
* attempts run out (both: not delivered — the message stays owed). */
|
|
231
|
+
export const INJECT_ACK_POLL_ATTEMPTS = 30;
|
|
232
|
+
export const INJECT_ACK_POLL_SECONDS = "0.5";
|
|
233
|
+
export const INJECT_ACK_EXEC_TIMEOUT_MS = 25_000;
|
|
234
|
+
|
|
235
|
+
/** The in-guest feeder fragment, prepended to the detached wrapper script.
|
|
236
|
+
* Runs inside the setsid session (group-kill reaps it) and keys its loop to
|
|
237
|
+
* the wrapper's own pid (`$$`), like the heartbeat subshell. Exported for
|
|
238
|
+
* tests. */
|
|
239
|
+
export function streamInputFeederFragment(paths: {
|
|
240
|
+
promptPath: string; fifoPath: string; inboxPath: string;
|
|
241
|
+
deliveredPath: string; outPath: string;
|
|
242
|
+
}): string {
|
|
243
|
+
const q = shellQuote;
|
|
244
|
+
return `rm -f ${q(paths.fifoPath)}; mkfifo ${q(paths.fifoPath)}; `
|
|
245
|
+
+ `: > ${q(paths.inboxPath)}; echo 0 > ${q(paths.deliveredPath)}; `
|
|
246
|
+
+ `( cat ${q(paths.promptPath)}; ac_fed=0; while kill -0 "$$" 2>/dev/null; do `
|
|
247
|
+
+ `if tail -c ${INJECT_FEEDER_RESULT_TAIL_BYTES} ${q(paths.outPath)} 2>/dev/null | grep -q '"type":"result"'; then break; fi; `
|
|
248
|
+
+ `ac_lines=$(wc -l < ${q(paths.inboxPath)} 2>/dev/null || echo 0); ac_lines=\${ac_lines:-0}; `
|
|
249
|
+
+ `if [ "$ac_lines" -gt "$ac_fed" ]; then `
|
|
250
|
+
+ `sed -n "$((ac_fed+1)),$((ac_lines))p" ${q(paths.inboxPath)}; ac_fed=$ac_lines; `
|
|
251
|
+
+ `echo "$ac_fed" > ${q(paths.deliveredPath)}; fi; `
|
|
252
|
+
+ `sleep ${INJECT_FEEDER_POLL_SECONDS}; done ) > ${q(paths.fifoPath)} 2>/dev/null </dev/null & `;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** The instructional envelope around a user message delivered INTO a
|
|
256
|
+
* running turn. Claude Code's own interactive harness wraps queued
|
|
257
|
+
* mid-turn input with an explicit "address this as you continue"
|
|
258
|
+
* instruction; a bare `--input-format stream-json` line carries no such
|
|
259
|
+
* framing, so an injected message reads like any other transcript text
|
|
260
|
+
* and the model tends to continue its work narration without
|
|
261
|
+
* acknowledging it. Every mid-turn inject producer (the runner's
|
|
262
|
+
* `injectUserMessage` and the workflow lane's `deliverInjectedMessage`
|
|
263
|
+
* activity) MUST wrap through here — the wrapper rides only the wire to
|
|
264
|
+
* the model; the stored conversation row keeps the user's raw text.
|
|
265
|
+
* The text is pinned by tests: change it deliberately or not at all. */
|
|
266
|
+
export function wrapMidTurnUserMessage(text: string): string {
|
|
267
|
+
return "The user sent a new message while you were working:\n\n"
|
|
268
|
+
+ text
|
|
269
|
+
+ "\n\nAddress the message above as you continue this turn.";
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** The one delivery-ack exec `injectUserMessage` runs: append the message
|
|
273
|
+
* line to the durable inbox, then wait for the feeder's delivered counter
|
|
274
|
+
* to cover it. Exit 0 = delivered into the CLI's stdin pre-result; 4 = the
|
|
275
|
+
* runner died first; 5 = not delivered within the ack window (feeder
|
|
276
|
+
* stopped on the result, or the guest is crawling) — both non-zero exits
|
|
277
|
+
* mean "leave the message owed". Exported for tests. */
|
|
278
|
+
export function injectAppendAndAckCommand(args: {
|
|
279
|
+
line: string; target: number; pid: number;
|
|
280
|
+
inboxPath: string; deliveredPath: string;
|
|
281
|
+
}): string {
|
|
282
|
+
const q = shellQuote;
|
|
283
|
+
return `printf '%s\\n' ${q(args.line)} >> ${q(args.inboxPath)}; i=0; `
|
|
284
|
+
+ `while [ "$i" -lt ${INJECT_ACK_POLL_ATTEMPTS} ]; do `
|
|
285
|
+
+ `d=$(cat ${q(args.deliveredPath)} 2>/dev/null || echo 0); `
|
|
286
|
+
+ `[ "\${d:-0}" -ge ${args.target} ] && exit 0; `
|
|
287
|
+
+ `kill -0 ${args.pid} 2>/dev/null || exit 4; `
|
|
288
|
+
+ `sleep ${INJECT_ACK_POLL_SECONDS}; i=$((i+1)); done; exit 5`;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// ── Reap = kill AND CONFIRM (2026-08-18 incident: codex writer lock) ────────
|
|
292
|
+
// A reaped runner must be CONFIRMED dead before the turn unwinds. codex holds
|
|
293
|
+
// a kernel flock on its thread store (`~/.codex/thread-writer-locks/<id>.lock`)
|
|
294
|
+
// for exactly as long as the process lives; the old fire-and-forget SIGTERM
|
|
295
|
+
// gave a slow-dying (or TERM-handling) CLI no deadline and never escalated,
|
|
296
|
+
// so a canceled turn could leave the writer alive — and the NEXT turn's
|
|
297
|
+
// `codex exec resume` died with `thread-store conflict: … already has an
|
|
298
|
+
// active writer`. The reap is now ONE guest exec that TERMs the detached
|
|
299
|
+
// group, waits bounded for the group leader to exit, escalates to SIGKILL,
|
|
300
|
+
// and confirms — and the transport AWAITS that confirmation in its unwind
|
|
301
|
+
// path, so the generator (and the executor's turn settle behind it) never
|
|
302
|
+
// completes while the guest tree may still be running.
|
|
303
|
+
|
|
304
|
+
/** SIGTERM grace before escalation: attempts × sleep = 5s. */
|
|
305
|
+
export const REAP_TERM_WAIT_ATTEMPTS = 20;
|
|
306
|
+
/** Post-SIGKILL confirm window: attempts × sleep = 2s (a KILLed process only
|
|
307
|
+
* lingers as an unreaped zombie, which `kill -0` still sees — brief). */
|
|
308
|
+
export const REAP_KILL_WAIT_ATTEMPTS = 8;
|
|
309
|
+
export const REAP_POLL_SECONDS = "0.25";
|
|
310
|
+
/** The confirm exec's "still alive after SIGKILL" exit code (surfaced as a
|
|
311
|
+
* warn — the launch preflight is the backstop for whatever survived). */
|
|
312
|
+
export const REAP_STILL_ALIVE_EXIT = 7;
|
|
313
|
+
/** Ceiling on the whole kill-and-confirm exec (TERM 5s + KILL 2s + slack). */
|
|
314
|
+
export const REAP_EXEC_TIMEOUT_MS = 15_000;
|
|
315
|
+
|
|
316
|
+
/** The one kill-and-confirm exec: TERM the group (setsid made `pid` the
|
|
317
|
+
* leader) and the leader itself, poll for exit, escalate to KILL, poll
|
|
318
|
+
* again. Exit 0 = confirmed dead; REAP_STILL_ALIVE_EXIT = it survived
|
|
319
|
+
* SIGKILL's confirm window. Exported for tests. */
|
|
320
|
+
export function reapKillAndConfirmCommand(pid: number): string {
|
|
321
|
+
const alive = `kill -0 ${pid} 2>/dev/null`;
|
|
322
|
+
return `kill -TERM -- -${pid} 2>/dev/null; kill -TERM ${pid} 2>/dev/null; ac_ri=0; `
|
|
323
|
+
+ `while [ "$ac_ri" -lt ${REAP_TERM_WAIT_ATTEMPTS} ]; do ${alive} || exit 0; `
|
|
324
|
+
+ `sleep ${REAP_POLL_SECONDS}; ac_ri=$((ac_ri+1)); done; `
|
|
325
|
+
+ `kill -KILL -- -${pid} 2>/dev/null; kill -KILL ${pid} 2>/dev/null; ac_ri=0; `
|
|
326
|
+
+ `while [ "$ac_ri" -lt ${REAP_KILL_WAIT_ATTEMPTS} ]; do ${alive} || exit 0; `
|
|
327
|
+
+ `sleep ${REAP_POLL_SECONDS}; ac_ri=$((ac_ri+1)); done; `
|
|
328
|
+
+ `exit ${REAP_STILL_ALIVE_EXIT}`;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// ── Resident harness (docs/specs/resident-harness-turns.md, P1) ─────────────
|
|
332
|
+
// A RESIDENT keeps one live CLI process serving consecutive turns inside the
|
|
333
|
+
// hot window: the feeder's result-break is replaced by an END-FILE break
|
|
334
|
+
// (`<prompt>.end` → close the FIFO → the CLI EOFs stdin and exits with a
|
|
335
|
+
// clean sentinel — the probe measured 625ms), and a guest idle watchdog
|
|
336
|
+
// reaps the group if the server's durable record is ever lost. Turn
|
|
337
|
+
// accounting rides two guest numbers next to the inbox:
|
|
338
|
+
// `<prompt>.served` — turns DELIVERED into this process (1 at launch,
|
|
339
|
+
// +1 per adoption; the server mirrors it durably
|
|
340
|
+
// as the runner stamp's `turnsServed`);
|
|
341
|
+
// result lines in .out — turns FINISHED (anchored grep — hint-grade; the
|
|
342
|
+
// server-side stream-json parse is authoritative).
|
|
343
|
+
// idle ⇔ results ≥ served AND the inbox is fully fed. Mid-turn injects ride
|
|
344
|
+
// the same inbox WITHOUT bumping served, and fold into the running turn
|
|
345
|
+
// producing no result of their own — the parity holds.
|
|
346
|
+
|
|
347
|
+
/** Guest idle TTL for a resident with no new inbox line: hot window (180s)
|
|
348
|
+
* + margin, so the park gate's graceful `.end` normally wins and the
|
|
349
|
+
* watchdog only answers a dead server / lost record. */
|
|
350
|
+
export const RESIDENT_IDLE_TTL_S = 240;
|
|
351
|
+
/** Watchdog poll cadence (seconds). */
|
|
352
|
+
export const RESIDENT_WATCHDOG_POLL_S = 15;
|
|
353
|
+
/** Anchored prefix of the CLI's terminal result event line. Anchoring keeps
|
|
354
|
+
* the guest-side counts honest against tool output that merely CONTAINS
|
|
355
|
+
* the substring; a serialization-order change fails SAFE (parity check
|
|
356
|
+
* refuses adoption → cold launch — a latency cost, never correctness). */
|
|
357
|
+
export const RESIDENT_RESULT_LINE_PREFIX = '{"type":"result"';
|
|
358
|
+
|
|
359
|
+
/** One bounded exec printing the count of result lines at or past
|
|
360
|
+
* `fromByte` (0-based) in the durable .out. Always exits 0; stdout is the
|
|
361
|
+
* count (a missing file reads as 0). Hint-grade by design. */
|
|
362
|
+
export function residentResultCountCommand(outPath: string, fromByte = 0): string {
|
|
363
|
+
const q = shellQuote;
|
|
364
|
+
const src = fromByte > 0
|
|
365
|
+
? `tail -c +${fromByte + 1} ${q(outPath)} 2>/dev/null`
|
|
366
|
+
: `cat ${q(outPath)} 2>/dev/null`;
|
|
367
|
+
return `ac_res=$(${src} | grep -c ${q(`^${RESIDENT_RESULT_LINE_PREFIX}`)}); echo "\${ac_res:-0}"`;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/** The resident feeder: `streamInputFeederFragment` with the result-break
|
|
371
|
+
* REPLACED by the end-file break — the ONLY guest-contract change the
|
|
372
|
+
* resident shape needs (verified live by the spec's probe: two messages,
|
|
373
|
+
* one process, one session file, graceful exit on `.end`). */
|
|
374
|
+
export function residentFeederFragment(paths: {
|
|
375
|
+
promptPath: string; fifoPath: string; inboxPath: string;
|
|
376
|
+
deliveredPath: string; endPath: string;
|
|
377
|
+
}): string {
|
|
378
|
+
const q = shellQuote;
|
|
379
|
+
return `rm -f ${q(paths.fifoPath)}; mkfifo ${q(paths.fifoPath)}; `
|
|
380
|
+
+ `: > ${q(paths.inboxPath)}; echo 0 > ${q(paths.deliveredPath)}; rm -f ${q(paths.endPath)}; `
|
|
381
|
+
+ `( cat ${q(paths.promptPath)}; ac_fed=0; while kill -0 "$$" 2>/dev/null; do `
|
|
382
|
+
+ `if [ -f ${q(paths.endPath)} ]; then break; fi; `
|
|
383
|
+
+ `ac_lines=$(wc -l < ${q(paths.inboxPath)} 2>/dev/null || echo 0); ac_lines=\${ac_lines:-0}; `
|
|
384
|
+
+ `if [ "$ac_lines" -gt "$ac_fed" ]; then `
|
|
385
|
+
+ `sed -n "$((ac_fed+1)),$((ac_lines))p" ${q(paths.inboxPath)}; ac_fed=$ac_lines; `
|
|
386
|
+
+ `echo "$ac_fed" > ${q(paths.deliveredPath)}; fi; `
|
|
387
|
+
+ `sleep ${INJECT_FEEDER_POLL_SECONDS}; done ) > ${q(paths.fifoPath)} 2>/dev/null </dev/null & `;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/** Sets `ac_idle` (1 = between turns: every delivered turn has its result
|
|
391
|
+
* and the inbox is fully fed). Embedded by the watchdog and the resident
|
|
392
|
+
* heartbeat gate — one idle predicate, stated once. */
|
|
393
|
+
export function residentIdleCheckFragment(paths: {
|
|
394
|
+
outPath: string; inboxPath: string; deliveredPath: string; servedPath: string;
|
|
395
|
+
}): string {
|
|
396
|
+
const q = shellQuote;
|
|
397
|
+
return `ac_res=$(grep -c ${q(`^${RESIDENT_RESULT_LINE_PREFIX}`)} ${q(paths.outPath)} 2>/dev/null); ac_res=\${ac_res:-0}; `
|
|
398
|
+
+ `ac_srv=$(cat ${q(paths.servedPath)} 2>/dev/null); ac_srv=\${ac_srv:-1}; `
|
|
399
|
+
+ `ac_in=$(wc -l < ${q(paths.inboxPath)} 2>/dev/null); ac_in=\${ac_in:-0}; `
|
|
400
|
+
+ `ac_del=$(cat ${q(paths.deliveredPath)} 2>/dev/null); ac_del=\${ac_del:-0}; `
|
|
401
|
+
+ `if [ "$ac_res" -ge "$ac_srv" ] && [ "$ac_in" -eq "$ac_del" ]; then ac_idle=1; else ac_idle=0; fi; `;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/** Guest idle watchdog: reap the whole process group after
|
|
405
|
+
* RESIDENT_IDLE_TTL_S of CONTINUOUS idleness (any busy reading resets the
|
|
406
|
+
* clock — a long-running TURN is never reaped; the parity check reads
|
|
407
|
+
* busy mid-turn). The park gate's graceful `.end` normally wins; this is
|
|
408
|
+
* the orphan backstop for a dead server or a lost durable record.
|
|
409
|
+
*
|
|
410
|
+
* BACKGROUND TASKS ARE NOT IDLE (2026-08-19 forensics: a user's
|
|
411
|
+
* between-turns background search died to a harness teardown while its
|
|
412
|
+
* journal was still advancing). The turn-parity predicate alone reads a
|
|
413
|
+
* resident BETWEEN turns as idle even while harness background tasks it
|
|
414
|
+
* tracks are hard at work — and this watchdog's group-kill would take
|
|
415
|
+
* those children with the tree. So idle additionally requires the CLI's
|
|
416
|
+
* own task journal (`$HOME/.claude/tasks`) to be QUIET: any file touched
|
|
417
|
+
* within RUNNER_BUSY_TASK_FRESH_MINUTES resets the clock — the exact
|
|
418
|
+
* journal-freshness license the server's park gate honors
|
|
419
|
+
* (guestBusyProbeCommand). A stalled task goes stale within the window
|
|
420
|
+
* and the TTL resumes on its own; liveness alone never counts (the
|
|
421
|
+
* 2026-08-17 wake-loop damper is a SERVER rule about waking — this is a
|
|
422
|
+
* guest rule about not killing, and journal advance is required, not mere
|
|
423
|
+
* process existence). */
|
|
424
|
+
export function residentIdleWatchdogFragment(paths: {
|
|
425
|
+
outPath: string; inboxPath: string; deliveredPath: string; servedPath: string;
|
|
426
|
+
}): string {
|
|
427
|
+
return `( ac_idle_s=0; while kill -0 "$$" 2>/dev/null; do `
|
|
428
|
+
+ residentIdleCheckFragment(paths)
|
|
429
|
+
+ `if [ "$ac_idle" = 1 ] && find "$HOME/.claude/tasks" -type f -mmin -${RUNNER_BUSY_TASK_FRESH_MINUTES} 2>/dev/null | grep -q .; then ac_idle=0; fi; `
|
|
430
|
+
+ `if [ "$ac_idle" = 1 ]; then ac_idle_s=$((ac_idle_s+${RESIDENT_WATCHDOG_POLL_S})); else ac_idle_s=0; fi; `
|
|
431
|
+
+ `if [ "$ac_idle_s" -ge ${RESIDENT_IDLE_TTL_S} ]; then kill -- -$$ 2>/dev/null; exit 0; fi; `
|
|
432
|
+
+ `sleep ${RESIDENT_WATCHDOG_POLL_S}; done ) >/dev/null 2>&1 </dev/null & `;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/** Exit-event doorbell with the turnId read from a guest FILE at push time
|
|
436
|
+
* (a resident serves many turns; each adoption rewrites `<prompt>.turnid`
|
|
437
|
+
* so a mid-turn death rings the turn actually being served). Same
|
|
438
|
+
* validation posture as `exitEventPushCommand`; an empty turnid file
|
|
439
|
+
* pushes nothing. */
|
|
440
|
+
export function deferredExitEventPushCommand(
|
|
441
|
+
notify: { url: string; tokenEnv: string }, turnIdFile: string,
|
|
442
|
+
): string {
|
|
443
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(notify.tokenEnv)) return "";
|
|
444
|
+
const q = shellQuote;
|
|
445
|
+
return `( ac_tid=$(cat ${q(turnIdFile)} 2>/dev/null | tr -cd 'A-Za-z0-9-'); `
|
|
446
|
+
+ `if [ -n "$ac_tid" ] && [ -n "$${notify.tokenEnv}" ] && command -v curl >/dev/null 2>&1; then `
|
|
447
|
+
+ `curl -fsS -m ${EXIT_PUSH_ATTEMPT_TIMEOUT_SECONDS} -o /dev/null -X POST `
|
|
448
|
+
+ `-H "Authorization: Bearer $${notify.tokenEnv}" `
|
|
449
|
+
+ `-H 'Content-Type: application/json' --data "{\\"turnId\\":\\"$ac_tid\\",\\"exitCode\\":$ac_rc}" ${q(notify.url)}; `
|
|
450
|
+
+ `fi ) >/dev/null 2>&1 || true`;
|
|
451
|
+
}
|
|
452
|
+
|
|
51
453
|
/** Provider deadline on the detached LAUNCH exec. The launch shell only
|
|
52
454
|
* truncates the durable files, forks the detached runner (stdio fully
|
|
53
455
|
* redirected — see the launch command), writes the pidfile, and echoes the
|
|
@@ -71,6 +473,13 @@ export const LAUNCH_PID_RECOVERY_DELAY_MS = 1_000;
|
|
|
71
473
|
* decide a turn's fate. Exported for tests. */
|
|
72
474
|
export const INSTALL_PROBE_TIMEOUT_MS = 30_000;
|
|
73
475
|
|
|
476
|
+
/** Deadline on the on-demand CLI install itself (`spec.install`). Shared by
|
|
477
|
+
* this transport's ensureInstalled and the server's CloudTurnWorkflow launch
|
|
478
|
+
* path (turn-workflow/tailer.ts), which provisions the same way before a
|
|
479
|
+
* detached launch — the session images bake only `claude`, so codex/
|
|
480
|
+
* opencode/cursor/droid MUST be installable at launch on a fresh machine. */
|
|
481
|
+
export const CLI_INSTALL_TIMEOUT_MS = 300_000;
|
|
482
|
+
|
|
74
483
|
/** Parse a pid out of captured stdout (last non-empty line). NaN when absent. */
|
|
75
484
|
function parsePid(raw: string): number {
|
|
76
485
|
const pid = Number.parseInt(raw.trim().split("\n").pop() ?? "", 10);
|
|
@@ -99,6 +508,80 @@ export function shellQuote(value: string): string {
|
|
|
99
508
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
100
509
|
}
|
|
101
510
|
|
|
511
|
+
// ── Session env file (session secrets) ──────────────────────────────────────
|
|
512
|
+
|
|
513
|
+
/** $HOME-relative path of the platform-managed session env file. The server
|
|
514
|
+
* writes it at sandbox acquire (session secrets → `export KEY='…'` lines);
|
|
515
|
+
* the detached turn launch sources it fresh EVERY turn, so a secret set or
|
|
516
|
+
* rotated between turns lands on the next turn with no VM recycle. Shared
|
|
517
|
+
* constant so the writer (server) and the reader (this runtime) can never
|
|
518
|
+
* drift. */
|
|
519
|
+
export const SESSION_ENV_FILE_RELPATH = ".agent-compose/session-env.sh";
|
|
520
|
+
|
|
521
|
+
/** Only plain relative paths make it into the script — no quotes/spaces/`..`
|
|
522
|
+
* (a malformed option must not break the launch or escape $HOME). */
|
|
523
|
+
const SESSION_ENV_FILE_SAFE = /^[A-Za-z0-9._\/-]+$/;
|
|
524
|
+
|
|
525
|
+
/** The `. $HOME/<file>;` fragment sourced ahead of the CLI command, or ""
|
|
526
|
+
* when no env file is configured. Errors are swallowed — a missing or
|
|
527
|
+
* unreadable file must never fail a turn. */
|
|
528
|
+
export function sessionEnvSourceFragment(relPath: string | undefined): string {
|
|
529
|
+
if (!relPath || !SESSION_ENV_FILE_SAFE.test(relPath) || relPath.includes("..")) return "";
|
|
530
|
+
return `[ -f "$HOME/${relPath}" ] && . "$HOME/${relPath}" >/dev/null 2>&1 || true; `;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/** The per-turn model-credential source fragment (`RuntimeOptions.
|
|
534
|
+
* credEnvFile` — an ABSOLUTE guest path), or "" when none is configured.
|
|
535
|
+
* Sourced AFTER the session env file so the turn actor's credential wins.
|
|
536
|
+
* Same swallow-errors posture: a missing file must never fail a turn. */
|
|
537
|
+
export function credEnvSourceFragment(absPath: string | undefined): string {
|
|
538
|
+
if (!absPath || !absPath.startsWith("/") || !SESSION_ENV_FILE_SAFE.test(absPath.slice(1))
|
|
539
|
+
|| absPath.includes("..")) return "";
|
|
540
|
+
return `[ -f "${absPath}" ] && . "${absPath}" >/dev/null 2>&1 || true; `;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// ── Exit-event push (the completion doorbell, v0.10.43) ─────────────────────
|
|
544
|
+
// THE PUSH IS A DOORBELL, NEVER A VERDICT. The detached wrapper announces
|
|
545
|
+
// {turnId, exitCode} to the server with ONE best-effort POST immediately
|
|
546
|
+
// AFTER the exit sentinel is durably written, so the server's watchdog can
|
|
547
|
+
// run its verification pass at event latency (~1-2s) instead of its poll
|
|
548
|
+
// cadence. The durable file stays the only truth: a lost push costs nothing
|
|
549
|
+
// (the poll ladder is the unchanged backstop), and the server treats every
|
|
550
|
+
// push as "go verify the durable state now", never as an outcome.
|
|
551
|
+
|
|
552
|
+
/** Ceiling on ONE curl attempt (seconds — it is `curl -m`). Short: the file
|
|
553
|
+
* is the truth and a slow push must not keep the wrapper alive for long. */
|
|
554
|
+
export const EXIT_PUSH_ATTEMPT_TIMEOUT_SECONDS = 5;
|
|
555
|
+
/** Pause before the single retry. */
|
|
556
|
+
export const EXIT_PUSH_RETRY_DELAY_SECONDS = 1;
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* The sh fragment appended to the detached wrapper AFTER the sentinel
|
|
560
|
+
* `printf` — never before it, and never in a way that can fail the wrapper:
|
|
561
|
+
* - reads the credential from the guest env at push time (`$<tokenEnv>`;
|
|
562
|
+
* the token never appears in the script text, argv-visible command
|
|
563
|
+
* lines the server logs, or any log line);
|
|
564
|
+
* - silently skips when the credential or curl is absent;
|
|
565
|
+
* - two attempts max (`-m 5`, one retry after 1s), then gives up silently
|
|
566
|
+
* — all output discarded, exit status swallowed (`|| true`).
|
|
567
|
+
* `$ac_rc` is the wrapper-local capture of the guarded pipeline's exit code
|
|
568
|
+
* (the same value the sentinel line carries). Returns "" — push disabled —
|
|
569
|
+
* for any `tokenEnv`/`turnId` that is not a plain identifier/uuid shape, so
|
|
570
|
+
* hostile config can never become shell injection.
|
|
571
|
+
*/
|
|
572
|
+
export function exitEventPushCommand(notify: TurnExitNotify): string {
|
|
573
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(notify.tokenEnv)) return "";
|
|
574
|
+
if (!/^[A-Za-z0-9-]{1,64}$/.test(notify.turnId)) return "";
|
|
575
|
+
const token = `"$${notify.tokenEnv}"`;
|
|
576
|
+
const body = `"{\\"turnId\\":\\"${notify.turnId}\\",\\"exitCode\\":$ac_rc}"`;
|
|
577
|
+
const attempt = `curl -fsS -m ${EXIT_PUSH_ATTEMPT_TIMEOUT_SECONDS} -o /dev/null -X POST`
|
|
578
|
+
+ ` -H "Authorization: Bearer $${notify.tokenEnv}"`
|
|
579
|
+
+ ` -H 'Content-Type: application/json' --data ${body} ${shellQuote(notify.url)}`;
|
|
580
|
+
return `( if [ -n ${token} ] && command -v curl >/dev/null 2>&1; then `
|
|
581
|
+
+ `${attempt} || { sleep ${EXIT_PUSH_RETRY_DELAY_SECONDS}; ${attempt}; }; `
|
|
582
|
+
+ `fi ) >/dev/null 2>&1 || true`;
|
|
583
|
+
}
|
|
584
|
+
|
|
102
585
|
/** Monotonic discriminator for prompt-file names within this process. */
|
|
103
586
|
let promptFileCounter = 0;
|
|
104
587
|
|
|
@@ -208,8 +691,12 @@ export interface CliAgentSpec {
|
|
|
208
691
|
/** Build the one-shot shell command for a turn. `promptPath` is a file in the
|
|
209
692
|
* sandbox holding `promptPayload(prompt)`; `sessionId` continues a thread.
|
|
210
693
|
* `effort` is present only when the caller configured a reasoning effort
|
|
211
|
-
* AND the spec has a real knob for it (see `CliReasoningEffort`).
|
|
212
|
-
|
|
694
|
+
* AND the spec has a real knob for it (see `CliReasoningEffort`).
|
|
695
|
+
* `streamInput` is true when the durable transport is feeding stdin as a
|
|
696
|
+
* live message stream (see `CliAgentSpec.streamInput`): `promptPath` is
|
|
697
|
+
* then the FIFO the feeder writes, and the CLI must be invoked in its
|
|
698
|
+
* stream-input mode (claude: `--input-format stream-json`). */
|
|
699
|
+
buildCommand(args: { promptPath: string; sessionId?: string; model?: string; cwd?: string; effort?: CliReasoningEffort; streamInput?: boolean }): string;
|
|
213
700
|
/** Map one parsed JSONL stdout event to `AgentMessage`s. The base emits
|
|
214
701
|
* `init`/`done`/`error` lifecycle itself, so a spec maps only content +
|
|
215
702
|
* usage (text / thinking / tool_use / tool_result / usage).
|
|
@@ -223,12 +710,43 @@ export interface CliAgentSpec {
|
|
|
223
710
|
* resume it (codex `thread.started.thread_id`, amp `session_id`).
|
|
224
711
|
* LEGACY FALLBACK PATH (see `mapEvent`). */
|
|
225
712
|
extractSessionId(parsed: Record<string, unknown>): string | undefined;
|
|
713
|
+
/** The runtime's thread persistence admits only ONE live writer per thread
|
|
714
|
+
* (codex: an OS advisory flock per thread id, held for the holding
|
|
715
|
+
* process's whole lifetime — codex-rs thread-store writer_lock.rs). A
|
|
716
|
+
* platform that wants a SUCCESSOR turn to resume the same thread must
|
|
717
|
+
* therefore KILL the predecessor's process tree first — a merely-detached
|
|
718
|
+
* live predecessor blocks every `thread/resume` with "already has an
|
|
719
|
+
* active writer" (the 2026-08-22 superseded-codex-turn incident). Absent
|
|
720
|
+
* ⇒ concurrent resume is safe (claude-code: append-only session JSONL)
|
|
721
|
+
* and a superseded runner may be detached alive as an asset. */
|
|
722
|
+
exclusiveSessionWriter?: boolean;
|
|
226
723
|
/** ACP-mode invocation (the Zed `agent_servers` shape). When present, the
|
|
227
724
|
* runner spawns the CLI in ACP agent mode and delegates the whole wire
|
|
228
725
|
* protocol to `AcpClientPeer`; the legacy `promptPayload`/`buildCommand`/
|
|
229
726
|
* `extractSessionId`/`mapEvent` members are used only as the version-mismatch
|
|
230
727
|
* fallback. Absent → the spec is JSONL-only (legacy path always). */
|
|
231
728
|
acp?: { command: string; args: string[]; env?: Record<string, string> };
|
|
729
|
+
/** Mid-turn STREAM-INPUT support (the delivery half of
|
|
730
|
+
* `injectUserMessage`). When present AND the durable detached transport
|
|
731
|
+
* is in play, the CLI's stdin is fed from a FIFO by an in-guest feeder:
|
|
732
|
+
* the prompt line first, then any lines appended to the turn's durable
|
|
733
|
+
* inbox file — so a steering-capable CLI (claude: queued user input is
|
|
734
|
+
* folded into the RUNNING turn at the next tool boundary; verified live
|
|
735
|
+
* against claude 2.1.233) receives user messages while the turn runs.
|
|
736
|
+
* The feeder stops at the CLI's terminal result line (a message that
|
|
737
|
+
* races the result is NOT forwarded — it stays owed) and closes the
|
|
738
|
+
* FIFO, which is what ends the CLI process (stream-input CLIs exit on
|
|
739
|
+
* stdin EOF, not after a result). Absent → the prompt file is the whole
|
|
740
|
+
* stdin, exactly as before. */
|
|
741
|
+
streamInput?: {
|
|
742
|
+
/** Serialise the opening prompt into ONE stream-input stdin line
|
|
743
|
+
* (claude: a stream-json user message). No trailing newline — the
|
|
744
|
+
* transport owns line framing. */
|
|
745
|
+
promptLine(prompt: string): string;
|
|
746
|
+
/** Serialise one mid-turn user message into ONE stdin line. No
|
|
747
|
+
* trailing newline. */
|
|
748
|
+
messageLine(text: string): string;
|
|
749
|
+
};
|
|
232
750
|
}
|
|
233
751
|
|
|
234
752
|
/** A spawned ACP-mode CLI: the read side of its stdout and a delivery for its
|
|
@@ -294,6 +812,111 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
294
812
|
* dormant. Not a public API. */
|
|
295
813
|
private acpTransportReady = ACP_TRANSPORT_READY;
|
|
296
814
|
|
|
815
|
+
/** The CURRENT turn's durable-trio reader (set once the detached runner is
|
|
816
|
+
* launched, replaced by the next launch). Null = no durable transport is
|
|
817
|
+
* active for this runner (boot phase, ACP path, single-exec providers) —
|
|
818
|
+
* `probeTurnLiveness` then answers null and the executor keeps its own
|
|
819
|
+
* fallback probe. */
|
|
820
|
+
private turnProbe: (() => Promise<TurnProbeReading | null>) | null = null;
|
|
821
|
+
|
|
822
|
+
/** The CURRENT turn's mid-turn injection state (durable stream-input
|
|
823
|
+
* transport only). Null = no live stream-input turn — `injectUserMessage`
|
|
824
|
+
* answers "unsupported" and the caller leaves the message owed. `chain`
|
|
825
|
+
* serializes appends so each delivery targets a deterministic line count. */
|
|
826
|
+
private turnInject: {
|
|
827
|
+
inboxPath: string; deliveredPath: string; pid: number;
|
|
828
|
+
appended: number; chain: Promise<unknown>;
|
|
829
|
+
} | null = null;
|
|
830
|
+
|
|
831
|
+
/** Deliver one user message INTO the live turn — see the contract doc
|
|
832
|
+
* (types/runtime.ts). Appends a stream-input line to the turn's durable
|
|
833
|
+
* inbox and waits for the in-guest feeder's delivered-counter ack; only
|
|
834
|
+
* an acked forward (pre-result, into the CLI's stdin) reports
|
|
835
|
+
* "delivered". Guest exit 5 (ack window exhausted with the runner still
|
|
836
|
+
* alive — a CLI that is not draining stdin mid-step) reports "pending":
|
|
837
|
+
* the appended line may still be read when the current step finishes,
|
|
838
|
+
* but it was NOT seen yet. Never throws. */
|
|
839
|
+
async injectUserMessage(text: string): Promise<"delivered" | "pending" | "closed" | "unsupported"> {
|
|
840
|
+
const inject = this.turnInject;
|
|
841
|
+
const streamInput = this.spec.streamInput;
|
|
842
|
+
if (!inject || !streamInput || this.guestDetached) return "unsupported";
|
|
843
|
+
// The address-this envelope (wrapMidTurnUserMessage): a raw injected
|
|
844
|
+
// line carries no framing, and the model continues its narration
|
|
845
|
+
// without acknowledging the message.
|
|
846
|
+
const line = streamInput.messageLine(wrapMidTurnUserMessage(text));
|
|
847
|
+
const attempt = inject.chain.then(async (): Promise<"delivered" | "pending" | "closed"> => {
|
|
848
|
+
if (this.turnInject !== inject) return "closed";
|
|
849
|
+
// The append lands whether or not the ack times out, so the target
|
|
850
|
+
// counter advances unconditionally: a later injection must never aim
|
|
851
|
+
// at a line number the feeder has already passed.
|
|
852
|
+
const target = inject.appended + 1;
|
|
853
|
+
inject.appended = target;
|
|
854
|
+
try {
|
|
855
|
+
const res = await this.sandbox.commands.run(
|
|
856
|
+
injectAppendAndAckCommand({
|
|
857
|
+
line, target, pid: inject.pid,
|
|
858
|
+
inboxPath: inject.inboxPath, deliveredPath: inject.deliveredPath,
|
|
859
|
+
}),
|
|
860
|
+
{ timeoutMs: INJECT_ACK_EXEC_TIMEOUT_MS });
|
|
861
|
+
if (res.exitCode === 0) return "delivered";
|
|
862
|
+
// 5 = ack window exhausted, runner alive: the line is in the inbox
|
|
863
|
+
// and may be fed when the CLI next drains stdin (the mid-step wall).
|
|
864
|
+
// 4 = the runner died first; anything else is a fault — both closed.
|
|
865
|
+
return res.exitCode === 5 ? "pending" : "closed";
|
|
866
|
+
} catch { return "closed"; }
|
|
867
|
+
});
|
|
868
|
+
inject.chain = attempt.catch(() => undefined);
|
|
869
|
+
return attempt;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
/** Durable three-valued liveness for the current turn — see the contract
|
|
873
|
+
* doc (types/runtime.ts). Never throws; never blocks past the probe's
|
|
874
|
+
* explicit exec timeout. */
|
|
875
|
+
async probeTurnLiveness(): Promise<RunnerLivenessVerdict | null> {
|
|
876
|
+
const probe = this.turnProbe;
|
|
877
|
+
if (!probe) return null;
|
|
878
|
+
return livenessVerdictFromReading(await probe());
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
/** Wakes the CURRENT tail-watchdog interval early. Armed only while the
|
|
882
|
+
* watchdog's wake timer is pending; null between arms / between turns. */
|
|
883
|
+
private wakeTailWatchdog: (() => void) | null = null;
|
|
884
|
+
/** A doorbell that rang while no watchdog wake was armed — consumed at the
|
|
885
|
+
* next arm (immediate wake), dropped at the start of a new turn. */
|
|
886
|
+
private nudgePending = false;
|
|
887
|
+
|
|
888
|
+
/** DOORBELL, NEVER A VERDICT (contract doc: types/runtime.ts). Wake the
|
|
889
|
+
* tail watchdog NOW so its normal verification pass — durable probe, then
|
|
890
|
+
* harvest-on-sentinel / honest no-sentinel death — runs immediately
|
|
891
|
+
* instead of at the next TAIL_WATCHDOG_INTERVAL_MS tick. Carries no
|
|
892
|
+
* information: a nudge for a live mid-turn runner probes alive and is a
|
|
893
|
+
* no-op; a spurious/duplicate nudge costs at most one probe exec. No-op
|
|
894
|
+
* when no durable watchdog is armed (boot phase, ACP path, single-exec
|
|
895
|
+
* transports, abandoned-streaming polling) — those phases already carry
|
|
896
|
+
* their own bounds. */
|
|
897
|
+
nudgeTurnProbe(): void {
|
|
898
|
+
const wake = this.wakeTailWatchdog;
|
|
899
|
+
if (wake) {
|
|
900
|
+
this.wakeTailWatchdog = null;
|
|
901
|
+
wake();
|
|
902
|
+
} else {
|
|
903
|
+
this.nudgePending = true;
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
/** RESUME HANDOFF, NEVER A KILL (contract doc: types/runtime.ts). Once
|
|
908
|
+
* set, abort/early-exit unwinds the transport WITHOUT reaping the
|
|
909
|
+
* detached guest tree and WITHOUT the opportunistic durable-file cleanup
|
|
910
|
+
* — the successor turn resumes the same guest session, and the park
|
|
911
|
+
* path's busy signal keeps reading the surviving heartbeat/busy files.
|
|
912
|
+
* One-way for this runner instance (a runner is per-turn on the cloud
|
|
913
|
+
* path). */
|
|
914
|
+
private guestDetached = false;
|
|
915
|
+
|
|
916
|
+
detachGuest(): void {
|
|
917
|
+
this.guestDetached = true;
|
|
918
|
+
}
|
|
919
|
+
|
|
297
920
|
constructor(
|
|
298
921
|
private readonly sandbox: SandboxProvider,
|
|
299
922
|
private readonly options: RuntimeOptions,
|
|
@@ -358,7 +981,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
358
981
|
const probe = await this.sandbox.commands.run(
|
|
359
982
|
`command -v ${this.spec.bin}`, { timeoutMs: INSTALL_PROBE_TIMEOUT_MS });
|
|
360
983
|
if (probe.exitCode === 0) { this.installed = true; return; }
|
|
361
|
-
const res = await this.sandbox.commands.run(this.spec.install, { timeoutMs:
|
|
984
|
+
const res = await this.sandbox.commands.run(this.spec.install, { timeoutMs: CLI_INSTALL_TIMEOUT_MS });
|
|
362
985
|
if (res.exitCode !== 0) {
|
|
363
986
|
throw new Error(`failed to install ${this.spec.bin}: ${(res.stderr || res.stdout || "").slice(-500)}`);
|
|
364
987
|
}
|
|
@@ -662,13 +1285,33 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
662
1285
|
await this.ensureInstalled();
|
|
663
1286
|
|
|
664
1287
|
const promptPath = uniquePromptPath(this.spec.kind, this.options.agentId ?? "agent", opts.iteration ?? 0);
|
|
665
|
-
|
|
1288
|
+
// A NEW turn invalidates any previous turn's durable probe — re-armed
|
|
1289
|
+
// below once (and only if) this turn's detached runner launches. A
|
|
1290
|
+
// stale doorbell is dropped with it: a new turn owes nothing to the
|
|
1291
|
+
// previous turn's push.
|
|
1292
|
+
this.turnProbe = null;
|
|
1293
|
+
this.turnInject = null;
|
|
1294
|
+
this.nudgePending = false;
|
|
1295
|
+
this.wakeTailWatchdog = null;
|
|
1296
|
+
// Mid-turn stream input engages only where BOTH halves exist: a spec
|
|
1297
|
+
// that can serialise stream-input lines AND the durable detached
|
|
1298
|
+
// transport (the feeder, inbox, and result-watch live in the guest;
|
|
1299
|
+
// single-exec transports keep the plain prompt-file stdin).
|
|
1300
|
+
const streamInput = this.spec.streamInput !== undefined
|
|
1301
|
+
&& this.sandbox.commands.runBackground !== undefined;
|
|
1302
|
+
const fifoPath = `${promptPath}.fifo`;
|
|
1303
|
+
const promptWrite = this.sandbox.files.write(promptPath, streamInput
|
|
1304
|
+
? `${this.spec.streamInput!.promptLine(opts.prompt)}\n`
|
|
1305
|
+
: this.spec.promptPayload(opts.prompt));
|
|
666
1306
|
const cmd = this.spec.buildCommand({
|
|
667
|
-
|
|
1307
|
+
// Stream-input turns read stdin from the feeder's FIFO; the prompt
|
|
1308
|
+
// file is the feeder's first delivery, not the CLI's stdin.
|
|
1309
|
+
promptPath: streamInput ? fifoPath : promptPath,
|
|
668
1310
|
sessionId: opts.sessionId,
|
|
669
1311
|
model: this.model,
|
|
670
1312
|
cwd: this.options.cwd,
|
|
671
1313
|
effort: this.configEffort,
|
|
1314
|
+
...(streamInput ? { streamInput: true } : {}),
|
|
672
1315
|
});
|
|
673
1316
|
|
|
674
1317
|
// Frame guard (see _jsonl-guard.ts): the CLI's stdout is piped through
|
|
@@ -687,7 +1330,11 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
687
1330
|
// uniquePromptPath) and the same opportunistic cleanup below.
|
|
688
1331
|
const guardPath = `${promptPath}.guard.js`;
|
|
689
1332
|
const sentinel = `__AC_JSONL_EXIT_${Math.random().toString(36).slice(2, 10)}__`;
|
|
690
|
-
|
|
1333
|
+
// Both file-transport writes ride in parallel: each is a full envd
|
|
1334
|
+
// HTTP round trip (~65ms measured against E2B from the dev host), and
|
|
1335
|
+
// they are independent — the launch below is the only consumer of
|
|
1336
|
+
// either path. One RTT shaved off every turn's first-token path.
|
|
1337
|
+
await Promise.all([promptWrite, this.sandbox.files.write(guardPath, jsonlGuardScript())]);
|
|
691
1338
|
const guardedCmd = wrapJsonlCommand({
|
|
692
1339
|
cmd, guardPath, sentinel,
|
|
693
1340
|
maxBytes: JSONL_GUARD_LINE_CAP_BYTES,
|
|
@@ -723,6 +1370,10 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
723
1370
|
let transportError: unknown = null;
|
|
724
1371
|
let consumerStopped = false;
|
|
725
1372
|
let reap: () => void = () => { /* set per transport */ };
|
|
1373
|
+
// The reap's kill-and-confirm exec, latched on first fire (reap runs
|
|
1374
|
+
// from BOTH the abort listener and the finally — one exec, not two).
|
|
1375
|
+
// The unwind awaits it so the turn never settles over a live tree.
|
|
1376
|
+
let reapConfirm: Promise<void> | null = null;
|
|
726
1377
|
let transport: Promise<void>;
|
|
727
1378
|
|
|
728
1379
|
const durableRead = this.sandbox.files.read?.bind(this.sandbox.files);
|
|
@@ -735,9 +1386,65 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
735
1386
|
// every server↔sandbox stream is dead by then. In node-guard mode the
|
|
736
1387
|
// pipeline's exit code is already the CLI's (the guard re-raises the
|
|
737
1388
|
// inner sentinel, including NO_SENTINEL_EXIT for a mid-pipe death).
|
|
1389
|
+
// The heartbeat subshell rides INSIDE the setsid session (group kill
|
|
1390
|
+
// reaps it with the tree). It is keyed to the wrapper sh's own pid
|
|
1391
|
+
// (`$$` — the pidfile pid) and DIES WITH it: heartbeat and busy
|
|
1392
|
+
// marker both stop within one interval of the wrapper exiting (see
|
|
1393
|
+
// the loop comment below for why outliving the wrapper was a bug).
|
|
1394
|
+
// Its writes are the durable "the runner tree is alive" signals the
|
|
1395
|
+
// probes read — signals no server↔sandbox stream can wedge.
|
|
1396
|
+
const hbPath = `${promptPath}.hb`;
|
|
1397
|
+
// Exit-event doorbell (see exitEventPushCommand): SENTINEL FIRST,
|
|
1398
|
+
// PUSH AFTER. The push rides behind the sentinel append so the
|
|
1399
|
+
// durable truth exists before anything is announced, and its whole
|
|
1400
|
+
// failure surface is swallowed — the runner lifecycle cannot see it.
|
|
1401
|
+
const exitPush = this.options.turnExitNotify
|
|
1402
|
+
? exitEventPushCommand(this.options.turnExitNotify)
|
|
1403
|
+
: "";
|
|
1404
|
+
// Busy sentinel (park-defers-while-busy): each beat also writes the
|
|
1405
|
+
// background-work marker next to the heartbeat — task count
|
|
1406
|
+
// (<prompt>.busy) and task ids (<prompt>.tasks). The server's park
|
|
1407
|
+
// probe reads the task JOURNALS (~/.claude/tasks) directly plus the
|
|
1408
|
+
// heartbeat/pid pair; the marker files are in-guest receipts.
|
|
1409
|
+
//
|
|
1410
|
+
// The subshell DIES WITH the wrapper. It used to outlive it while
|
|
1411
|
+
// fresh task files existed (so a clean turn end would not take the
|
|
1412
|
+
// busy marker down with running background tasks) — but the server
|
|
1413
|
+
// probe stopped reading the marker files (journal mtimes carry the
|
|
1414
|
+
// busy signal), and the lingering subshell kept the setsid GROUP
|
|
1415
|
+
// alive minutes past turn end, which the probe's live-group gating
|
|
1416
|
+
// reads as work: every turn that touched its task list became its
|
|
1417
|
+
// own "background work" and armed a spurious wake (the 2026-08-17
|
|
1418
|
+
// loop). Genuine background children launched by the CLI live in
|
|
1419
|
+
// this same setsid group and keep it alive by THEMSELVES — that,
|
|
1420
|
+
// plus their fresh journals, is the honest busy signal. A
|
|
1421
|
+
// supersede group-kill still reaps this subshell with the tree.
|
|
1422
|
+
const busyPath = `${promptPath}.busy`;
|
|
1423
|
+
const tasksPath = `${promptPath}.tasks`;
|
|
1424
|
+
// Mid-turn stream input: the feeder subshell (see
|
|
1425
|
+
// streamInputFeederFragment) rides at the FRONT of the detached
|
|
1426
|
+
// script so the FIFO exists before the CLI opens it as stdin.
|
|
1427
|
+
const inboxPath = `${promptPath}.inbox`;
|
|
1428
|
+
const deliveredPath = `${promptPath}.delivered`;
|
|
1429
|
+
const feeder = streamInput
|
|
1430
|
+
? streamInputFeederFragment({ promptPath, fifoPath, inboxPath, deliveredPath, outPath })
|
|
1431
|
+
: "";
|
|
1432
|
+
// Perf sampling rides the same beat: every 6th beat (~60s) the
|
|
1433
|
+
// subshell burst-reads /proc and rewrites the `.perf` token the
|
|
1434
|
+
// durable-trio probe carries as its trailing field (perf-sampler.ts).
|
|
1435
|
+
const perfPath = `${promptPath}.perf`;
|
|
738
1436
|
const detachedScript =
|
|
739
|
-
|
|
740
|
-
+ `
|
|
1437
|
+
feeder
|
|
1438
|
+
+ `( ${perfSamplerFunctionFragment({ perfPath })}while kill -0 "$$" 2>/dev/null; do `
|
|
1439
|
+
+ `date +%s%3N > ${shellQuote(hbPath)}; `
|
|
1440
|
+
+ `${runnerBusySentinelFragment(busyPath, tasksPath)}; `
|
|
1441
|
+
+ `ac_perf_tick; `
|
|
1442
|
+
+ `sleep ${RUNNER_HEARTBEAT_INTERVAL_SECONDS}; done ) >/dev/null 2>&1 </dev/null & `
|
|
1443
|
+
+ sessionEnvSourceFragment(this.options.sessionEnvFile)
|
|
1444
|
+
+ credEnvSourceFragment(this.options.credEnvFile)
|
|
1445
|
+
+ `{ ${guardedCmd}; } >> ${shellQuote(outPath)} 2>> ${shellQuote(errPath)} </dev/null; ac_rc=$?; `
|
|
1446
|
+
+ `printf '\\n%s %s\\n' ${shellQuote(sentinel)} "$ac_rc" >> ${shellQuote(outPath)}`
|
|
1447
|
+
+ (exitPush ? `; ${exitPush}` : "");
|
|
741
1448
|
// `setsid` detaches the runner into its own session/process group so
|
|
742
1449
|
// it survives the launch exec ending AND gives `kill -- -pid` a whole
|
|
743
1450
|
// tree to terminate. The fallback keeps working where setsid is
|
|
@@ -761,7 +1468,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
761
1468
|
// below.
|
|
762
1469
|
const pidPath = `${promptPath}.pid`;
|
|
763
1470
|
const launchCmd =
|
|
764
|
-
`: > ${shellQuote(outPath)}; : > ${shellQuote(errPath)}; `
|
|
1471
|
+
`: > ${shellQuote(outPath)}; : > ${shellQuote(errPath)}; date +%s%3N > ${shellQuote(hbPath)}; `
|
|
765
1472
|
+ `if command -v setsid >/dev/null 2>&1; then setsid sh -c ${shellQuote(detachedScript)} >/dev/null 2>&1 </dev/null & `
|
|
766
1473
|
+ `else sh -c ${shellQuote(detachedScript)} >/dev/null 2>&1 </dev/null & fi; `
|
|
767
1474
|
+ `echo "$!" > ${shellQuote(pidPath)}; echo "$!"`;
|
|
@@ -791,6 +1498,24 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
791
1498
|
+ `(pid ${pid} via ${pidPath}) — continuing on the durable transport: ${formatError(launchErr)}`);
|
|
792
1499
|
}
|
|
793
1500
|
|
|
1501
|
+
// Arm the durable-trio probe for this turn: the watchdog below and the
|
|
1502
|
+
// executor's evidence ticker (`probeTurnLiveness`) both read through
|
|
1503
|
+
// it — one fresh short exec per call, explicit timeout, faults map to
|
|
1504
|
+
// null ("probe-failed"), never to a verdict.
|
|
1505
|
+
const probeCmd = turnLivenessProbeCommand({ outPath, hbPath, perfPath, pid, sentinel });
|
|
1506
|
+
const runTurnProbe = async (): Promise<TurnProbeReading | null> => {
|
|
1507
|
+
try {
|
|
1508
|
+
const res = await this.sandbox.commands.run(probeCmd, { timeoutMs: TURN_PROBE_TIMEOUT_MS });
|
|
1509
|
+
return parseTurnProbeOutput(res.stdout ?? "");
|
|
1510
|
+
} catch { return null; }
|
|
1511
|
+
};
|
|
1512
|
+
this.turnProbe = runTurnProbe;
|
|
1513
|
+
// Arm mid-turn injection now that the detached runner (and its
|
|
1514
|
+
// feeder) exist; cleared with the turn in the finally below.
|
|
1515
|
+
if (streamInput) {
|
|
1516
|
+
this.turnInject = { inboxPath, deliveredPath, pid, appended: 0, chain: Promise.resolve() };
|
|
1517
|
+
}
|
|
1518
|
+
|
|
794
1519
|
// Bytes of COMPLETE lines already consumed from outPath — a re-attach
|
|
795
1520
|
// tails from here, so a dead stream never duplicates or drops lines.
|
|
796
1521
|
let offset = 0;
|
|
@@ -808,16 +1533,31 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
808
1533
|
};
|
|
809
1534
|
|
|
810
1535
|
reap = () => {
|
|
811
|
-
// Kill the detached tree (group first — setsid
|
|
812
|
-
// leader
|
|
813
|
-
//
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
1536
|
+
// Kill the detached tree AND CONFIRM it died (group first — setsid
|
|
1537
|
+
// made pid the group leader; TERM → bounded wait → KILL escalation,
|
|
1538
|
+
// see reapKillAndConfirmCommand), then whatever tail is currently
|
|
1539
|
+
// attached. Fresh execs, deliberately independent of any
|
|
1540
|
+
// possibly-dead stream. The confirm promise is latched so the
|
|
1541
|
+
// finally below can await it — the turn must not settle while the
|
|
1542
|
+
// tree may still be alive (it holds e.g. codex's thread-store
|
|
1543
|
+
// writer flock until it actually exits).
|
|
1544
|
+
reapConfirm ??= this.sandbox.commands.run(
|
|
1545
|
+
reapKillAndConfirmCommand(pid), { timeoutMs: REAP_EXEC_TIMEOUT_MS })
|
|
1546
|
+
.then((res) => {
|
|
1547
|
+
if (res.exitCode === REAP_STILL_ALIVE_EXIT) {
|
|
1548
|
+
console.warn(`[cli-agent] ${this.spec.kind} runner ${pid} survived SIGKILL's confirm window `
|
|
1549
|
+
+ "— the next launch's preflight is the backstop");
|
|
1550
|
+
}
|
|
1551
|
+
})
|
|
1552
|
+
.catch(() => { /* best-effort exec — the launch preflight covers a missed kill */ });
|
|
817
1553
|
void currentTail?.kill().catch(() => { /* already gone */ });
|
|
818
1554
|
};
|
|
819
1555
|
|
|
820
1556
|
transport = (async () => {
|
|
1557
|
+
// Stream deaths this turn — past the abandon bound the turn stops
|
|
1558
|
+
// re-attaching tails and finishes on durable polling alone (a wire
|
|
1559
|
+
// that killed three streams will kill the fourth).
|
|
1560
|
+
let streamDeaths = 0;
|
|
821
1561
|
while (exitCode === null && !consumerStopped && !opts.signal?.aborted) {
|
|
822
1562
|
let buf = "";
|
|
823
1563
|
let handle: SandboxBackgroundProcess | null = null;
|
|
@@ -830,15 +1570,98 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
830
1570
|
if (exitCode !== null) { void handle?.kill().catch(() => { /* racing exit */ }); return; }
|
|
831
1571
|
}
|
|
832
1572
|
};
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
`
|
|
836
|
-
|
|
837
|
-
|
|
1573
|
+
if (streamDeaths < TAIL_ABANDON_AFTER_STREAM_DEATHS) {
|
|
1574
|
+
try {
|
|
1575
|
+
// `-s 0.05` tightens ONLY tail's polling fallback: the guest's
|
|
1576
|
+
// GNU tail (coreutils 9.1, verified on the agent-env image)
|
|
1577
|
+
// follows /tmp via inotify — measured delivery is event-driven
|
|
1578
|
+
// (~46ms p50 write→server, one line per callback, S2 bench
|
|
1579
|
+
// 2026-08-14) — but if inotify were ever unavailable, the
|
|
1580
|
+
// default fallback polls at 1s, which WOULD become the stream's
|
|
1581
|
+
// burstiness. GNU tail accepts fractional seconds; this
|
|
1582
|
+
// transport only runs on providers with runBackground (E2B),
|
|
1583
|
+
// whose image is Debian/GNU — never busybox.
|
|
1584
|
+
handle = await this.sandbox.commands.runBackground!(
|
|
1585
|
+
`tail -s 0.05 -c +${offset + 1} -f ${shellQuote(outPath)}`,
|
|
1586
|
+
{ timeoutMs: 0, onStdout });
|
|
1587
|
+
} catch { /* attach failed at the wire — durable catch-up below */ }
|
|
1588
|
+
}
|
|
838
1589
|
if (handle) {
|
|
839
1590
|
if (exitCode !== null) void handle.kill().catch(() => { /* done */ });
|
|
840
1591
|
currentTail = handle;
|
|
841
|
-
|
|
1592
|
+
// ── The watchdog: never TRUST an attached stream ─────────────
|
|
1593
|
+
// (2026-08-15 incident, turn 18dc5261: the tail's connect-web
|
|
1594
|
+
// stream wedged silently — wait() never settled, no error, no
|
|
1595
|
+
// EOF — while the finished answer + exit sentinel sat in the
|
|
1596
|
+
// durable file for ten minutes.) wait() is raced against a
|
|
1597
|
+
// fixed-cadence durable probe; the probe's DURABLE reading —
|
|
1598
|
+
// never the stream's health — decides:
|
|
1599
|
+
// - bytes in the file the stream failed to deliver across a
|
|
1600
|
+
// full interval (offset frozen since the previous tick)
|
|
1601
|
+
// prove the stream dead → kill the TAIL (never the runner)
|
|
1602
|
+
// and fall through to the durable catch-up + re-attach;
|
|
1603
|
+
// - runner process gone → nothing more will be written: kill
|
|
1604
|
+
// the tail and let the catch-up read the sentinel (or
|
|
1605
|
+
// surface the honest no-sentinel death);
|
|
1606
|
+
// - probe fault/timeout → NO verdict: keep streaming (the
|
|
1607
|
+
// executor's evidence machinery bounds an unreachable
|
|
1608
|
+
// sandbox, not this loop);
|
|
1609
|
+
// - abort/consumer-stop wakes the race immediately, so a
|
|
1610
|
+
// wedged wait() can no longer leak the whole transport.
|
|
1611
|
+
await (async (tail: SandboxBackgroundProcess): Promise<void> => {
|
|
1612
|
+
let waitSettled = false;
|
|
1613
|
+
const waitP = tail.wait().catch(() => undefined)
|
|
1614
|
+
.then(() => { waitSettled = true; });
|
|
1615
|
+
let lastTickOffset = -1;
|
|
1616
|
+
while (!waitSettled) {
|
|
1617
|
+
let onAbort: (() => void) | undefined;
|
|
1618
|
+
const wake = new Promise<void>((resolve) => {
|
|
1619
|
+
const t = setTimeout(resolve, TAIL_WATCHDOG_INTERVAL_MS);
|
|
1620
|
+
(t as { unref?: () => void }).unref?.();
|
|
1621
|
+
const fire = () => { clearTimeout(t); resolve(); };
|
|
1622
|
+
// Exit-event doorbell (nudgeTurnProbe): a nudge wakes
|
|
1623
|
+
// THIS interval immediately — its only effect is running
|
|
1624
|
+
// the durable probe below NOW. One that rang while no
|
|
1625
|
+
// wake was armed is consumed here.
|
|
1626
|
+
if (this.nudgePending) {
|
|
1627
|
+
this.nudgePending = false;
|
|
1628
|
+
fire();
|
|
1629
|
+
} else {
|
|
1630
|
+
this.wakeTailWatchdog = fire;
|
|
1631
|
+
}
|
|
1632
|
+
if (opts.signal) {
|
|
1633
|
+
onAbort = fire;
|
|
1634
|
+
opts.signal.addEventListener("abort", onAbort, { once: true });
|
|
1635
|
+
}
|
|
1636
|
+
});
|
|
1637
|
+
await Promise.race([waitP, wake]);
|
|
1638
|
+
this.wakeTailWatchdog = null;
|
|
1639
|
+
if (onAbort) opts.signal?.removeEventListener("abort", onAbort);
|
|
1640
|
+
if (waitSettled || exitCode !== null || consumerStopped || opts.signal?.aborted) return;
|
|
1641
|
+
const tickOffset = offset;
|
|
1642
|
+
const reading = await runTurnProbe();
|
|
1643
|
+
if (offset !== tickOffset) { lastTickOffset = offset; continue; } // stream delivered meanwhile — healthy
|
|
1644
|
+
if (reading === null) { lastTickOffset = tickOffset; continue; } // probe-failed — no verdict
|
|
1645
|
+
// HARVEST-ON-PROBE (one strike — THE fix for the incident's
|
|
1646
|
+
// ten-minute blindness): a durable exit sentinel means the
|
|
1647
|
+
// turn is FINISHED and no more bytes are coming; any stream
|
|
1648
|
+
// that hasn't delivered it is done being useful. Likewise a
|
|
1649
|
+
// runner process that is gone. Mid-turn starvation keeps a
|
|
1650
|
+
// two-tick grace (offset frozen since the previous tick) so
|
|
1651
|
+
// a byte-burst racing the probe never kills a healthy tail.
|
|
1652
|
+
const streamStarved = reading.size > offset && tickOffset === lastTickOffset;
|
|
1653
|
+
if (reading.sentinelSeen || !reading.alive || streamStarved) {
|
|
1654
|
+
console.warn(`[cli-agent] ${this.spec.kind} tail stream ${reading.sentinelSeen
|
|
1655
|
+
? "outlived the FINISHED turn (durable sentinel unharvested)"
|
|
1656
|
+
: !reading.alive ? "outlived the runner"
|
|
1657
|
+
: `starved (durable file at ${reading.size} bytes, stream stuck at ${offset})`}`
|
|
1658
|
+
+ " — killing the tail and catching up from the durable file");
|
|
1659
|
+
void tail.kill().catch(() => { /* best-effort; catch-up follows regardless */ });
|
|
1660
|
+
return;
|
|
1661
|
+
}
|
|
1662
|
+
lastTickOffset = tickOffset;
|
|
1663
|
+
}
|
|
1664
|
+
})(handle);
|
|
842
1665
|
currentTail = null;
|
|
843
1666
|
}
|
|
844
1667
|
if (exitCode !== null || consumerStopped || opts.signal?.aborted) break;
|
|
@@ -873,8 +1696,17 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
873
1696
|
exitCode = JSONL_GUARD_NO_SENTINEL_EXIT;
|
|
874
1697
|
break;
|
|
875
1698
|
}
|
|
876
|
-
|
|
877
|
-
|
|
1699
|
+
streamDeaths += 1;
|
|
1700
|
+
if (streamDeaths < TAIL_ABANDON_AFTER_STREAM_DEATHS) {
|
|
1701
|
+
console.warn(`[cli-agent] ${this.spec.kind} turn stream lost with the runner alive — re-tailing ${outPath} from byte ${offset}`);
|
|
1702
|
+
await new Promise((r) => setTimeout(r, TAIL_REATTACH_DELAY_MS));
|
|
1703
|
+
} else {
|
|
1704
|
+
if (streamDeaths === TAIL_ABANDON_AFTER_STREAM_DEATHS) {
|
|
1705
|
+
console.warn(`[cli-agent] ${this.spec.kind} turn abandoning streaming after `
|
|
1706
|
+
+ `${streamDeaths} stream deaths — finishing on durable polling from byte ${offset}`);
|
|
1707
|
+
}
|
|
1708
|
+
await new Promise((r) => setTimeout(r, DURABLE_POLL_INTERVAL_MS));
|
|
1709
|
+
}
|
|
878
1710
|
}
|
|
879
1711
|
if (exitCode !== null && exitCode !== 0 && durableRead) {
|
|
880
1712
|
try { exitStderr = (await durableRead(errPath)).slice(-2000); }
|
|
@@ -898,7 +1730,8 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
898
1730
|
onStdout,
|
|
899
1731
|
timeoutMs: 0, // no provider deadline — see the doc comment above
|
|
900
1732
|
};
|
|
901
|
-
transport = this.sandbox.commands.run(
|
|
1733
|
+
transport = this.sandbox.commands.run(
|
|
1734
|
+
credEnvSourceFragment(this.options.credEnvFile) + guardedCmd, runOpts).then(
|
|
902
1735
|
(res) => {
|
|
903
1736
|
const tail = buf.trim();
|
|
904
1737
|
if (tail) lines.push(tail);
|
|
@@ -912,11 +1745,22 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
912
1745
|
// observed so an aborted turn can never surface an unhandled rejection.
|
|
913
1746
|
void transport.catch(() => { /* observed via `await transport` when consumed */ });
|
|
914
1747
|
|
|
1748
|
+
// Detach-aware reap seam: a runner marked for resume handoff
|
|
1749
|
+
// (`detachGuest`) unwinds WITHOUT killing the guest tree — the
|
|
1750
|
+
// successor turn resumes that exact guest session.
|
|
1751
|
+
const reapUnlessDetached = (): void => { if (!this.guestDetached) reap(); };
|
|
915
1752
|
if (opts.signal) {
|
|
916
|
-
if (opts.signal.aborted)
|
|
917
|
-
else opts.signal.addEventListener("abort",
|
|
1753
|
+
if (opts.signal.aborted) reapUnlessDetached();
|
|
1754
|
+
else opts.signal.addEventListener("abort", reapUnlessDetached, { once: true });
|
|
918
1755
|
}
|
|
919
1756
|
try {
|
|
1757
|
+
// The guest session id ALREADY ANNOUNCED to the consumer (the
|
|
1758
|
+
// synthesized init above carried opts.sessionId). A CLI that forks a
|
|
1759
|
+
// NEW session id (claude --resume forks; codex threads) announces it
|
|
1760
|
+
// mid-stream below, so the executor can durably record the id even
|
|
1761
|
+
// when the turn never reaches its `done` (a reaped/superseded turn
|
|
1762
|
+
// must still be resumable by its redispatch).
|
|
1763
|
+
let announcedSessionId = opts.sessionId ?? "";
|
|
920
1764
|
for await (const line of lines) {
|
|
921
1765
|
let parsed: Record<string, unknown>;
|
|
922
1766
|
try {
|
|
@@ -926,6 +1770,10 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
926
1770
|
}
|
|
927
1771
|
const sid = this.spec.extractSessionId(parsed);
|
|
928
1772
|
if (sid) sessionId = sid;
|
|
1773
|
+
if (sid && sid !== announcedSessionId) {
|
|
1774
|
+
announcedSessionId = sid;
|
|
1775
|
+
yield { type: "init", sessionId: sid, timestamp: now() };
|
|
1776
|
+
}
|
|
929
1777
|
for (const msg of this.spec.mapEvent(parsed)) {
|
|
930
1778
|
if (msg.type === "error") sawError = true;
|
|
931
1779
|
yield msg;
|
|
@@ -948,23 +1796,51 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
948
1796
|
yield { type: "error", text: `${this.spec.kind} exited with code ${exitCode}${detail}${exitStderr ? `: ${exitStderr}` : ""}`, timestamp: now() };
|
|
949
1797
|
return;
|
|
950
1798
|
}
|
|
1799
|
+
if (exitCode !== 0 && sawError && RESUME_TARGET_MISSING_STDERR.test(exitStderr)) {
|
|
1800
|
+
// The CLI reported a bare result error on stdout (claude's
|
|
1801
|
+
// `error_during_execution` with num_turns 0) while its REAL
|
|
1802
|
+
// complaint — the missing `--resume` target — went to stderr
|
|
1803
|
+
// (the 2026-08-18 split-brain forensic). Surface the stderr as
|
|
1804
|
+
// the LAST error event: the consumer's terminal close keeps the
|
|
1805
|
+
// last error text, so the classifier sees the truth instead of a
|
|
1806
|
+
// generic runner death it would call transient forever.
|
|
1807
|
+
yield { type: "error", text: `${this.spec.kind} exited with code ${exitCode}: ${exitStderr}`, timestamp: now() };
|
|
1808
|
+
return;
|
|
1809
|
+
}
|
|
951
1810
|
if (!sawError) yield { type: "done", sessionId: sessionId ?? "", timestamp: now() };
|
|
952
1811
|
} finally {
|
|
953
1812
|
// Runs on completion AND on early generator exit (the caller broke
|
|
954
1813
|
// out of its for-await: abort, turn closed under the executor,
|
|
955
|
-
// supersede). Reap only while the runner hasn't reported an exit
|
|
956
|
-
|
|
1814
|
+
// supersede). Reap only while the runner hasn't reported an exit —
|
|
1815
|
+
// and NEVER when the guest was detached for a resume handoff: the
|
|
1816
|
+
// successor turn resumes this exact guest session, and the park
|
|
1817
|
+
// path's busy signal reads the surviving heartbeat/busy files.
|
|
1818
|
+
opts.signal?.removeEventListener("abort", reapUnlessDetached);
|
|
957
1819
|
consumerStopped = true;
|
|
958
|
-
if (exitCode === null) reap();
|
|
1820
|
+
if (exitCode === null && !this.guestDetached) reap();
|
|
1821
|
+
// Teardown hygiene (2026-08-18 incident): a reaped turn's unwind
|
|
1822
|
+
// WAITS for the kill-and-confirm exec, so the generator's return —
|
|
1823
|
+
// and the executor's turn settle behind it — never completes while
|
|
1824
|
+
// the guest tree may still be alive holding e.g. codex's
|
|
1825
|
+
// thread-store writer flock. Bounded by the exec's own ceiling
|
|
1826
|
+
// (REAP_EXEC_TIMEOUT_MS); a clean exit never armed it.
|
|
1827
|
+
if (reapConfirm) await reapConfirm;
|
|
959
1828
|
// Opportunistic prompt-file + guard-script + transcript cleanup:
|
|
960
1829
|
// paths are never reused (see uniquePromptPath), so this is hygiene,
|
|
961
1830
|
// not correctness — fire and forget, and a dead sandbox / failed rm
|
|
962
1831
|
// is fine. The shell holds an open fd on the redirect, so unlinking
|
|
963
|
-
// under a still-exiting CLI is harmless.
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
1832
|
+
// under a still-exiting CLI is harmless. Skipped on a detach: the
|
|
1833
|
+
// handed-off runner's durable trio must survive it.
|
|
1834
|
+
// The turn's injection window is over either way: a caller landing
|
|
1835
|
+
// after this point must get "unsupported", never a write into a
|
|
1836
|
+
// dead (or handed-off) turn's inbox.
|
|
1837
|
+
this.turnInject = null;
|
|
1838
|
+
if (!this.guestDetached) {
|
|
1839
|
+
void this.sandbox.commands.run(
|
|
1840
|
+
`rm -f ${shellQuote(promptPath)} ${shellQuote(guardPath)} ${shellQuote(`${promptPath}.out`)} ${shellQuote(`${promptPath}.err`)} ${shellQuote(`${promptPath}.pid`)} ${shellQuote(`${promptPath}.hb`)} ${shellQuote(`${promptPath}.busy`)} ${shellQuote(`${promptPath}.tasks`)} ${shellQuote(`${promptPath}.fifo`)} ${shellQuote(`${promptPath}.inbox`)} ${shellQuote(`${promptPath}.delivered`)}`,
|
|
1841
|
+
{ timeoutMs: 10_000 })
|
|
1842
|
+
.catch(() => { /* best-effort */ });
|
|
1843
|
+
}
|
|
968
1844
|
}
|
|
969
1845
|
} catch (err) {
|
|
970
1846
|
yield { type: "error", text: formatError(err), timestamp: now() };
|