@agent-compose/sdk 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/__tests__/perf-sampler.test.d.ts +10 -0
- package/dist/agent/agent-context.d.ts +1 -1
- package/dist/agent/agent-loop.d.ts +5 -1
- package/dist/agent/desktop-open.d.ts +184 -0
- package/dist/agent/perf-sampler.d.ts +99 -0
- package/dist/agent/services-manifest.d.ts +88 -0
- package/dist/agent/services-restore.d.ts +58 -0
- package/dist/client.d.ts +189 -15
- package/dist/display.d.ts +17 -0
- package/dist/index.d.ts +14 -5
- package/dist/index.js +1625 -120
- package/dist/runtimes/_cli-agent.d.ts +372 -2
- package/dist/runtimes/claude-code.d.ts +12 -0
- package/dist/runtimes/codex.buildcommand.test.d.ts +9 -0
- package/dist/runtimes/codex.d.ts +8 -0
- package/dist/runtimes/openai-desktop.js +1555 -120
- package/dist/runtimes/session-env.test.d.ts +14 -0
- package/dist/sandbox/sizes.d.ts +120 -30
- package/dist/sandbox.d.ts +1 -1
- package/dist/types/api-conversations.d.ts +476 -1
- package/dist/types/api-factory.d.ts +164 -7
- package/dist/types/api-runs.d.ts +23 -1
- package/dist/types/protocol.d.ts +32 -1
- package/dist/types/runtime.d.ts +120 -0
- package/dist/types/workflow-metadata.d.ts +6 -5
- package/package.json +1 -1
- package/src/agent/agent-context.ts +128 -28
- package/src/agent/agent-loop.ts +10 -3
- package/src/agent/desktop-open.ts +418 -0
- package/src/agent/perf-sampler.ts +202 -0
- package/src/agent/services-manifest.ts +356 -0
- package/src/agent/services-restore.ts +195 -0
- package/src/client.ts +384 -32
- package/src/display.ts +44 -1
- package/src/index.ts +74 -7
- package/src/runtimes/_cli-agent.ts +1160 -67
- package/src/runtimes/claude-code.ts +187 -12
- package/src/runtimes/codex.ts +65 -2
- package/src/sandbox/providers/e2b.ts +8 -4
- package/src/sandbox/providers/local.ts +16 -4
- package/src/sandbox/sizes.ts +127 -44
- package/src/sandbox.ts +8 -0
- package/src/types/api-conversations.ts +461 -2
- package/src/types/api-factory.ts +165 -7
- package/src/types/api-runs.ts +25 -1
- package/src/types/protocol.ts +30 -1
- package/src/types/runtime.ts +122 -0
- package/src/types/workflow-metadata.ts +6 -5
|
@@ -24,9 +24,15 @@
|
|
|
24
24
|
* values set via `agentc secrets set` are visible to the CLI.
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
|
-
import type { AgentMessage, ModelExecutionContract,
|
|
27
|
+
import type { AgentMessage, ModelExecutionContract, RunnerLivenessVerdict, RuntimeOptions, SandboxProvider, ToolCallGateResult, TurnExitNotify } from "../index.js";
|
|
28
28
|
import { defineRuntime } from "../types/runtime.js";
|
|
29
29
|
import { AsyncQueue } from "../agent/async-queue.js";
|
|
30
|
+
// Function-only cycle with perf-sampler.ts (it imports `shellQuote` from
|
|
31
|
+
// here) — safe: both sides are hoisted declarations used at call time.
|
|
32
|
+
import {
|
|
33
|
+
parsePerfToken, perfSamplerFunctionFragment, PERF_TOKEN_MAX_CHARS,
|
|
34
|
+
} from "../agent/perf-sampler.js";
|
|
35
|
+
import type { GuestPerfSample } from "../agent/perf-sampler.js";
|
|
30
36
|
import { formatError } from "../utils/errors.js";
|
|
31
37
|
import { ndJsonStream, type Stream } from "@agentclientprotocol/sdk";
|
|
32
38
|
import { runProcessorChain } from "../processors/runner.js";
|
|
@@ -35,17 +41,530 @@ import { RequestContext } from "../request-context/request-context.js";
|
|
|
35
41
|
import { AcpClientPeer, ACP_PROTOCOL_VERSION } from "./_acp-client.js";
|
|
36
42
|
import { isPauseSignal, boundProcessorPause } from "../pause/pause-core.js";
|
|
37
43
|
import {
|
|
38
|
-
JSONL_GUARD_LINE_CAP_BYTES, JSONL_GUARD_KEEP_BYTES,
|
|
44
|
+
JSONL_GUARD_LINE_CAP_BYTES, JSONL_GUARD_KEEP_BYTES, JSONL_GUARD_NO_SENTINEL_EXIT,
|
|
39
45
|
jsonlGuardScript, wrapJsonlCommand,
|
|
40
46
|
} from "./_jsonl-guard.js";
|
|
47
|
+
import type { SandboxBackgroundProcess } from "../types/sandbox.js";
|
|
41
48
|
|
|
42
49
|
function now(): string { return new Date().toISOString(); }
|
|
43
50
|
|
|
51
|
+
/** Backoff between tail re-attach attempts after a mid-turn stream fault
|
|
52
|
+
* (durable detached transport). Short: the runner is alive and producing,
|
|
53
|
+
* and each retry costs one exec; the executor's evidence machinery — not
|
|
54
|
+
* this loop — bounds a truly dead sandbox. Exported for tests. */
|
|
55
|
+
export const TAIL_REATTACH_DELAY_MS = 2_000;
|
|
56
|
+
|
|
57
|
+
// ── Durable-truth liveness (2026-08-15 incident, turn 18dc5261) ─────────────
|
|
58
|
+
// DURABLE STATE IS THE ONLY TRUTH; STREAMS ARE A LATENCY OPTIMIZATION.
|
|
59
|
+
// The incident: the attached tail's connect-web stream wedged SILENTLY —
|
|
60
|
+
// `wait()` never settled, no error, no EOF — while the runner finished its
|
|
61
|
+
// answer into the durable file (full transcript + exit sentinel on disk at
|
|
62
|
+
// +87s). Every recovery arm lived AFTER `await handle.wait()`, so the
|
|
63
|
+
// transport was blind forever; the finished turn sat unharvested for ten
|
|
64
|
+
// minutes until the next dispatch superseded it. The pieces below make the
|
|
65
|
+
// durable plane authoritative:
|
|
66
|
+
// - the detached runner writes a HEARTBEAT file (epoch ms, every ~10s,
|
|
67
|
+
// from a subshell inside the setsid session that self-terminates with
|
|
68
|
+
// the wrapper) next to the stdout/pid/sentinel files;
|
|
69
|
+
// - an attached tail is WATCHED, never trusted: a fixed-cadence probe (one
|
|
70
|
+
// fresh short exec) reads the durable trio, and durable bytes the stream
|
|
71
|
+
// failed to deliver across a full interval prove the stream dead — kill
|
|
72
|
+
// the tail (never the runner) and catch up from the byte offset;
|
|
73
|
+
// - `probeTurnLiveness` serves the same durable reading to the executor's
|
|
74
|
+
// evidence ticker as a three-valued verdict (alive / dead /
|
|
75
|
+
// probe-failed), replacing process-grep guesswork.
|
|
76
|
+
|
|
77
|
+
/** Cadence of the in-guest heartbeat writer (seconds — it is a shell loop). */
|
|
78
|
+
export const RUNNER_HEARTBEAT_INTERVAL_SECONDS = 10;
|
|
79
|
+
/** How recently a file under the CLI's own task state dir (~/.claude/tasks)
|
|
80
|
+
* must have been touched to count as a LIVE background task in the busy
|
|
81
|
+
* sentinel (minutes — it is `find -mmin`). A running task's state/output
|
|
82
|
+
* files are written continuously, so two minutes is generous; a finished
|
|
83
|
+
* task's files go quiet and age out on their own. */
|
|
84
|
+
export const RUNNER_BUSY_TASK_FRESH_MINUTES = 2;
|
|
85
|
+
/** Cap on the task-id list the busy sentinel publishes — the ids are for
|
|
86
|
+
* server-side log attribution, not an inventory. */
|
|
87
|
+
export const RUNNER_BUSY_TASK_ID_LIMIT = 16;
|
|
88
|
+
/** The sh fragment the heartbeat subshell runs each beat to maintain the
|
|
89
|
+
* busy sentinel — the guest half of the background-work marker contract:
|
|
90
|
+
* - `<promptPath>.busy` — the COUNT of recently-active files under the
|
|
91
|
+
* CLI's task state dir (`$ac_busy` stays set in-shell for the loop's
|
|
92
|
+
* own exit decision);
|
|
93
|
+
* - `<promptPath>.tasks` — the task IDS those files belong to (first
|
|
94
|
+
* path segment under tasks/, deduped, capped).
|
|
95
|
+
* The marker's freshness stamp is the files' own mtime (each beat rewrites
|
|
96
|
+
* them; the server probe gates on `-mmin -1`). Cheap (one bounded find
|
|
97
|
+
* over a small dir), and read by the server's park path so a VM hosting
|
|
98
|
+
* live background work is never frozen mid-flight (2026-08-15 forensics:
|
|
99
|
+
* a 3-min idle park suspended a VM with three research agents mid-flight).
|
|
100
|
+
* A missing tasks dir counts 0. Exported for tests. */
|
|
101
|
+
export function runnerBusySentinelFragment(busyPath: string, tasksPath: string): string {
|
|
102
|
+
return `ac_task_files=$(find "$HOME/.claude/tasks" -type f -mmin -${RUNNER_BUSY_TASK_FRESH_MINUTES} 2>/dev/null); `
|
|
103
|
+
+ `if [ -n "$ac_task_files" ]; then ac_busy=$(printf '%s\\n' "$ac_task_files" | wc -l | tr -cd 0-9); else ac_busy=0; fi; `
|
|
104
|
+
+ `printf '%s' "$ac_busy" > ${shellQuote(busyPath)} 2>/dev/null; `
|
|
105
|
+
+ `printf '%s\\n' "$ac_task_files" | sed 's|.*/tasks/||; s|/.*||' | grep . | sort -u `
|
|
106
|
+
+ `| head -${RUNNER_BUSY_TASK_ID_LIMIT} > ${shellQuote(tasksPath)} 2>/dev/null`;
|
|
107
|
+
}
|
|
108
|
+
/** A heartbeat older than this (by the GUEST's own clock — the probe reads
|
|
109
|
+
* `date +%s%3N` in the same exec, so host clock skew is irrelevant) is
|
|
110
|
+
* stale: three missed writes. */
|
|
111
|
+
export const RUNNER_HEARTBEAT_STALE_MS = RUNNER_HEARTBEAT_INTERVAL_SECONDS * 1000 * 3;
|
|
112
|
+
/** Watchdog cadence while a tail stream is attached. Each tick is one fresh
|
|
113
|
+
* short exec; a healthy stream makes every tick a no-op. */
|
|
114
|
+
export const TAIL_WATCHDOG_INTERVAL_MS = 15_000;
|
|
115
|
+
/** Explicit ceiling on one durable-trio probe exec. A probe that can't
|
|
116
|
+
* answer is "probe-failed" — never a verdict. */
|
|
117
|
+
export const TURN_PROBE_TIMEOUT_MS = 10_000;
|
|
118
|
+
/** After this many stream deaths in ONE turn, stop re-attaching tails and
|
|
119
|
+
* finish the turn on durable-file polling alone (the GHA permanent-
|
|
120
|
+
* fallback pattern): a wire that killed three streams will kill the
|
|
121
|
+
* fourth, and the durable plane already delivers everything. */
|
|
122
|
+
export const TAIL_ABANDON_AFTER_STREAM_DEATHS = 3;
|
|
123
|
+
/** Poll cadence once streaming is abandoned — a full durable read per poll
|
|
124
|
+
* (envd HTTP, ~65ms RTT), consumed from the byte offset. Latency-tuned:
|
|
125
|
+
* this is the degraded path, correctness never depends on it being fast. */
|
|
126
|
+
export const DURABLE_POLL_INTERVAL_MS = 1_000;
|
|
127
|
+
/** The claude CLI's stderr complaint when `--resume <id>` names a thread
|
|
128
|
+
* that does not exist on this machine (the 2026-08-18 split-brain
|
|
129
|
+
* forensic). When a nonzero exit's stderr carries it, the transport
|
|
130
|
+
* surfaces the stderr as the turn's LAST error event even though a bare
|
|
131
|
+
* result error already streamed — the consumer's classifier needs the
|
|
132
|
+
* specific complaint, not the generic `error_during_execution` token. */
|
|
133
|
+
export const RESUME_TARGET_MISSING_STDERR = /no conversation found with session id/i;
|
|
134
|
+
|
|
135
|
+
/** One durable-trio reading, parsed from the probe exec's single line. */
|
|
136
|
+
export interface TurnProbeReading {
|
|
137
|
+
/** Bytes currently in the durable stdout file (-1: unreadable/absent). */
|
|
138
|
+
size: number;
|
|
139
|
+
/** Is the detached runner's process group leader alive (kill -0)? */
|
|
140
|
+
alive: boolean;
|
|
141
|
+
/** Last heartbeat value (epoch ms, guest clock; 0 = no heartbeat yet). */
|
|
142
|
+
hbMs: number;
|
|
143
|
+
/** The guest's own clock at probe time (epoch ms). */
|
|
144
|
+
nowMs: number;
|
|
145
|
+
/** Is the exit sentinel already in the durable file (turn FINISHED)? */
|
|
146
|
+
sentinelSeen: boolean;
|
|
147
|
+
/** Latest guest perf sample (the trailing `.perf` token field), when the
|
|
148
|
+
* guest wrote one AND it parsed clean. Absent = older guest image, no
|
|
149
|
+
* sample yet, or a garbled token — evidence-absent, never an error.
|
|
150
|
+
* Telemetry-only: no liveness verdict may ever read it. */
|
|
151
|
+
perf?: GuestPerfSample;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** The probe exec: one line, six fields, always exit 0 — faults surface as
|
|
155
|
+
* parse failures, not exec failures. Reads the guest's own clock in the
|
|
156
|
+
* same exec so staleness math never involves the host clock. The sixth
|
|
157
|
+
* field is the perf token (perf-sampler.ts), `-` when absent; the char
|
|
158
|
+
* whitelist (`tr -cd`) collapses any garbled write to at most one token —
|
|
159
|
+
* it can never add whitespace and desync the field positions. */
|
|
160
|
+
export function turnLivenessProbeCommand(
|
|
161
|
+
paths: { outPath: string; hbPath: string; perfPath: string; pid: number; sentinel: string },
|
|
162
|
+
): string {
|
|
163
|
+
return `sz=$(wc -c < ${shellQuote(paths.outPath)} 2>/dev/null || echo -1); `
|
|
164
|
+
+ `alive=0; kill -0 ${paths.pid} 2>/dev/null && alive=1; `
|
|
165
|
+
+ `hb=$(cat ${shellQuote(paths.hbPath)} 2>/dev/null || echo 0); `
|
|
166
|
+
+ `now=$(date +%s%3N); `
|
|
167
|
+
+ `sent=$(grep -c -F ${shellQuote(paths.sentinel)} ${shellQuote(paths.outPath)} 2>/dev/null || echo 0); `
|
|
168
|
+
+ `pf=$(head -c ${PERF_TOKEN_MAX_CHARS} ${shellQuote(paths.perfPath)} 2>/dev/null | tr -cd 'A-Za-z0-9=,.-'); `
|
|
169
|
+
+ `printf '%s %s %s %s %s %s\\n' "$sz" "$alive" "$hb" "$now" "$sent" "\${pf:--}"`;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Parse the probe's line. Null = unusable output (a wedged or faulted exec)
|
|
173
|
+
* — the caller treats that as "probe-failed", never as a verdict. The
|
|
174
|
+
* sixth field (perf token) is OPTIONAL both ways: a five-field line from
|
|
175
|
+
* an old guest parses fine, and a token that fails its own clamp simply
|
|
176
|
+
* leaves `perf` absent — perf can never fail a liveness reading. */
|
|
177
|
+
export function parseTurnProbeOutput(raw: string): TurnProbeReading | null {
|
|
178
|
+
const fields = raw.trim().split(/\s+/);
|
|
179
|
+
if (fields.length < 5) return null;
|
|
180
|
+
const size = Number.parseInt(fields[0]!, 10);
|
|
181
|
+
const alive = fields[1] === "1" ? true : fields[1] === "0" ? false : null;
|
|
182
|
+
const hbMs = Number.parseInt(fields[2]!, 10);
|
|
183
|
+
const nowMs = Number.parseInt(fields[3]!, 10);
|
|
184
|
+
const sentinelCount = Number.parseInt(fields[4]!, 10);
|
|
185
|
+
if (!Number.isFinite(size) || alive === null || !Number.isFinite(hbMs)
|
|
186
|
+
|| !Number.isFinite(nowMs) || !Number.isFinite(sentinelCount)) return null;
|
|
187
|
+
const perfTok = fields[5];
|
|
188
|
+
const perf = perfTok !== undefined && perfTok !== "-" ? parsePerfToken(perfTok) : null;
|
|
189
|
+
return {
|
|
190
|
+
size, alive, hbMs, nowMs, sentinelSeen: sentinelCount > 0,
|
|
191
|
+
...(perf ? { perf } : {}),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** The three-valued verdict from one durable reading. A FINISHED turn
|
|
196
|
+
* (sentinel durable in the file) is ALIVE — the transport's watchdog
|
|
197
|
+
* harvests it within one interval; reporting it dead is exactly the
|
|
198
|
+
* incident's ten-minute blindness. */
|
|
199
|
+
export function livenessVerdictFromReading(reading: TurnProbeReading | null): RunnerLivenessVerdict {
|
|
200
|
+
if (reading === null) return "probe-failed";
|
|
201
|
+
if (reading.alive) return "alive";
|
|
202
|
+
if (reading.sentinelSeen) return "alive";
|
|
203
|
+
if (reading.hbMs > 0 && reading.nowMs - reading.hbMs <= RUNNER_HEARTBEAT_STALE_MS) return "alive";
|
|
204
|
+
return "dead";
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// ── Mid-turn stream input (the delivery half of injectUserMessage) ──────────
|
|
208
|
+
// The durable transport's stdin, when the spec declares `streamInput`, is a
|
|
209
|
+
// FIFO fed by an in-guest subshell: the prompt line first, then any complete
|
|
210
|
+
// lines appended to the turn's durable INBOX file. A steering-capable CLI
|
|
211
|
+
// (claude -p --input-format stream-json) folds a message that arrives while
|
|
212
|
+
// the turn runs into the RUNNING turn at its next tool boundary — verified
|
|
213
|
+
// live against claude 2.1.233. The feeder checks for the CLI's terminal
|
|
214
|
+
// `result` line BEFORE each delivery (a message that races the result stays
|
|
215
|
+
// owed for the follow-up turn, never half-consumed) and stops on it; closing
|
|
216
|
+
// the FIFO is what EOFs the CLI's stdin and lets it exit. The DELIVERED
|
|
217
|
+
// counter file is the ack the server trusts: it advances only after the
|
|
218
|
+
// feeder forwarded those inbox lines into the CLI's stdin.
|
|
219
|
+
|
|
220
|
+
/** Feeder poll cadence (seconds — it is a shell `sleep`; GNU sleep accepts
|
|
221
|
+
* fractions and the durable transport only runs on the GNU guest image). */
|
|
222
|
+
export const INJECT_FEEDER_POLL_SECONDS = "0.3";
|
|
223
|
+
/** How much of the durable stdout tail the feeder greps for the terminal
|
|
224
|
+
* result line each poll — bounded so a long transcript never turns the
|
|
225
|
+
* poll into a full-file scan. Comfortably above the JSONL guard's line cap,
|
|
226
|
+
* so a capped result line still fits whole. */
|
|
227
|
+
export const INJECT_FEEDER_RESULT_TAIL_BYTES = 262_144;
|
|
228
|
+
/** One in-guest delivery-ack exec: append the line, then poll the delivered
|
|
229
|
+
* counter until it covers this append (delivered), the runner dies, or the
|
|
230
|
+
* attempts run out (both: not delivered — the message stays owed). */
|
|
231
|
+
export const INJECT_ACK_POLL_ATTEMPTS = 30;
|
|
232
|
+
export const INJECT_ACK_POLL_SECONDS = "0.5";
|
|
233
|
+
export const INJECT_ACK_EXEC_TIMEOUT_MS = 25_000;
|
|
234
|
+
|
|
235
|
+
/** The in-guest feeder fragment, prepended to the detached wrapper script.
|
|
236
|
+
* Runs inside the setsid session (group-kill reaps it) and keys its loop to
|
|
237
|
+
* the wrapper's own pid (`$$`), like the heartbeat subshell. Exported for
|
|
238
|
+
* tests. */
|
|
239
|
+
export function streamInputFeederFragment(paths: {
|
|
240
|
+
promptPath: string; fifoPath: string; inboxPath: string;
|
|
241
|
+
deliveredPath: string; outPath: string;
|
|
242
|
+
}): string {
|
|
243
|
+
const q = shellQuote;
|
|
244
|
+
return `rm -f ${q(paths.fifoPath)}; mkfifo ${q(paths.fifoPath)}; `
|
|
245
|
+
+ `: > ${q(paths.inboxPath)}; echo 0 > ${q(paths.deliveredPath)}; `
|
|
246
|
+
+ `( cat ${q(paths.promptPath)}; ac_fed=0; while kill -0 "$$" 2>/dev/null; do `
|
|
247
|
+
+ `if tail -c ${INJECT_FEEDER_RESULT_TAIL_BYTES} ${q(paths.outPath)} 2>/dev/null | grep -q '"type":"result"'; then break; fi; `
|
|
248
|
+
+ `ac_lines=$(wc -l < ${q(paths.inboxPath)} 2>/dev/null || echo 0); ac_lines=\${ac_lines:-0}; `
|
|
249
|
+
+ `if [ "$ac_lines" -gt "$ac_fed" ]; then `
|
|
250
|
+
+ `sed -n "$((ac_fed+1)),$((ac_lines))p" ${q(paths.inboxPath)}; ac_fed=$ac_lines; `
|
|
251
|
+
+ `echo "$ac_fed" > ${q(paths.deliveredPath)}; fi; `
|
|
252
|
+
+ `sleep ${INJECT_FEEDER_POLL_SECONDS}; done ) > ${q(paths.fifoPath)} 2>/dev/null </dev/null & `;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** The one delivery-ack exec `injectUserMessage` runs: append the message
|
|
256
|
+
* line to the durable inbox, then wait for the feeder's delivered counter
|
|
257
|
+
* to cover it. Exit 0 = delivered into the CLI's stdin pre-result; 4 = the
|
|
258
|
+
* runner died first; 5 = not delivered within the ack window (feeder
|
|
259
|
+
* stopped on the result, or the guest is crawling) — both non-zero exits
|
|
260
|
+
* mean "leave the message owed". Exported for tests. */
|
|
261
|
+
export function injectAppendAndAckCommand(args: {
|
|
262
|
+
line: string; target: number; pid: number;
|
|
263
|
+
inboxPath: string; deliveredPath: string;
|
|
264
|
+
}): string {
|
|
265
|
+
const q = shellQuote;
|
|
266
|
+
return `printf '%s\\n' ${q(args.line)} >> ${q(args.inboxPath)}; i=0; `
|
|
267
|
+
+ `while [ "$i" -lt ${INJECT_ACK_POLL_ATTEMPTS} ]; do `
|
|
268
|
+
+ `d=$(cat ${q(args.deliveredPath)} 2>/dev/null || echo 0); `
|
|
269
|
+
+ `[ "\${d:-0}" -ge ${args.target} ] && exit 0; `
|
|
270
|
+
+ `kill -0 ${args.pid} 2>/dev/null || exit 4; `
|
|
271
|
+
+ `sleep ${INJECT_ACK_POLL_SECONDS}; i=$((i+1)); done; exit 5`;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// ── Reap = kill AND CONFIRM (2026-08-18 incident: codex writer lock) ────────
|
|
275
|
+
// A reaped runner must be CONFIRMED dead before the turn unwinds. codex holds
|
|
276
|
+
// a kernel flock on its thread store (`~/.codex/thread-writer-locks/<id>.lock`)
|
|
277
|
+
// for exactly as long as the process lives; the old fire-and-forget SIGTERM
|
|
278
|
+
// gave a slow-dying (or TERM-handling) CLI no deadline and never escalated,
|
|
279
|
+
// so a canceled turn could leave the writer alive — and the NEXT turn's
|
|
280
|
+
// `codex exec resume` died with `thread-store conflict: … already has an
|
|
281
|
+
// active writer`. The reap is now ONE guest exec that TERMs the detached
|
|
282
|
+
// group, waits bounded for the group leader to exit, escalates to SIGKILL,
|
|
283
|
+
// and confirms — and the transport AWAITS that confirmation in its unwind
|
|
284
|
+
// path, so the generator (and the executor's turn settle behind it) never
|
|
285
|
+
// completes while the guest tree may still be running.
|
|
286
|
+
|
|
287
|
+
/** SIGTERM grace before escalation: attempts × sleep = 5s. */
|
|
288
|
+
export const REAP_TERM_WAIT_ATTEMPTS = 20;
|
|
289
|
+
/** Post-SIGKILL confirm window: attempts × sleep = 2s (a KILLed process only
|
|
290
|
+
* lingers as an unreaped zombie, which `kill -0` still sees — brief). */
|
|
291
|
+
export const REAP_KILL_WAIT_ATTEMPTS = 8;
|
|
292
|
+
export const REAP_POLL_SECONDS = "0.25";
|
|
293
|
+
/** The confirm exec's "still alive after SIGKILL" exit code (surfaced as a
|
|
294
|
+
* warn — the launch preflight is the backstop for whatever survived). */
|
|
295
|
+
export const REAP_STILL_ALIVE_EXIT = 7;
|
|
296
|
+
/** Ceiling on the whole kill-and-confirm exec (TERM 5s + KILL 2s + slack). */
|
|
297
|
+
export const REAP_EXEC_TIMEOUT_MS = 15_000;
|
|
298
|
+
|
|
299
|
+
/** The one kill-and-confirm exec: TERM the group (setsid made `pid` the
|
|
300
|
+
* leader) and the leader itself, poll for exit, escalate to KILL, poll
|
|
301
|
+
* again. Exit 0 = confirmed dead; REAP_STILL_ALIVE_EXIT = it survived
|
|
302
|
+
* SIGKILL's confirm window. Exported for tests. */
|
|
303
|
+
export function reapKillAndConfirmCommand(pid: number): string {
|
|
304
|
+
const alive = `kill -0 ${pid} 2>/dev/null`;
|
|
305
|
+
return `kill -TERM -- -${pid} 2>/dev/null; kill -TERM ${pid} 2>/dev/null; ac_ri=0; `
|
|
306
|
+
+ `while [ "$ac_ri" -lt ${REAP_TERM_WAIT_ATTEMPTS} ]; do ${alive} || exit 0; `
|
|
307
|
+
+ `sleep ${REAP_POLL_SECONDS}; ac_ri=$((ac_ri+1)); done; `
|
|
308
|
+
+ `kill -KILL -- -${pid} 2>/dev/null; kill -KILL ${pid} 2>/dev/null; ac_ri=0; `
|
|
309
|
+
+ `while [ "$ac_ri" -lt ${REAP_KILL_WAIT_ATTEMPTS} ]; do ${alive} || exit 0; `
|
|
310
|
+
+ `sleep ${REAP_POLL_SECONDS}; ac_ri=$((ac_ri+1)); done; `
|
|
311
|
+
+ `exit ${REAP_STILL_ALIVE_EXIT}`;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// ── Resident harness (docs/specs/resident-harness-turns.md, P1) ─────────────
|
|
315
|
+
// A RESIDENT keeps one live CLI process serving consecutive turns inside the
|
|
316
|
+
// hot window: the feeder's result-break is replaced by an END-FILE break
|
|
317
|
+
// (`<prompt>.end` → close the FIFO → the CLI EOFs stdin and exits with a
|
|
318
|
+
// clean sentinel — the probe measured 625ms), and a guest idle watchdog
|
|
319
|
+
// reaps the group if the server's durable record is ever lost. Turn
|
|
320
|
+
// accounting rides two guest numbers next to the inbox:
|
|
321
|
+
// `<prompt>.served` — turns DELIVERED into this process (1 at launch,
|
|
322
|
+
// +1 per adoption; the server mirrors it durably
|
|
323
|
+
// as the runner stamp's `turnsServed`);
|
|
324
|
+
// result lines in .out — turns FINISHED (anchored grep — hint-grade; the
|
|
325
|
+
// server-side stream-json parse is authoritative).
|
|
326
|
+
// idle ⇔ results ≥ served AND the inbox is fully fed. Mid-turn injects ride
|
|
327
|
+
// the same inbox WITHOUT bumping served, and fold into the running turn
|
|
328
|
+
// producing no result of their own — the parity holds.
|
|
329
|
+
|
|
330
|
+
/** Guest idle TTL for a resident with no new inbox line: hot window (180s)
|
|
331
|
+
* + margin, so the park gate's graceful `.end` normally wins and the
|
|
332
|
+
* watchdog only answers a dead server / lost record. */
|
|
333
|
+
export const RESIDENT_IDLE_TTL_S = 240;
|
|
334
|
+
/** Watchdog poll cadence (seconds). */
|
|
335
|
+
export const RESIDENT_WATCHDOG_POLL_S = 15;
|
|
336
|
+
/** Anchored prefix of the CLI's terminal result event line. Anchoring keeps
|
|
337
|
+
* the guest-side counts honest against tool output that merely CONTAINS
|
|
338
|
+
* the substring; a serialization-order change fails SAFE (parity check
|
|
339
|
+
* refuses adoption → cold launch — a latency cost, never correctness). */
|
|
340
|
+
export const RESIDENT_RESULT_LINE_PREFIX = '{"type":"result"';
|
|
341
|
+
|
|
342
|
+
/** One bounded exec printing the count of result lines at or past
|
|
343
|
+
* `fromByte` (0-based) in the durable .out. Always exits 0; stdout is the
|
|
344
|
+
* count (a missing file reads as 0). Hint-grade by design. */
|
|
345
|
+
export function residentResultCountCommand(outPath: string, fromByte = 0): string {
|
|
346
|
+
const q = shellQuote;
|
|
347
|
+
const src = fromByte > 0
|
|
348
|
+
? `tail -c +${fromByte + 1} ${q(outPath)} 2>/dev/null`
|
|
349
|
+
: `cat ${q(outPath)} 2>/dev/null`;
|
|
350
|
+
return `ac_res=$(${src} | grep -c ${q(`^${RESIDENT_RESULT_LINE_PREFIX}`)}); echo "\${ac_res:-0}"`;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/** The resident feeder: `streamInputFeederFragment` with the result-break
|
|
354
|
+
* REPLACED by the end-file break — the ONLY guest-contract change the
|
|
355
|
+
* resident shape needs (verified live by the spec's probe: two messages,
|
|
356
|
+
* one process, one session file, graceful exit on `.end`). */
|
|
357
|
+
export function residentFeederFragment(paths: {
|
|
358
|
+
promptPath: string; fifoPath: string; inboxPath: string;
|
|
359
|
+
deliveredPath: string; endPath: string;
|
|
360
|
+
}): string {
|
|
361
|
+
const q = shellQuote;
|
|
362
|
+
return `rm -f ${q(paths.fifoPath)}; mkfifo ${q(paths.fifoPath)}; `
|
|
363
|
+
+ `: > ${q(paths.inboxPath)}; echo 0 > ${q(paths.deliveredPath)}; rm -f ${q(paths.endPath)}; `
|
|
364
|
+
+ `( cat ${q(paths.promptPath)}; ac_fed=0; while kill -0 "$$" 2>/dev/null; do `
|
|
365
|
+
+ `if [ -f ${q(paths.endPath)} ]; then break; fi; `
|
|
366
|
+
+ `ac_lines=$(wc -l < ${q(paths.inboxPath)} 2>/dev/null || echo 0); ac_lines=\${ac_lines:-0}; `
|
|
367
|
+
+ `if [ "$ac_lines" -gt "$ac_fed" ]; then `
|
|
368
|
+
+ `sed -n "$((ac_fed+1)),$((ac_lines))p" ${q(paths.inboxPath)}; ac_fed=$ac_lines; `
|
|
369
|
+
+ `echo "$ac_fed" > ${q(paths.deliveredPath)}; fi; `
|
|
370
|
+
+ `sleep ${INJECT_FEEDER_POLL_SECONDS}; done ) > ${q(paths.fifoPath)} 2>/dev/null </dev/null & `;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/** Sets `ac_idle` (1 = between turns: every delivered turn has its result
|
|
374
|
+
* and the inbox is fully fed). Embedded by the watchdog and the resident
|
|
375
|
+
* heartbeat gate — one idle predicate, stated once. */
|
|
376
|
+
export function residentIdleCheckFragment(paths: {
|
|
377
|
+
outPath: string; inboxPath: string; deliveredPath: string; servedPath: string;
|
|
378
|
+
}): string {
|
|
379
|
+
const q = shellQuote;
|
|
380
|
+
return `ac_res=$(grep -c ${q(`^${RESIDENT_RESULT_LINE_PREFIX}`)} ${q(paths.outPath)} 2>/dev/null); ac_res=\${ac_res:-0}; `
|
|
381
|
+
+ `ac_srv=$(cat ${q(paths.servedPath)} 2>/dev/null); ac_srv=\${ac_srv:-1}; `
|
|
382
|
+
+ `ac_in=$(wc -l < ${q(paths.inboxPath)} 2>/dev/null); ac_in=\${ac_in:-0}; `
|
|
383
|
+
+ `ac_del=$(cat ${q(paths.deliveredPath)} 2>/dev/null); ac_del=\${ac_del:-0}; `
|
|
384
|
+
+ `if [ "$ac_res" -ge "$ac_srv" ] && [ "$ac_in" -eq "$ac_del" ]; then ac_idle=1; else ac_idle=0; fi; `;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** Guest idle watchdog: reap the whole process group after
|
|
388
|
+
* RESIDENT_IDLE_TTL_S of CONTINUOUS idleness (any busy reading resets the
|
|
389
|
+
* clock — a long-running TURN is never reaped; the parity check reads
|
|
390
|
+
* busy mid-turn). The park gate's graceful `.end` normally wins; this is
|
|
391
|
+
* the orphan backstop for a dead server or a lost durable record.
|
|
392
|
+
*
|
|
393
|
+
* BACKGROUND TASKS ARE NOT IDLE (2026-08-19 forensics: a user's
|
|
394
|
+
* between-turns background search died to a harness teardown while its
|
|
395
|
+
* journal was still advancing). The turn-parity predicate alone reads a
|
|
396
|
+
* resident BETWEEN turns as idle even while harness background tasks it
|
|
397
|
+
* tracks are hard at work — and this watchdog's group-kill would take
|
|
398
|
+
* those children with the tree. So idle additionally requires the CLI's
|
|
399
|
+
* own task journal (`$HOME/.claude/tasks`) to be QUIET: any file touched
|
|
400
|
+
* within RUNNER_BUSY_TASK_FRESH_MINUTES resets the clock — the exact
|
|
401
|
+
* journal-freshness license the server's park gate honors
|
|
402
|
+
* (guestBusyProbeCommand). A stalled task goes stale within the window
|
|
403
|
+
* and the TTL resumes on its own; liveness alone never counts (the
|
|
404
|
+
* 2026-08-17 wake-loop damper is a SERVER rule about waking — this is a
|
|
405
|
+
* guest rule about not killing, and journal advance is required, not mere
|
|
406
|
+
* process existence). */
|
|
407
|
+
export function residentIdleWatchdogFragment(paths: {
|
|
408
|
+
outPath: string; inboxPath: string; deliveredPath: string; servedPath: string;
|
|
409
|
+
}): string {
|
|
410
|
+
return `( ac_idle_s=0; while kill -0 "$$" 2>/dev/null; do `
|
|
411
|
+
+ residentIdleCheckFragment(paths)
|
|
412
|
+
+ `if [ "$ac_idle" = 1 ] && find "$HOME/.claude/tasks" -type f -mmin -${RUNNER_BUSY_TASK_FRESH_MINUTES} 2>/dev/null | grep -q .; then ac_idle=0; fi; `
|
|
413
|
+
+ `if [ "$ac_idle" = 1 ]; then ac_idle_s=$((ac_idle_s+${RESIDENT_WATCHDOG_POLL_S})); else ac_idle_s=0; fi; `
|
|
414
|
+
+ `if [ "$ac_idle_s" -ge ${RESIDENT_IDLE_TTL_S} ]; then kill -- -$$ 2>/dev/null; exit 0; fi; `
|
|
415
|
+
+ `sleep ${RESIDENT_WATCHDOG_POLL_S}; done ) >/dev/null 2>&1 </dev/null & `;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/** Exit-event doorbell with the turnId read from a guest FILE at push time
|
|
419
|
+
* (a resident serves many turns; each adoption rewrites `<prompt>.turnid`
|
|
420
|
+
* so a mid-turn death rings the turn actually being served). Same
|
|
421
|
+
* validation posture as `exitEventPushCommand`; an empty turnid file
|
|
422
|
+
* pushes nothing. */
|
|
423
|
+
export function deferredExitEventPushCommand(
|
|
424
|
+
notify: { url: string; tokenEnv: string }, turnIdFile: string,
|
|
425
|
+
): string {
|
|
426
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(notify.tokenEnv)) return "";
|
|
427
|
+
const q = shellQuote;
|
|
428
|
+
return `( ac_tid=$(cat ${q(turnIdFile)} 2>/dev/null | tr -cd 'A-Za-z0-9-'); `
|
|
429
|
+
+ `if [ -n "$ac_tid" ] && [ -n "$${notify.tokenEnv}" ] && command -v curl >/dev/null 2>&1; then `
|
|
430
|
+
+ `curl -fsS -m ${EXIT_PUSH_ATTEMPT_TIMEOUT_SECONDS} -o /dev/null -X POST `
|
|
431
|
+
+ `-H "Authorization: Bearer $${notify.tokenEnv}" `
|
|
432
|
+
+ `-H 'Content-Type: application/json' --data "{\\"turnId\\":\\"$ac_tid\\",\\"exitCode\\":$ac_rc}" ${q(notify.url)}; `
|
|
433
|
+
+ `fi ) >/dev/null 2>&1 || true`;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/** Provider deadline on the detached LAUNCH exec. The launch shell only
|
|
437
|
+
* truncates the durable files, forks the detached runner (stdio fully
|
|
438
|
+
* redirected — see the launch command), writes the pidfile, and echoes the
|
|
439
|
+
* pid — sub-second work, so 30s is generous headroom for a slow envd, not a
|
|
440
|
+
* budget the runner ever consumes. Exported for tests. */
|
|
441
|
+
export const LAUNCH_EXEC_TIMEOUT_MS = 30_000;
|
|
442
|
+
|
|
443
|
+
/** How many times (and how spaced) a THROWN launch exec falls back to reading
|
|
444
|
+
* the durable pidfile before the fault is surfaced. A deadlined/canceled
|
|
445
|
+
* launch STREAM does not mean the launch failed — the detached tree may be
|
|
446
|
+
* up and working — so the pid is recovered over the envd HTTP file
|
|
447
|
+
* transport (immune to the command-stream fault) and the turn proceeds.
|
|
448
|
+
* Only "no pidfile after these attempts" is a real launch failure.
|
|
449
|
+
* Exported for tests. */
|
|
450
|
+
export const LAUNCH_PID_RECOVERY_ATTEMPTS = 3;
|
|
451
|
+
export const LAUNCH_PID_RECOVERY_DELAY_MS = 1_000;
|
|
452
|
+
|
|
453
|
+
/** Explicit deadline on the `command -v` install probe. E2B's default command
|
|
454
|
+
* deadline (60s) throws the deadline_exceeded TimeoutError — every provider
|
|
455
|
+
* call in the turn path carries an explicit timeout so no SDK default can
|
|
456
|
+
* decide a turn's fate. Exported for tests. */
|
|
457
|
+
export const INSTALL_PROBE_TIMEOUT_MS = 30_000;
|
|
458
|
+
|
|
459
|
+
/** Deadline on the on-demand CLI install itself (`spec.install`). Shared by
|
|
460
|
+
* this transport's ensureInstalled and the server's CloudTurnWorkflow launch
|
|
461
|
+
* path (turn-workflow/tailer.ts), which provisions the same way before a
|
|
462
|
+
* detached launch — the session images bake only `claude`, so codex/
|
|
463
|
+
* opencode/cursor/droid MUST be installable at launch on a fresh machine. */
|
|
464
|
+
export const CLI_INSTALL_TIMEOUT_MS = 300_000;
|
|
465
|
+
|
|
466
|
+
/** Parse a pid out of captured stdout (last non-empty line). NaN when absent. */
|
|
467
|
+
function parsePid(raw: string): number {
|
|
468
|
+
const pid = Number.parseInt(raw.trim().split("\n").pop() ?? "", 10);
|
|
469
|
+
return Number.isInteger(pid) && pid > 0 ? pid : Number.NaN;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/** Recover the detached runner's pid from its durable pidfile after the
|
|
473
|
+
* launch exec's STREAM died (deadline_exceeded, canceled context, any wire
|
|
474
|
+
* fault). Reads over the file transport with a short retry so the race
|
|
475
|
+
* where the launch shell is still writing the pidfile is absorbed. */
|
|
476
|
+
async function recoverLaunchPid(read: (path: string) => Promise<string>, pidPath: string): Promise<number> {
|
|
477
|
+
for (let attempt = 0; attempt < LAUNCH_PID_RECOVERY_ATTEMPTS; attempt++) {
|
|
478
|
+
try {
|
|
479
|
+
const pid = parsePid(await read(pidPath));
|
|
480
|
+
if (!Number.isNaN(pid)) return pid;
|
|
481
|
+
} catch { /* file plane briefly unreachable — retried */ }
|
|
482
|
+
if (attempt < LAUNCH_PID_RECOVERY_ATTEMPTS - 1) {
|
|
483
|
+
await new Promise((r) => setTimeout(r, LAUNCH_PID_RECOVERY_DELAY_MS));
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
return Number.NaN;
|
|
487
|
+
}
|
|
488
|
+
|
|
44
489
|
/** Single-quote a value for safe interpolation into a `sh -c` command line. */
|
|
45
490
|
export function shellQuote(value: string): string {
|
|
46
491
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
47
492
|
}
|
|
48
493
|
|
|
494
|
+
// ── Session env file (session secrets) ──────────────────────────────────────
|
|
495
|
+
|
|
496
|
+
/** $HOME-relative path of the platform-managed session env file. The server
|
|
497
|
+
* writes it at sandbox acquire (session secrets → `export KEY='…'` lines);
|
|
498
|
+
* the detached turn launch sources it fresh EVERY turn, so a secret set or
|
|
499
|
+
* rotated between turns lands on the next turn with no VM recycle. Shared
|
|
500
|
+
* constant so the writer (server) and the reader (this runtime) can never
|
|
501
|
+
* drift. */
|
|
502
|
+
export const SESSION_ENV_FILE_RELPATH = ".agent-compose/session-env.sh";
|
|
503
|
+
|
|
504
|
+
/** Only plain relative paths make it into the script — no quotes/spaces/`..`
|
|
505
|
+
* (a malformed option must not break the launch or escape $HOME). */
|
|
506
|
+
const SESSION_ENV_FILE_SAFE = /^[A-Za-z0-9._\/-]+$/;
|
|
507
|
+
|
|
508
|
+
/** The `. $HOME/<file>;` fragment sourced ahead of the CLI command, or ""
|
|
509
|
+
* when no env file is configured. Errors are swallowed — a missing or
|
|
510
|
+
* unreadable file must never fail a turn. */
|
|
511
|
+
export function sessionEnvSourceFragment(relPath: string | undefined): string {
|
|
512
|
+
if (!relPath || !SESSION_ENV_FILE_SAFE.test(relPath) || relPath.includes("..")) return "";
|
|
513
|
+
return `[ -f "$HOME/${relPath}" ] && . "$HOME/${relPath}" >/dev/null 2>&1 || true; `;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/** The per-turn model-credential source fragment (`RuntimeOptions.
|
|
517
|
+
* credEnvFile` — an ABSOLUTE guest path), or "" when none is configured.
|
|
518
|
+
* Sourced AFTER the session env file so the turn actor's credential wins.
|
|
519
|
+
* Same swallow-errors posture: a missing file must never fail a turn. */
|
|
520
|
+
export function credEnvSourceFragment(absPath: string | undefined): string {
|
|
521
|
+
if (!absPath || !absPath.startsWith("/") || !SESSION_ENV_FILE_SAFE.test(absPath.slice(1))
|
|
522
|
+
|| absPath.includes("..")) return "";
|
|
523
|
+
return `[ -f "${absPath}" ] && . "${absPath}" >/dev/null 2>&1 || true; `;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// ── Exit-event push (the completion doorbell, v0.10.43) ─────────────────────
|
|
527
|
+
// THE PUSH IS A DOORBELL, NEVER A VERDICT. The detached wrapper announces
|
|
528
|
+
// {turnId, exitCode} to the server with ONE best-effort POST immediately
|
|
529
|
+
// AFTER the exit sentinel is durably written, so the server's watchdog can
|
|
530
|
+
// run its verification pass at event latency (~1-2s) instead of its poll
|
|
531
|
+
// cadence. The durable file stays the only truth: a lost push costs nothing
|
|
532
|
+
// (the poll ladder is the unchanged backstop), and the server treats every
|
|
533
|
+
// push as "go verify the durable state now", never as an outcome.
|
|
534
|
+
|
|
535
|
+
/** Ceiling on ONE curl attempt (seconds — it is `curl -m`). Short: the file
|
|
536
|
+
* is the truth and a slow push must not keep the wrapper alive for long. */
|
|
537
|
+
export const EXIT_PUSH_ATTEMPT_TIMEOUT_SECONDS = 5;
|
|
538
|
+
/** Pause before the single retry. */
|
|
539
|
+
export const EXIT_PUSH_RETRY_DELAY_SECONDS = 1;
|
|
540
|
+
|
|
541
|
+
/**
|
|
542
|
+
* The sh fragment appended to the detached wrapper AFTER the sentinel
|
|
543
|
+
* `printf` — never before it, and never in a way that can fail the wrapper:
|
|
544
|
+
* - reads the credential from the guest env at push time (`$<tokenEnv>`;
|
|
545
|
+
* the token never appears in the script text, argv-visible command
|
|
546
|
+
* lines the server logs, or any log line);
|
|
547
|
+
* - silently skips when the credential or curl is absent;
|
|
548
|
+
* - two attempts max (`-m 5`, one retry after 1s), then gives up silently
|
|
549
|
+
* — all output discarded, exit status swallowed (`|| true`).
|
|
550
|
+
* `$ac_rc` is the wrapper-local capture of the guarded pipeline's exit code
|
|
551
|
+
* (the same value the sentinel line carries). Returns "" — push disabled —
|
|
552
|
+
* for any `tokenEnv`/`turnId` that is not a plain identifier/uuid shape, so
|
|
553
|
+
* hostile config can never become shell injection.
|
|
554
|
+
*/
|
|
555
|
+
export function exitEventPushCommand(notify: TurnExitNotify): string {
|
|
556
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(notify.tokenEnv)) return "";
|
|
557
|
+
if (!/^[A-Za-z0-9-]{1,64}$/.test(notify.turnId)) return "";
|
|
558
|
+
const token = `"$${notify.tokenEnv}"`;
|
|
559
|
+
const body = `"{\\"turnId\\":\\"${notify.turnId}\\",\\"exitCode\\":$ac_rc}"`;
|
|
560
|
+
const attempt = `curl -fsS -m ${EXIT_PUSH_ATTEMPT_TIMEOUT_SECONDS} -o /dev/null -X POST`
|
|
561
|
+
+ ` -H "Authorization: Bearer $${notify.tokenEnv}"`
|
|
562
|
+
+ ` -H 'Content-Type: application/json' --data ${body} ${shellQuote(notify.url)}`;
|
|
563
|
+
return `( if [ -n ${token} ] && command -v curl >/dev/null 2>&1; then `
|
|
564
|
+
+ `${attempt} || { sleep ${EXIT_PUSH_RETRY_DELAY_SECONDS}; ${attempt}; }; `
|
|
565
|
+
+ `fi ) >/dev/null 2>&1 || true`;
|
|
566
|
+
}
|
|
567
|
+
|
|
49
568
|
/** Monotonic discriminator for prompt-file names within this process. */
|
|
50
569
|
let promptFileCounter = 0;
|
|
51
570
|
|
|
@@ -155,8 +674,12 @@ export interface CliAgentSpec {
|
|
|
155
674
|
/** Build the one-shot shell command for a turn. `promptPath` is a file in the
|
|
156
675
|
* sandbox holding `promptPayload(prompt)`; `sessionId` continues a thread.
|
|
157
676
|
* `effort` is present only when the caller configured a reasoning effort
|
|
158
|
-
* AND the spec has a real knob for it (see `CliReasoningEffort`).
|
|
159
|
-
|
|
677
|
+
* AND the spec has a real knob for it (see `CliReasoningEffort`).
|
|
678
|
+
* `streamInput` is true when the durable transport is feeding stdin as a
|
|
679
|
+
* live message stream (see `CliAgentSpec.streamInput`): `promptPath` is
|
|
680
|
+
* then the FIFO the feeder writes, and the CLI must be invoked in its
|
|
681
|
+
* stream-input mode (claude: `--input-format stream-json`). */
|
|
682
|
+
buildCommand(args: { promptPath: string; sessionId?: string; model?: string; cwd?: string; effort?: CliReasoningEffort; streamInput?: boolean }): string;
|
|
160
683
|
/** Map one parsed JSONL stdout event to `AgentMessage`s. The base emits
|
|
161
684
|
* `init`/`done`/`error` lifecycle itself, so a spec maps only content +
|
|
162
685
|
* usage (text / thinking / tool_use / tool_result / usage).
|
|
@@ -170,12 +693,43 @@ export interface CliAgentSpec {
|
|
|
170
693
|
* resume it (codex `thread.started.thread_id`, amp `session_id`).
|
|
171
694
|
* LEGACY FALLBACK PATH (see `mapEvent`). */
|
|
172
695
|
extractSessionId(parsed: Record<string, unknown>): string | undefined;
|
|
696
|
+
/** The runtime's thread persistence admits only ONE live writer per thread
|
|
697
|
+
* (codex: an OS advisory flock per thread id, held for the holding
|
|
698
|
+
* process's whole lifetime — codex-rs thread-store writer_lock.rs). A
|
|
699
|
+
* platform that wants a SUCCESSOR turn to resume the same thread must
|
|
700
|
+
* therefore KILL the predecessor's process tree first — a merely-detached
|
|
701
|
+
* live predecessor blocks every `thread/resume` with "already has an
|
|
702
|
+
* active writer" (the 2026-08-22 superseded-codex-turn incident). Absent
|
|
703
|
+
* ⇒ concurrent resume is safe (claude-code: append-only session JSONL)
|
|
704
|
+
* and a superseded runner may be detached alive as an asset. */
|
|
705
|
+
exclusiveSessionWriter?: boolean;
|
|
173
706
|
/** ACP-mode invocation (the Zed `agent_servers` shape). When present, the
|
|
174
707
|
* runner spawns the CLI in ACP agent mode and delegates the whole wire
|
|
175
708
|
* protocol to `AcpClientPeer`; the legacy `promptPayload`/`buildCommand`/
|
|
176
709
|
* `extractSessionId`/`mapEvent` members are used only as the version-mismatch
|
|
177
710
|
* fallback. Absent → the spec is JSONL-only (legacy path always). */
|
|
178
711
|
acp?: { command: string; args: string[]; env?: Record<string, string> };
|
|
712
|
+
/** Mid-turn STREAM-INPUT support (the delivery half of
|
|
713
|
+
* `injectUserMessage`). When present AND the durable detached transport
|
|
714
|
+
* is in play, the CLI's stdin is fed from a FIFO by an in-guest feeder:
|
|
715
|
+
* the prompt line first, then any lines appended to the turn's durable
|
|
716
|
+
* inbox file — so a steering-capable CLI (claude: queued user input is
|
|
717
|
+
* folded into the RUNNING turn at the next tool boundary; verified live
|
|
718
|
+
* against claude 2.1.233) receives user messages while the turn runs.
|
|
719
|
+
* The feeder stops at the CLI's terminal result line (a message that
|
|
720
|
+
* races the result is NOT forwarded — it stays owed) and closes the
|
|
721
|
+
* FIFO, which is what ends the CLI process (stream-input CLIs exit on
|
|
722
|
+
* stdin EOF, not after a result). Absent → the prompt file is the whole
|
|
723
|
+
* stdin, exactly as before. */
|
|
724
|
+
streamInput?: {
|
|
725
|
+
/** Serialise the opening prompt into ONE stream-input stdin line
|
|
726
|
+
* (claude: a stream-json user message). No trailing newline — the
|
|
727
|
+
* transport owns line framing. */
|
|
728
|
+
promptLine(prompt: string): string;
|
|
729
|
+
/** Serialise one mid-turn user message into ONE stdin line. No
|
|
730
|
+
* trailing newline. */
|
|
731
|
+
messageLine(text: string): string;
|
|
732
|
+
};
|
|
179
733
|
}
|
|
180
734
|
|
|
181
735
|
/** A spawned ACP-mode CLI: the read side of its stdout and a delivery for its
|
|
@@ -241,6 +795,108 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
241
795
|
* dormant. Not a public API. */
|
|
242
796
|
private acpTransportReady = ACP_TRANSPORT_READY;
|
|
243
797
|
|
|
798
|
+
/** The CURRENT turn's durable-trio reader (set once the detached runner is
|
|
799
|
+
* launched, replaced by the next launch). Null = no durable transport is
|
|
800
|
+
* active for this runner (boot phase, ACP path, single-exec providers) —
|
|
801
|
+
* `probeTurnLiveness` then answers null and the executor keeps its own
|
|
802
|
+
* fallback probe. */
|
|
803
|
+
private turnProbe: (() => Promise<TurnProbeReading | null>) | null = null;
|
|
804
|
+
|
|
805
|
+
/** The CURRENT turn's mid-turn injection state (durable stream-input
|
|
806
|
+
* transport only). Null = no live stream-input turn — `injectUserMessage`
|
|
807
|
+
* answers "unsupported" and the caller leaves the message owed. `chain`
|
|
808
|
+
* serializes appends so each delivery targets a deterministic line count. */
|
|
809
|
+
private turnInject: {
|
|
810
|
+
inboxPath: string; deliveredPath: string; pid: number;
|
|
811
|
+
appended: number; chain: Promise<unknown>;
|
|
812
|
+
} | null = null;
|
|
813
|
+
|
|
814
|
+
/** Deliver one user message INTO the live turn — see the contract doc
|
|
815
|
+
* (types/runtime.ts). Appends a stream-input line to the turn's durable
|
|
816
|
+
* inbox and waits for the in-guest feeder's delivered-counter ack; only
|
|
817
|
+
* an acked forward (pre-result, into the CLI's stdin) reports
|
|
818
|
+
* "delivered". Guest exit 5 (ack window exhausted with the runner still
|
|
819
|
+
* alive — a CLI that is not draining stdin mid-step) reports "pending":
|
|
820
|
+
* the appended line may still be read when the current step finishes,
|
|
821
|
+
* but it was NOT seen yet. Never throws. */
|
|
822
|
+
async injectUserMessage(text: string): Promise<"delivered" | "pending" | "closed" | "unsupported"> {
|
|
823
|
+
const inject = this.turnInject;
|
|
824
|
+
const streamInput = this.spec.streamInput;
|
|
825
|
+
if (!inject || !streamInput || this.guestDetached) return "unsupported";
|
|
826
|
+
const line = streamInput.messageLine(text);
|
|
827
|
+
const attempt = inject.chain.then(async (): Promise<"delivered" | "pending" | "closed"> => {
|
|
828
|
+
if (this.turnInject !== inject) return "closed";
|
|
829
|
+
// The append lands whether or not the ack times out, so the target
|
|
830
|
+
// counter advances unconditionally: a later injection must never aim
|
|
831
|
+
// at a line number the feeder has already passed.
|
|
832
|
+
const target = inject.appended + 1;
|
|
833
|
+
inject.appended = target;
|
|
834
|
+
try {
|
|
835
|
+
const res = await this.sandbox.commands.run(
|
|
836
|
+
injectAppendAndAckCommand({
|
|
837
|
+
line, target, pid: inject.pid,
|
|
838
|
+
inboxPath: inject.inboxPath, deliveredPath: inject.deliveredPath,
|
|
839
|
+
}),
|
|
840
|
+
{ timeoutMs: INJECT_ACK_EXEC_TIMEOUT_MS });
|
|
841
|
+
if (res.exitCode === 0) return "delivered";
|
|
842
|
+
// 5 = ack window exhausted, runner alive: the line is in the inbox
|
|
843
|
+
// and may be fed when the CLI next drains stdin (the mid-step wall).
|
|
844
|
+
// 4 = the runner died first; anything else is a fault — both closed.
|
|
845
|
+
return res.exitCode === 5 ? "pending" : "closed";
|
|
846
|
+
} catch { return "closed"; }
|
|
847
|
+
});
|
|
848
|
+
inject.chain = attempt.catch(() => undefined);
|
|
849
|
+
return attempt;
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/** Durable three-valued liveness for the current turn — see the contract
|
|
853
|
+
* doc (types/runtime.ts). Never throws; never blocks past the probe's
|
|
854
|
+
* explicit exec timeout. */
|
|
855
|
+
async probeTurnLiveness(): Promise<RunnerLivenessVerdict | null> {
|
|
856
|
+
const probe = this.turnProbe;
|
|
857
|
+
if (!probe) return null;
|
|
858
|
+
return livenessVerdictFromReading(await probe());
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
/** Wakes the CURRENT tail-watchdog interval early. Armed only while the
|
|
862
|
+
* watchdog's wake timer is pending; null between arms / between turns. */
|
|
863
|
+
private wakeTailWatchdog: (() => void) | null = null;
|
|
864
|
+
/** A doorbell that rang while no watchdog wake was armed — consumed at the
|
|
865
|
+
* next arm (immediate wake), dropped at the start of a new turn. */
|
|
866
|
+
private nudgePending = false;
|
|
867
|
+
|
|
868
|
+
/** DOORBELL, NEVER A VERDICT (contract doc: types/runtime.ts). Wake the
|
|
869
|
+
* tail watchdog NOW so its normal verification pass — durable probe, then
|
|
870
|
+
* harvest-on-sentinel / honest no-sentinel death — runs immediately
|
|
871
|
+
* instead of at the next TAIL_WATCHDOG_INTERVAL_MS tick. Carries no
|
|
872
|
+
* information: a nudge for a live mid-turn runner probes alive and is a
|
|
873
|
+
* no-op; a spurious/duplicate nudge costs at most one probe exec. No-op
|
|
874
|
+
* when no durable watchdog is armed (boot phase, ACP path, single-exec
|
|
875
|
+
* transports, abandoned-streaming polling) — those phases already carry
|
|
876
|
+
* their own bounds. */
|
|
877
|
+
nudgeTurnProbe(): void {
|
|
878
|
+
const wake = this.wakeTailWatchdog;
|
|
879
|
+
if (wake) {
|
|
880
|
+
this.wakeTailWatchdog = null;
|
|
881
|
+
wake();
|
|
882
|
+
} else {
|
|
883
|
+
this.nudgePending = true;
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
/** RESUME HANDOFF, NEVER A KILL (contract doc: types/runtime.ts). Once
|
|
888
|
+
* set, abort/early-exit unwinds the transport WITHOUT reaping the
|
|
889
|
+
* detached guest tree and WITHOUT the opportunistic durable-file cleanup
|
|
890
|
+
* — the successor turn resumes the same guest session, and the park
|
|
891
|
+
* path's busy signal keeps reading the surviving heartbeat/busy files.
|
|
892
|
+
* One-way for this runner instance (a runner is per-turn on the cloud
|
|
893
|
+
* path). */
|
|
894
|
+
private guestDetached = false;
|
|
895
|
+
|
|
896
|
+
detachGuest(): void {
|
|
897
|
+
this.guestDetached = true;
|
|
898
|
+
}
|
|
899
|
+
|
|
244
900
|
constructor(
|
|
245
901
|
private readonly sandbox: SandboxProvider,
|
|
246
902
|
private readonly options: RuntimeOptions,
|
|
@@ -302,9 +958,10 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
302
958
|
* install command runs exactly once: on the first run of a content hash. */
|
|
303
959
|
private async ensureInstalled(): Promise<void> {
|
|
304
960
|
if (this.installed) return;
|
|
305
|
-
const probe = await this.sandbox.commands.run(
|
|
961
|
+
const probe = await this.sandbox.commands.run(
|
|
962
|
+
`command -v ${this.spec.bin}`, { timeoutMs: INSTALL_PROBE_TIMEOUT_MS });
|
|
306
963
|
if (probe.exitCode === 0) { this.installed = true; return; }
|
|
307
|
-
const res = await this.sandbox.commands.run(this.spec.install, { timeoutMs:
|
|
964
|
+
const res = await this.sandbox.commands.run(this.spec.install, { timeoutMs: CLI_INSTALL_TIMEOUT_MS });
|
|
308
965
|
if (res.exitCode !== 0) {
|
|
309
966
|
throw new Error(`failed to install ${this.spec.bin}: ${(res.stderr || res.stdout || "").slice(-500)}`);
|
|
310
967
|
}
|
|
@@ -608,13 +1265,33 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
608
1265
|
await this.ensureInstalled();
|
|
609
1266
|
|
|
610
1267
|
const promptPath = uniquePromptPath(this.spec.kind, this.options.agentId ?? "agent", opts.iteration ?? 0);
|
|
611
|
-
|
|
1268
|
+
// A NEW turn invalidates any previous turn's durable probe — re-armed
|
|
1269
|
+
// below once (and only if) this turn's detached runner launches. A
|
|
1270
|
+
// stale doorbell is dropped with it: a new turn owes nothing to the
|
|
1271
|
+
// previous turn's push.
|
|
1272
|
+
this.turnProbe = null;
|
|
1273
|
+
this.turnInject = null;
|
|
1274
|
+
this.nudgePending = false;
|
|
1275
|
+
this.wakeTailWatchdog = null;
|
|
1276
|
+
// Mid-turn stream input engages only where BOTH halves exist: a spec
|
|
1277
|
+
// that can serialise stream-input lines AND the durable detached
|
|
1278
|
+
// transport (the feeder, inbox, and result-watch live in the guest;
|
|
1279
|
+
// single-exec transports keep the plain prompt-file stdin).
|
|
1280
|
+
const streamInput = this.spec.streamInput !== undefined
|
|
1281
|
+
&& this.sandbox.commands.runBackground !== undefined;
|
|
1282
|
+
const fifoPath = `${promptPath}.fifo`;
|
|
1283
|
+
const promptWrite = this.sandbox.files.write(promptPath, streamInput
|
|
1284
|
+
? `${this.spec.streamInput!.promptLine(opts.prompt)}\n`
|
|
1285
|
+
: this.spec.promptPayload(opts.prompt));
|
|
612
1286
|
const cmd = this.spec.buildCommand({
|
|
613
|
-
|
|
1287
|
+
// Stream-input turns read stdin from the feeder's FIFO; the prompt
|
|
1288
|
+
// file is the feeder's first delivery, not the CLI's stdin.
|
|
1289
|
+
promptPath: streamInput ? fifoPath : promptPath,
|
|
614
1290
|
sessionId: opts.sessionId,
|
|
615
1291
|
model: this.model,
|
|
616
1292
|
cwd: this.options.cwd,
|
|
617
1293
|
effort: this.configEffort,
|
|
1294
|
+
...(streamInput ? { streamInput: true } : {}),
|
|
618
1295
|
});
|
|
619
1296
|
|
|
620
1297
|
// Frame guard (see _jsonl-guard.ts): the CLI's stdout is piped through
|
|
@@ -633,65 +1310,437 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
633
1310
|
// uniquePromptPath) and the same opportunistic cleanup below.
|
|
634
1311
|
const guardPath = `${promptPath}.guard.js`;
|
|
635
1312
|
const sentinel = `__AC_JSONL_EXIT_${Math.random().toString(36).slice(2, 10)}__`;
|
|
636
|
-
|
|
1313
|
+
// Both file-transport writes ride in parallel: each is a full envd
|
|
1314
|
+
// HTTP round trip (~65ms measured against E2B from the dev host), and
|
|
1315
|
+
// they are independent — the launch below is the only consumer of
|
|
1316
|
+
// either path. One RTT shaved off every turn's first-token path.
|
|
1317
|
+
await Promise.all([promptWrite, this.sandbox.files.write(guardPath, jsonlGuardScript())]);
|
|
637
1318
|
const guardedCmd = wrapJsonlCommand({
|
|
638
1319
|
cmd, guardPath, sentinel,
|
|
639
1320
|
maxBytes: JSONL_GUARD_LINE_CAP_BYTES,
|
|
640
1321
|
keepBytes: JSONL_GUARD_KEEP_BYTES,
|
|
641
1322
|
});
|
|
642
1323
|
|
|
643
|
-
// Bridge the
|
|
644
|
-
// JSONL lines.
|
|
1324
|
+
// Bridge the transport's output into an async-iterable of complete
|
|
1325
|
+
// JSONL lines. Two transports feed it (chosen below):
|
|
1326
|
+
//
|
|
1327
|
+
// - DURABLE DETACHED (providers with `runBackground` + `files.read`,
|
|
1328
|
+
// i.e. E2B — every cloud session): the CLI runs as a DETACHED
|
|
1329
|
+
// in-guest process whose stdout is a durable file; the server only
|
|
1330
|
+
// ever TAILS that file, and a dead tail is re-attached (or the file
|
|
1331
|
+
// re-read over the envd HTTP file transport) without the runner
|
|
1332
|
+
// noticing. The exit code travels IN-BAND as a final sentinel line.
|
|
1333
|
+
// This exists because of the 2026-08-14 prod incident (conversation
|
|
1334
|
+
// 8bc082f0, turn bfcb318f): the turn's single streaming exec was the
|
|
1335
|
+
// CLI's lifeline — when the connect-web stream's context was
|
|
1336
|
+
// canceled at the wire 50s in (envd logged `context canceled`), envd
|
|
1337
|
+
// tore down the exec's stdout, the guard died on EPIPE, the CLI took
|
|
1338
|
+
// SIGPIPE mid-work, the finished output was lost, and `wait()` never
|
|
1339
|
+
// settled — the turn wedged silently until the next dispatch
|
|
1340
|
+
// superseded it as a phantom "session restarted".
|
|
1341
|
+
//
|
|
1342
|
+
// - SINGLE-EXEC STREAMING (everything else: local child_process,
|
|
1343
|
+
// Vercel): the pre-incident shape — one exec, stdout streamed,
|
|
1344
|
+
// exit code from the exec result.
|
|
645
1345
|
const lines = new AsyncQueue<string>();
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
};
|
|
661
|
-
|
|
662
|
-
// The command resolves when the process exits. Kick it off (don't await
|
|
663
|
-
// yet); flush the trailing buffer + close the queue on completion so the
|
|
664
|
-
// for-await below drains and we can read the exit code. Prefer the
|
|
665
|
-
// kill-capable background handle so abort/early-exit can terminate the
|
|
666
|
-
// process itself, not just stop reading its stream.
|
|
667
|
-
let exited = false;
|
|
668
|
-
const settle = (res: SandboxCommandResult): SandboxCommandResult => {
|
|
669
|
-
exited = true;
|
|
670
|
-
const tail = buf.trim();
|
|
671
|
-
if (tail) lines.push(tail);
|
|
672
|
-
lines.close();
|
|
673
|
-
return res;
|
|
674
|
-
};
|
|
675
|
-
const fail = (err: unknown): never => { exited = true; lines.close(); throw err; };
|
|
676
|
-
let kill: (() => Promise<void>) | undefined;
|
|
677
|
-
let runPromise: Promise<SandboxCommandResult>;
|
|
1346
|
+
// How the turn ended: the CLI's exit code once known (null while
|
|
1347
|
+
// running), plus a best-effort stderr tail for the error message.
|
|
1348
|
+
let exitCode: number | null = null;
|
|
1349
|
+
let exitStderr = "";
|
|
1350
|
+
let transportError: unknown = null;
|
|
1351
|
+
let consumerStopped = false;
|
|
1352
|
+
let reap: () => void = () => { /* set per transport */ };
|
|
1353
|
+
// The reap's kill-and-confirm exec, latched on first fire (reap runs
|
|
1354
|
+
// from BOTH the abort listener and the finally — one exec, not two).
|
|
1355
|
+
// The unwind awaits it so the turn never settles over a live tree.
|
|
1356
|
+
let reapConfirm: Promise<void> | null = null;
|
|
1357
|
+
let transport: Promise<void>;
|
|
1358
|
+
|
|
1359
|
+
const durableRead = this.sandbox.files.read?.bind(this.sandbox.files);
|
|
678
1360
|
if (this.sandbox.commands.runBackground) {
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
1361
|
+
// ── Durable detached transport ─────────────────────────────────────
|
|
1362
|
+
const outPath = `${promptPath}.out`;
|
|
1363
|
+
const errPath = `${promptPath}.err`;
|
|
1364
|
+
// The exit sentinel is appended by the DETACHED script itself after
|
|
1365
|
+
// the guarded pipeline exits, so it lands in the durable file even if
|
|
1366
|
+
// every server↔sandbox stream is dead by then. In node-guard mode the
|
|
1367
|
+
// pipeline's exit code is already the CLI's (the guard re-raises the
|
|
1368
|
+
// inner sentinel, including NO_SENTINEL_EXIT for a mid-pipe death).
|
|
1369
|
+
// The heartbeat subshell rides INSIDE the setsid session (group kill
|
|
1370
|
+
// reaps it with the tree). It is keyed to the wrapper sh's own pid
|
|
1371
|
+
// (`$$` — the pidfile pid) and DIES WITH it: heartbeat and busy
|
|
1372
|
+
// marker both stop within one interval of the wrapper exiting (see
|
|
1373
|
+
// the loop comment below for why outliving the wrapper was a bug).
|
|
1374
|
+
// Its writes are the durable "the runner tree is alive" signals the
|
|
1375
|
+
// probes read — signals no server↔sandbox stream can wedge.
|
|
1376
|
+
const hbPath = `${promptPath}.hb`;
|
|
1377
|
+
// Exit-event doorbell (see exitEventPushCommand): SENTINEL FIRST,
|
|
1378
|
+
// PUSH AFTER. The push rides behind the sentinel append so the
|
|
1379
|
+
// durable truth exists before anything is announced, and its whole
|
|
1380
|
+
// failure surface is swallowed — the runner lifecycle cannot see it.
|
|
1381
|
+
const exitPush = this.options.turnExitNotify
|
|
1382
|
+
? exitEventPushCommand(this.options.turnExitNotify)
|
|
1383
|
+
: "";
|
|
1384
|
+
// Busy sentinel (park-defers-while-busy): each beat also writes the
|
|
1385
|
+
// background-work marker next to the heartbeat — task count
|
|
1386
|
+
// (<prompt>.busy) and task ids (<prompt>.tasks). The server's park
|
|
1387
|
+
// probe reads the task JOURNALS (~/.claude/tasks) directly plus the
|
|
1388
|
+
// heartbeat/pid pair; the marker files are in-guest receipts.
|
|
1389
|
+
//
|
|
1390
|
+
// The subshell DIES WITH the wrapper. It used to outlive it while
|
|
1391
|
+
// fresh task files existed (so a clean turn end would not take the
|
|
1392
|
+
// busy marker down with running background tasks) — but the server
|
|
1393
|
+
// probe stopped reading the marker files (journal mtimes carry the
|
|
1394
|
+
// busy signal), and the lingering subshell kept the setsid GROUP
|
|
1395
|
+
// alive minutes past turn end, which the probe's live-group gating
|
|
1396
|
+
// reads as work: every turn that touched its task list became its
|
|
1397
|
+
// own "background work" and armed a spurious wake (the 2026-08-17
|
|
1398
|
+
// loop). Genuine background children launched by the CLI live in
|
|
1399
|
+
// this same setsid group and keep it alive by THEMSELVES — that,
|
|
1400
|
+
// plus their fresh journals, is the honest busy signal. A
|
|
1401
|
+
// supersede group-kill still reaps this subshell with the tree.
|
|
1402
|
+
const busyPath = `${promptPath}.busy`;
|
|
1403
|
+
const tasksPath = `${promptPath}.tasks`;
|
|
1404
|
+
// Mid-turn stream input: the feeder subshell (see
|
|
1405
|
+
// streamInputFeederFragment) rides at the FRONT of the detached
|
|
1406
|
+
// script so the FIFO exists before the CLI opens it as stdin.
|
|
1407
|
+
const inboxPath = `${promptPath}.inbox`;
|
|
1408
|
+
const deliveredPath = `${promptPath}.delivered`;
|
|
1409
|
+
const feeder = streamInput
|
|
1410
|
+
? streamInputFeederFragment({ promptPath, fifoPath, inboxPath, deliveredPath, outPath })
|
|
1411
|
+
: "";
|
|
1412
|
+
// Perf sampling rides the same beat: every 6th beat (~60s) the
|
|
1413
|
+
// subshell burst-reads /proc and rewrites the `.perf` token the
|
|
1414
|
+
// durable-trio probe carries as its trailing field (perf-sampler.ts).
|
|
1415
|
+
const perfPath = `${promptPath}.perf`;
|
|
1416
|
+
const detachedScript =
|
|
1417
|
+
feeder
|
|
1418
|
+
+ `( ${perfSamplerFunctionFragment({ perfPath })}while kill -0 "$$" 2>/dev/null; do `
|
|
1419
|
+
+ `date +%s%3N > ${shellQuote(hbPath)}; `
|
|
1420
|
+
+ `${runnerBusySentinelFragment(busyPath, tasksPath)}; `
|
|
1421
|
+
+ `ac_perf_tick; `
|
|
1422
|
+
+ `sleep ${RUNNER_HEARTBEAT_INTERVAL_SECONDS}; done ) >/dev/null 2>&1 </dev/null & `
|
|
1423
|
+
+ sessionEnvSourceFragment(this.options.sessionEnvFile)
|
|
1424
|
+
+ credEnvSourceFragment(this.options.credEnvFile)
|
|
1425
|
+
+ `{ ${guardedCmd}; } >> ${shellQuote(outPath)} 2>> ${shellQuote(errPath)} </dev/null; ac_rc=$?; `
|
|
1426
|
+
+ `printf '\\n%s %s\\n' ${shellQuote(sentinel)} "$ac_rc" >> ${shellQuote(outPath)}`
|
|
1427
|
+
+ (exitPush ? `; ${exitPush}` : "");
|
|
1428
|
+
// `setsid` detaches the runner into its own session/process group so
|
|
1429
|
+
// it survives the launch exec ending AND gives `kill -- -pid` a whole
|
|
1430
|
+
// tree to terminate. The fallback keeps working where setsid is
|
|
1431
|
+
// absent; a runner that dies anyway surfaces as NO_SENTINEL below.
|
|
1432
|
+
//
|
|
1433
|
+
// The `>/dev/null 2>&1 </dev/null` on the forked child is LOAD-BEARING,
|
|
1434
|
+
// not hygiene (2026-08-14 prod regression, v0.10.39): without it the
|
|
1435
|
+
// detached `sh` inherits the LAUNCH exec's stdout/stderr pipes, and
|
|
1436
|
+
// envd only ends a command's event stream on pipe EOF — so the launch
|
|
1437
|
+
// exec's wait() couldn't settle until the entire detached CLI tree
|
|
1438
|
+
// exited, the 30s launch deadline fired first, and EVERY turn died
|
|
1439
|
+
// with E2B's `[deadline_exceeded] the operation timed out: … exceeding
|
|
1440
|
+
// 'timeoutMs' …` while the orphaned runner kept working unobserved.
|
|
1441
|
+
// (The runner's real stdio is redirected INSIDE detachedScript — into
|
|
1442
|
+
// the durable out/err files — so the child needs nothing from the
|
|
1443
|
+
// launch exec's pipes.)
|
|
1444
|
+
//
|
|
1445
|
+
// The pid ALSO lands in a durable pidfile so a launch whose STREAM
|
|
1446
|
+
// dies (the same wire faults the tail survives) is recovered over the
|
|
1447
|
+
// file transport instead of surfacing as turn death — see the catch
|
|
1448
|
+
// below.
|
|
1449
|
+
const pidPath = `${promptPath}.pid`;
|
|
1450
|
+
const launchCmd =
|
|
1451
|
+
`: > ${shellQuote(outPath)}; : > ${shellQuote(errPath)}; date +%s%3N > ${shellQuote(hbPath)}; `
|
|
1452
|
+
+ `if command -v setsid >/dev/null 2>&1; then setsid sh -c ${shellQuote(detachedScript)} >/dev/null 2>&1 </dev/null & `
|
|
1453
|
+
+ `else sh -c ${shellQuote(detachedScript)} >/dev/null 2>&1 </dev/null & fi; `
|
|
1454
|
+
+ `echo "$!" > ${shellQuote(pidPath)}; echo "$!"`;
|
|
1455
|
+
let pid = Number.NaN;
|
|
1456
|
+
let launched: { exitCode: number; stdout?: string; stderr?: string } | null = null;
|
|
1457
|
+
let launchErr: unknown = null;
|
|
1458
|
+
try {
|
|
1459
|
+
launched = await this.sandbox.commands.run(launchCmd, {
|
|
1460
|
+
...(this.options.cwd ? { cwd: this.options.cwd } : {}), timeoutMs: LAUNCH_EXEC_TIMEOUT_MS });
|
|
1461
|
+
} catch (err) { launchErr = err; }
|
|
1462
|
+
if (launched) {
|
|
1463
|
+
// The exec ROUND-TRIPPED: its exit code and stdout are authoritative.
|
|
1464
|
+
pid = parsePid(launched.stdout ?? "");
|
|
1465
|
+
if (launched.exitCode !== 0 || Number.isNaN(pid)) {
|
|
1466
|
+
throw new Error(`failed to launch ${this.spec.kind}: `
|
|
1467
|
+
+ `exit ${launched.exitCode}${launched.stderr ? `: ${launched.stderr.slice(-500)}` : ""}`);
|
|
1468
|
+
}
|
|
1469
|
+
} else {
|
|
1470
|
+
// Structural rule of this transport: a dead STREAM is never treated
|
|
1471
|
+
// as a dead RUNNER. The launch may well have run to completion with
|
|
1472
|
+
// only its exec stream lost (deadline_exceeded, canceled context) —
|
|
1473
|
+
// recover the pid from the durable pidfile and continue; only an
|
|
1474
|
+
// absent pidfile makes the original fault real.
|
|
1475
|
+
pid = durableRead ? await recoverLaunchPid(durableRead, pidPath) : Number.NaN;
|
|
1476
|
+
if (Number.isNaN(pid)) throw launchErr;
|
|
1477
|
+
console.warn(`[cli-agent] ${this.spec.kind} launch exec stream died but the detached runner is up `
|
|
1478
|
+
+ `(pid ${pid} via ${pidPath}) — continuing on the durable transport: ${formatError(launchErr)}`);
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
// Arm the durable-trio probe for this turn: the watchdog below and the
|
|
1482
|
+
// executor's evidence ticker (`probeTurnLiveness`) both read through
|
|
1483
|
+
// it — one fresh short exec per call, explicit timeout, faults map to
|
|
1484
|
+
// null ("probe-failed"), never to a verdict.
|
|
1485
|
+
const probeCmd = turnLivenessProbeCommand({ outPath, hbPath, perfPath, pid, sentinel });
|
|
1486
|
+
const runTurnProbe = async (): Promise<TurnProbeReading | null> => {
|
|
1487
|
+
try {
|
|
1488
|
+
const res = await this.sandbox.commands.run(probeCmd, { timeoutMs: TURN_PROBE_TIMEOUT_MS });
|
|
1489
|
+
return parseTurnProbeOutput(res.stdout ?? "");
|
|
1490
|
+
} catch { return null; }
|
|
1491
|
+
};
|
|
1492
|
+
this.turnProbe = runTurnProbe;
|
|
1493
|
+
// Arm mid-turn injection now that the detached runner (and its
|
|
1494
|
+
// feeder) exist; cleared with the turn in the finally below.
|
|
1495
|
+
if (streamInput) {
|
|
1496
|
+
this.turnInject = { inboxPath, deliveredPath, pid, appended: 0, chain: Promise.resolve() };
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
// Bytes of COMPLETE lines already consumed from outPath — a re-attach
|
|
1500
|
+
// tails from here, so a dead stream never duplicates or drops lines.
|
|
1501
|
+
let offset = 0;
|
|
1502
|
+
let currentTail: SandboxBackgroundProcess | null = null;
|
|
1503
|
+
const consumeLine = (raw: string): void => {
|
|
1504
|
+
offset += Buffer.byteLength(raw, "utf8") + 1;
|
|
1505
|
+
const line = raw.trim();
|
|
1506
|
+
if (!line) return;
|
|
1507
|
+
if (line.startsWith(sentinel)) {
|
|
1508
|
+
const code = Number(line.slice(sentinel.length).trim());
|
|
1509
|
+
exitCode = Number.isFinite(code) ? code : 0;
|
|
1510
|
+
return;
|
|
1511
|
+
}
|
|
1512
|
+
lines.push(line);
|
|
1513
|
+
};
|
|
1514
|
+
|
|
1515
|
+
reap = () => {
|
|
1516
|
+
// Kill the detached tree AND CONFIRM it died (group first — setsid
|
|
1517
|
+
// made pid the group leader; TERM → bounded wait → KILL escalation,
|
|
1518
|
+
// see reapKillAndConfirmCommand), then whatever tail is currently
|
|
1519
|
+
// attached. Fresh execs, deliberately independent of any
|
|
1520
|
+
// possibly-dead stream. The confirm promise is latched so the
|
|
1521
|
+
// finally below can await it — the turn must not settle while the
|
|
1522
|
+
// tree may still be alive (it holds e.g. codex's thread-store
|
|
1523
|
+
// writer flock until it actually exits).
|
|
1524
|
+
reapConfirm ??= this.sandbox.commands.run(
|
|
1525
|
+
reapKillAndConfirmCommand(pid), { timeoutMs: REAP_EXEC_TIMEOUT_MS })
|
|
1526
|
+
.then((res) => {
|
|
1527
|
+
if (res.exitCode === REAP_STILL_ALIVE_EXIT) {
|
|
1528
|
+
console.warn(`[cli-agent] ${this.spec.kind} runner ${pid} survived SIGKILL's confirm window `
|
|
1529
|
+
+ "— the next launch's preflight is the backstop");
|
|
1530
|
+
}
|
|
1531
|
+
})
|
|
1532
|
+
.catch(() => { /* best-effort exec — the launch preflight covers a missed kill */ });
|
|
1533
|
+
void currentTail?.kill().catch(() => { /* already gone */ });
|
|
1534
|
+
};
|
|
1535
|
+
|
|
1536
|
+
transport = (async () => {
|
|
1537
|
+
// Stream deaths this turn — past the abandon bound the turn stops
|
|
1538
|
+
// re-attaching tails and finishes on durable polling alone (a wire
|
|
1539
|
+
// that killed three streams will kill the fourth).
|
|
1540
|
+
let streamDeaths = 0;
|
|
1541
|
+
while (exitCode === null && !consumerStopped && !opts.signal?.aborted) {
|
|
1542
|
+
let buf = "";
|
|
1543
|
+
let handle: SandboxBackgroundProcess | null = null;
|
|
1544
|
+
const onStdout = (data: string): void => {
|
|
1545
|
+
buf += data;
|
|
1546
|
+
let nl: number;
|
|
1547
|
+
while ((nl = buf.indexOf("\n")) >= 0) {
|
|
1548
|
+
consumeLine(buf.slice(0, nl));
|
|
1549
|
+
buf = buf.slice(nl + 1);
|
|
1550
|
+
if (exitCode !== null) { void handle?.kill().catch(() => { /* racing exit */ }); return; }
|
|
1551
|
+
}
|
|
1552
|
+
};
|
|
1553
|
+
if (streamDeaths < TAIL_ABANDON_AFTER_STREAM_DEATHS) {
|
|
1554
|
+
try {
|
|
1555
|
+
// `-s 0.05` tightens ONLY tail's polling fallback: the guest's
|
|
1556
|
+
// GNU tail (coreutils 9.1, verified on the agent-env image)
|
|
1557
|
+
// follows /tmp via inotify — measured delivery is event-driven
|
|
1558
|
+
// (~46ms p50 write→server, one line per callback, S2 bench
|
|
1559
|
+
// 2026-08-14) — but if inotify were ever unavailable, the
|
|
1560
|
+
// default fallback polls at 1s, which WOULD become the stream's
|
|
1561
|
+
// burstiness. GNU tail accepts fractional seconds; this
|
|
1562
|
+
// transport only runs on providers with runBackground (E2B),
|
|
1563
|
+
// whose image is Debian/GNU — never busybox.
|
|
1564
|
+
handle = await this.sandbox.commands.runBackground!(
|
|
1565
|
+
`tail -s 0.05 -c +${offset + 1} -f ${shellQuote(outPath)}`,
|
|
1566
|
+
{ timeoutMs: 0, onStdout });
|
|
1567
|
+
} catch { /* attach failed at the wire — durable catch-up below */ }
|
|
1568
|
+
}
|
|
1569
|
+
if (handle) {
|
|
1570
|
+
if (exitCode !== null) void handle.kill().catch(() => { /* done */ });
|
|
1571
|
+
currentTail = handle;
|
|
1572
|
+
// ── The watchdog: never TRUST an attached stream ─────────────
|
|
1573
|
+
// (2026-08-15 incident, turn 18dc5261: the tail's connect-web
|
|
1574
|
+
// stream wedged silently — wait() never settled, no error, no
|
|
1575
|
+
// EOF — while the finished answer + exit sentinel sat in the
|
|
1576
|
+
// durable file for ten minutes.) wait() is raced against a
|
|
1577
|
+
// fixed-cadence durable probe; the probe's DURABLE reading —
|
|
1578
|
+
// never the stream's health — decides:
|
|
1579
|
+
// - bytes in the file the stream failed to deliver across a
|
|
1580
|
+
// full interval (offset frozen since the previous tick)
|
|
1581
|
+
// prove the stream dead → kill the TAIL (never the runner)
|
|
1582
|
+
// and fall through to the durable catch-up + re-attach;
|
|
1583
|
+
// - runner process gone → nothing more will be written: kill
|
|
1584
|
+
// the tail and let the catch-up read the sentinel (or
|
|
1585
|
+
// surface the honest no-sentinel death);
|
|
1586
|
+
// - probe fault/timeout → NO verdict: keep streaming (the
|
|
1587
|
+
// executor's evidence machinery bounds an unreachable
|
|
1588
|
+
// sandbox, not this loop);
|
|
1589
|
+
// - abort/consumer-stop wakes the race immediately, so a
|
|
1590
|
+
// wedged wait() can no longer leak the whole transport.
|
|
1591
|
+
await (async (tail: SandboxBackgroundProcess): Promise<void> => {
|
|
1592
|
+
let waitSettled = false;
|
|
1593
|
+
const waitP = tail.wait().catch(() => undefined)
|
|
1594
|
+
.then(() => { waitSettled = true; });
|
|
1595
|
+
let lastTickOffset = -1;
|
|
1596
|
+
while (!waitSettled) {
|
|
1597
|
+
let onAbort: (() => void) | undefined;
|
|
1598
|
+
const wake = new Promise<void>((resolve) => {
|
|
1599
|
+
const t = setTimeout(resolve, TAIL_WATCHDOG_INTERVAL_MS);
|
|
1600
|
+
(t as { unref?: () => void }).unref?.();
|
|
1601
|
+
const fire = () => { clearTimeout(t); resolve(); };
|
|
1602
|
+
// Exit-event doorbell (nudgeTurnProbe): a nudge wakes
|
|
1603
|
+
// THIS interval immediately — its only effect is running
|
|
1604
|
+
// the durable probe below NOW. One that rang while no
|
|
1605
|
+
// wake was armed is consumed here.
|
|
1606
|
+
if (this.nudgePending) {
|
|
1607
|
+
this.nudgePending = false;
|
|
1608
|
+
fire();
|
|
1609
|
+
} else {
|
|
1610
|
+
this.wakeTailWatchdog = fire;
|
|
1611
|
+
}
|
|
1612
|
+
if (opts.signal) {
|
|
1613
|
+
onAbort = fire;
|
|
1614
|
+
opts.signal.addEventListener("abort", onAbort, { once: true });
|
|
1615
|
+
}
|
|
1616
|
+
});
|
|
1617
|
+
await Promise.race([waitP, wake]);
|
|
1618
|
+
this.wakeTailWatchdog = null;
|
|
1619
|
+
if (onAbort) opts.signal?.removeEventListener("abort", onAbort);
|
|
1620
|
+
if (waitSettled || exitCode !== null || consumerStopped || opts.signal?.aborted) return;
|
|
1621
|
+
const tickOffset = offset;
|
|
1622
|
+
const reading = await runTurnProbe();
|
|
1623
|
+
if (offset !== tickOffset) { lastTickOffset = offset; continue; } // stream delivered meanwhile — healthy
|
|
1624
|
+
if (reading === null) { lastTickOffset = tickOffset; continue; } // probe-failed — no verdict
|
|
1625
|
+
// HARVEST-ON-PROBE (one strike — THE fix for the incident's
|
|
1626
|
+
// ten-minute blindness): a durable exit sentinel means the
|
|
1627
|
+
// turn is FINISHED and no more bytes are coming; any stream
|
|
1628
|
+
// that hasn't delivered it is done being useful. Likewise a
|
|
1629
|
+
// runner process that is gone. Mid-turn starvation keeps a
|
|
1630
|
+
// two-tick grace (offset frozen since the previous tick) so
|
|
1631
|
+
// a byte-burst racing the probe never kills a healthy tail.
|
|
1632
|
+
const streamStarved = reading.size > offset && tickOffset === lastTickOffset;
|
|
1633
|
+
if (reading.sentinelSeen || !reading.alive || streamStarved) {
|
|
1634
|
+
console.warn(`[cli-agent] ${this.spec.kind} tail stream ${reading.sentinelSeen
|
|
1635
|
+
? "outlived the FINISHED turn (durable sentinel unharvested)"
|
|
1636
|
+
: !reading.alive ? "outlived the runner"
|
|
1637
|
+
: `starved (durable file at ${reading.size} bytes, stream stuck at ${offset})`}`
|
|
1638
|
+
+ " — killing the tail and catching up from the durable file");
|
|
1639
|
+
void tail.kill().catch(() => { /* best-effort; catch-up follows regardless */ });
|
|
1640
|
+
return;
|
|
1641
|
+
}
|
|
1642
|
+
lastTickOffset = tickOffset;
|
|
1643
|
+
}
|
|
1644
|
+
})(handle);
|
|
1645
|
+
currentTail = null;
|
|
1646
|
+
}
|
|
1647
|
+
if (exitCode !== null || consumerStopped || opts.signal?.aborted) break;
|
|
1648
|
+
// The tail stream died without delivering the exit line. Catch up
|
|
1649
|
+
// over the FILE transport (envd HTTP — immune to the command-
|
|
1650
|
+
// stream fault): this is also what recovers an already-finished
|
|
1651
|
+
// turn's result when every stream is dead.
|
|
1652
|
+
if (durableRead) {
|
|
1653
|
+
try {
|
|
1654
|
+
const bytes = Buffer.from(await durableRead(outPath), "utf8");
|
|
1655
|
+
let rest = bytes.subarray(offset).toString("utf8");
|
|
1656
|
+
let nl: number;
|
|
1657
|
+
while (exitCode === null && (nl = rest.indexOf("\n")) >= 0) {
|
|
1658
|
+
consumeLine(rest.slice(0, nl));
|
|
1659
|
+
rest = rest.slice(nl + 1);
|
|
1660
|
+
}
|
|
1661
|
+
} catch { /* file plane briefly unreachable — retried below */ }
|
|
1662
|
+
}
|
|
1663
|
+
if (exitCode !== null) break;
|
|
1664
|
+
// Runner still alive? A FRESH exec answers; unknown (wire fault)
|
|
1665
|
+
// keeps retrying — the executor's evidence machinery bounds a
|
|
1666
|
+
// truly dead sandbox, not this loop.
|
|
1667
|
+
let alive: boolean | null = null;
|
|
1668
|
+
try {
|
|
1669
|
+
const probe = await this.sandbox.commands.run(
|
|
1670
|
+
`kill -0 ${pid} 2>/dev/null`, { timeoutMs: 10_000 });
|
|
1671
|
+
alive = probe.exitCode === 0;
|
|
1672
|
+
} catch { alive = null; }
|
|
1673
|
+
if (alive === false) {
|
|
1674
|
+
// Dead with no exit line even in the durable file: abnormal
|
|
1675
|
+
// death — surfaces as an error, never a clean done.
|
|
1676
|
+
exitCode = JSONL_GUARD_NO_SENTINEL_EXIT;
|
|
1677
|
+
break;
|
|
1678
|
+
}
|
|
1679
|
+
streamDeaths += 1;
|
|
1680
|
+
if (streamDeaths < TAIL_ABANDON_AFTER_STREAM_DEATHS) {
|
|
1681
|
+
console.warn(`[cli-agent] ${this.spec.kind} turn stream lost with the runner alive — re-tailing ${outPath} from byte ${offset}`);
|
|
1682
|
+
await new Promise((r) => setTimeout(r, TAIL_REATTACH_DELAY_MS));
|
|
1683
|
+
} else {
|
|
1684
|
+
if (streamDeaths === TAIL_ABANDON_AFTER_STREAM_DEATHS) {
|
|
1685
|
+
console.warn(`[cli-agent] ${this.spec.kind} turn abandoning streaming after `
|
|
1686
|
+
+ `${streamDeaths} stream deaths — finishing on durable polling from byte ${offset}`);
|
|
1687
|
+
}
|
|
1688
|
+
await new Promise((r) => setTimeout(r, DURABLE_POLL_INTERVAL_MS));
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
if (exitCode !== null && exitCode !== 0 && durableRead) {
|
|
1692
|
+
try { exitStderr = (await durableRead(errPath)).slice(-2000); }
|
|
1693
|
+
catch { /* best-effort */ }
|
|
1694
|
+
}
|
|
1695
|
+
})().finally(() => lines.close());
|
|
682
1696
|
} else {
|
|
683
|
-
|
|
1697
|
+
// ── Single-exec streaming transport (no runBackground/files.read) ──
|
|
1698
|
+
let buf = "";
|
|
1699
|
+
const onStdout = (data: string) => {
|
|
1700
|
+
buf += data;
|
|
1701
|
+
let nl: number;
|
|
1702
|
+
while ((nl = buf.indexOf("\n")) >= 0) {
|
|
1703
|
+
const line = buf.slice(0, nl).trim();
|
|
1704
|
+
buf = buf.slice(nl + 1);
|
|
1705
|
+
if (line) lines.push(line);
|
|
1706
|
+
}
|
|
1707
|
+
};
|
|
1708
|
+
const runOpts = {
|
|
1709
|
+
...(this.options.cwd ? { cwd: this.options.cwd } : {}),
|
|
1710
|
+
onStdout,
|
|
1711
|
+
timeoutMs: 0, // no provider deadline — see the doc comment above
|
|
1712
|
+
};
|
|
1713
|
+
transport = this.sandbox.commands.run(
|
|
1714
|
+
credEnvSourceFragment(this.options.credEnvFile) + guardedCmd, runOpts).then(
|
|
1715
|
+
(res) => {
|
|
1716
|
+
const tail = buf.trim();
|
|
1717
|
+
if (tail) lines.push(tail);
|
|
1718
|
+
exitCode = res.exitCode;
|
|
1719
|
+
exitStderr = (res.stderr ?? "").slice(-2000);
|
|
1720
|
+
},
|
|
1721
|
+
(err) => { transportError = err; },
|
|
1722
|
+
).finally(() => lines.close());
|
|
684
1723
|
}
|
|
685
|
-
// An early generator exit stops awaiting
|
|
1724
|
+
// An early generator exit stops awaiting the transport — keep it
|
|
686
1725
|
// observed so an aborted turn can never surface an unhandled rejection.
|
|
687
|
-
|
|
1726
|
+
void transport.catch(() => { /* observed via `await transport` when consumed */ });
|
|
688
1727
|
|
|
689
|
-
|
|
1728
|
+
// Detach-aware reap seam: a runner marked for resume handoff
|
|
1729
|
+
// (`detachGuest`) unwinds WITHOUT killing the guest tree — the
|
|
1730
|
+
// successor turn resumes that exact guest session.
|
|
1731
|
+
const reapUnlessDetached = (): void => { if (!this.guestDetached) reap(); };
|
|
690
1732
|
if (opts.signal) {
|
|
691
|
-
if (opts.signal.aborted)
|
|
692
|
-
else opts.signal.addEventListener("abort",
|
|
1733
|
+
if (opts.signal.aborted) reapUnlessDetached();
|
|
1734
|
+
else opts.signal.addEventListener("abort", reapUnlessDetached, { once: true });
|
|
693
1735
|
}
|
|
694
1736
|
try {
|
|
1737
|
+
// The guest session id ALREADY ANNOUNCED to the consumer (the
|
|
1738
|
+
// synthesized init above carried opts.sessionId). A CLI that forks a
|
|
1739
|
+
// NEW session id (claude --resume forks; codex threads) announces it
|
|
1740
|
+
// mid-stream below, so the executor can durably record the id even
|
|
1741
|
+
// when the turn never reaches its `done` (a reaped/superseded turn
|
|
1742
|
+
// must still be resumable by its redispatch).
|
|
1743
|
+
let announcedSessionId = opts.sessionId ?? "";
|
|
695
1744
|
for await (const line of lines) {
|
|
696
1745
|
let parsed: Record<string, unknown>;
|
|
697
1746
|
try {
|
|
@@ -701,33 +1750,77 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
701
1750
|
}
|
|
702
1751
|
const sid = this.spec.extractSessionId(parsed);
|
|
703
1752
|
if (sid) sessionId = sid;
|
|
1753
|
+
if (sid && sid !== announcedSessionId) {
|
|
1754
|
+
announcedSessionId = sid;
|
|
1755
|
+
yield { type: "init", sessionId: sid, timestamp: now() };
|
|
1756
|
+
}
|
|
704
1757
|
for (const msg of this.spec.mapEvent(parsed)) {
|
|
705
1758
|
if (msg.type === "error") sawError = true;
|
|
706
1759
|
yield msg;
|
|
707
1760
|
}
|
|
708
1761
|
}
|
|
709
1762
|
|
|
710
|
-
|
|
711
|
-
if (
|
|
712
|
-
|
|
713
|
-
|
|
1763
|
+
await transport;
|
|
1764
|
+
if (transportError) throw transportError;
|
|
1765
|
+
if (exitCode === null) {
|
|
1766
|
+
// Aborted/reaped before an exit line existed — honest terminal,
|
|
1767
|
+
// never a hang and never a clean done.
|
|
1768
|
+
if (!sawError) {
|
|
1769
|
+
yield { type: "error", text: `${this.spec.kind} aborted — the in-sandbox runner was terminated`, timestamp: now() };
|
|
1770
|
+
}
|
|
1771
|
+
return;
|
|
1772
|
+
}
|
|
1773
|
+
if (exitCode !== 0 && !sawError) {
|
|
1774
|
+
const detail = exitCode === JSONL_GUARD_NO_SENTINEL_EXIT
|
|
1775
|
+
? " (the runner died without reporting an exit code)" : "";
|
|
1776
|
+
yield { type: "error", text: `${this.spec.kind} exited with code ${exitCode}${detail}${exitStderr ? `: ${exitStderr}` : ""}`, timestamp: now() };
|
|
1777
|
+
return;
|
|
1778
|
+
}
|
|
1779
|
+
if (exitCode !== 0 && sawError && RESUME_TARGET_MISSING_STDERR.test(exitStderr)) {
|
|
1780
|
+
// The CLI reported a bare result error on stdout (claude's
|
|
1781
|
+
// `error_during_execution` with num_turns 0) while its REAL
|
|
1782
|
+
// complaint — the missing `--resume` target — went to stderr
|
|
1783
|
+
// (the 2026-08-18 split-brain forensic). Surface the stderr as
|
|
1784
|
+
// the LAST error event: the consumer's terminal close keeps the
|
|
1785
|
+
// last error text, so the classifier sees the truth instead of a
|
|
1786
|
+
// generic runner death it would call transient forever.
|
|
1787
|
+
yield { type: "error", text: `${this.spec.kind} exited with code ${exitCode}: ${exitStderr}`, timestamp: now() };
|
|
714
1788
|
return;
|
|
715
1789
|
}
|
|
716
1790
|
if (!sawError) yield { type: "done", sessionId: sessionId ?? "", timestamp: now() };
|
|
717
1791
|
} finally {
|
|
718
1792
|
// Runs on completion AND on early generator exit (the caller broke
|
|
719
1793
|
// out of its for-await: abort, turn closed under the executor,
|
|
720
|
-
// supersede).
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
//
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
//
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
1794
|
+
// supersede). Reap only while the runner hasn't reported an exit —
|
|
1795
|
+
// and NEVER when the guest was detached for a resume handoff: the
|
|
1796
|
+
// successor turn resumes this exact guest session, and the park
|
|
1797
|
+
// path's busy signal reads the surviving heartbeat/busy files.
|
|
1798
|
+
opts.signal?.removeEventListener("abort", reapUnlessDetached);
|
|
1799
|
+
consumerStopped = true;
|
|
1800
|
+
if (exitCode === null && !this.guestDetached) reap();
|
|
1801
|
+
// Teardown hygiene (2026-08-18 incident): a reaped turn's unwind
|
|
1802
|
+
// WAITS for the kill-and-confirm exec, so the generator's return —
|
|
1803
|
+
// and the executor's turn settle behind it — never completes while
|
|
1804
|
+
// the guest tree may still be alive holding e.g. codex's
|
|
1805
|
+
// thread-store writer flock. Bounded by the exec's own ceiling
|
|
1806
|
+
// (REAP_EXEC_TIMEOUT_MS); a clean exit never armed it.
|
|
1807
|
+
if (reapConfirm) await reapConfirm;
|
|
1808
|
+
// Opportunistic prompt-file + guard-script + transcript cleanup:
|
|
1809
|
+
// paths are never reused (see uniquePromptPath), so this is hygiene,
|
|
1810
|
+
// not correctness — fire and forget, and a dead sandbox / failed rm
|
|
1811
|
+
// is fine. The shell holds an open fd on the redirect, so unlinking
|
|
1812
|
+
// under a still-exiting CLI is harmless. Skipped on a detach: the
|
|
1813
|
+
// handed-off runner's durable trio must survive it.
|
|
1814
|
+
// The turn's injection window is over either way: a caller landing
|
|
1815
|
+
// after this point must get "unsupported", never a write into a
|
|
1816
|
+
// dead (or handed-off) turn's inbox.
|
|
1817
|
+
this.turnInject = null;
|
|
1818
|
+
if (!this.guestDetached) {
|
|
1819
|
+
void this.sandbox.commands.run(
|
|
1820
|
+
`rm -f ${shellQuote(promptPath)} ${shellQuote(guardPath)} ${shellQuote(`${promptPath}.out`)} ${shellQuote(`${promptPath}.err`)} ${shellQuote(`${promptPath}.pid`)} ${shellQuote(`${promptPath}.hb`)} ${shellQuote(`${promptPath}.busy`)} ${shellQuote(`${promptPath}.tasks`)} ${shellQuote(`${promptPath}.fifo`)} ${shellQuote(`${promptPath}.inbox`)} ${shellQuote(`${promptPath}.delivered`)}`,
|
|
1821
|
+
{ timeoutMs: 10_000 })
|
|
1822
|
+
.catch(() => { /* best-effort */ });
|
|
1823
|
+
}
|
|
731
1824
|
}
|
|
732
1825
|
} catch (err) {
|
|
733
1826
|
yield { type: "error", text: formatError(err), timestamp: now() };
|