@agent-compose/sdk 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +213 -189
- package/dist/agent/agent-context.d.ts +9 -1
- package/dist/agent/agent-loop.d.ts +14 -6
- package/dist/agent/perf-sampler.d.ts +27 -2
- package/dist/agent/run-agent.d.ts +1 -1
- package/dist/client.d.ts +250 -59
- package/dist/directives.d.ts +14 -0
- package/dist/display.d.ts +7 -0
- package/dist/errors.d.ts +1 -1
- package/dist/generated/agentc-commands.d.ts +34 -0
- package/dist/index.d.ts +13 -11
- package/dist/index.js +1692 -194
- package/dist/request-context/request-context.d.ts +1 -1
- package/dist/runtimes/_cli-agent.d.ts +278 -58
- package/dist/runtimes/claude-code.d.ts +90 -1
- package/dist/runtimes/claude.d.ts +1 -1
- package/dist/runtimes/codex.d.ts +94 -6
- package/dist/runtimes/codex.mid-turn-hook.test.d.ts +10 -0
- package/dist/runtimes/openai-desktop.d.ts +50 -0
- package/dist/runtimes/openai-desktop.js +1689 -211
- package/dist/runtimes/openai-desktop.test.d.ts +20 -0
- package/dist/runtimes/opencode.d.ts +48 -11
- package/dist/runtimes/opencode.test.d.ts +14 -0
- package/dist/runtimes/tool-pulse.test.d.ts +17 -0
- package/dist/sandbox/baked-clis.d.ts +75 -0
- package/dist/sandbox/devbox.d.ts +5 -5
- package/dist/sandbox/exec-stream.d.ts +1 -2
- package/dist/sandbox/network-policy.d.ts +23 -5
- package/dist/sandbox/registry.d.ts +12 -0
- package/dist/sandbox/sizes.d.ts +11 -5
- package/dist/sandbox.d.ts +5 -3
- package/dist/step-invocation/protocol.d.ts +3 -4
- package/dist/step-invocation/server.d.ts +2 -2
- package/dist/step-invocation/types.d.ts +2 -2
- package/dist/types/api-conversations.d.ts +513 -27
- package/dist/types/api-factory.d.ts +183 -3
- package/dist/types/api-projects.d.ts +480 -0
- package/dist/types/api-runs.d.ts +8 -0
- package/dist/types/api-scopes.d.ts +32 -3
- package/dist/types/conversation-stream.d.ts +27 -1
- package/dist/types/execution-context.d.ts +1 -1
- package/dist/types/protocol.d.ts +182 -2
- package/dist/types/runtime.d.ts +80 -2
- package/dist/types/workflow-metadata.d.ts +2 -4
- package/dist/types/workflow-plan.d.ts +1 -3
- package/dist/utils/bundler.d.ts +23 -0
- package/dist/workflow-steps/observability.d.ts +2 -3
- package/dist/workflow-steps/runner.d.ts +5 -8
- package/dist/workflow-steps/types.d.ts +8 -10
- package/dist/workflow-steps/workflow.d.ts +2 -1
- package/dist/workflows/engine.d.ts +3 -5
- package/dist/workflows/invoke-child.d.ts +2 -2
- package/package.json +2 -2
- package/src/agent/agent-context.ts +193 -116
- package/src/agent/agent-loop.ts +16 -9
- package/src/agent/desktop-open.ts +13 -1
- package/src/agent/perf-sampler.ts +54 -3
- package/src/agent/run-agent.ts +1 -1
- package/src/client.ts +418 -80
- package/src/directives.ts +21 -1
- package/src/display.ts +12 -0
- package/src/errors.ts +1 -0
- package/src/generated/agentc-commands.ts +571 -0
- package/src/index.ts +65 -18
- package/src/pause/pause-core.ts +2 -1
- package/src/request-context/request-context.ts +1 -1
- package/src/runtimes/_cli-agent.ts +607 -132
- package/src/runtimes/claude-code.ts +427 -20
- package/src/runtimes/claude.ts +1 -1
- package/src/runtimes/codex.ts +188 -19
- package/src/runtimes/openai-desktop.ts +82 -19
- package/src/runtimes/opencode.ts +195 -26
- package/src/sandbox/baked-clis.ts +86 -0
- package/src/sandbox/devbox.ts +5 -5
- package/src/sandbox/exec-stream.ts +1 -2
- package/src/sandbox/network-policy.ts +51 -7
- package/src/sandbox/providers/e2b.ts +63 -19
- package/src/sandbox/providers/vercel.ts +6 -6
- package/src/sandbox/registry.ts +19 -1
- package/src/sandbox/sizes.ts +11 -5
- package/src/sandbox.ts +9 -2
- package/src/step-invocation/invoker.ts +2 -6
- package/src/step-invocation/protocol.ts +3 -4
- package/src/step-invocation/server.ts +2 -2
- package/src/types/api-conversations.ts +424 -29
- package/src/types/api-factory.ts +189 -3
- package/src/types/api-projects.ts +443 -0
- package/src/types/api-runs.ts +5 -0
- package/src/types/api-scopes.ts +32 -3
- package/src/types/conversation-stream.ts +29 -1
- package/src/types/execution-context.ts +1 -1
- package/src/types/protocol.ts +180 -2
- package/src/types/runtime.ts +71 -2
- package/src/types/sandbox-environment.ts +1 -2
- package/src/types/workflow-metadata.ts +2 -4
- package/src/types/workflow-plan.ts +1 -3
- package/src/utils/bundler.ts +88 -19
- package/src/workflow-steps/observability.ts +2 -3
- package/src/workflow-steps/runner.ts +5 -8
- package/src/workflow-steps/types.ts +8 -10
- package/src/workflow-steps/workflow.ts +2 -1
- package/src/workflows/engine.ts +3 -5
- package/src/workflows/invoke-child.ts +2 -2
- package/dist/pause/__tests__/errors.test.d.ts +0 -1
- package/dist/pause/__tests__/wrappers.test.d.ts +0 -1
- package/dist/step-invocation/__tests__/protocol.test.d.ts +0 -1
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* contract.
|
|
5
5
|
*
|
|
6
6
|
* This is a different *mechanism* from the other runtimes: `claudeRuntime`
|
|
7
|
-
* drives the Anthropic Agent SDK and `
|
|
7
|
+
* drives the Anthropic Agent SDK and `createVercelRuntime` drives the Vercel AI SDK,
|
|
8
8
|
* but a CLI-agent runtime spawns the provider's own CLI (`codex exec --json`,
|
|
9
9
|
* `amp -x --stream-json`) — the CLI brings its own agent loop + tools, and we
|
|
10
10
|
* only stream-parse the events it prints. Codex and Amp are the first two; this
|
|
@@ -109,6 +109,67 @@ export function runnerBusySentinelFragment(busyPath: string, tasksPath: string):
|
|
|
109
109
|
* `date +%s%3N` in the same exec, so host clock skew is irrelevant) is
|
|
110
110
|
* stale: three missed writes. */
|
|
111
111
|
export const RUNNER_HEARTBEAT_STALE_MS = RUNNER_HEARTBEAT_INTERVAL_SECONDS * 1000 * 3;
|
|
112
|
+
|
|
113
|
+
// ── The tool-run pulse (child-telemetry contract, 2026-08-29) ───────────────
|
|
114
|
+
// The "no signal 4m" report: a long foreground tool (`bun install`) emits
|
|
115
|
+
// NOTHING on the stream — the CLI only speaks at tool boundaries — so the
|
|
116
|
+
// transcript decayed to "no signal" over a provably-working machine. The
|
|
117
|
+
// heartbeat subshell already beats every 10s; this fragment rides the same
|
|
118
|
+
// beat and samples the RUNNER'S OWN SESSION (every process the setsid tree
|
|
119
|
+
// holds): aggregate CPU jiffies + written bytes + live process count. The
|
|
120
|
+
// durable-trio probe carries the newest sample as its trailing field; the
|
|
121
|
+
// executor compares successive samples — counters ADVANCING is proof the
|
|
122
|
+
// foreground tool is working, fanned to the client as a cheap live
|
|
123
|
+
// `tool_pulse` frame (~one per activity tick, no output content). Absence
|
|
124
|
+
// of advancing pulses while a tool is nominally running is then the REAL
|
|
125
|
+
// no-signal. Cost: one `ps` + a few bounded /proc reads per beat; every
|
|
126
|
+
// read degrades quiet (a guest without procps/proc writes nothing, and
|
|
127
|
+
// the field parses absent — evidence-absent, never an error).
|
|
128
|
+
|
|
129
|
+
/** The sh fragment the heartbeat subshell runs each beat to rewrite the
|
|
130
|
+
* pulse file: `<guest epoch ms>,<cpu jiffies>,<written bytes>,<procs>`
|
|
131
|
+
* for every process in the runner's own session (the setsid tree — the
|
|
132
|
+
* CLI, its foreground tool, their children). Comma-separated so the
|
|
133
|
+
* probe can carry it as ONE whitespace-delimited field. Exported for
|
|
134
|
+
* tests. */
|
|
135
|
+
export function toolPulseFragment(pulsePath: string): string {
|
|
136
|
+
return `ac_sid=$(ps -o sess= -p $$ 2>/dev/null | tr -cd 0-9); `
|
|
137
|
+
// sid "0" would match every process on a platform without real session
|
|
138
|
+
// ids — no sample beats a wrong one.
|
|
139
|
+
+ `if [ -n "$ac_sid" ] && [ "$ac_sid" != "0" ]; then `
|
|
140
|
+
+ `ac_pp=$(ps -eo pid=,sess= 2>/dev/null | awk -v s="$ac_sid" '$2==s{print $1}'); `
|
|
141
|
+
+ `ac_cpu=$(for ac_p in $ac_pp; do sed 's/^.*) //' "/proc/$ac_p/stat" 2>/dev/null; done | awk '{s+=$12+$13} END{printf "%d", s}'); `
|
|
142
|
+
+ `ac_io=$(for ac_p in $ac_pp; do cat "/proc/$ac_p/io" 2>/dev/null; done | awk '/^wchar:/{s+=$2} END{printf "%d", s}'); `
|
|
143
|
+
+ `ac_np=$(printf '%s\\n' "$ac_pp" | grep -c .); `
|
|
144
|
+
+ `printf '%s,%s,%s,%s' "$(date +%s%3N)" "\${ac_cpu:-0}" "\${ac_io:-0}" "\${ac_np:-0}" > ${shellQuote(pulsePath)} 2>/dev/null; `
|
|
145
|
+
+ `fi`;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** One tool-run pulse sample, parsed off the probe's trailing field. */
|
|
149
|
+
export interface TurnToolPulse {
|
|
150
|
+
/** Guest epoch ms when the sample was written. */
|
|
151
|
+
atMs: number;
|
|
152
|
+
/** Aggregate utime+stime jiffies across the runner's session. */
|
|
153
|
+
cpuJiffies: number;
|
|
154
|
+
/** Aggregate written bytes (wchar) across the runner's session. */
|
|
155
|
+
ioBytes: number;
|
|
156
|
+
/** Live processes in the session. */
|
|
157
|
+
procs: number;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Parse the pulse token (`at,cpu,io,procs`). Null on anything malformed —
|
|
161
|
+
* evidence-absent, never an error. Exported for tests. */
|
|
162
|
+
export function parseToolPulseToken(token: string): TurnToolPulse | null {
|
|
163
|
+
const fields = token.split(",");
|
|
164
|
+
if (fields.length !== 4) return null;
|
|
165
|
+
const [atMs, cpuJiffies, ioBytes, procs] = fields.map((f) => Number.parseInt(f, 10));
|
|
166
|
+
if (!Number.isFinite(atMs) || atMs <= 0 || !Number.isFinite(cpuJiffies)
|
|
167
|
+
|| !Number.isFinite(ioBytes) || !Number.isFinite(procs)) return null;
|
|
168
|
+
return { atMs: atMs!, cpuJiffies: cpuJiffies!, ioBytes: ioBytes!, procs: procs! };
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Bound on the pulse token read (4 numeric fields + commas). */
|
|
172
|
+
export const TOOL_PULSE_TOKEN_MAX_CHARS = 96;
|
|
112
173
|
/** Watchdog cadence while a tail stream is attached. Each tick is one fresh
|
|
113
174
|
* short exec; a healthy stream makes every tick a no-op. */
|
|
114
175
|
export const TAIL_WATCHDOG_INTERVAL_MS = 15_000;
|
|
@@ -124,13 +185,72 @@ export const TAIL_ABANDON_AFTER_STREAM_DEATHS = 3;
|
|
|
124
185
|
* (envd HTTP, ~65ms RTT), consumed from the byte offset. Latency-tuned:
|
|
125
186
|
* this is the degraded path, correctness never depends on it being fast. */
|
|
126
187
|
export const DURABLE_POLL_INTERVAL_MS = 1_000;
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
188
|
+
// ── Durable-poll death door (chaos S13 run 3, 2026-08-31) ───────────────────
|
|
189
|
+
// The poll loop's `kill -0` probe answers "pid gone" ONLY through a
|
|
190
|
+
// SUCCESSFUL exec; a DEAD sandbox's exec THROWS, which used to leave the
|
|
191
|
+
// verdict at "unknown" forever — the transport polled a corpse until
|
|
192
|
+
// something outside the SDK killed the turn. These bounds are the honest
|
|
193
|
+
// exit. The K8s-Unknown philosophy still holds: a transport blip is never a
|
|
194
|
+
// death verdict — the generic streak is deliberately generous (a count AND a
|
|
195
|
+
// time window must BOTH be met, and any successful probe or durable read
|
|
196
|
+
// resets everything), while provider-attested not-found evidence
|
|
197
|
+
// (SandboxNotFoundError / "sandbox … not found") is trusted on a much
|
|
198
|
+
// shorter streak because it names the sandbox, not the wire.
|
|
199
|
+
|
|
200
|
+
/** Consecutive provider-attested "sandbox gone" probe failures ⇒ dead. */
|
|
201
|
+
export const PROBE_DEATH_GONE_STREAK = 3;
|
|
202
|
+
/** Not-found evidence must also persist this long (guards a momentary
|
|
203
|
+
* control-plane 404 blip from killing a live turn). */
|
|
204
|
+
export const PROBE_DEATH_GONE_WINDOW_MS = 15_000;
|
|
205
|
+
/** Consecutive GENERIC probe-transport failures (timeouts, connection
|
|
206
|
+
* faults) ⇒ dead — only together with the window below. */
|
|
207
|
+
export const PROBE_DEATH_FAIL_STREAK = 20;
|
|
208
|
+
/** …and the failure run must span at least this long with zero durable
|
|
209
|
+
* evidence of any kind. */
|
|
210
|
+
export const PROBE_DEATH_FAIL_WINDOW_MS = 120_000;
|
|
211
|
+
|
|
212
|
+
/** Provider evidence that the SANDBOX itself is gone — E2B's
|
|
213
|
+
* SandboxNotFoundError / NotFoundError classes (matched by name so no
|
|
214
|
+
* provider import rides in the runtime layer), or their messages after any
|
|
215
|
+
* wrapping ("Paused sandbox X not found" was run 3's exact shape). */
|
|
216
|
+
export function isSandboxGoneError(err: unknown): boolean {
|
|
217
|
+
const name = (err as { name?: unknown } | null)?.name;
|
|
218
|
+
if (name === "SandboxNotFoundError" || name === "NotFoundError") return true;
|
|
219
|
+
const msg = err instanceof Error ? err.message : typeof err === "string" ? err : "";
|
|
220
|
+
return /sandbox[^\n]{0,120}not found|not found[^\n]{0,120}sandbox/i.test(msg);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** One probe-failure run's bookkeeping (reset to zeros on ANY evidence). */
|
|
224
|
+
export interface ProbeDeathState {
|
|
225
|
+
/** Consecutive probe execs that THREW (no verdict at all). */
|
|
226
|
+
failStreak: number;
|
|
227
|
+
/** Consecutive throws that were provider "sandbox gone" evidence. */
|
|
228
|
+
goneStreak: number;
|
|
229
|
+
/** Wall-clock start of the current failure run (0 = no run). */
|
|
230
|
+
firstFailAtMs: number;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** The death-door decision, pure for tests. Null = keep polling. */
|
|
234
|
+
export function probeDeathVerdict(
|
|
235
|
+
s: ProbeDeathState, nowMs: number,
|
|
236
|
+
): "dead-gone" | "dead-silent" | null {
|
|
237
|
+
if (s.firstFailAtMs <= 0) return null;
|
|
238
|
+
const windowMs = nowMs - s.firstFailAtMs;
|
|
239
|
+
if (s.goneStreak >= PROBE_DEATH_GONE_STREAK && windowMs >= PROBE_DEATH_GONE_WINDOW_MS) return "dead-gone";
|
|
240
|
+
if (s.failStreak >= PROBE_DEATH_FAIL_STREAK && windowMs >= PROBE_DEATH_FAIL_WINDOW_MS) return "dead-silent";
|
|
241
|
+
return null;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** The CLI's stderr complaint when the thread it was told to resume does
|
|
245
|
+
* not exist on this machine: claude's `--resume <id>` "No conversation
|
|
246
|
+
* found with session ID" (the 2026-08-18 split-brain forensic) and
|
|
247
|
+
* opencode's `--session <id>` "Error: Session not found" (in the CLI's
|
|
248
|
+
* red-bold ANSI dressing; 1.18.34). When a nonzero exit's stderr carries
|
|
249
|
+
* it, the transport surfaces the stderr as the turn's LAST error event
|
|
250
|
+
* even though a bare result error already streamed — the consumer's
|
|
251
|
+
* classifier needs the specific complaint, not the generic
|
|
252
|
+
* `error_during_execution` token. */
|
|
253
|
+
export const RESUME_TARGET_MISSING_STDERR = /no conversation found with session id|error:\s*(?:\x1b\[[0-9;]*m)*\s*session not found/i;
|
|
134
254
|
|
|
135
255
|
/** One durable-trio reading, parsed from the probe exec's single line. */
|
|
136
256
|
export interface TurnProbeReading {
|
|
@@ -149,6 +269,10 @@ export interface TurnProbeReading {
|
|
|
149
269
|
* sample yet, or a garbled token — evidence-absent, never an error.
|
|
150
270
|
* Telemetry-only: no liveness verdict may ever read it. */
|
|
151
271
|
perf?: GuestPerfSample;
|
|
272
|
+
/** Latest tool-run pulse (the `.pulse` token field, 2026-08-29), when the
|
|
273
|
+
* guest wrote one AND it parsed clean. Telemetry-only, same rule as
|
|
274
|
+
* `perf`: no liveness verdict may ever read it. */
|
|
275
|
+
pulse?: TurnToolPulse;
|
|
152
276
|
}
|
|
153
277
|
|
|
154
278
|
/** The probe exec: one line, six fields, always exit 0 — faults surface as
|
|
@@ -158,15 +282,23 @@ export interface TurnProbeReading {
|
|
|
158
282
|
* whitelist (`tr -cd`) collapses any garbled write to at most one token —
|
|
159
283
|
* it can never add whitespace and desync the field positions. */
|
|
160
284
|
export function turnLivenessProbeCommand(
|
|
161
|
-
paths: { outPath: string; hbPath: string; perfPath: string; pid: number; sentinel: string
|
|
285
|
+
paths: { outPath: string; hbPath: string; perfPath: string; pid: number; sentinel: string;
|
|
286
|
+
pulsePath?: string },
|
|
162
287
|
): string {
|
|
288
|
+
// The 7th field is the tool-run pulse token (`.pulse`, 2026-08-29), `-`
|
|
289
|
+
// when absent — same optional-trailing-field contract as perf; the char
|
|
290
|
+
// whitelist collapses garbled writes to at most one token.
|
|
291
|
+
const pulseRead = paths.pulsePath
|
|
292
|
+
? `pu=$(head -c ${TOOL_PULSE_TOKEN_MAX_CHARS} ${shellQuote(paths.pulsePath)} 2>/dev/null | tr -cd '0-9,'); `
|
|
293
|
+
: `pu=; `;
|
|
163
294
|
return `sz=$(wc -c < ${shellQuote(paths.outPath)} 2>/dev/null || echo -1); `
|
|
164
295
|
+ `alive=0; kill -0 ${paths.pid} 2>/dev/null && alive=1; `
|
|
165
296
|
+ `hb=$(cat ${shellQuote(paths.hbPath)} 2>/dev/null || echo 0); `
|
|
166
297
|
+ `now=$(date +%s%3N); `
|
|
167
298
|
+ `sent=$(grep -c -F ${shellQuote(paths.sentinel)} ${shellQuote(paths.outPath)} 2>/dev/null || echo 0); `
|
|
168
299
|
+ `pf=$(head -c ${PERF_TOKEN_MAX_CHARS} ${shellQuote(paths.perfPath)} 2>/dev/null | tr -cd 'A-Za-z0-9=,.-'); `
|
|
169
|
-
+
|
|
300
|
+
+ pulseRead
|
|
301
|
+
+ `printf '%s %s %s %s %s %s %s\\n' "$sz" "$alive" "$hb" "$now" "$sent" "\${pf:--}" "\${pu:--}"`;
|
|
170
302
|
}
|
|
171
303
|
|
|
172
304
|
/** Parse the probe's line. Null = unusable output (a wedged or faulted exec)
|
|
@@ -186,9 +318,12 @@ export function parseTurnProbeOutput(raw: string): TurnProbeReading | null {
|
|
|
186
318
|
|| !Number.isFinite(nowMs) || !Number.isFinite(sentinelCount)) return null;
|
|
187
319
|
const perfTok = fields[5];
|
|
188
320
|
const perf = perfTok !== undefined && perfTok !== "-" ? parsePerfToken(perfTok) : null;
|
|
321
|
+
const pulseTok = fields[6];
|
|
322
|
+
const pulse = pulseTok !== undefined && pulseTok !== "-" ? parseToolPulseToken(pulseTok) : null;
|
|
189
323
|
return {
|
|
190
324
|
size, alive, hbMs, nowMs, sentinelSeen: sentinelCount > 0,
|
|
191
325
|
...(perf ? { perf } : {}),
|
|
326
|
+
...(pulse ? { pulse } : {}),
|
|
192
327
|
};
|
|
193
328
|
}
|
|
194
329
|
|
|
@@ -204,18 +339,24 @@ export function livenessVerdictFromReading(reading: TurnProbeReading | null): Ru
|
|
|
204
339
|
return "dead";
|
|
205
340
|
}
|
|
206
341
|
|
|
207
|
-
// ── Mid-turn
|
|
208
|
-
//
|
|
209
|
-
//
|
|
210
|
-
//
|
|
211
|
-
//
|
|
212
|
-
//
|
|
213
|
-
//
|
|
214
|
-
//
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
//
|
|
342
|
+
// ── Mid-turn input (the delivery half of injectUserMessage) ─────────────────
|
|
343
|
+
// A message for a RUNNING turn is appended as one line to the turn's durable
|
|
344
|
+
// INBOX file; the DELIVERED counter file beside it is the ack the server
|
|
345
|
+
// trusts, advancing only once the line was handed to the CLI. Two transports
|
|
346
|
+
// hand it over (CliAgentSpec.midTurnInput):
|
|
347
|
+
// - "stdin-stream": the durable transport's stdin is a FIFO fed by an
|
|
348
|
+
// in-guest subshell — the prompt line first, then any complete inbox
|
|
349
|
+
// lines. A steering-capable CLI (claude -p --input-format stream-json)
|
|
350
|
+
// folds a message that arrives while the turn runs into the RUNNING turn
|
|
351
|
+
// at its next tool boundary — verified live against claude 2.1.233. The
|
|
352
|
+
// feeder checks for the CLI's terminal `result` line BEFORE each delivery
|
|
353
|
+
// (a message that races the result stays owed for the follow-up turn,
|
|
354
|
+
// never half-consumed) and stops on it; closing the FIFO is what EOFs the
|
|
355
|
+
// CLI's stdin and lets it exit.
|
|
356
|
+
// - "tool-hook": the CLI runs a platform hook after every tool call it
|
|
357
|
+
// completes (codex: a PostToolUse command hook); the hook forwards the
|
|
358
|
+
// unfed inbox lines to the model as additional context and advances the
|
|
359
|
+
// counter. The CLI's stdin stays the plain prompt file.
|
|
219
360
|
|
|
220
361
|
/** Feeder poll cadence (seconds — it is a shell `sleep`; GNU sleep accepts
|
|
221
362
|
* fractions and the durable transport only runs on the GNU guest image). */
|
|
@@ -232,10 +373,40 @@ export const INJECT_ACK_POLL_ATTEMPTS = 30;
|
|
|
232
373
|
export const INJECT_ACK_POLL_SECONDS = "0.5";
|
|
233
374
|
export const INJECT_ACK_EXEC_TIMEOUT_MS = 25_000;
|
|
234
375
|
|
|
376
|
+
/** How the guest recognises the CLI's terminal `result` event in the durable
|
|
377
|
+
* .out: a line that carries the `"type":"result"` pair and does not open
|
|
378
|
+
* with another event's type, as an awk condition over `$0` (POSIX awk, so
|
|
379
|
+
* dash, busybox and the macOS test hosts read it alike). The CLI does not
|
|
380
|
+
* keep its key order: 2.1.212 wrote `{"type":"result",…` but 2.1.278 and
|
|
381
|
+
* later write the pair near the END of the object
|
|
382
|
+
* (`{"duration_api_ms":…,"result":"…","type":"result","duration_ms":…}`), so
|
|
383
|
+
* the anchored prefix the resident contract used matched nothing on the
|
|
384
|
+
* pinned 2.1.283. Every other event the CLI writes opens with its own type
|
|
385
|
+
* (`{"type":"assistant",…`, `{"type":"stream_event",…`), which keeps a tool
|
|
386
|
+
* call whose input holds the same pair from counting; tool output and model
|
|
387
|
+
* text ride inside JSON strings, where the quotes are escaped. */
|
|
388
|
+
export const CLI_RESULT_LINE_AWK =
|
|
389
|
+
'index($0, "\\"type\\":\\"result\\"") > 0 && '
|
|
390
|
+
+ '(index($0, "{\\"type\\":\\"") != 1 || index($0, "{\\"type\\":\\"result\\"") == 1)';
|
|
391
|
+
|
|
392
|
+
/** An awk program over the lines on stdin (or the files after it) that
|
|
393
|
+
* applies `action` to each result line and runs `end` at the end. */
|
|
394
|
+
function resultLineAwk(action: string, end: string): string {
|
|
395
|
+
return `awk ${shellQuote(`${CLI_RESULT_LINE_AWK} { ${action} } END { ${end} }`)}`;
|
|
396
|
+
}
|
|
397
|
+
|
|
235
398
|
/** The in-guest feeder fragment, prepended to the detached wrapper script.
|
|
236
399
|
* Runs inside the setsid session (group-kill reaps it) and keys its loop to
|
|
237
400
|
* the wrapper's own pid (`$$`), like the heartbeat subshell. Exported for
|
|
238
|
-
* tests.
|
|
401
|
+
* tests.
|
|
402
|
+
*
|
|
403
|
+
* The forward is GUARDED: the fed and delivered counters advance only
|
|
404
|
+
* when `sed` wrote the lines. The loop keys on the wrapper, which outlives
|
|
405
|
+
* a CLI that died without a result line by the sentinel write and the
|
|
406
|
+
* exit push; in that window the FIFO has no reader, `sed` dies of SIGPIPE,
|
|
407
|
+
* and an unguarded counter still advanced, so the ack exited 0 for a line
|
|
408
|
+
* nobody could read and the settle sealed it (review C1). Guarded, the ack
|
|
409
|
+
* sees no advance and exits 4 or 5, and the message stays owed. */
|
|
239
410
|
export function streamInputFeederFragment(paths: {
|
|
240
411
|
promptPath: string; fifoPath: string; inboxPath: string;
|
|
241
412
|
deliveredPath: string; outPath: string;
|
|
@@ -244,11 +415,11 @@ export function streamInputFeederFragment(paths: {
|
|
|
244
415
|
return `rm -f ${q(paths.fifoPath)}; mkfifo ${q(paths.fifoPath)}; `
|
|
245
416
|
+ `: > ${q(paths.inboxPath)}; echo 0 > ${q(paths.deliveredPath)}; `
|
|
246
417
|
+ `( cat ${q(paths.promptPath)}; ac_fed=0; while kill -0 "$$" 2>/dev/null; do `
|
|
247
|
-
+ `if tail -c ${INJECT_FEEDER_RESULT_TAIL_BYTES} ${q(paths.outPath)} 2>/dev/null |
|
|
418
|
+
+ `if tail -c ${INJECT_FEEDER_RESULT_TAIL_BYTES} ${q(paths.outPath)} 2>/dev/null | ${resultLineAwk("f = 1", "exit f ? 0 : 1")}; then break; fi; `
|
|
248
419
|
+ `ac_lines=$(wc -l < ${q(paths.inboxPath)} 2>/dev/null || echo 0); ac_lines=\${ac_lines:-0}; `
|
|
249
420
|
+ `if [ "$ac_lines" -gt "$ac_fed" ]; then `
|
|
250
|
-
+ `sed -n "$((ac_fed+1)),$((ac_lines))p" ${q(paths.inboxPath)}
|
|
251
|
-
+ `echo "$ac_fed" > ${q(paths.deliveredPath)}; fi; `
|
|
421
|
+
+ `sed -n "$((ac_fed+1)),$((ac_lines))p" ${q(paths.inboxPath)} && { ac_fed=$ac_lines; `
|
|
422
|
+
+ `echo "$ac_fed" > ${q(paths.deliveredPath)}; }; fi; `
|
|
252
423
|
+ `sleep ${INJECT_FEEDER_POLL_SECONDS}; done ) > ${q(paths.fifoPath)} 2>/dev/null </dev/null & `;
|
|
253
424
|
}
|
|
254
425
|
|
|
@@ -263,18 +434,44 @@ export function streamInputFeederFragment(paths: {
|
|
|
263
434
|
* activity) MUST wrap through here — the wrapper rides only the wire to
|
|
264
435
|
* the model; the stored conversation row keeps the user's raw text.
|
|
265
436
|
* The text is pinned by tests: change it deliberately or not at all. */
|
|
266
|
-
export function wrapMidTurnUserMessage(text: string): string {
|
|
267
|
-
|
|
437
|
+
export function wrapMidTurnUserMessage(text: string, opts?: MidTurnEnvelopeOptions): string {
|
|
438
|
+
const relayedFrom = opts?.relayedFrom?.trim();
|
|
439
|
+
// A person's own words RELAYED by the agent that owns the worker's thread
|
|
440
|
+
// (review D2, 2026-09-25): the envelope names who said them, because they
|
|
441
|
+
// are not the session user's words and "The user sent" would misattribute
|
|
442
|
+
// them. The relayed row's own text carries the SAID marker and its frame.
|
|
443
|
+
// The thread agent that owns this worker, in its own words (owner decision
|
|
444
|
+
// 2026-09-27): not the session user either. The row's own text closes with
|
|
445
|
+
// its trust frame.
|
|
446
|
+
const head = relayedFrom
|
|
447
|
+
? `A message relaying ${relayedFrom}'s own words arrived while you were working:\n\n`
|
|
448
|
+
: opts?.fromOwnerAgent
|
|
449
|
+
? "The agent that owns your thread sent a message while you were working:\n\n"
|
|
450
|
+
: "The user sent a new message while you were working:\n\n";
|
|
451
|
+
return head
|
|
268
452
|
+ text
|
|
269
453
|
+ "\n\nAddress the message above as you continue this turn.";
|
|
270
454
|
}
|
|
271
455
|
|
|
456
|
+
/** Who a mid-turn message speaks for, when it is not the session user.
|
|
457
|
+
* `relayedFrom` = the display name of the person whose own words an agent
|
|
458
|
+
* relayed verbatim; `fromOwnerAgent` = the thread agent that owns this
|
|
459
|
+
* worker, in its own words; both absent = the session user's own message. */
|
|
460
|
+
export interface MidTurnEnvelopeOptions {
|
|
461
|
+
relayedFrom?: string | null;
|
|
462
|
+
fromOwnerAgent?: boolean;
|
|
463
|
+
}
|
|
464
|
+
|
|
272
465
|
/** The one delivery-ack exec `injectUserMessage` runs: append the message
|
|
273
466
|
* line to the durable inbox, then wait for the feeder's delivered counter
|
|
274
|
-
* to cover it. Exit 0 =
|
|
275
|
-
*
|
|
276
|
-
*
|
|
277
|
-
*
|
|
467
|
+
* to cover it. Exit 0 = written into the CLI's stdin pipe pre-result,
|
|
468
|
+
* which is NOT "the model saw it": the CLI surfaces the line at its next
|
|
469
|
+
* tool boundary, so a kill before then loses it, and the settle's kill
|
|
470
|
+
* terminals hold the anchor at the pre-inject dispatch head for that
|
|
471
|
+
* reason (review D8). 4 = the runner died first; 5 = not forwarded within
|
|
472
|
+
* the ack window (the feeder is not forwarding: it stopped on the result,
|
|
473
|
+
* the resident turn gate is closed, or the guest is crawling). Both
|
|
474
|
+
* non-zero exits mean "leave the message owed". Exported for tests. */
|
|
278
475
|
export function injectAppendAndAckCommand(args: {
|
|
279
476
|
line: string; target: number; pid: number;
|
|
280
477
|
inboxPath: string; deliveredPath: string;
|
|
@@ -288,6 +485,71 @@ export function injectAppendAndAckCommand(args: {
|
|
|
288
485
|
+ `sleep ${INJECT_ACK_POLL_SECONDS}; i=$((i+1)); done; exit 5`;
|
|
289
486
|
}
|
|
290
487
|
|
|
488
|
+
// ── Mid-turn input transports (CliAgentSpec.midTurnInput) ───────────────────
|
|
489
|
+
|
|
490
|
+
/** stdin is a live message stream: the FIFO feeder above delivers the prompt
|
|
491
|
+
* line first and inbox lines after (claude: `--input-format stream-json`). */
|
|
492
|
+
export interface StdinStreamInputSpec {
|
|
493
|
+
transport: "stdin-stream";
|
|
494
|
+
/** Serialise the opening prompt into ONE stream-input stdin line
|
|
495
|
+
* (claude: a stream-json user message). No trailing newline — the
|
|
496
|
+
* transport owns line framing. */
|
|
497
|
+
promptLine(prompt: string): string;
|
|
498
|
+
/** Serialise one mid-turn user message into ONE stdin line. No
|
|
499
|
+
* trailing newline. */
|
|
500
|
+
messageLine(text: string): string;
|
|
501
|
+
/** Serialise ONE in-band interrupt control line (claude: a stream-json
|
|
502
|
+
* `control_request` with subtype "interrupt") — the ESC equivalent.
|
|
503
|
+
* The CLI's control layer handles these immediately, MID-STEP
|
|
504
|
+
* included: the running tool call aborts (its tool_result records the
|
|
505
|
+
* harness's own rejection text), the run ends with an
|
|
506
|
+
* `error_during_execution` result within ~100ms, and the session file
|
|
507
|
+
* stays `--resume`-able with the whole turn context. Verified live
|
|
508
|
+
* against claude 2.1.236. Absent → the runtime has no in-band
|
|
509
|
+
* interrupt; callers fall back to kill semantics. */
|
|
510
|
+
interruptLine?(requestId: string): string;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/** The CLI hands the inbox to the model itself, through a hook it runs after
|
|
514
|
+
* each tool call it completes (codex: the platform's PostToolUse command
|
|
515
|
+
* hook, CODEX_MID_TURN_HOOK_COMMAND in codex.ts). The launch creates the
|
|
516
|
+
* inbox and exports its paths into the CLI's environment
|
|
517
|
+
* (toolHookInboxFragment); the hook reads them back, prints the unfed lines
|
|
518
|
+
* as its additional context for the model and advances the delivered
|
|
519
|
+
* counter. stdin stays the plain prompt file, so there is no in-band
|
|
520
|
+
* interrupt on this transport. */
|
|
521
|
+
export interface ToolHookInputSpec {
|
|
522
|
+
transport: "tool-hook";
|
|
523
|
+
/** Serialise one mid-turn user message into ONE inbox line the hook can
|
|
524
|
+
* splice into its JSON output without decoding: a JSON string literal. */
|
|
525
|
+
messageLine(text: string): string;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
export type MidTurnInputSpec = StdinStreamInputSpec | ToolHookInputSpec;
|
|
529
|
+
|
|
530
|
+
/** The stdin-stream transport of a spec, or undefined. The FIFO feeder, the
|
|
531
|
+
* resident harness, the prewarm lane and the in-band interrupt exist only
|
|
532
|
+
* for this transport; the inject lane itself reads `midTurnInput`. */
|
|
533
|
+
export function stdinStreamInputOf(spec: Pick<CliAgentSpec, "midTurnInput">): StdinStreamInputSpec | undefined {
|
|
534
|
+
return spec.midTurnInput?.transport === "stdin-stream" ? spec.midTurnInput : undefined;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
/** The environment the tool-hook lane hands the CLI, and so the hook: codex
|
|
538
|
+
* runs a command hook with the CLI process's own environment (codex-rs
|
|
539
|
+
* hooks/src/registry.rs Hooks::new, engine/command_runner.rs at
|
|
540
|
+
* rust-v0.160.0). The turn's inbox and delivered-counter paths. */
|
|
541
|
+
export const MID_TURN_INBOX_ENV = "AC_MID_TURN_INBOX";
|
|
542
|
+
export const MID_TURN_DELIVERED_ENV = "AC_MID_TURN_DELIVERED";
|
|
543
|
+
|
|
544
|
+
/** The tool-hook lane's launch fragment, prepended to the detached wrapper
|
|
545
|
+
* script (the shell the CLI starts in): a fresh inbox, the counter at 0,
|
|
546
|
+
* and both paths exported for the hook. Exported for tests. */
|
|
547
|
+
export function toolHookInboxFragment(paths: { inboxPath: string; deliveredPath: string }): string {
|
|
548
|
+
const q = shellQuote;
|
|
549
|
+
return `: > ${q(paths.inboxPath)}; echo 0 > ${q(paths.deliveredPath)}; `
|
|
550
|
+
+ `export ${MID_TURN_INBOX_ENV}=${q(paths.inboxPath)} ${MID_TURN_DELIVERED_ENV}=${q(paths.deliveredPath)}; `;
|
|
551
|
+
}
|
|
552
|
+
|
|
291
553
|
// ── Reap = kill AND CONFIRM (2026-08-18 incident: codex writer lock) ────────
|
|
292
554
|
// A reaped runner must be CONFIRMED dead before the turn unwinds. codex holds
|
|
293
555
|
// a kernel flock on its thread store (`~/.codex/thread-writer-locks/<id>.lock`)
|
|
@@ -335,14 +597,19 @@ export function reapKillAndConfirmCommand(pid: number): string {
|
|
|
335
597
|
// clean sentinel — the probe measured 625ms), and a guest idle watchdog
|
|
336
598
|
// reaps the group if the server's durable record is ever lost. Turn
|
|
337
599
|
// accounting rides two guest numbers next to the inbox:
|
|
338
|
-
// `<prompt>.served` — turns DELIVERED into this process (1 at launch
|
|
339
|
-
//
|
|
600
|
+
// `<prompt>.served` — turns DELIVERED into this process (1 at launch;
|
|
601
|
+
// each adoption rebases it to the result lines it
|
|
602
|
+
// counted + 1, so a result the CLI wrote on its
|
|
603
|
+
// own, such as a background task's wake turn,
|
|
604
|
+
// never desyncs it; the server mirrors it durably
|
|
340
605
|
// as the runner stamp's `turnsServed`);
|
|
341
|
-
// result lines in .out — turns FINISHED (
|
|
342
|
-
// server-side stream-json parse is
|
|
343
|
-
//
|
|
344
|
-
//
|
|
345
|
-
//
|
|
606
|
+
// result lines in .out — turns FINISHED (CLI_RESULT_LINE_AWK — hint-
|
|
607
|
+
// grade; the server-side stream-json parse is
|
|
608
|
+
// authoritative).
|
|
609
|
+
// idle ⇔ results ≥ served. Mid-turn injects ride the same inbox WITHOUT
|
|
610
|
+
// bumping served, and fold into the running turn producing no result of
|
|
611
|
+
// their own. The feeder forwards inbox lines only while results < served,
|
|
612
|
+
// so a line that raced the result is never fed into an idle CLI.
|
|
346
613
|
|
|
347
614
|
/** Guest idle TTL for a resident with no new inbox line: hot window (180s)
|
|
348
615
|
* + margin, so the park gate's graceful `.end` normally wins and the
|
|
@@ -350,11 +617,14 @@ export function reapKillAndConfirmCommand(pid: number): string {
|
|
|
350
617
|
export const RESIDENT_IDLE_TTL_S = 240;
|
|
351
618
|
/** Watchdog poll cadence (seconds). */
|
|
352
619
|
export const RESIDENT_WATCHDOG_POLL_S = 15;
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
* the
|
|
356
|
-
*
|
|
357
|
-
|
|
620
|
+
|
|
621
|
+
/** Shell command printing how many result lines the durable .out holds —
|
|
622
|
+
* the guest's turn accounting (feeder gate, idle check, adoption parity).
|
|
623
|
+
* Prints nothing when the file is missing: callers default an empty count
|
|
624
|
+
* to 0. */
|
|
625
|
+
export function cliResultCountCommand(outPath: string): string {
|
|
626
|
+
return `${resultLineAwk("n++", "print n + 0")} ${shellQuote(outPath)} 2>/dev/null`;
|
|
627
|
+
}
|
|
358
628
|
|
|
359
629
|
/** One bounded exec printing the count of result lines at or past
|
|
360
630
|
* `fromByte` (0-based) in the durable .out. Always exits 0; stdout is the
|
|
@@ -364,41 +634,82 @@ export function residentResultCountCommand(outPath: string, fromByte = 0): strin
|
|
|
364
634
|
const src = fromByte > 0
|
|
365
635
|
? `tail -c +${fromByte + 1} ${q(outPath)} 2>/dev/null`
|
|
366
636
|
: `cat ${q(outPath)} 2>/dev/null`;
|
|
367
|
-
return `ac_res=$(${src} |
|
|
637
|
+
return `ac_res=$(${src} | ${resultLineAwk("n++", "print n + 0")}); echo "\${ac_res:-0}"`;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/** One bounded exec reporting the LAST result line in the durable .out
|
|
641
|
+
* window [`fromByte`, `toByte`): prints `none` (no result line), `error`
|
|
642
|
+
* (its `is_error` is true) or `ok`. Always exits 0 and never prints the
|
|
643
|
+
* line itself. The harvest's confirm for a resident result the tailer never
|
|
644
|
+
* parsed (review C2): a count alone cannot say whether the turn failed. The
|
|
645
|
+
* whole line is searched for `"is_error":true`, wherever the CLI put it
|
|
646
|
+
* (key order is not fixed). `toByte` bounds the window to bytes a drain
|
|
647
|
+
* already walked past, so a result the drain never reached (or a line still
|
|
648
|
+
* being written past its line-aligned offset) is not confirmed; omitted,
|
|
649
|
+
* the window runs to the end of the file. A window with `toByte <= fromByte`
|
|
650
|
+
* is empty. */
|
|
651
|
+
export function residentLastResultCommand(outPath: string, fromByte = 0, toByte?: number): string {
|
|
652
|
+
const q = shellQuote;
|
|
653
|
+
const read = fromByte > 0
|
|
654
|
+
? `tail -c +${fromByte + 1} ${q(outPath)} 2>/dev/null`
|
|
655
|
+
: `cat ${q(outPath)} 2>/dev/null`;
|
|
656
|
+
const src = toByte === undefined ? read : `${read} | head -c ${Math.max(0, toByte - fromByte)}`;
|
|
657
|
+
return `${src} | ${resultLineAwk("last = $0",
|
|
658
|
+
'if (last == "") print "none"; else if (index(last, "\\"is_error\\":true") > 0) print "error"; else print "ok"')}`;
|
|
368
659
|
}
|
|
369
660
|
|
|
370
661
|
/** The resident feeder: `streamInputFeederFragment` with the result-break
|
|
371
|
-
* REPLACED by the end-file break
|
|
372
|
-
*
|
|
373
|
-
*
|
|
662
|
+
* REPLACED by the end-file break (verified live by the spec's probe: two
|
|
663
|
+
* messages, one process, one session file, graceful exit on `.end`), plus
|
|
664
|
+
* the TURN GATE: inbox lines are forwarded only while a delivered turn is
|
|
665
|
+
* still unanswered (the result count in .out is below `.served`),
|
|
666
|
+
* the same two numbers the idle predicate reads. Without the gate, a line
|
|
667
|
+
* appended after the turn's result line but before the server settled the
|
|
668
|
+
* turn was fed into the idle CLI, started a turn nobody tails, and was
|
|
669
|
+
* acked as delivered: the message was never answered, and the extra result
|
|
670
|
+
* line broke the next adoption's parity. With the gate that line stays
|
|
671
|
+
* unfed, the inject ack exits 5, and the message stays owed for the
|
|
672
|
+
* follow-up turn. Adoption bumps `.served` BEFORE it appends, so the next
|
|
673
|
+
* turn's prompt still flows. The count is a full-file grep, re-run
|
|
674
|
+
* only on a poll that has unfed lines AND a changed (.out size, .served)
|
|
675
|
+
* pair, so an idle resident holding a gated line never rescans. The
|
|
676
|
+
* forward is guarded like the plain feeder's: the counters advance only
|
|
677
|
+
* when `sed` wrote the lines into the FIFO. */
|
|
374
678
|
export function residentFeederFragment(paths: {
|
|
375
679
|
promptPath: string; fifoPath: string; inboxPath: string;
|
|
376
|
-
deliveredPath: string; endPath: string;
|
|
680
|
+
deliveredPath: string; endPath: string; outPath: string; servedPath: string;
|
|
377
681
|
}): string {
|
|
378
682
|
const q = shellQuote;
|
|
379
683
|
return `rm -f ${q(paths.fifoPath)}; mkfifo ${q(paths.fifoPath)}; `
|
|
380
684
|
+ `: > ${q(paths.inboxPath)}; echo 0 > ${q(paths.deliveredPath)}; rm -f ${q(paths.endPath)}; `
|
|
381
|
-
+ `( cat ${q(paths.promptPath)}; ac_fed=0; while kill -0 "$$" 2>/dev/null; do `
|
|
685
|
+
+ `( cat ${q(paths.promptPath)}; ac_fed=0; ac_gk=; ac_gr=0; while kill -0 "$$" 2>/dev/null; do `
|
|
382
686
|
+ `if [ -f ${q(paths.endPath)} ]; then break; fi; `
|
|
383
687
|
+ `ac_lines=$(wc -l < ${q(paths.inboxPath)} 2>/dev/null || echo 0); ac_lines=\${ac_lines:-0}; `
|
|
384
688
|
+ `if [ "$ac_lines" -gt "$ac_fed" ]; then `
|
|
385
|
-
+ `
|
|
386
|
-
+ `
|
|
689
|
+
+ `ac_gs=$(wc -c < ${q(paths.outPath)} 2>/dev/null); ac_gv=$(cat ${q(paths.servedPath)} 2>/dev/null); ac_gv=\${ac_gv:-1}; `
|
|
690
|
+
+ `if [ "$ac_gs:$ac_gv" != "$ac_gk" ]; then ac_gk="$ac_gs:$ac_gv"; `
|
|
691
|
+
+ `ac_gr=$(${cliResultCountCommand(paths.outPath)}); ac_gr=\${ac_gr:-0}; fi; `
|
|
692
|
+
+ `if [ "$ac_gr" -lt "$ac_gv" ]; then `
|
|
693
|
+
+ `sed -n "$((ac_fed+1)),$((ac_lines))p" ${q(paths.inboxPath)} && { ac_fed=$ac_lines; `
|
|
694
|
+
+ `echo "$ac_fed" > ${q(paths.deliveredPath)}; }; fi; fi; `
|
|
387
695
|
+ `sleep ${INJECT_FEEDER_POLL_SECONDS}; done ) > ${q(paths.fifoPath)} 2>/dev/null </dev/null & `;
|
|
388
696
|
}
|
|
389
697
|
|
|
390
|
-
/** Sets `ac_idle` (1 = between turns: every delivered turn has its result
|
|
391
|
-
*
|
|
392
|
-
*
|
|
698
|
+
/** Sets `ac_idle` (1 = between turns: every delivered turn has its result).
|
|
699
|
+
* Embedded by the watchdog and the resident heartbeat gate: one idle
|
|
700
|
+
* predicate, stated once. Unfed inbox lines do not make a resident busy:
|
|
701
|
+
* the feeder's turn gate forwards a line only while results < served, so
|
|
702
|
+
* a line still unfed once results >= served is held until the next
|
|
703
|
+
* adoption bumps `.served` (a message that raced the result, owed on the
|
|
704
|
+
* server). Reading it as work would keep the heartbeat fresh and the
|
|
705
|
+
* watchdog quiet for as long as the line sat there. */
|
|
393
706
|
export function residentIdleCheckFragment(paths: {
|
|
394
|
-
outPath: string;
|
|
707
|
+
outPath: string; servedPath: string;
|
|
395
708
|
}): string {
|
|
396
709
|
const q = shellQuote;
|
|
397
|
-
return `ac_res=$(
|
|
710
|
+
return `ac_res=$(${cliResultCountCommand(paths.outPath)}); ac_res=\${ac_res:-0}; `
|
|
398
711
|
+ `ac_srv=$(cat ${q(paths.servedPath)} 2>/dev/null); ac_srv=\${ac_srv:-1}; `
|
|
399
|
-
+ `
|
|
400
|
-
+ `ac_del=$(cat ${q(paths.deliveredPath)} 2>/dev/null); ac_del=\${ac_del:-0}; `
|
|
401
|
-
+ `if [ "$ac_res" -ge "$ac_srv" ] && [ "$ac_in" -eq "$ac_del" ]; then ac_idle=1; else ac_idle=0; fi; `;
|
|
712
|
+
+ `if [ "$ac_res" -ge "$ac_srv" ]; then ac_idle=1; else ac_idle=0; fi; `;
|
|
402
713
|
}
|
|
403
714
|
|
|
404
715
|
/** Guest idle watchdog: reap the whole process group after
|
|
@@ -422,7 +733,7 @@ export function residentIdleCheckFragment(paths: {
|
|
|
422
733
|
* guest rule about not killing, and journal advance is required, not mere
|
|
423
734
|
* process existence). */
|
|
424
735
|
export function residentIdleWatchdogFragment(paths: {
|
|
425
|
-
outPath: string;
|
|
736
|
+
outPath: string; servedPath: string;
|
|
426
737
|
}): string {
|
|
427
738
|
return `( ac_idle_s=0; while kill -0 "$$" 2>/dev/null; do `
|
|
428
739
|
+ residentIdleCheckFragment(paths)
|
|
@@ -489,8 +800,10 @@ function parsePid(raw: string): number {
|
|
|
489
800
|
/** Recover the detached runner's pid from its durable pidfile after the
|
|
490
801
|
* launch exec's STREAM died (deadline_exceeded, canceled context, any wire
|
|
491
802
|
* fault). Reads over the file transport with a short retry so the race
|
|
492
|
-
* where the launch shell is still writing the pidfile is absorbed.
|
|
493
|
-
|
|
803
|
+
* where the launch shell is still writing the pidfile is absorbed.
|
|
804
|
+
* Exported for the server's workflow-lane launch (turn-workflow/tailer.ts
|
|
805
|
+
* launchDetachedRunner), which recovers the same way. */
|
|
806
|
+
export async function recoverLaunchPid(read: (path: string) => Promise<string>, pidPath: string): Promise<number> {
|
|
494
807
|
for (let attempt = 0; attempt < LAUNCH_PID_RECOVERY_ATTEMPTS; attempt++) {
|
|
495
808
|
try {
|
|
496
809
|
const pid = parsePid(await read(pidPath));
|
|
@@ -530,6 +843,23 @@ export function sessionEnvSourceFragment(relPath: string | undefined): string {
|
|
|
530
843
|
return `[ -f "$HOME/${relPath}" ] && . "$HOME/${relPath}" >/dev/null 2>&1 || true; `;
|
|
531
844
|
}
|
|
532
845
|
|
|
846
|
+
/** Shell command printing a fingerprint of the session env file's current
|
|
847
|
+
* bytes (POSIX `cksum`), or `absent` when there is no file. A resident
|
|
848
|
+
* records it next to its prompt immediately BEFORE sourcing the file, and
|
|
849
|
+
* an adoption compares the file's fingerprint at that moment with it: a
|
|
850
|
+
* fresh launch would source the file as it is now, the resident only ever
|
|
851
|
+
* saw the one it started with. Recording before the source keeps the
|
|
852
|
+
* comparison conservative — a write that lands between the two reads as a
|
|
853
|
+
* change. The guest file is the one truth every replica writes to, so the
|
|
854
|
+
* verdict never depends on which server process wrote it. */
|
|
855
|
+
export function sessionEnvFingerprintCommand(relPath: string): string {
|
|
856
|
+
if (!SESSION_ENV_FILE_SAFE.test(relPath) || relPath.includes("..")) {
|
|
857
|
+
throw new Error(`unsafe session env path: ${relPath}`);
|
|
858
|
+
}
|
|
859
|
+
// One group, so a redirect after it (`> <prompt>.envsum`) takes either arm.
|
|
860
|
+
return `{ { cksum < "$HOME/${relPath}"; } 2>/dev/null || echo absent; }`;
|
|
861
|
+
}
|
|
862
|
+
|
|
533
863
|
/** The per-turn model-credential source fragment (`RuntimeOptions.
|
|
534
864
|
* credEnvFile` — an ABSOLUTE guest path), or "" when none is configured.
|
|
535
865
|
* Sourced AFTER the session env file so the turn actor's credential wins.
|
|
@@ -693,9 +1023,9 @@ export interface CliAgentSpec {
|
|
|
693
1023
|
* `effort` is present only when the caller configured a reasoning effort
|
|
694
1024
|
* AND the spec has a real knob for it (see `CliReasoningEffort`).
|
|
695
1025
|
* `streamInput` is true when the durable transport is feeding stdin as a
|
|
696
|
-
* live message stream (
|
|
697
|
-
* then the FIFO the feeder writes, and the CLI must be
|
|
698
|
-
* stream-input mode (claude: `--input-format stream-json`). */
|
|
1026
|
+
* live message stream (the "stdin-stream" `midTurnInput` transport):
|
|
1027
|
+
* `promptPath` is then the FIFO the feeder writes, and the CLI must be
|
|
1028
|
+
* invoked in its stream-input mode (claude: `--input-format stream-json`). */
|
|
699
1029
|
buildCommand(args: { promptPath: string; sessionId?: string; model?: string; cwd?: string; effort?: CliReasoningEffort; streamInput?: boolean }): string;
|
|
700
1030
|
/** Map one parsed JSONL stdout event to `AgentMessage`s. The base emits
|
|
701
1031
|
* `init`/`done`/`error` lifecycle itself, so a spec maps only content +
|
|
@@ -726,27 +1056,22 @@ export interface CliAgentSpec {
|
|
|
726
1056
|
* `extractSessionId`/`mapEvent` members are used only as the version-mismatch
|
|
727
1057
|
* fallback. Absent → the spec is JSONL-only (legacy path always). */
|
|
728
1058
|
acp?: { command: string; args: string[]; env?: Record<string, string> };
|
|
729
|
-
/** Mid-turn
|
|
730
|
-
*
|
|
731
|
-
*
|
|
732
|
-
*
|
|
733
|
-
*
|
|
734
|
-
*
|
|
735
|
-
*
|
|
736
|
-
*
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
promptLine(prompt: string): string;
|
|
746
|
-
/** Serialise one mid-turn user message into ONE stdin line. No
|
|
747
|
-
* trailing newline. */
|
|
748
|
-
messageLine(text: string): string;
|
|
749
|
-
};
|
|
1059
|
+
/** Mid-turn INPUT support (the delivery half of `injectUserMessage`): how
|
|
1060
|
+
* a line appended to the turn's durable inbox reaches the CLI while its
|
|
1061
|
+
* turn runs — see the transports above. Engages only on the durable
|
|
1062
|
+
* detached transport (the inbox, the delivered counter and the feeder or
|
|
1063
|
+
* hook live in the guest; single-exec transports keep the plain
|
|
1064
|
+
* prompt-file stdin). Absent → the runtime cannot take a message
|
|
1065
|
+
* mid-turn: every inject answers "unsupported" and the message waits for
|
|
1066
|
+
* the turn boundary. */
|
|
1067
|
+
midTurnInput?: MidTurnInputSpec;
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
/** The CURRENT turn's mid-turn injection state (durable stream-input
|
|
1071
|
+
* transport only) — see the `turnInject` field. */
|
|
1072
|
+
interface TurnInjectState {
|
|
1073
|
+
inboxPath: string; deliveredPath: string; pid: number;
|
|
1074
|
+
appended: number; chain: Promise<unknown>;
|
|
750
1075
|
}
|
|
751
1076
|
|
|
752
1077
|
/** A spawned ACP-mode CLI: the read side of its stdout and a delivery for its
|
|
@@ -823,27 +1148,74 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
823
1148
|
* transport only). Null = no live stream-input turn — `injectUserMessage`
|
|
824
1149
|
* answers "unsupported" and the caller leaves the message owed. `chain`
|
|
825
1150
|
* serializes appends so each delivery targets a deterministic line count. */
|
|
826
|
-
private turnInject:
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
1151
|
+
private turnInject: TurnInjectState | null = null;
|
|
1152
|
+
|
|
1153
|
+
/** The CURRENT turn's detached-runner wrapper pid (the setsid group
|
|
1154
|
+
* leader echoed by the launch / recovered from the durable pidfile) —
|
|
1155
|
+
* durable-transport turns only. Null between turns and on transports
|
|
1156
|
+
* without a detached guest. See the contract doc (types/runtime.ts). */
|
|
1157
|
+
private turnPid: number | null = null;
|
|
1158
|
+
|
|
1159
|
+
/** Advisory guest-pid identity for the live turn's runner tree — see the
|
|
1160
|
+
* contract doc (types/runtime.ts). Never a liveness verdict. */
|
|
1161
|
+
currentRunnerPid(): number | null {
|
|
1162
|
+
return this.turnPid;
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
/** Safe-to-persist durable byte offset — end of the last line whose
|
|
1166
|
+
* messages have ALL been yielded (boot-time turn adoption; contract in
|
|
1167
|
+
* types/runtime.ts). One line behind the parse cursor by design. */
|
|
1168
|
+
private turnConsumedOffset: number | null = null;
|
|
1169
|
+
/** End offset of the line currently being yielded from — promoted to
|
|
1170
|
+
* `turnConsumedOffset` when the NEXT line is pulled (or at stream end). */
|
|
1171
|
+
private turnPendingLineEnd: number | null = null;
|
|
1172
|
+
|
|
1173
|
+
currentTurnDurableOffset(): number | null {
|
|
1174
|
+
return this.turnConsumedOffset;
|
|
1175
|
+
}
|
|
830
1176
|
|
|
831
1177
|
/** Deliver one user message INTO the live turn — see the contract doc
|
|
832
1178
|
* (types/runtime.ts). Appends a stream-input line to the turn's durable
|
|
833
1179
|
* inbox and waits for the in-guest feeder's delivered-counter ack; only
|
|
834
|
-
* an acked forward (pre-result, into the CLI's stdin)
|
|
835
|
-
* "delivered"
|
|
836
|
-
*
|
|
837
|
-
* the
|
|
838
|
-
*
|
|
839
|
-
|
|
1180
|
+
* an acked forward (pre-result, written into the CLI's stdin pipe)
|
|
1181
|
+
* reports "delivered", and even then the model reads the line only at
|
|
1182
|
+
* its next tool boundary (review D8). Guest exit 5 (ack window exhausted
|
|
1183
|
+
* with the runner still alive: the feeder is not forwarding) reports
|
|
1184
|
+
* "pending": the appended line may still be forwarded later, but it was
|
|
1185
|
+
* NOT seen yet. Never throws. */
|
|
1186
|
+
async injectUserMessage(text: string, opts?: MidTurnEnvelopeOptions): Promise<"delivered" | "pending" | "closed" | "unsupported"> {
|
|
840
1187
|
const inject = this.turnInject;
|
|
841
|
-
const
|
|
842
|
-
if (!inject || !
|
|
1188
|
+
const input = this.spec.midTurnInput;
|
|
1189
|
+
if (!inject || !input || this.guestDetached) return "unsupported";
|
|
843
1190
|
// The address-this envelope (wrapMidTurnUserMessage): a raw injected
|
|
844
1191
|
// line carries no framing, and the model continues its narration
|
|
845
1192
|
// without acknowledging the message.
|
|
846
|
-
|
|
1193
|
+
return this.appendTurnStreamLine(inject, input.messageLine(wrapMidTurnUserMessage(text, opts)));
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
/** Request an in-band step interrupt of the LIVE turn — the ESC
|
|
1197
|
+
* equivalent; see the contract doc (types/runtime.ts). Appends the
|
|
1198
|
+
* spec's interrupt control line to the turn's durable inbox; the guest
|
|
1199
|
+
* feeder forwards it into the CLI's stdin, whose control layer handles
|
|
1200
|
+
* it immediately — MID-STEP included, unlike queued user messages (the
|
|
1201
|
+
* inject "pending" wall applies to turn-loop consumption, not the
|
|
1202
|
+
* control layer). Only "delivered" means the control line reached the
|
|
1203
|
+
* CLI's stdin; callers escalate anything else to kill semantics. Never
|
|
1204
|
+
* throws. */
|
|
1205
|
+
async interruptTurn(): Promise<"delivered" | "pending" | "closed" | "unsupported"> {
|
|
1206
|
+
const inject = this.turnInject;
|
|
1207
|
+
const interruptLine = stdinStreamInputOf(this.spec)?.interruptLine;
|
|
1208
|
+
if (!inject || !interruptLine || this.guestDetached) return "unsupported";
|
|
1209
|
+
return this.appendTurnStreamLine(inject, interruptLine(`itr-${Date.now().toString(36)}`));
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
/** The one durable append+ack path both mid-turn producers share
|
|
1213
|
+
* (`injectUserMessage`'s enveloped message line, `interruptTurn`'s raw
|
|
1214
|
+
* control line). Serialized on the inject chain so every append targets
|
|
1215
|
+
* a deterministic inbox line count. */
|
|
1216
|
+
private appendTurnStreamLine(
|
|
1217
|
+
inject: TurnInjectState, line: string,
|
|
1218
|
+
): Promise<"delivered" | "pending" | "closed"> {
|
|
847
1219
|
const attempt = inject.chain.then(async (): Promise<"delivered" | "pending" | "closed"> => {
|
|
848
1220
|
if (this.turnInject !== inject) return "closed";
|
|
849
1221
|
// The append lands whether or not the ack times out, so the target
|
|
@@ -860,7 +1232,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
860
1232
|
{ timeoutMs: INJECT_ACK_EXEC_TIMEOUT_MS });
|
|
861
1233
|
if (res.exitCode === 0) return "delivered";
|
|
862
1234
|
// 5 = ack window exhausted, runner alive: the line is in the inbox
|
|
863
|
-
// and
|
|
1235
|
+
// and the feeder is not forwarding it (stopped, or gated).
|
|
864
1236
|
// 4 = the runner died first; anything else is a fault — both closed.
|
|
865
1237
|
return res.exitCode === 5 ? "pending" : "closed";
|
|
866
1238
|
} catch { return "closed"; }
|
|
@@ -869,13 +1241,29 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
869
1241
|
return attempt;
|
|
870
1242
|
}
|
|
871
1243
|
|
|
1244
|
+
/** The most recent probe reading's tool-run pulse (2026-08-29), plus the
|
|
1245
|
+
* guest clock it was read against — telemetry only, refreshed by each
|
|
1246
|
+
* `probeTurnLiveness` call. Null between turns / before any probe. */
|
|
1247
|
+
private lastTurnPulse: (TurnToolPulse & { guestNowMs: number }) | null = null;
|
|
1248
|
+
|
|
1249
|
+
/** Peek the newest tool-run pulse the durable probe carried — the
|
|
1250
|
+
* executor's evidence ticker reads it AFTER its liveness check to fan
|
|
1251
|
+
* the tool-progress fact to clients. Never probes on its own. */
|
|
1252
|
+
peekTurnPulse(): (TurnToolPulse & { guestNowMs: number }) | null {
|
|
1253
|
+
return this.lastTurnPulse;
|
|
1254
|
+
}
|
|
1255
|
+
|
|
872
1256
|
/** Durable three-valued liveness for the current turn — see the contract
|
|
873
1257
|
* doc (types/runtime.ts). Never throws; never blocks past the probe's
|
|
874
1258
|
* explicit exec timeout. */
|
|
875
1259
|
async probeTurnLiveness(): Promise<RunnerLivenessVerdict | null> {
|
|
876
1260
|
const probe = this.turnProbe;
|
|
877
1261
|
if (!probe) return null;
|
|
878
|
-
|
|
1262
|
+
const reading = await probe();
|
|
1263
|
+
if (reading?.pulse) {
|
|
1264
|
+
this.lastTurnPulse = { ...reading.pulse, guestNowMs: reading.nowMs };
|
|
1265
|
+
}
|
|
1266
|
+
return livenessVerdictFromReading(reading);
|
|
879
1267
|
}
|
|
880
1268
|
|
|
881
1269
|
/** Wakes the CURRENT tail-watchdog interval early. Armed only while the
|
|
@@ -1291,27 +1679,32 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1291
1679
|
// previous turn's push.
|
|
1292
1680
|
this.turnProbe = null;
|
|
1293
1681
|
this.turnInject = null;
|
|
1682
|
+
this.turnPid = null;
|
|
1683
|
+
this.turnConsumedOffset = null;
|
|
1684
|
+
this.turnPendingLineEnd = null;
|
|
1294
1685
|
this.nudgePending = false;
|
|
1295
1686
|
this.wakeTailWatchdog = null;
|
|
1296
|
-
// Mid-turn
|
|
1297
|
-
//
|
|
1298
|
-
// transport (the
|
|
1299
|
-
// single-exec transports keep the plain
|
|
1300
|
-
|
|
1301
|
-
|
|
1687
|
+
// Mid-turn input engages only where BOTH halves exist: a spec that
|
|
1688
|
+
// declares a mid-turn input transport AND the durable detached
|
|
1689
|
+
// transport (the inbox, the delivered counter and the feeder or hook
|
|
1690
|
+
// live in the guest; single-exec transports keep the plain
|
|
1691
|
+
// prompt-file stdin).
|
|
1692
|
+
const durable = this.sandbox.commands.runBackground !== undefined;
|
|
1693
|
+
const stdinStream = durable ? stdinStreamInputOf(this.spec) : undefined;
|
|
1694
|
+
const toolHook = durable && this.spec.midTurnInput?.transport === "tool-hook";
|
|
1302
1695
|
const fifoPath = `${promptPath}.fifo`;
|
|
1303
|
-
const promptWrite = this.sandbox.files.write(promptPath,
|
|
1304
|
-
? `${
|
|
1696
|
+
const promptWrite = this.sandbox.files.write(promptPath, stdinStream
|
|
1697
|
+
? `${stdinStream.promptLine(opts.prompt)}\n`
|
|
1305
1698
|
: this.spec.promptPayload(opts.prompt));
|
|
1306
1699
|
const cmd = this.spec.buildCommand({
|
|
1307
1700
|
// Stream-input turns read stdin from the feeder's FIFO; the prompt
|
|
1308
1701
|
// file is the feeder's first delivery, not the CLI's stdin.
|
|
1309
|
-
promptPath:
|
|
1702
|
+
promptPath: stdinStream ? fifoPath : promptPath,
|
|
1310
1703
|
sessionId: opts.sessionId,
|
|
1311
1704
|
model: this.model,
|
|
1312
1705
|
cwd: this.options.cwd,
|
|
1313
1706
|
effort: this.configEffort,
|
|
1314
|
-
...(
|
|
1707
|
+
...(stdinStream ? { streamInput: true } : {}),
|
|
1315
1708
|
});
|
|
1316
1709
|
|
|
1317
1710
|
// Frame guard (see _jsonl-guard.ts): the CLI's stdout is piped through
|
|
@@ -1362,12 +1755,20 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1362
1755
|
// - SINGLE-EXEC STREAMING (everything else: local child_process,
|
|
1363
1756
|
// Vercel): the pre-incident shape — one exec, stdout streamed,
|
|
1364
1757
|
// exit code from the exec result.
|
|
1365
|
-
const lines = new AsyncQueue<string>();
|
|
1758
|
+
const lines = new AsyncQueue<{ line: string; end: number | null }>();
|
|
1366
1759
|
// How the turn ended: the CLI's exit code once known (null while
|
|
1367
1760
|
// running), plus a best-effort stderr tail for the error message.
|
|
1368
1761
|
let exitCode: number | null = null;
|
|
1369
1762
|
let exitStderr = "";
|
|
1370
1763
|
let transportError: unknown = null;
|
|
1764
|
+
/** True when the death door closed on PROVIDER-ATTESTED "sandbox
|
|
1765
|
+
* gone" evidence (dead-gone) — the machine itself died, not the CLI.
|
|
1766
|
+
* The error detail then says so in the classifier's sandbox_gone
|
|
1767
|
+
* vocabulary (turn-failure-class.ts), which routes the settle to the
|
|
1768
|
+
* unconditional machine-loss auto-continue lane; an answered
|
|
1769
|
+
* "pid gone" on a LIVE machine (a CLI that crashed on its own) stays
|
|
1770
|
+
* the evidence-gated runner_exit family. */
|
|
1771
|
+
let sandboxGoneDeath = false;
|
|
1371
1772
|
let consumerStopped = false;
|
|
1372
1773
|
let reap: () => void = () => { /* set per transport */ };
|
|
1373
1774
|
// The reap's kill-and-confirm exec, latched on first fire (reap runs
|
|
@@ -1421,23 +1822,32 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1421
1822
|
// supersede group-kill still reaps this subshell with the tree.
|
|
1422
1823
|
const busyPath = `${promptPath}.busy`;
|
|
1423
1824
|
const tasksPath = `${promptPath}.tasks`;
|
|
1424
|
-
// Mid-turn
|
|
1825
|
+
// Mid-turn input: the stdin-stream feeder subshell (see
|
|
1425
1826
|
// streamInputFeederFragment) rides at the FRONT of the detached
|
|
1426
|
-
// script so the FIFO exists before the CLI opens it as stdin
|
|
1827
|
+
// script so the FIFO exists before the CLI opens it as stdin; the
|
|
1828
|
+
// tool-hook lane creates the inbox here and exports its paths into
|
|
1829
|
+
// the CLI's environment, for the hook the CLI runs at each tool step.
|
|
1427
1830
|
const inboxPath = `${promptPath}.inbox`;
|
|
1428
1831
|
const deliveredPath = `${promptPath}.delivered`;
|
|
1429
|
-
const feeder =
|
|
1832
|
+
const feeder = stdinStream
|
|
1430
1833
|
? streamInputFeederFragment({ promptPath, fifoPath, inboxPath, deliveredPath, outPath })
|
|
1431
|
-
:
|
|
1834
|
+
: toolHook
|
|
1835
|
+
? toolHookInboxFragment({ inboxPath, deliveredPath })
|
|
1836
|
+
: "";
|
|
1432
1837
|
// Perf sampling rides the same beat: every 6th beat (~60s) the
|
|
1433
1838
|
// subshell burst-reads /proc and rewrites the `.perf` token the
|
|
1434
1839
|
// durable-trio probe carries as its trailing field (perf-sampler.ts).
|
|
1435
1840
|
const perfPath = `${promptPath}.perf`;
|
|
1841
|
+
// The tool-run pulse (2026-08-29, child-telemetry contract): the
|
|
1842
|
+
// same beat samples the session's own process tree so a long
|
|
1843
|
+
// foreground tool proves it is working — see toolPulseFragment.
|
|
1844
|
+
const pulsePath = `${promptPath}.pulse`;
|
|
1436
1845
|
const detachedScript =
|
|
1437
1846
|
feeder
|
|
1438
1847
|
+ `( ${perfSamplerFunctionFragment({ perfPath })}while kill -0 "$$" 2>/dev/null; do `
|
|
1439
1848
|
+ `date +%s%3N > ${shellQuote(hbPath)}; `
|
|
1440
1849
|
+ `${runnerBusySentinelFragment(busyPath, tasksPath)}; `
|
|
1850
|
+
+ `${toolPulseFragment(pulsePath)}; `
|
|
1441
1851
|
+ `ac_perf_tick; `
|
|
1442
1852
|
+ `sleep ${RUNNER_HEARTBEAT_INTERVAL_SECONDS}; done ) >/dev/null 2>&1 </dev/null & `
|
|
1443
1853
|
+ sessionEnvSourceFragment(this.options.sessionEnvFile)
|
|
@@ -1502,7 +1912,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1502
1912
|
// executor's evidence ticker (`probeTurnLiveness`) both read through
|
|
1503
1913
|
// it — one fresh short exec per call, explicit timeout, faults map to
|
|
1504
1914
|
// null ("probe-failed"), never to a verdict.
|
|
1505
|
-
const probeCmd = turnLivenessProbeCommand({ outPath, hbPath, perfPath, pid, sentinel });
|
|
1915
|
+
const probeCmd = turnLivenessProbeCommand({ outPath, hbPath, perfPath, pid, sentinel, pulsePath });
|
|
1506
1916
|
const runTurnProbe = async (): Promise<TurnProbeReading | null> => {
|
|
1507
1917
|
try {
|
|
1508
1918
|
const res = await this.sandbox.commands.run(probeCmd, { timeoutMs: TURN_PROBE_TIMEOUT_MS });
|
|
@@ -1510,9 +1920,19 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1510
1920
|
} catch { return null; }
|
|
1511
1921
|
};
|
|
1512
1922
|
this.turnProbe = runTurnProbe;
|
|
1923
|
+
// Advisory pid identity for the platform's background-work declare
|
|
1924
|
+
// (in-harness Workflow tasks) — armed with the probe, cleared at the
|
|
1925
|
+
// next turn's start (the tree may legitimately outlive this turn).
|
|
1926
|
+
this.turnPid = pid;
|
|
1927
|
+
// Durable launch report (boot-time turn adoption): hand the caller
|
|
1928
|
+
// the guest prompt path + sentinel + pid so it can stamp them on
|
|
1929
|
+
// the turn row — a successor process can then find the runner's
|
|
1930
|
+
// durable files without this process's memory.
|
|
1931
|
+
this.options.onDetachedLaunch?.({ promptPath, sentinel, pid });
|
|
1513
1932
|
// Arm mid-turn injection now that the detached runner (and its
|
|
1514
|
-
// feeder) exist; cleared with the turn in the finally
|
|
1515
|
-
|
|
1933
|
+
// feeder or hook inbox) exist; cleared with the turn in the finally
|
|
1934
|
+
// below.
|
|
1935
|
+
if (stdinStream !== undefined || toolHook) {
|
|
1516
1936
|
this.turnInject = { inboxPath, deliveredPath, pid, appended: 0, chain: Promise.resolve() };
|
|
1517
1937
|
}
|
|
1518
1938
|
|
|
@@ -1529,7 +1949,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1529
1949
|
exitCode = Number.isFinite(code) ? code : 0;
|
|
1530
1950
|
return;
|
|
1531
1951
|
}
|
|
1532
|
-
lines.push(line);
|
|
1952
|
+
lines.push({ line, end: offset });
|
|
1533
1953
|
};
|
|
1534
1954
|
|
|
1535
1955
|
reap = () => {
|
|
@@ -1558,6 +1978,13 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1558
1978
|
// re-attaching tails and finishes on durable polling alone (a wire
|
|
1559
1979
|
// that killed three streams will kill the fourth).
|
|
1560
1980
|
let streamDeaths = 0;
|
|
1981
|
+
// Death-door bookkeeping (see probeDeathVerdict): the current
|
|
1982
|
+
// run of consecutive probe failures. ANY evidence — a probe exec
|
|
1983
|
+
// that answered, a durable read that succeeded — resets it.
|
|
1984
|
+
const probeDeath: ProbeDeathState = { failStreak: 0, goneStreak: 0, firstFailAtMs: 0 };
|
|
1985
|
+
const resetProbeDeath = (): void => {
|
|
1986
|
+
probeDeath.failStreak = 0; probeDeath.goneStreak = 0; probeDeath.firstFailAtMs = 0;
|
|
1987
|
+
};
|
|
1561
1988
|
while (exitCode === null && !consumerStopped && !opts.signal?.aborted) {
|
|
1562
1989
|
let buf = "";
|
|
1563
1990
|
let handle: SandboxBackgroundProcess | null = null;
|
|
@@ -1672,6 +2099,9 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1672
2099
|
if (durableRead) {
|
|
1673
2100
|
try {
|
|
1674
2101
|
const bytes = Buffer.from(await durableRead(outPath), "utf8");
|
|
2102
|
+
// The file plane answered — the sandbox is reachable, so no
|
|
2103
|
+
// probe-failure run may accumulate toward the death door.
|
|
2104
|
+
resetProbeDeath();
|
|
1675
2105
|
let rest = bytes.subarray(offset).toString("utf8");
|
|
1676
2106
|
let nl: number;
|
|
1677
2107
|
while (exitCode === null && (nl = rest.indexOf("\n")) >= 0) {
|
|
@@ -1681,18 +2111,43 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1681
2111
|
} catch { /* file plane briefly unreachable — retried below */ }
|
|
1682
2112
|
}
|
|
1683
2113
|
if (exitCode !== null) break;
|
|
1684
|
-
// Runner still alive? A FRESH exec answers;
|
|
1685
|
-
// keeps retrying —
|
|
1686
|
-
//
|
|
2114
|
+
// Runner still alive? A FRESH exec answers; a wire-fault throw
|
|
2115
|
+
// is no verdict and keeps retrying — but never FOREVER (chaos
|
|
2116
|
+
// S13 run 3, 2026-08-31: a DEAD sandbox's probe exec THROWS, so
|
|
2117
|
+
// `alive` stayed null and this loop polled the corpse blind
|
|
2118
|
+
// until the executor closed the turn from outside). The death
|
|
2119
|
+
// door: provider-attested "sandbox gone" evidence on a short
|
|
2120
|
+
// streak, or a long generous window of nothing-but-failures
|
|
2121
|
+
// (probeDeathVerdict) ⇒ the same honest abnormal-death terminal
|
|
2122
|
+
// as an answered "pid gone" — never a clean done, never an
|
|
2123
|
+
// eternal poll. Any evidence (an answered probe, a durable
|
|
2124
|
+
// read) resets the run, so a live sandbox behind a flaky wire
|
|
2125
|
+
// is never killed (the K8s-Unknown rule).
|
|
1687
2126
|
let alive: boolean | null = null;
|
|
1688
2127
|
try {
|
|
1689
2128
|
const probe = await this.sandbox.commands.run(
|
|
1690
|
-
`kill -0 ${pid} 2>/dev/null`, { timeoutMs:
|
|
2129
|
+
`kill -0 ${pid} 2>/dev/null`, { timeoutMs: TURN_PROBE_TIMEOUT_MS });
|
|
1691
2130
|
alive = probe.exitCode === 0;
|
|
1692
|
-
|
|
1693
|
-
|
|
2131
|
+
resetProbeDeath();
|
|
2132
|
+
} catch (probeErr) {
|
|
2133
|
+
alive = null;
|
|
2134
|
+
probeDeath.failStreak += 1;
|
|
2135
|
+
if (probeDeath.firstFailAtMs === 0) probeDeath.firstFailAtMs = Date.now();
|
|
2136
|
+
probeDeath.goneStreak = isSandboxGoneError(probeErr) ? probeDeath.goneStreak + 1 : 0;
|
|
2137
|
+
}
|
|
2138
|
+
const deathVerdict = probeDeathVerdict(probeDeath, Date.now());
|
|
2139
|
+
if (deathVerdict !== null) {
|
|
2140
|
+
console.warn(`[cli-agent] ${this.spec.kind} durable-poll death door: ${deathVerdict === "dead-gone"
|
|
2141
|
+
? `the provider reports the sandbox gone (${probeDeath.goneStreak} consecutive not-found probes over ${Math.round((Date.now() - probeDeath.firstFailAtMs) / 1000)}s)`
|
|
2142
|
+
: `${probeDeath.failStreak} consecutive probe-transport failures over ${Math.round((Date.now() - probeDeath.firstFailAtMs) / 1000)}s with no durable evidence`
|
|
2143
|
+
} — treating the runner as dead`);
|
|
2144
|
+
}
|
|
2145
|
+
if (alive === false || deathVerdict !== null) {
|
|
1694
2146
|
// Dead with no exit line even in the durable file: abnormal
|
|
1695
|
-
// death — surfaces as an error, never a clean done.
|
|
2147
|
+
// death — surfaces as an error, never a clean done. A
|
|
2148
|
+
// provider-attested gone verdict marks the death as the
|
|
2149
|
+
// MACHINE's (sandbox_gone), not the CLI's.
|
|
2150
|
+
if (deathVerdict === "dead-gone") sandboxGoneDeath = true;
|
|
1696
2151
|
exitCode = JSONL_GUARD_NO_SENTINEL_EXIT;
|
|
1697
2152
|
break;
|
|
1698
2153
|
}
|
|
@@ -1722,7 +2177,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1722
2177
|
while ((nl = buf.indexOf("\n")) >= 0) {
|
|
1723
2178
|
const line = buf.slice(0, nl).trim();
|
|
1724
2179
|
buf = buf.slice(nl + 1);
|
|
1725
|
-
if (line) lines.push(line);
|
|
2180
|
+
if (line) lines.push({ line, end: null });
|
|
1726
2181
|
}
|
|
1727
2182
|
};
|
|
1728
2183
|
const runOpts = {
|
|
@@ -1734,7 +2189,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1734
2189
|
credEnvSourceFragment(this.options.credEnvFile) + guardedCmd, runOpts).then(
|
|
1735
2190
|
(res) => {
|
|
1736
2191
|
const tail = buf.trim();
|
|
1737
|
-
if (tail) lines.push(tail);
|
|
2192
|
+
if (tail) lines.push({ line: tail, end: null });
|
|
1738
2193
|
exitCode = res.exitCode;
|
|
1739
2194
|
exitStderr = (res.stderr ?? "").slice(-2000);
|
|
1740
2195
|
},
|
|
@@ -1761,7 +2216,16 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1761
2216
|
// when the turn never reaches its `done` (a reaped/superseded turn
|
|
1762
2217
|
// must still be resumable by its redispatch).
|
|
1763
2218
|
let announcedSessionId = opts.sessionId ?? "";
|
|
1764
|
-
for await (const
|
|
2219
|
+
for await (const item of lines) {
|
|
2220
|
+
// Consumed-offset bookkeeping (boot-time turn adoption): pulling
|
|
2221
|
+
// line N proves lines 1..N-1 are FULLY yielded, so the byte
|
|
2222
|
+
// offset just past line N-1 is the safe harvest watermark — see
|
|
2223
|
+
// `currentTurnDurableOffset` in types/runtime.ts.
|
|
2224
|
+
if (item.end !== null) {
|
|
2225
|
+
this.turnConsumedOffset = this.turnPendingLineEnd;
|
|
2226
|
+
this.turnPendingLineEnd = item.end;
|
|
2227
|
+
}
|
|
2228
|
+
const line = item.line;
|
|
1765
2229
|
let parsed: Record<string, unknown>;
|
|
1766
2230
|
try {
|
|
1767
2231
|
parsed = JSON.parse(line) as Record<string, unknown>;
|
|
@@ -1781,6 +2245,9 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1781
2245
|
}
|
|
1782
2246
|
|
|
1783
2247
|
await transport;
|
|
2248
|
+
// Stream fully consumed — the final line's messages are all
|
|
2249
|
+
// yielded, so its end offset graduates to the safe watermark.
|
|
2250
|
+
this.turnConsumedOffset = this.turnPendingLineEnd;
|
|
1784
2251
|
if (transportError) throw transportError;
|
|
1785
2252
|
if (exitCode === null) {
|
|
1786
2253
|
// Aborted/reaped before an exit line existed — honest terminal,
|
|
@@ -1792,7 +2259,15 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1792
2259
|
}
|
|
1793
2260
|
if (exitCode !== 0 && !sawError) {
|
|
1794
2261
|
const detail = exitCode === JSONL_GUARD_NO_SENTINEL_EXIT
|
|
1795
|
-
?
|
|
2262
|
+
? sandboxGoneDeath
|
|
2263
|
+
// The classifier's sandbox_gone vocabulary, verbatim on
|
|
2264
|
+
// purpose (turn-failure-class.ts:"machine went away" /
|
|
2265
|
+
// "sandbox gone") — this is what routes the settle to the
|
|
2266
|
+
// machine-loss auto-continue lane instead of the
|
|
2267
|
+
// evidence-gated runner_exit family.
|
|
2268
|
+
? " (the machine went away mid-turn — the provider reports the sandbox gone)"
|
|
2269
|
+
: " (the runner died without reporting an exit code)"
|
|
2270
|
+
: "";
|
|
1796
2271
|
yield { type: "error", text: `${this.spec.kind} exited with code ${exitCode}${detail}${exitStderr ? `: ${exitStderr}` : ""}`, timestamp: now() };
|
|
1797
2272
|
return;
|
|
1798
2273
|
}
|