@agent-compose/sdk 0.8.4 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent-context.d.ts +9 -1
- package/dist/agent/agent-loop.d.ts +10 -1
- package/dist/client.d.ts +171 -33
- package/dist/directives.d.ts +14 -0
- package/dist/generated/verb-synopsis.d.ts +34 -0
- package/dist/index.d.ts +6 -4
- package/dist/index.js +1024 -39
- package/dist/runtimes/_cli-agent.d.ts +106 -0
- package/dist/runtimes/claude-code.d.ts +31 -1
- package/dist/runtimes/openai-desktop.d.ts +50 -0
- package/dist/runtimes/openai-desktop.js +1048 -57
- package/dist/runtimes/openai-desktop.test.d.ts +20 -0
- package/dist/runtimes/tool-pulse.test.d.ts +17 -0
- package/dist/sandbox/devbox.d.ts +5 -5
- package/dist/sandbox/registry.d.ts +12 -0
- package/dist/sandbox/sizes.d.ts +11 -5
- package/dist/sandbox.d.ts +1 -1
- package/dist/step-invocation/types.d.ts +1 -1
- package/dist/types/api-conversations.d.ts +85 -12
- package/dist/types/api-factory.d.ts +111 -1
- package/dist/types/conversation-stream.d.ts +22 -1
- package/dist/types/protocol.d.ts +118 -1
- package/dist/types/runtime.d.ts +71 -0
- package/package.json +1 -1
- package/src/agent/agent-context.ts +43 -9
- package/src/agent/agent-loop.ts +11 -5
- package/src/agent/desktop-open.ts +13 -1
- package/src/client.ts +256 -38
- package/src/directives.ts +21 -1
- package/src/generated/verb-synopsis.ts +544 -0
- package/src/index.ts +17 -3
- package/src/runtimes/_cli-agent.ts +313 -22
- package/src/runtimes/claude-code.ts +249 -12
- package/src/runtimes/openai-desktop.ts +82 -19
- package/src/sandbox/devbox.ts +5 -5
- package/src/sandbox/providers/e2b.ts +60 -16
- package/src/sandbox/registry.ts +19 -1
- package/src/sandbox/sizes.ts +11 -5
- package/src/sandbox.ts +1 -0
- package/src/types/api-conversations.ts +65 -13
- package/src/types/api-factory.ts +121 -1
- package/src/types/conversation-stream.ts +24 -1
- package/src/types/protocol.ts +113 -1
- package/src/types/runtime.ts +63 -0
|
@@ -109,6 +109,67 @@ export function runnerBusySentinelFragment(busyPath: string, tasksPath: string):
|
|
|
109
109
|
* `date +%s%3N` in the same exec, so host clock skew is irrelevant) is
|
|
110
110
|
* stale: three missed writes. */
|
|
111
111
|
export const RUNNER_HEARTBEAT_STALE_MS = RUNNER_HEARTBEAT_INTERVAL_SECONDS * 1000 * 3;
|
|
112
|
+
|
|
113
|
+
// ── The tool-run pulse (child-telemetry contract, 2026-08-29) ───────────────
|
|
114
|
+
// The "no signal 4m" report: a long foreground tool (`bun install`) emits
|
|
115
|
+
// NOTHING on the stream — the CLI only speaks at tool boundaries — so the
|
|
116
|
+
// transcript decayed to "no signal" over a provably-working machine. The
|
|
117
|
+
// heartbeat subshell already beats every 10s; this fragment rides the same
|
|
118
|
+
// beat and samples the RUNNER'S OWN SESSION (every process the setsid tree
|
|
119
|
+
// holds): aggregate CPU jiffies + written bytes + live process count. The
|
|
120
|
+
// durable-trio probe carries the newest sample as its trailing field; the
|
|
121
|
+
// executor compares successive samples — counters ADVANCING is proof the
|
|
122
|
+
// foreground tool is working, fanned to the client as a cheap live
|
|
123
|
+
// `tool_pulse` frame (~one per activity tick, no output content). Absence
|
|
124
|
+
// of advancing pulses while a tool is nominally running is then the REAL
|
|
125
|
+
// no-signal. Cost: one `ps` + a few bounded /proc reads per beat; every
|
|
126
|
+
// read degrades quiet (a guest without procps/proc writes nothing, and
|
|
127
|
+
// the field parses absent — evidence-absent, never an error).
|
|
128
|
+
|
|
129
|
+
/** The sh fragment the heartbeat subshell runs each beat to rewrite the
|
|
130
|
+
* pulse file: `<guest epoch ms>,<cpu jiffies>,<written bytes>,<procs>`
|
|
131
|
+
* for every process in the runner's own session (the setsid tree — the
|
|
132
|
+
* CLI, its foreground tool, their children). Comma-separated so the
|
|
133
|
+
* probe can carry it as ONE whitespace-delimited field. Exported for
|
|
134
|
+
* tests. */
|
|
135
|
+
export function toolPulseFragment(pulsePath: string): string {
|
|
136
|
+
return `ac_sid=$(ps -o sess= -p $$ 2>/dev/null | tr -cd 0-9); `
|
|
137
|
+
// sid "0" would match every process on a platform without real session
|
|
138
|
+
// ids — no sample beats a wrong one.
|
|
139
|
+
+ `if [ -n "$ac_sid" ] && [ "$ac_sid" != "0" ]; then `
|
|
140
|
+
+ `ac_pp=$(ps -eo pid=,sess= 2>/dev/null | awk -v s="$ac_sid" '$2==s{print $1}'); `
|
|
141
|
+
+ `ac_cpu=$(for ac_p in $ac_pp; do sed 's/^.*) //' "/proc/$ac_p/stat" 2>/dev/null; done | awk '{s+=$12+$13} END{printf "%d", s}'); `
|
|
142
|
+
+ `ac_io=$(for ac_p in $ac_pp; do cat "/proc/$ac_p/io" 2>/dev/null; done | awk '/^wchar:/{s+=$2} END{printf "%d", s}'); `
|
|
143
|
+
+ `ac_np=$(printf '%s\\n' "$ac_pp" | grep -c .); `
|
|
144
|
+
+ `printf '%s,%s,%s,%s' "$(date +%s%3N)" "\${ac_cpu:-0}" "\${ac_io:-0}" "\${ac_np:-0}" > ${shellQuote(pulsePath)} 2>/dev/null; `
|
|
145
|
+
+ `fi`;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** One tool-run pulse sample, parsed off the probe's trailing field. */
|
|
149
|
+
export interface TurnToolPulse {
|
|
150
|
+
/** Guest epoch ms when the sample was written. */
|
|
151
|
+
atMs: number;
|
|
152
|
+
/** Aggregate utime+stime jiffies across the runner's session. */
|
|
153
|
+
cpuJiffies: number;
|
|
154
|
+
/** Aggregate written bytes (wchar) across the runner's session. */
|
|
155
|
+
ioBytes: number;
|
|
156
|
+
/** Live processes in the session. */
|
|
157
|
+
procs: number;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Parse the pulse token (`at,cpu,io,procs`). Null on anything malformed —
|
|
161
|
+
* evidence-absent, never an error. Exported for tests. */
|
|
162
|
+
export function parseToolPulseToken(token: string): TurnToolPulse | null {
|
|
163
|
+
const fields = token.split(",");
|
|
164
|
+
if (fields.length !== 4) return null;
|
|
165
|
+
const [atMs, cpuJiffies, ioBytes, procs] = fields.map((f) => Number.parseInt(f, 10));
|
|
166
|
+
if (!Number.isFinite(atMs) || atMs <= 0 || !Number.isFinite(cpuJiffies)
|
|
167
|
+
|| !Number.isFinite(ioBytes) || !Number.isFinite(procs)) return null;
|
|
168
|
+
return { atMs: atMs!, cpuJiffies: cpuJiffies!, ioBytes: ioBytes!, procs: procs! };
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Bound on the pulse token read (4 numeric fields + commas). */
|
|
172
|
+
export const TOOL_PULSE_TOKEN_MAX_CHARS = 96;
|
|
112
173
|
/** Watchdog cadence while a tail stream is attached. Each tick is one fresh
|
|
113
174
|
* short exec; a healthy stream makes every tick a no-op. */
|
|
114
175
|
export const TAIL_WATCHDOG_INTERVAL_MS = 15_000;
|
|
@@ -124,6 +185,62 @@ export const TAIL_ABANDON_AFTER_STREAM_DEATHS = 3;
|
|
|
124
185
|
* (envd HTTP, ~65ms RTT), consumed from the byte offset. Latency-tuned:
|
|
125
186
|
* this is the degraded path, correctness never depends on it being fast. */
|
|
126
187
|
export const DURABLE_POLL_INTERVAL_MS = 1_000;
|
|
188
|
+
// ── Durable-poll death door (chaos S13 run 3, 2026-08-31) ───────────────────
|
|
189
|
+
// The poll loop's `kill -0` probe answers "pid gone" ONLY through a
|
|
190
|
+
// SUCCESSFUL exec; a DEAD sandbox's exec THROWS, which used to leave the
|
|
191
|
+
// verdict at "unknown" forever — the transport polled a corpse until
|
|
192
|
+
// something outside the SDK killed the turn. These bounds are the honest
|
|
193
|
+
// exit. The K8s-Unknown philosophy still holds: a transport blip is never a
|
|
194
|
+
// death verdict — the generic streak is deliberately generous (a count AND a
|
|
195
|
+
// time window must BOTH be met, and any successful probe or durable read
|
|
196
|
+
// resets everything), while provider-attested not-found evidence
|
|
197
|
+
// (SandboxNotFoundError / "sandbox … not found") is trusted on a much
|
|
198
|
+
// shorter streak because it names the sandbox, not the wire.
|
|
199
|
+
|
|
200
|
+
/** Consecutive provider-attested "sandbox gone" probe failures ⇒ dead. */
|
|
201
|
+
export const PROBE_DEATH_GONE_STREAK = 3;
|
|
202
|
+
/** Not-found evidence must also persist this long (guards a momentary
|
|
203
|
+
* control-plane 404 blip from killing a live turn). */
|
|
204
|
+
export const PROBE_DEATH_GONE_WINDOW_MS = 15_000;
|
|
205
|
+
/** Consecutive GENERIC probe-transport failures (timeouts, connection
|
|
206
|
+
* faults) ⇒ dead — only together with the window below. */
|
|
207
|
+
export const PROBE_DEATH_FAIL_STREAK = 20;
|
|
208
|
+
/** …and the failure run must span at least this long with zero durable
|
|
209
|
+
* evidence of any kind. */
|
|
210
|
+
export const PROBE_DEATH_FAIL_WINDOW_MS = 120_000;
|
|
211
|
+
|
|
212
|
+
/** Provider evidence that the SANDBOX itself is gone — E2B's
|
|
213
|
+
* SandboxNotFoundError / NotFoundError classes (matched by name so no
|
|
214
|
+
* provider import rides in the runtime layer), or their messages after any
|
|
215
|
+
* wrapping ("Paused sandbox X not found" was run 3's exact shape). */
|
|
216
|
+
export function isSandboxGoneError(err: unknown): boolean {
|
|
217
|
+
const name = (err as { name?: unknown } | null)?.name;
|
|
218
|
+
if (name === "SandboxNotFoundError" || name === "NotFoundError") return true;
|
|
219
|
+
const msg = err instanceof Error ? err.message : typeof err === "string" ? err : "";
|
|
220
|
+
return /sandbox[^\n]{0,120}not found|not found[^\n]{0,120}sandbox/i.test(msg);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** One probe-failure run's bookkeeping (reset to zeros on ANY evidence). */
|
|
224
|
+
export interface ProbeDeathState {
|
|
225
|
+
/** Consecutive probe execs that THREW (no verdict at all). */
|
|
226
|
+
failStreak: number;
|
|
227
|
+
/** Consecutive throws that were provider "sandbox gone" evidence. */
|
|
228
|
+
goneStreak: number;
|
|
229
|
+
/** Wall-clock start of the current failure run (0 = no run). */
|
|
230
|
+
firstFailAtMs: number;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** The death-door decision, pure for tests. Null = keep polling. */
|
|
234
|
+
export function probeDeathVerdict(
|
|
235
|
+
s: ProbeDeathState, nowMs: number,
|
|
236
|
+
): "dead-gone" | "dead-silent" | null {
|
|
237
|
+
if (s.firstFailAtMs <= 0) return null;
|
|
238
|
+
const windowMs = nowMs - s.firstFailAtMs;
|
|
239
|
+
if (s.goneStreak >= PROBE_DEATH_GONE_STREAK && windowMs >= PROBE_DEATH_GONE_WINDOW_MS) return "dead-gone";
|
|
240
|
+
if (s.failStreak >= PROBE_DEATH_FAIL_STREAK && windowMs >= PROBE_DEATH_FAIL_WINDOW_MS) return "dead-silent";
|
|
241
|
+
return null;
|
|
242
|
+
}
|
|
243
|
+
|
|
127
244
|
/** The claude CLI's stderr complaint when `--resume <id>` names a thread
|
|
128
245
|
* that does not exist on this machine (the 2026-08-18 split-brain
|
|
129
246
|
* forensic). When a nonzero exit's stderr carries it, the transport
|
|
@@ -149,6 +266,10 @@ export interface TurnProbeReading {
|
|
|
149
266
|
* sample yet, or a garbled token — evidence-absent, never an error.
|
|
150
267
|
* Telemetry-only: no liveness verdict may ever read it. */
|
|
151
268
|
perf?: GuestPerfSample;
|
|
269
|
+
/** Latest tool-run pulse (the `.pulse` token field, 2026-08-29), when the
|
|
270
|
+
* guest wrote one AND it parsed clean. Telemetry-only, same rule as
|
|
271
|
+
* `perf`: no liveness verdict may ever read it. */
|
|
272
|
+
pulse?: TurnToolPulse;
|
|
152
273
|
}
|
|
153
274
|
|
|
154
275
|
/** The probe exec: one line, six fields, always exit 0 — faults surface as
|
|
@@ -158,15 +279,23 @@ export interface TurnProbeReading {
|
|
|
158
279
|
* whitelist (`tr -cd`) collapses any garbled write to at most one token —
|
|
159
280
|
* it can never add whitespace and desync the field positions. */
|
|
160
281
|
export function turnLivenessProbeCommand(
|
|
161
|
-
paths: { outPath: string; hbPath: string; perfPath: string; pid: number; sentinel: string
|
|
282
|
+
paths: { outPath: string; hbPath: string; perfPath: string; pid: number; sentinel: string;
|
|
283
|
+
pulsePath?: string },
|
|
162
284
|
): string {
|
|
285
|
+
// The 7th field is the tool-run pulse token (`.pulse`, 2026-08-29), `-`
|
|
286
|
+
// when absent — same optional-trailing-field contract as perf; the char
|
|
287
|
+
// whitelist collapses garbled writes to at most one token.
|
|
288
|
+
const pulseRead = paths.pulsePath
|
|
289
|
+
? `pu=$(head -c ${TOOL_PULSE_TOKEN_MAX_CHARS} ${shellQuote(paths.pulsePath)} 2>/dev/null | tr -cd '0-9,'); `
|
|
290
|
+
: `pu=; `;
|
|
163
291
|
return `sz=$(wc -c < ${shellQuote(paths.outPath)} 2>/dev/null || echo -1); `
|
|
164
292
|
+ `alive=0; kill -0 ${paths.pid} 2>/dev/null && alive=1; `
|
|
165
293
|
+ `hb=$(cat ${shellQuote(paths.hbPath)} 2>/dev/null || echo 0); `
|
|
166
294
|
+ `now=$(date +%s%3N); `
|
|
167
295
|
+ `sent=$(grep -c -F ${shellQuote(paths.sentinel)} ${shellQuote(paths.outPath)} 2>/dev/null || echo 0); `
|
|
168
296
|
+ `pf=$(head -c ${PERF_TOKEN_MAX_CHARS} ${shellQuote(paths.perfPath)} 2>/dev/null | tr -cd 'A-Za-z0-9=,.-'); `
|
|
169
|
-
+
|
|
297
|
+
+ pulseRead
|
|
298
|
+
+ `printf '%s %s %s %s %s %s %s\\n' "$sz" "$alive" "$hb" "$now" "$sent" "\${pf:--}" "\${pu:--}"`;
|
|
170
299
|
}
|
|
171
300
|
|
|
172
301
|
/** Parse the probe's line. Null = unusable output (a wedged or faulted exec)
|
|
@@ -186,9 +315,12 @@ export function parseTurnProbeOutput(raw: string): TurnProbeReading | null {
|
|
|
186
315
|
|| !Number.isFinite(nowMs) || !Number.isFinite(sentinelCount)) return null;
|
|
187
316
|
const perfTok = fields[5];
|
|
188
317
|
const perf = perfTok !== undefined && perfTok !== "-" ? parsePerfToken(perfTok) : null;
|
|
318
|
+
const pulseTok = fields[6];
|
|
319
|
+
const pulse = pulseTok !== undefined && pulseTok !== "-" ? parseToolPulseToken(pulseTok) : null;
|
|
189
320
|
return {
|
|
190
321
|
size, alive, hbMs, nowMs, sentinelSeen: sentinelCount > 0,
|
|
191
322
|
...(perf ? { perf } : {}),
|
|
323
|
+
...(pulse ? { pulse } : {}),
|
|
192
324
|
};
|
|
193
325
|
}
|
|
194
326
|
|
|
@@ -746,9 +878,26 @@ export interface CliAgentSpec {
|
|
|
746
878
|
/** Serialise one mid-turn user message into ONE stdin line. No
|
|
747
879
|
* trailing newline. */
|
|
748
880
|
messageLine(text: string): string;
|
|
881
|
+
/** Serialise ONE in-band interrupt control line (claude: a stream-json
|
|
882
|
+
* `control_request` with subtype "interrupt") — the ESC equivalent.
|
|
883
|
+
* The CLI's control layer handles these immediately, MID-STEP
|
|
884
|
+
* included: the running tool call aborts (its tool_result records the
|
|
885
|
+
* harness's own rejection text), the run ends with an
|
|
886
|
+
* `error_during_execution` result within ~100ms, and the session file
|
|
887
|
+
* stays `--resume`-able with the whole turn context. Verified live
|
|
888
|
+
* against claude 2.1.236. Absent → the runtime has no in-band
|
|
889
|
+
* interrupt; callers fall back to kill semantics. */
|
|
890
|
+
interruptLine?(requestId: string): string;
|
|
749
891
|
};
|
|
750
892
|
}
|
|
751
893
|
|
|
894
|
+
/** The CURRENT turn's mid-turn injection state (durable stream-input
|
|
895
|
+
* transport only) — see the `turnInject` field. */
|
|
896
|
+
interface TurnInjectState {
|
|
897
|
+
inboxPath: string; deliveredPath: string; pid: number;
|
|
898
|
+
appended: number; chain: Promise<unknown>;
|
|
899
|
+
}
|
|
900
|
+
|
|
752
901
|
/** A spawned ACP-mode CLI: the read side of its stdout and a delivery for its
|
|
753
902
|
* stdin, wrapped as byte web-streams for `ndJsonStream`. Producing this is a
|
|
754
903
|
* below-ACP transport concern (the duplex command primitive) — see
|
|
@@ -823,10 +972,31 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
823
972
|
* transport only). Null = no live stream-input turn — `injectUserMessage`
|
|
824
973
|
* answers "unsupported" and the caller leaves the message owed. `chain`
|
|
825
974
|
* serializes appends so each delivery targets a deterministic line count. */
|
|
826
|
-
private turnInject:
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
975
|
+
private turnInject: TurnInjectState | null = null;
|
|
976
|
+
|
|
977
|
+
/** The CURRENT turn's detached-runner wrapper pid (the setsid group
|
|
978
|
+
* leader echoed by the launch / recovered from the durable pidfile) —
|
|
979
|
+
* durable-transport turns only. Null between turns and on transports
|
|
980
|
+
* without a detached guest. See the contract doc (types/runtime.ts). */
|
|
981
|
+
private turnPid: number | null = null;
|
|
982
|
+
|
|
983
|
+
/** Advisory guest-pid identity for the live turn's runner tree — see the
|
|
984
|
+
* contract doc (types/runtime.ts). Never a liveness verdict. */
|
|
985
|
+
currentRunnerPid(): number | null {
|
|
986
|
+
return this.turnPid;
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
/** Safe-to-persist durable byte offset — end of the last line whose
|
|
990
|
+
* messages have ALL been yielded (boot-time turn adoption; contract in
|
|
991
|
+
* types/runtime.ts). One line behind the parse cursor by design. */
|
|
992
|
+
private turnConsumedOffset: number | null = null;
|
|
993
|
+
/** End offset of the line currently being yielded from — promoted to
|
|
994
|
+
* `turnConsumedOffset` when the NEXT line is pulled (or at stream end). */
|
|
995
|
+
private turnPendingLineEnd: number | null = null;
|
|
996
|
+
|
|
997
|
+
currentTurnDurableOffset(): number | null {
|
|
998
|
+
return this.turnConsumedOffset;
|
|
999
|
+
}
|
|
830
1000
|
|
|
831
1001
|
/** Deliver one user message INTO the live turn — see the contract doc
|
|
832
1002
|
* (types/runtime.ts). Appends a stream-input line to the turn's durable
|
|
@@ -843,7 +1013,32 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
843
1013
|
// The address-this envelope (wrapMidTurnUserMessage): a raw injected
|
|
844
1014
|
// line carries no framing, and the model continues its narration
|
|
845
1015
|
// without acknowledging the message.
|
|
846
|
-
|
|
1016
|
+
return this.appendTurnStreamLine(inject, streamInput.messageLine(wrapMidTurnUserMessage(text)));
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
/** Request an in-band step interrupt of the LIVE turn — the ESC
|
|
1020
|
+
* equivalent; see the contract doc (types/runtime.ts). Appends the
|
|
1021
|
+
* spec's interrupt control line to the turn's durable inbox; the guest
|
|
1022
|
+
* feeder forwards it into the CLI's stdin, whose control layer handles
|
|
1023
|
+
* it immediately — MID-STEP included, unlike queued user messages (the
|
|
1024
|
+
* inject "pending" wall applies to turn-loop consumption, not the
|
|
1025
|
+
* control layer). Only "delivered" means the control line reached the
|
|
1026
|
+
* CLI's stdin; callers escalate anything else to kill semantics. Never
|
|
1027
|
+
* throws. */
|
|
1028
|
+
async interruptTurn(): Promise<"delivered" | "pending" | "closed" | "unsupported"> {
|
|
1029
|
+
const inject = this.turnInject;
|
|
1030
|
+
const interruptLine = this.spec.streamInput?.interruptLine;
|
|
1031
|
+
if (!inject || !interruptLine || this.guestDetached) return "unsupported";
|
|
1032
|
+
return this.appendTurnStreamLine(inject, interruptLine(`itr-${Date.now().toString(36)}`));
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
/** The one durable append+ack path both mid-turn producers share
|
|
1036
|
+
* (`injectUserMessage`'s enveloped message line, `interruptTurn`'s raw
|
|
1037
|
+
* control line). Serialized on the inject chain so every append targets
|
|
1038
|
+
* a deterministic inbox line count. */
|
|
1039
|
+
private appendTurnStreamLine(
|
|
1040
|
+
inject: TurnInjectState, line: string,
|
|
1041
|
+
): Promise<"delivered" | "pending" | "closed"> {
|
|
847
1042
|
const attempt = inject.chain.then(async (): Promise<"delivered" | "pending" | "closed"> => {
|
|
848
1043
|
if (this.turnInject !== inject) return "closed";
|
|
849
1044
|
// The append lands whether or not the ack times out, so the target
|
|
@@ -869,13 +1064,29 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
869
1064
|
return attempt;
|
|
870
1065
|
}
|
|
871
1066
|
|
|
1067
|
+
/** The most recent probe reading's tool-run pulse (2026-08-29), plus the
|
|
1068
|
+
* guest clock it was read against — telemetry only, refreshed by each
|
|
1069
|
+
* `probeTurnLiveness` call. Null between turns / before any probe. */
|
|
1070
|
+
private lastTurnPulse: (TurnToolPulse & { guestNowMs: number }) | null = null;
|
|
1071
|
+
|
|
1072
|
+
/** Peek the newest tool-run pulse the durable probe carried — the
|
|
1073
|
+
* executor's evidence ticker reads it AFTER its liveness check to fan
|
|
1074
|
+
* the tool-progress fact to clients. Never probes on its own. */
|
|
1075
|
+
peekTurnPulse(): (TurnToolPulse & { guestNowMs: number }) | null {
|
|
1076
|
+
return this.lastTurnPulse;
|
|
1077
|
+
}
|
|
1078
|
+
|
|
872
1079
|
/** Durable three-valued liveness for the current turn — see the contract
|
|
873
1080
|
* doc (types/runtime.ts). Never throws; never blocks past the probe's
|
|
874
1081
|
* explicit exec timeout. */
|
|
875
1082
|
async probeTurnLiveness(): Promise<RunnerLivenessVerdict | null> {
|
|
876
1083
|
const probe = this.turnProbe;
|
|
877
1084
|
if (!probe) return null;
|
|
878
|
-
|
|
1085
|
+
const reading = await probe();
|
|
1086
|
+
if (reading?.pulse) {
|
|
1087
|
+
this.lastTurnPulse = { ...reading.pulse, guestNowMs: reading.nowMs };
|
|
1088
|
+
}
|
|
1089
|
+
return livenessVerdictFromReading(reading);
|
|
879
1090
|
}
|
|
880
1091
|
|
|
881
1092
|
/** Wakes the CURRENT tail-watchdog interval early. Armed only while the
|
|
@@ -1291,6 +1502,9 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1291
1502
|
// previous turn's push.
|
|
1292
1503
|
this.turnProbe = null;
|
|
1293
1504
|
this.turnInject = null;
|
|
1505
|
+
this.turnPid = null;
|
|
1506
|
+
this.turnConsumedOffset = null;
|
|
1507
|
+
this.turnPendingLineEnd = null;
|
|
1294
1508
|
this.nudgePending = false;
|
|
1295
1509
|
this.wakeTailWatchdog = null;
|
|
1296
1510
|
// Mid-turn stream input engages only where BOTH halves exist: a spec
|
|
@@ -1362,12 +1576,20 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1362
1576
|
// - SINGLE-EXEC STREAMING (everything else: local child_process,
|
|
1363
1577
|
// Vercel): the pre-incident shape — one exec, stdout streamed,
|
|
1364
1578
|
// exit code from the exec result.
|
|
1365
|
-
const lines = new AsyncQueue<string>();
|
|
1579
|
+
const lines = new AsyncQueue<{ line: string; end: number | null }>();
|
|
1366
1580
|
// How the turn ended: the CLI's exit code once known (null while
|
|
1367
1581
|
// running), plus a best-effort stderr tail for the error message.
|
|
1368
1582
|
let exitCode: number | null = null;
|
|
1369
1583
|
let exitStderr = "";
|
|
1370
1584
|
let transportError: unknown = null;
|
|
1585
|
+
/** True when the death door closed on PROVIDER-ATTESTED "sandbox
|
|
1586
|
+
* gone" evidence (dead-gone) — the machine itself died, not the CLI.
|
|
1587
|
+
* The error detail then says so in the classifier's sandbox_gone
|
|
1588
|
+
* vocabulary (turn-failure-class.ts), which routes the settle to the
|
|
1589
|
+
* unconditional machine-loss auto-continue lane; an answered
|
|
1590
|
+
* "pid gone" on a LIVE machine (a CLI that crashed on its own) stays
|
|
1591
|
+
* the evidence-gated runner_exit family. */
|
|
1592
|
+
let sandboxGoneDeath = false;
|
|
1371
1593
|
let consumerStopped = false;
|
|
1372
1594
|
let reap: () => void = () => { /* set per transport */ };
|
|
1373
1595
|
// The reap's kill-and-confirm exec, latched on first fire (reap runs
|
|
@@ -1433,11 +1655,16 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1433
1655
|
// subshell burst-reads /proc and rewrites the `.perf` token the
|
|
1434
1656
|
// durable-trio probe carries as its trailing field (perf-sampler.ts).
|
|
1435
1657
|
const perfPath = `${promptPath}.perf`;
|
|
1658
|
+
// The tool-run pulse (2026-08-29, child-telemetry contract): the
|
|
1659
|
+
// same beat samples the session's own process tree so a long
|
|
1660
|
+
// foreground tool proves it is working — see toolPulseFragment.
|
|
1661
|
+
const pulsePath = `${promptPath}.pulse`;
|
|
1436
1662
|
const detachedScript =
|
|
1437
1663
|
feeder
|
|
1438
1664
|
+ `( ${perfSamplerFunctionFragment({ perfPath })}while kill -0 "$$" 2>/dev/null; do `
|
|
1439
1665
|
+ `date +%s%3N > ${shellQuote(hbPath)}; `
|
|
1440
1666
|
+ `${runnerBusySentinelFragment(busyPath, tasksPath)}; `
|
|
1667
|
+
+ `${toolPulseFragment(pulsePath)}; `
|
|
1441
1668
|
+ `ac_perf_tick; `
|
|
1442
1669
|
+ `sleep ${RUNNER_HEARTBEAT_INTERVAL_SECONDS}; done ) >/dev/null 2>&1 </dev/null & `
|
|
1443
1670
|
+ sessionEnvSourceFragment(this.options.sessionEnvFile)
|
|
@@ -1502,7 +1729,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1502
1729
|
// executor's evidence ticker (`probeTurnLiveness`) both read through
|
|
1503
1730
|
// it — one fresh short exec per call, explicit timeout, faults map to
|
|
1504
1731
|
// null ("probe-failed"), never to a verdict.
|
|
1505
|
-
const probeCmd = turnLivenessProbeCommand({ outPath, hbPath, perfPath, pid, sentinel });
|
|
1732
|
+
const probeCmd = turnLivenessProbeCommand({ outPath, hbPath, perfPath, pid, sentinel, pulsePath });
|
|
1506
1733
|
const runTurnProbe = async (): Promise<TurnProbeReading | null> => {
|
|
1507
1734
|
try {
|
|
1508
1735
|
const res = await this.sandbox.commands.run(probeCmd, { timeoutMs: TURN_PROBE_TIMEOUT_MS });
|
|
@@ -1510,6 +1737,15 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1510
1737
|
} catch { return null; }
|
|
1511
1738
|
};
|
|
1512
1739
|
this.turnProbe = runTurnProbe;
|
|
1740
|
+
// Advisory pid identity for the platform's background-work declare
|
|
1741
|
+
// (in-harness Workflow tasks) — armed with the probe, cleared at the
|
|
1742
|
+
// next turn's start (the tree may legitimately outlive this turn).
|
|
1743
|
+
this.turnPid = pid;
|
|
1744
|
+
// Durable launch report (boot-time turn adoption): hand the caller
|
|
1745
|
+
// the guest prompt path + sentinel + pid so it can stamp them on
|
|
1746
|
+
// the turn row — a successor process can then find the runner's
|
|
1747
|
+
// durable files without this process's memory.
|
|
1748
|
+
this.options.onDetachedLaunch?.({ promptPath, sentinel, pid });
|
|
1513
1749
|
// Arm mid-turn injection now that the detached runner (and its
|
|
1514
1750
|
// feeder) exist; cleared with the turn in the finally below.
|
|
1515
1751
|
if (streamInput) {
|
|
@@ -1529,7 +1765,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1529
1765
|
exitCode = Number.isFinite(code) ? code : 0;
|
|
1530
1766
|
return;
|
|
1531
1767
|
}
|
|
1532
|
-
lines.push(line);
|
|
1768
|
+
lines.push({ line, end: offset });
|
|
1533
1769
|
};
|
|
1534
1770
|
|
|
1535
1771
|
reap = () => {
|
|
@@ -1558,6 +1794,13 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1558
1794
|
// re-attaching tails and finishes on durable polling alone (a wire
|
|
1559
1795
|
// that killed three streams will kill the fourth).
|
|
1560
1796
|
let streamDeaths = 0;
|
|
1797
|
+
// Death-door bookkeeping (see probeDeathVerdict): the current
|
|
1798
|
+
// run of consecutive probe failures. ANY evidence — a probe exec
|
|
1799
|
+
// that answered, a durable read that succeeded — resets it.
|
|
1800
|
+
const probeDeath: ProbeDeathState = { failStreak: 0, goneStreak: 0, firstFailAtMs: 0 };
|
|
1801
|
+
const resetProbeDeath = (): void => {
|
|
1802
|
+
probeDeath.failStreak = 0; probeDeath.goneStreak = 0; probeDeath.firstFailAtMs = 0;
|
|
1803
|
+
};
|
|
1561
1804
|
while (exitCode === null && !consumerStopped && !opts.signal?.aborted) {
|
|
1562
1805
|
let buf = "";
|
|
1563
1806
|
let handle: SandboxBackgroundProcess | null = null;
|
|
@@ -1672,6 +1915,9 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1672
1915
|
if (durableRead) {
|
|
1673
1916
|
try {
|
|
1674
1917
|
const bytes = Buffer.from(await durableRead(outPath), "utf8");
|
|
1918
|
+
// The file plane answered — the sandbox is reachable, so no
|
|
1919
|
+
// probe-failure run may accumulate toward the death door.
|
|
1920
|
+
resetProbeDeath();
|
|
1675
1921
|
let rest = bytes.subarray(offset).toString("utf8");
|
|
1676
1922
|
let nl: number;
|
|
1677
1923
|
while (exitCode === null && (nl = rest.indexOf("\n")) >= 0) {
|
|
@@ -1681,18 +1927,43 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1681
1927
|
} catch { /* file plane briefly unreachable — retried below */ }
|
|
1682
1928
|
}
|
|
1683
1929
|
if (exitCode !== null) break;
|
|
1684
|
-
// Runner still alive? A FRESH exec answers;
|
|
1685
|
-
// keeps retrying —
|
|
1686
|
-
//
|
|
1930
|
+
// Runner still alive? A FRESH exec answers; a wire-fault throw
|
|
1931
|
+
// is no verdict and keeps retrying — but never FOREVER (chaos
|
|
1932
|
+
// S13 run 3, 2026-08-31: a DEAD sandbox's probe exec THROWS, so
|
|
1933
|
+
// `alive` stayed null and this loop polled the corpse blind
|
|
1934
|
+
// until the executor closed the turn from outside). The death
|
|
1935
|
+
// door: provider-attested "sandbox gone" evidence on a short
|
|
1936
|
+
// streak, or a long generous window of nothing-but-failures
|
|
1937
|
+
// (probeDeathVerdict) ⇒ the same honest abnormal-death terminal
|
|
1938
|
+
// as an answered "pid gone" — never a clean done, never an
|
|
1939
|
+
// eternal poll. Any evidence (an answered probe, a durable
|
|
1940
|
+
// read) resets the run, so a live sandbox behind a flaky wire
|
|
1941
|
+
// is never killed (the K8s-Unknown rule).
|
|
1687
1942
|
let alive: boolean | null = null;
|
|
1688
1943
|
try {
|
|
1689
1944
|
const probe = await this.sandbox.commands.run(
|
|
1690
|
-
`kill -0 ${pid} 2>/dev/null`, { timeoutMs:
|
|
1945
|
+
`kill -0 ${pid} 2>/dev/null`, { timeoutMs: TURN_PROBE_TIMEOUT_MS });
|
|
1691
1946
|
alive = probe.exitCode === 0;
|
|
1692
|
-
|
|
1693
|
-
|
|
1947
|
+
resetProbeDeath();
|
|
1948
|
+
} catch (probeErr) {
|
|
1949
|
+
alive = null;
|
|
1950
|
+
probeDeath.failStreak += 1;
|
|
1951
|
+
if (probeDeath.firstFailAtMs === 0) probeDeath.firstFailAtMs = Date.now();
|
|
1952
|
+
probeDeath.goneStreak = isSandboxGoneError(probeErr) ? probeDeath.goneStreak + 1 : 0;
|
|
1953
|
+
}
|
|
1954
|
+
const deathVerdict = probeDeathVerdict(probeDeath, Date.now());
|
|
1955
|
+
if (deathVerdict !== null) {
|
|
1956
|
+
console.warn(`[cli-agent] ${this.spec.kind} durable-poll death door: ${deathVerdict === "dead-gone"
|
|
1957
|
+
? `the provider reports the sandbox gone (${probeDeath.goneStreak} consecutive not-found probes over ${Math.round((Date.now() - probeDeath.firstFailAtMs) / 1000)}s)`
|
|
1958
|
+
: `${probeDeath.failStreak} consecutive probe-transport failures over ${Math.round((Date.now() - probeDeath.firstFailAtMs) / 1000)}s with no durable evidence`
|
|
1959
|
+
} — treating the runner as dead`);
|
|
1960
|
+
}
|
|
1961
|
+
if (alive === false || deathVerdict !== null) {
|
|
1694
1962
|
// Dead with no exit line even in the durable file: abnormal
|
|
1695
|
-
// death — surfaces as an error, never a clean done.
|
|
1963
|
+
// death — surfaces as an error, never a clean done. A
|
|
1964
|
+
// provider-attested gone verdict marks the death as the
|
|
1965
|
+
// MACHINE's (sandbox_gone), not the CLI's.
|
|
1966
|
+
if (deathVerdict === "dead-gone") sandboxGoneDeath = true;
|
|
1696
1967
|
exitCode = JSONL_GUARD_NO_SENTINEL_EXIT;
|
|
1697
1968
|
break;
|
|
1698
1969
|
}
|
|
@@ -1722,7 +1993,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1722
1993
|
while ((nl = buf.indexOf("\n")) >= 0) {
|
|
1723
1994
|
const line = buf.slice(0, nl).trim();
|
|
1724
1995
|
buf = buf.slice(nl + 1);
|
|
1725
|
-
if (line) lines.push(line);
|
|
1996
|
+
if (line) lines.push({ line, end: null });
|
|
1726
1997
|
}
|
|
1727
1998
|
};
|
|
1728
1999
|
const runOpts = {
|
|
@@ -1734,7 +2005,7 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1734
2005
|
credEnvSourceFragment(this.options.credEnvFile) + guardedCmd, runOpts).then(
|
|
1735
2006
|
(res) => {
|
|
1736
2007
|
const tail = buf.trim();
|
|
1737
|
-
if (tail) lines.push(tail);
|
|
2008
|
+
if (tail) lines.push({ line: tail, end: null });
|
|
1738
2009
|
exitCode = res.exitCode;
|
|
1739
2010
|
exitStderr = (res.stderr ?? "").slice(-2000);
|
|
1740
2011
|
},
|
|
@@ -1761,7 +2032,16 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1761
2032
|
// when the turn never reaches its `done` (a reaped/superseded turn
|
|
1762
2033
|
// must still be resumable by its redispatch).
|
|
1763
2034
|
let announcedSessionId = opts.sessionId ?? "";
|
|
1764
|
-
for await (const
|
|
2035
|
+
for await (const item of lines) {
|
|
2036
|
+
// Consumed-offset bookkeeping (boot-time turn adoption): pulling
|
|
2037
|
+
// line N proves lines 1..N-1 are FULLY yielded, so the byte
|
|
2038
|
+
// offset just past line N-1 is the safe harvest watermark — see
|
|
2039
|
+
// `currentTurnDurableOffset` in types/runtime.ts.
|
|
2040
|
+
if (item.end !== null) {
|
|
2041
|
+
this.turnConsumedOffset = this.turnPendingLineEnd;
|
|
2042
|
+
this.turnPendingLineEnd = item.end;
|
|
2043
|
+
}
|
|
2044
|
+
const line = item.line;
|
|
1765
2045
|
let parsed: Record<string, unknown>;
|
|
1766
2046
|
try {
|
|
1767
2047
|
parsed = JSON.parse(line) as Record<string, unknown>;
|
|
@@ -1781,6 +2061,9 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1781
2061
|
}
|
|
1782
2062
|
|
|
1783
2063
|
await transport;
|
|
2064
|
+
// Stream fully consumed — the final line's messages are all
|
|
2065
|
+
// yielded, so its end offset graduates to the safe watermark.
|
|
2066
|
+
this.turnConsumedOffset = this.turnPendingLineEnd;
|
|
1784
2067
|
if (transportError) throw transportError;
|
|
1785
2068
|
if (exitCode === null) {
|
|
1786
2069
|
// Aborted/reaped before an exit line existed — honest terminal,
|
|
@@ -1792,7 +2075,15 @@ export class CliAgentRunner implements ModelExecutionContract {
|
|
|
1792
2075
|
}
|
|
1793
2076
|
if (exitCode !== 0 && !sawError) {
|
|
1794
2077
|
const detail = exitCode === JSONL_GUARD_NO_SENTINEL_EXIT
|
|
1795
|
-
?
|
|
2078
|
+
? sandboxGoneDeath
|
|
2079
|
+
// The classifier's sandbox_gone vocabulary, verbatim on
|
|
2080
|
+
// purpose (turn-failure-class.ts:"machine went away" /
|
|
2081
|
+
// "sandbox gone") — this is what routes the settle to the
|
|
2082
|
+
// machine-loss auto-continue lane instead of the
|
|
2083
|
+
// evidence-gated runner_exit family.
|
|
2084
|
+
? " (the machine went away mid-turn — the provider reports the sandbox gone)"
|
|
2085
|
+
: " (the runner died without reporting an exit code)"
|
|
2086
|
+
: "";
|
|
1796
2087
|
yield { type: "error", text: `${this.spec.kind} exited with code ${exitCode}${detail}${exitStderr ? `: ${exitStderr}` : ""}`, timestamp: now() };
|
|
1797
2088
|
return;
|
|
1798
2089
|
}
|