@coreplane/switchboard 1.227.0 → 1.229.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/assets/config/config.example.yaml +7 -13
  2. package/dist/assets/deploy/cloudflare-memory/worker.ts +157 -3
  3. package/dist/assets/deploy/cloudflare-resident/worker.ts +170 -30
  4. package/dist/assets/deploy/cloudflare-sandbox/Dockerfile +22 -18
  5. package/dist/assets/deploy/cloudflare-sandbox/package.json +1 -1
  6. package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.sh +11 -10
  7. package/dist/assets/deploy/cloudflare-sandbox/worker.ts +332 -299
  8. package/dist/assets/deploy/cloudflare-sandbox/wrangler.template.jsonc +2 -2
  9. package/dist/assets/package-lock.json +2542 -212
  10. package/dist/assets/package.json +2 -2
  11. package/dist/assets/project.json +2 -2
  12. package/dist/assets/source.json +3 -3
  13. package/dist/assets/src/agents/registry.ts +19 -34
  14. package/dist/assets/src/core/authz/policy.ts +8 -0
  15. package/dist/assets/src/core/authz/resource.ts +3 -1
  16. package/dist/assets/src/core/authz/types.ts +10 -1
  17. package/dist/assets/src/core/chatMessage.ts +1 -1
  18. package/dist/assets/src/core/coordinator/contract.ts +6 -0
  19. package/dist/assets/src/core/coordinator/driver.ts +8 -6
  20. package/dist/assets/src/core/runEvents.ts +53 -14
  21. package/dist/assets/src/core/runFriction.ts +3 -2
  22. package/dist/assets/src/core/runRecord.ts +46 -3
  23. package/dist/assets/src/core/runUsage.ts +199 -0
  24. package/dist/assets/src/core/ship/coordinator.ts +4 -2
  25. package/dist/assets/src/execution/residentDepCache.ts +34 -8
  26. package/dist/assets/src/execution/residentRebind.ts +84 -7
  27. package/dist/assets/src/execution/sandboxErrors.ts +14 -38
  28. package/dist/assets/src/execution/sandboxLifecycle.ts +78 -0
  29. package/dist/assets/web/dist/.vite/manifest.json +20 -20
  30. package/dist/assets/web/dist/assets/CostsPage-DQg30mHr.js +2 -0
  31. package/dist/assets/web/dist/assets/{ResidentDetailPage-Chvll3wy.js → ResidentDetailPage-CkVYktwT.js} +1 -1
  32. package/dist/assets/web/dist/assets/{ResidentsIndexPage-B5f8IwGF.js → ResidentsIndexPage-BLanxf27.js} +1 -1
  33. package/dist/assets/web/dist/assets/RunRoutePage-U3nwL8Df.js +13 -0
  34. package/dist/assets/web/dist/assets/{RunsIndexPage-BTJuKFTv.js → RunsIndexPage-DrIVxmpl.js} +1 -1
  35. package/dist/assets/web/dist/assets/{ScheduledPage-BVfgUBvP.js → ScheduledPage-DpqubmIm.js} +1 -1
  36. package/dist/assets/web/dist/assets/{StatusDot-CFXbAw7S.js → StatusDot-BpD9MRge.js} +1 -1
  37. package/dist/assets/web/dist/assets/{Tooltip-DcHMtbHJ.js → Tooltip-BYv0WSrA.js} +1 -1
  38. package/dist/assets/web/dist/assets/{dist-DKhqHu0V.js → dist-BZmA5qTt.js} +1 -1
  39. package/dist/assets/web/dist/assets/{main-CveRd2yk.js → main-C4GOEklV.js} +2 -2
  40. package/dist/assets/web/dist/assets/main-CAVqMbiX.css +1 -0
  41. package/dist/cli.js +9094 -9223
  42. package/package.json +1 -2
  43. package/dist/assets/src/execution/sandboxKeepalive.ts +0 -118
  44. package/dist/assets/web/dist/assets/CostsPage-5pl3HB2F.js +0 -2
  45. package/dist/assets/web/dist/assets/RunRoutePage-CRvmCuXh.js +0 -12
  46. package/dist/assets/web/dist/assets/main-xLAsdkfB.css +0 -1
@@ -13,10 +13,21 @@
13
13
  // for that one command — in the body, never in headers, because Workers Logs
14
14
  // record request headers (docs/reference/specs/execution.md item 5).
15
15
  //
16
- // The Sandbox SDK only runs inside Workers — that's why this proxy exists.
17
- // Verify method names against https://developers.cloudflare.com/sandbox/ on
18
- // first deploy; the SDK is young and its surface may shift.
19
- import { getSandbox, Sandbox, type ExecOptions, type ExecResult } from "@cloudflare/sandbox";
16
+ // The Sandbox SDK only runs inside Workers — that's why this proxy exists. The
17
+ // Durable Object below does the work (one supervised process per command, the
18
+ // file reads and writes) and answers the fetch handler over RPC with plain
19
+ // data; the handler authenticates, streams, and names what came back.
20
+ import {
21
+ OperationInterruptedError,
22
+ ProcessWaitTimeoutError,
23
+ RPCTransportError,
24
+ RuntimeIdentityInactiveError,
25
+ Sandbox,
26
+ StaleProcessHandleError,
27
+ getSandbox,
28
+ type SandboxCommand,
29
+ } from "@cloudflare/sandbox";
30
+ import { createExtensionProcessSandbox } from "@cloudflare/sandbox/extensions";
20
31
  import { BASH_TIMEOUT_MAX_MS, clampBashTimeout } from "../../src/execution/bashTimeout.js";
21
32
  import {
22
33
  base64ByteLength,
@@ -27,20 +38,16 @@ import {
27
38
  type Base64ReadAnswer,
28
39
  } from "../../src/execution/binaryRead.js";
29
40
  import {
30
- EXEC_KEEPALIVE_INTERVAL_MS,
31
41
  SANDBOX_SLEEP_AFTER,
32
42
  isRecycleError,
33
43
  recycledMidCommandMessage,
34
- withActivityKeepalive,
35
- } from "../../src/execution/sandboxKeepalive.js";
44
+ } from "../../src/execution/sandboxLifecycle.js";
36
45
  import { envFromRequest } from "../../src/execution/sandboxEnv.js";
37
- import { shellQuote } from "../../src/execution/shellQuote.js";
46
+ import { RUNTIME_REPLACEMENT_WORDING, isRuntimeUnreachableSignal } from "../../src/execution/residentRefresh.js";
38
47
  import {
39
48
  fleetBusyAnswer,
40
49
  fleetBusyExecAnswer,
41
- isContainerStarting,
42
50
  isFleetBusyError,
43
- isRuntimeProxyFailure,
44
51
  isRuntimeUnreachableError,
45
52
  runtimeUnreachableAnswer,
46
53
  runtimeUnreachableExecAnswer,
@@ -57,137 +64,267 @@ import sandboxPkg from "./package.json" with { type: "json" };
57
64
 
58
65
  /** The `@cloudflare/sandbox` version this Worker is built against — the pin
59
66
  * `check:sandbox-pair` holds equal to the Dockerfile's image tag, so it is
60
- * also the version a container on the CURRENT image reports. */
67
+ * also the version a container on the CURRENT image runs. */
61
68
  const SDK_PIN: string = sandboxPkg.dependencies["@cloudflare/sandbox"];
62
69
 
63
- export class SwitchboardSandbox extends Sandbox {
64
- // Idle lifetime of a thread's container (the SDK's own default is 10 min on
65
- // 0.12.x, 20 on 0.3.x). On the 0.0.28 containers base the activity clock was
66
- // renewed once per proxied fetch and the alarm loop SIGTERMed the container
67
- // the moment it expired, in-flight request or not — so a review's first
68
- // command (a 20-minute budget under a 20-minute default) was once
69
- // killed at 20:00 exactly, surfaced as "Command execution failed", and
70
- // the next command found a fresh container with an empty /workspace. `exec`
71
- // below renews the clock every minute while a command runs, which makes
72
- // this a true idle timeout (docs/reference/specs/execution.md item 2). The 0.3.x
73
- // containers base tracks in-flight requests itself, so the keepalive is now
74
- // belt-and-braces — kept until a live long-command receipt retires it. 5
75
- // minutes of idle frees the slot sooner while a prompt follow-up still
76
- // reuses the warm workspace.
70
+ const WORKDIR = "/workspace";
71
+
72
+ // Default per-command time limit, enforced by coreutils `timeout` inside the
73
+ // sandbox. The tuned 280s applies when the body carries no timeoutMs (an older
74
+ // bot); a caller-supplied timeoutMs is clamped server-side to the shared
75
+ // [1s, 20 min] bounds (clampBashTimeout — never trust the client's number).
76
+ // Whatever the effective limit, it must stay BELOW the SDK's own backstops so
77
+ // the real exit 124 wins: the per-process `timeout` this Worker launches with
78
+ // (the limit plus SDK_BACKSTOP_MARGIN_MS) and the container's
79
+ // COMMAND_TIMEOUT_MS (Dockerfile, above the 20-min ceiling).
80
+ const EXEC_TIMEOUT_SECS = 280;
81
+
82
+ /** How far above coreutils `timeout`'s deadline the SDK's own per-process
83
+ * limit sits: room for `-k 10`'s SIGKILL follow-up and the exit to land, so
84
+ * the shell-level 124 is always the deadline a command meets first. */
85
+ const SDK_BACKSTOP_MARGIN_MS = 40_000;
86
+
87
+ /** How long the Durable Object waits for the process's exit past the SDK's
88
+ * own limit before it kills the process itself and reports the timeout: the
89
+ * runtime should have ended it at the limit; a starved container ends it late. */
90
+ const OUTPUT_WAIT_MARGIN_MS = 30_000;
91
+
92
+ /** The interruption reasons that mean the container's runtime is not the one
93
+ * the command started on: the process, if it started, is gone with its
94
+ * output. `transport_disposed` is the Durable Object's own connection going
95
+ * away and is not one of them. */
96
+ const RUNTIME_REPLACED_REASONS = new Set([
97
+ "runtime_replaced",
98
+ "container_stopped",
99
+ "sandbox_destroyed",
100
+ "sandbox_lifetime_changed",
101
+ ]);
102
+
103
+ /** The transport losses that mean the same: the control connection's peer
104
+ * closed (a runtime crash or stop), the socket failed, the upgrade failed. */
105
+ const RPC_TRANSPORT_LOSS_KINDS = new Set(["peer_closed", "connection_failed", "upgrade_failed", "session_disposed"]);
106
+
107
+ /** `err` and its `cause` chain, bounded like the SDK's own walk. */
108
+ function* selfAndCauses(err: unknown): Generator<unknown> {
109
+ let link: unknown = err;
110
+ for (let depth = 0; link !== null && link !== undefined && depth < 8; depth++) {
111
+ yield link;
112
+ link = typeof link === "object" ? (link as { cause?: unknown }).cause : undefined;
113
+ }
114
+ }
115
+
116
+ /** Did the container's runtime change under the command? Typed first (the
117
+ * Durable Object sees the SDK's own classes, so `instanceof` holds here), the
118
+ * SDK's replacement wording second — the same list the resident Worker
119
+ * classifies its own execs with. */
120
+ function isRuntimeReplacement(err: unknown): boolean {
121
+ if (err instanceof StaleProcessHandleError) return true;
122
+ if (err instanceof RuntimeIdentityInactiveError) return true;
123
+ if (err instanceof OperationInterruptedError && RUNTIME_REPLACED_REASONS.has(err.reason)) return true;
124
+ if (err instanceof RPCTransportError && RPC_TRANSPORT_LOSS_KINDS.has(err.kind)) return true;
125
+ for (const link of selfAndCauses(err)) {
126
+ if (RUNTIME_REPLACEMENT_WORDING.test(thrownShape(link).message ?? "")) return true;
127
+ }
128
+ return false;
129
+ }
130
+
131
+ /** Did the container's control port never answer? The SDK's connect abort
132
+ * (30 s), anywhere in the cause chain. Asked only after `isRuntimeReplacement`. */
133
+ function isRuntimeUnreachable(err: unknown): boolean {
134
+ for (const link of selfAndCauses(err)) if (isRuntimeUnreachableSignal(link)) return true;
135
+ return false;
136
+ }
137
+
138
+ /** A finished command, as `/exec` answers it. `durationMs` is the command's
139
+ * wall time in the sandbox (docs/reference/specs/tracing.md item 19). */
140
+ export interface ExecAnswer {
141
+ stdout: string;
142
+ stderr: string;
143
+ exitCode: number;
144
+ durationMs: number;
145
+ }
146
+
147
+ /** A command the sandbox never answered for, in the dual in-body shape
148
+ * (docs/reference/specs/execution.md item 3): a new executor throws on `error`, an
149
+ * older one still renders `exit 127: <stderr>`. `reason` names the machine
150
+ * token when there is one (`fleet-busy`, `runtime-unreachable`). */
151
+ export interface ExecFailure {
152
+ error: string;
153
+ reason?: string;
154
+ stdout: "";
155
+ stderr: string;
156
+ exitCode: 127;
157
+ }
158
+
159
+ /** A file route's refusal, with the HTTP status the fetch handler answers and,
160
+ * when the refusal is a named condition the executor waits on (`fleet-busy`,
161
+ * `runtime-unreachable`), its machine token — the executor reads the token,
162
+ * never the text, so a refusal without it is a dead sandbox to it. */
163
+ interface FileRefusal {
164
+ error: string;
165
+ status: number;
166
+ reason?: string;
167
+ }
168
+
169
+ export class SwitchboardSandbox extends Sandbox<Env> {
170
+ // Idle lifetime of a thread's container (the SDK's own default is 10 min).
171
+ // Idle means idle: the SDK renews the activity timeout every second while a
172
+ // command's stream is open (its control connection's busy poll), so a
173
+ // running command never counts toward it and the shell-level `timeout` is
174
+ // the one deadline a command can hit (docs/reference/specs/execution.md item 2).
175
+ // 5 minutes frees the slot sooner while a prompt follow-up still reuses the
176
+ // warm workspace.
77
177
  sleepAfter = SANDBOX_SLEEP_AFTER;
78
178
 
79
- // A Worker and its image deploy as two artifacts; until the rollout finishes
80
- // this Worker can be handed a container still on the PREVIOUS image
81
- // (docs/reference/specs/execution.md item 6; seen live: a 0.3.7 container under the
82
- // 0.12.9 SDK, every command a message-less 400 for 90 s). The SDK's own
83
- // check logs `container=unknown` at info and does nothing else; this one
84
- // names the skew at warn so the rollout is visible in the logs.
85
- //
86
- // LOG ONLY — never `destroy()` here: onStart runs inside
87
- // `blockConcurrencyWhile`, `destroy()` is unbounded and coalesced callers
88
- // hang until eviction, a fresh placement during a gradual wave can land on
89
- // the old image again (the incident's DO was brand new), and the
90
- // healthy-but-not-running state after a destroy takes the SDK's stale-state
91
- // path, which can `ctx.abort()` the DO. A command that fails on a skewed
92
- // instance is named by `thrownText` with the rollout hint; the one-wave
93
- // rollout (`rollout_step_percentage: 100`) keeps the window to seconds.
94
- override async onStart(): Promise<void> {
95
- await super.onStart();
96
- const v = await this.client.utils.getVersion().catch(() => "unknown");
97
- if (v !== SDK_PIN) {
98
- console.warn(
99
- `sandbox.version-skew container=${v} sdk=${SDK_PIN} — this instance may be on a previous image (Worker/image rollout in progress)`,
100
- );
179
+ /** One command as one supervised process (docs/reference/specs/execution.md
180
+ * item 4): `timeout -k 10 <secs> bash -c 'mkdir -p /workspace && cd
181
+ * /workspace && <command>'`, the caller's env on the process alone (item 5),
182
+ * the SDK's own per-process limit a margin above the shell's. The process
183
+ * is started and collected here, inside the Durable Object, where the
184
+ * SDK's error classes are still themselves — so a full fleet, a silent
185
+ * control port and a runtime replaced under the command are named by type
186
+ * and answered as data; anything else propagates to the fetch handler. */
187
+ async runCommand(
188
+ command: string,
189
+ execTimeoutSecs: number,
190
+ envVars: Record<string, string>,
191
+ ): Promise<ExecAnswer | ExecFailure> {
192
+ const startedAt = systemClock();
193
+ const full = `mkdir -p ${WORKDIR} && cd ${WORKDIR} && ${command}`;
194
+ const argv: SandboxCommand = ["timeout", "-k", "10", String(execTimeoutSecs), "bash", "-c", full];
195
+ const backstopMs = execTimeoutSecs * 1000 + SDK_BACKSTOP_MARGIN_MS;
196
+ let proc: Awaited<ReturnType<ReturnType<typeof createExtensionProcessSandbox>["exec"]>>;
197
+ try {
198
+ proc = await createExtensionProcessSandbox(this).exec(argv, { env: envVars, timeout: backstopMs });
199
+ } catch (err) {
200
+ return this.execFailure(err, startedAt);
201
+ }
202
+ try {
203
+ const out = await proc.output({ encoding: "utf8", timeout: backstopMs + OUTPUT_WAIT_MARGIN_MS });
204
+ // coreutils `timeout` exits 124 when the deadline killed the command
205
+ // (137 when the follow-up SIGKILL had to); the SDK's own limit, if it
206
+ // ever wins, reports `timedOut`. All three are the one story.
207
+ const timedOut = out.timedOut || out.exitCode === 124 || out.exitCode === 137;
208
+ const notes: string[] = [];
209
+ if (timedOut) notes.push(timeoutNote(execTimeoutSecs));
210
+ // The SDK cut the process's log past its own retention: the output
211
+ // here is a prefix, whatever the executor's caps say.
212
+ if (out.truncated) notes.push("output truncated by the sandbox runtime — the streams above are a prefix");
213
+ return {
214
+ stdout: out.stdout,
215
+ stderr: [out.stderr, ...notes].filter(Boolean).join("\n"),
216
+ exitCode: timedOut ? 124 : out.exitCode,
217
+ durationMs: systemClock() - startedAt,
218
+ };
219
+ } catch (err) {
220
+ if (err instanceof ProcessWaitTimeoutError) {
221
+ // The runtime should have ended the process at the SDK's limit and
222
+ // did not; abandoning a live process would leave it running in the
223
+ // workspace. End it, then report the shell-level timeout it exceeded.
224
+ await proc.kill(9).catch(() => {});
225
+ return {
226
+ stdout: "",
227
+ stderr: `${timeoutNote(execTimeoutSecs)}\n(the process outlived the sandbox's own limit and was killed)`,
228
+ exitCode: 124,
229
+ durationMs: systemClock() - startedAt,
230
+ };
231
+ }
232
+ return this.execFailure(err, startedAt);
101
233
  }
102
234
  }
103
235
 
104
- // A proxied fetch that nothing answered (docs/reference/specs/execution.md
105
- // item 9): the `@cloudflare/containers` base class turns a failed fetch to
106
- // the container's port into a plain-text HTTP 500 — `Error proxying request
107
- // to container: …` when nothing listens (the SDK's server exited, or the
108
- // image's PID 1 has not started it again yet), `Container suddenly
109
- // disconnected` when the connection dropped mid-request — and the SDK's
110
- // client, expecting JSON, throws the bare `HTTP error! status: 500`. Read
111
- // the body here, before the client does, and throw the named error with the
112
- // facts a card and a log search need: this container's id (the Durable
113
- // Object's id is the container id in the `containers` dataset), the SDK
114
- // pin, and whether the platform still calls the container running. Every
115
- // other answer passes through untouched — a 500 body that is not the proxy's
116
- // is re-wrapped unread by the client.
117
- override async containerFetch(
118
- requestOrUrl: Request | string | URL,
119
- portOrInit?: number | RequestInit,
120
- portParam?: number,
121
- ): Promise<Response> {
122
- const res = await super.containerFetch(requestOrUrl, portOrInit, portParam);
123
- if (res.status !== 500) return res;
124
- const body = await res.text();
125
- if (isRuntimeProxyFailure(res.status, body)) {
126
- throw new SandboxRuntimeUnreachableError({
127
- containerId: this.ctx.id.toString(),
128
- running: this.ctx.container?.running,
129
- sdkVersion: SDK_PIN,
130
- cause: body,
131
- });
236
+ /** The named failures, as `/exec` data; anything else is thrown as it came. */
237
+ private execFailure(err: unknown, startedAt: number): ExecFailure {
238
+ const raw = thrownText(thrownShape(err));
239
+ // A full fleet (docs/reference/specs/execution.md item 14): no container
240
+ // instance for this thread, so nothing started — the executor waits.
241
+ if (isFleetBusyError(err)) return fleetBusyExecAnswer(raw);
242
+ // The runtime changed under the command (item 9): the process, if it
243
+ // started, is gone with its output. Certain — the SDK said so by type.
244
+ if (isRuntimeReplacement(err)) {
245
+ const msg = recycledMidCommandMessage(systemClock() - startedAt, raw, true);
246
+ return { error: msg, stdout: "", stderr: msg, exitCode: 127 };
132
247
  }
133
- return new Response(body, { status: res.status, statusText: res.statusText, headers: res.headers });
248
+ // The control port never answered (item 9): nothing ran.
249
+ if (isRuntimeUnreachable(err)) return runtimeUnreachableExecAnswer(this.runtimeUnreachable(raw).message);
250
+ throw err;
134
251
  }
135
252
 
136
- override async exec(command: string, options?: ExecOptions): Promise<ExecResult> {
137
- return withActivityKeepalive(
138
- () => this.renewActivityTimeout(),
139
- () => super.exec(command, options),
140
- EXEC_KEEPALIVE_INTERVAL_MS,
141
- );
253
+ /** The typed, named error for a silent control port, with this container's
254
+ * facts — thrown across the RPC boundary to the fetch handler on the file
255
+ * routes, where it is matched by name. */
256
+ private runtimeUnreachable(cause: string): SandboxRuntimeUnreachableError {
257
+ return new SandboxRuntimeUnreachableError({
258
+ containerId: this.ctx.id.toString(),
259
+ running: this.ctx.container?.running,
260
+ sdkVersion: SDK_PIN,
261
+ cause,
262
+ });
142
263
  }
143
264
 
144
- // A fence from the 0.3.x days, kept until a live container-restart receipt
145
- // retires it: 0.3.x cached its default ExecutionSession in Durable
146
- // Object memory while the session lived in the container, so a container
147
- // restart under a live DO (image rollout, crash, sleep/wake) made every later
148
- // call fail with "Session '<id>' not found" forever. 0.12.x persists the
149
- // session id in DO storage, clears it itself in `onStop`, and its container
150
- // recreates a missing session on the next exec — so this should never run;
151
- // if it does, nulling the cached id only makes the SDK recreate the session,
152
- // which is what it would have done anyway. The workspace disk is gone either
153
- // way; repos re-clone — the same graceful degradation as an expired E2B
154
- // sandbox.
155
- resetDefaultSession(): void {
156
- (this as unknown as { defaultSession: unknown }).defaultSession = null;
265
+ /** A file operation with its runtime failures named for the fetch handler:
266
+ * a silent control port becomes the typed error (item 9); a missing file
267
+ * is the refusal the route answers 404. The SDK's other errors propagate. */
268
+ private async fileOp<T>(op: () => Promise<T>): Promise<T | FileRefusal> {
269
+ try {
270
+ return await op();
271
+ } catch (err) {
272
+ const shape = thrownShape(err);
273
+ if (shape.name === "FileNotFoundError") return { error: `read-failed: ${thrownText(shape)}`, status: 404 };
274
+ if (!isRuntimeReplacement(err) && isRuntimeUnreachable(err)) throw this.runtimeUnreachable(thrownText(shape));
275
+ throw err;
276
+ }
157
277
  }
158
- }
159
278
 
160
- /** The 0.3.x stale-session text; see `resetDefaultSession`. */
161
- const STALE_SESSION = /session '[^']*' not found/i;
162
-
163
- /** Run a sandbox call and retry it ONCE when the failure says nothing ran: a
164
- * stale session (0.3.x; reset the cached id first) or a container still
165
- * booting (0.12.x's "Container is starting. Please retry in a moment.", after
166
- * a short pause). Both fail before the command or file op executes, so the
167
- * re-send is safe by construction. Anything else propagates: a failure whose
168
- * command MAY have run (a session shell that exited mid-command, a container
169
- * that stopped under the call) is never re-run here — /exec names it a
170
- * recycle instead (docs/reference/specs/execution.md item 9). */
171
- const CONTAINER_STARTING_RETRY_DELAY_MS = 3_000;
172
-
173
- async function withSessionRecovery<T>(
174
- sandbox: { resetDefaultSession(): void | Promise<void> },
175
- fn: () => Promise<T>,
176
- ): Promise<T> {
177
- try {
178
- return await fn();
179
- } catch (err) {
180
- const { message } = thrownShape(err);
181
- if (message && STALE_SESSION.test(message)) {
182
- await sandbox.resetDefaultSession();
183
- return await fn();
184
- }
185
- if (isContainerStarting(err)) {
186
- await new Promise((r) => setTimeout(r, CONTAINER_STARTING_RETRY_DELAY_MS));
187
- return await fn();
279
+ async readText(path: string): Promise<{ content: string } | FileRefusal> {
280
+ return this.fileOp(async () => ({ content: (await this.readFile(path, { encoding: "utf-8" })).content }));
281
+ }
282
+
283
+ /** `encoding: "base64"` (src/execution/binaryRead.ts): the size first, from
284
+ * `stat`, so the cap is judged before any read and the client can hold the
285
+ * decoded bytes to it — an SDK read that came back short would otherwise
286
+ * pass as the file. A file over the cap is refused by name inside a 200. */
287
+ async readBase64(path: string): Promise<Base64ReadAnswer | FileRefusal> {
288
+ const stat = await this.runCommand(statCommandFor(path), 60, {});
289
+ if ("error" in stat) {
290
+ // The stat's own named failure — a full fleet, a silent control port —
291
+ // is the read's, token included, so the executor waits as it would
292
+ // have for the text read.
293
+ return stat.reason ? { error: stat.error, status: 503, reason: stat.reason } : { error: stat.error, status: 500 };
188
294
  }
189
- throw err;
295
+ if (stat.exitCode !== 0) return { error: `read-failed: ${(stat.stderr || stat.stdout).trim()}`, status: 404 };
296
+ const size = parseByteSize(stat.stdout);
297
+ if (size === null) return { error: `read-failed: stat answered ${JSON.stringify(stat.stdout)}`, status: 500 };
298
+ if (size > MAX_READ_BYTES) return { encoding: "base64", tooLarge: true } satisfies Base64ReadAnswer;
299
+ return this.fileOp(async () => {
300
+ const content = (await this.readFile(path, { encoding: "base64" })).content;
301
+ const got = base64ByteLength(content);
302
+ // 409, not 5xx: the client retries a 5xx over 30 s, and a short read is
303
+ // answered by the caller re-reading, not by waiting.
304
+ if (got !== size) {
305
+ return { error: `read-inconsistent: ${path} is ${size} bytes but the read returned ${got}`, status: 409 };
306
+ }
307
+ return { encoding: "base64", content, size } satisfies Base64ReadAnswer;
308
+ });
190
309
  }
310
+
311
+ async write(path: string, content: string): Promise<{ ok: true } | FileRefusal> {
312
+ return this.fileOp(async () => {
313
+ await this.writeFile(path, content);
314
+ return { ok: true as const };
315
+ });
316
+ }
317
+ }
318
+
319
+ /** The stderr line an exit 124 carries: the limit, the knob, and the way to
320
+ * outlive a command (`setsid -f`: every /exec runs under `timeout … bash -c`,
321
+ * whose process group is reaped when the command returns, so a plain
322
+ * background job dies with it). */
323
+ function timeoutNote(execTimeoutSecs: number): string {
324
+ return (
325
+ `command timed out in the sandbox after ${execTimeoutSecs}s (pass the bash tool's timeoutMs for longer commands, max ${BASH_TIMEOUT_MAX_MS} ms); ` +
326
+ "re-run as smaller/faster steps, or start it detached with `setsid -f sh -c '<command> > /tmp/job.log 2>&1'` and poll the log on later calls"
327
+ );
191
328
  }
192
329
 
193
330
  interface Env {
@@ -195,31 +332,18 @@ interface Env {
195
332
  SANDBOX_TOKEN: string;
196
333
  }
197
334
 
198
- const WORKDIR = "/workspace";
199
-
200
- // Default per-command time limit, enforced by coreutils `timeout` inside the
201
- // sandbox. The tuned 280s applies when the body carries no timeoutMs (an older
202
- // bot); a caller-supplied timeoutMs is clamped server-side to the shared
203
- // [1s, 20 min] bounds (clampBashTimeout — never trust the client's number).
204
- // Whatever the effective limit, it must stay BELOW the SDK backstop
205
- // (COMMAND_TIMEOUT_MS in the Dockerfile, sized above the 20-min ceiling) so
206
- // the real exit 124 wins. undici's 300s no-headers ceiling stopped mattering
207
- // once /exec streamed heartbeats — headers go out immediately.
208
- const EXEC_TIMEOUT_SECS = 280;
209
-
210
335
  /** The commit this bundle was built from, injected by the deploy
211
336
  * (`deploy/bin/build-stamp.mjs`) and answered on GET /healthz as `build`. */
212
337
  const BUILD = injectedBuildStamp();
213
338
 
214
339
  // The Worker's own spans (docs/reference/specs/tracing.md item 22): one `sandbox.exec`
215
- // root per command, started at the attempt that produced the answer, joining
216
- // the bot's trace (the bearer checked out before the header is read).
340
+ // root per command, joining the bot's trace (the bearer checked out before the
341
+ // header is read).
217
342
  const tracer = createTracer({ clock: systemClock });
218
343
  const traceSinks = [workerLogSink((line) => console.log(line))];
219
344
 
220
- /** One command's root, started at the attempt that answered, joining the bot's trace. */
221
- function execRoot(attemptStartedAt: number, traceparent: string | undefined) {
222
- return startAdoptedRoot(tracer, "sandbox.exec", { sinks: traceSinks, startedAt: attemptStartedAt, traceparent });
345
+ function execRoot(startedAt: number, traceparent: string | undefined) {
346
+ return startAdoptedRoot(tracer, "sandbox.exec", { sinks: traceSinks, startedAt, traceparent });
223
347
  }
224
348
 
225
349
  export default {
@@ -240,57 +364,26 @@ export default {
240
364
  const threadKey = request.headers.get("x-thread-key");
241
365
  if (!threadKey) return json({ error: "missing X-Thread-Key" }, 400);
242
366
 
243
- // One sandbox per thread; the DO name is the thread key. getSandbox is
244
- // generic over the namespace's class since 0.12, so the subclass's
245
- // resetDefaultSession is callable over RPC without a cast.
367
+ // One sandbox per thread; the DO name is the thread key. The stub's own
368
+ // methods (`runCommand`, `readText`, …) are what the routes call — the
369
+ // work happens in the Durable Object, the data comes back over RPC.
246
370
  const sandbox = getSandbox(env.Sandbox, threadKey);
247
371
 
248
372
  const url = new URL(request.url);
249
373
  const body = (await request.json().catch(() => ({}))) as Record<string, unknown>;
250
374
 
251
375
  // Optional env passthrough (e.g. GH_TOKEN), read from the BODY's `env`
252
- // (docs/reference/specs/execution.md item 5). It then rides in the SDK's per-exec
253
- // `env` option, which the container applies to that one command and
254
- // restores afterwards (0.12.x; 0.3.7 ignored it, so the value used to be
255
- // an inline base64 `export` prefix — which put the live GH_TOKEN into every
256
- // "Command executed" line the SDK logs). Nothing persists in the
257
- // sandbox beyond the command's lifetime, and the command text the SDK
258
- // logs never carries a credential.
259
- //
260
- // Body, never headers: Workers Logs record this invocation's request
261
- // headers and redact them by a name heuristic only — a receipt probe's
262
- // per-variable env header was once logged in clear. Bodies are not
263
- // recorded. The one-release per-variable-header fallback that carried a
264
- // body-only bot against a header-only Worker during that rollout is
265
- // gone now that the body reader is live everywhere, so the
266
- // credential rides only in the body and no request header is read as env.
376
+ // (docs/reference/specs/execution.md item 5). It then rides in the SDK's per-process
377
+ // `env` option, which the runtime applies to that one command and
378
+ // nothing else. Body, never headers: Workers Logs record this
379
+ // invocation's request headers and redact them by a name heuristic only —
380
+ // a receipt probe's per-variable env header was once logged in clear.
381
+ // Bodies are not recorded.
267
382
  const envVars = envFromRequest({ body });
268
383
 
269
384
  try {
270
385
  switch (url.pathname) {
271
386
  case "/exec": {
272
- // Streamed with a whitespace heartbeat. A long command otherwise
273
- // holds a byteless HTTP request open for minutes, and some hop
274
- // between the bot and this Worker silently drops idle connections —
275
- // measured live: the container answered a 290s command at its 280s
276
- // timeout, but the bot's fetch died without ever seeing the
277
- // response (undici gives up 300s after sending a request that has
278
- // received no headers). Headers go out immediately and a heartbeat
279
- // byte flows every 15s, so no intermediary ever sees an idle
280
- // connection. Heartbeats are pure whitespace, which is legal around
281
- // a JSON document — the executor's res.json() on the full body
282
- // parses unchanged.
283
- //
284
- // The time limit is enforced with coreutils `timeout` INSIDE the
285
- // sandbox, not by the SDK: the SDK's COMMAND_TIMEOUT_MS rejection is
286
- // useless to callers — its exec handler wraps every failure as a
287
- // generic "Command execution failed" and buries the real message in
288
- // a field its client discards (measured live). Shell-level timeout
289
- // produces a real exit 124 through the normal result path, no error
290
- // classification needed. SIGKILL follows 10s after TERM for
291
- // stragglers. COMMAND_TIMEOUT_MS (Dockerfile) sits above this as a
292
- // pure backstop.
293
- //
294
387
  // Per-call budget (docs/reference/specs/execution.md item 11): a numeric
295
388
  // timeoutMs in the body is clamped server-side to the shared
296
389
  // [1s, 20 min] bounds; absent (an older bot) → the tuned 280s
@@ -300,69 +393,37 @@ export default {
300
393
  typeof requested === "number" && Number.isFinite(requested)
301
394
  ? Math.ceil(clampBashTimeout(requested) / 1000)
302
395
  : EXEC_TIMEOUT_SECS;
303
- const full = `mkdir -p ${WORKDIR} && cd ${WORKDIR} && ${String(body.command ?? "")}`;
304
396
  return streamExec(
305
- sandbox,
306
- `timeout -k 10 ${execTimeoutSecs} bash -c ${shellQuote(full)}`,
307
- { env: envVars },
308
- execTimeoutSecs,
397
+ () => sandbox.runCommand(String(body.command ?? ""), execTimeoutSecs, envVars),
309
398
  request.headers.get("traceparent") ?? undefined,
310
399
  );
311
400
  }
312
401
  case "/read": {
313
- // `encoding: "base64"` is a binary read (src/execution/binaryRead.ts):
314
- // the SDK encodes the bytes, and a file over the cap is refused by
315
- // name inside a 200 — the client counts a non-2xx as a sick Worker.
316
402
  const encoding = readEncodingOf(body);
317
403
  if (typeof encoding !== "string") return json({ error: encoding.error }, 400);
318
404
  const path = abs(String(body.path ?? ""));
319
- if (encoding === "base64") {
320
- // The size first, from `stat`, so the cap is judged before any
321
- // read and the client can hold the decoded bytes to it — an SDK
322
- // read that came back short would otherwise pass as the file.
323
- const stat = await withSessionRecovery(sandbox, () => sandbox.exec(statCommandFor(path)));
324
- if ((stat.exitCode ?? 0) !== 0) {
325
- return json({ error: `read-failed: ${String(stat.stderr ?? stat.stdout ?? "").trim()}` }, 404);
326
- }
327
- const size = parseByteSize(String(stat.stdout ?? ""));
328
- if (size === null) return json({ error: `read-failed: stat answered ${JSON.stringify(stat.stdout)}` }, 500);
329
- if (size > MAX_READ_BYTES) return json({ encoding: "base64", tooLarge: true } satisfies Base64ReadAnswer);
330
- const file = await withSessionRecovery(sandbox, () => sandbox.readFile(path, { encoding: "base64" }));
331
- const content = typeof file === "string" ? file : (file?.content ?? "");
332
- const got = base64ByteLength(content);
333
- if (got !== size) {
334
- // 409, not 5xx: the client retries a 5xx twice over 30 s, and a
335
- // short read is answered by the caller re-reading, not by waiting.
336
- return json({ error: `read-inconsistent: ${path} is ${size} bytes but the read returned ${got}` }, 409);
337
- }
338
- return json({ encoding: "base64", content, size } satisfies Base64ReadAnswer);
339
- }
340
- const file = await withSessionRecovery(sandbox, () => sandbox.readFile(path));
341
- return json({ content: typeof file === "string" ? file : (file?.content ?? "") });
405
+ const answer = encoding === "base64" ? await sandbox.readBase64(path) : await sandbox.readText(path);
406
+ return "status" in answer ? refused(answer) : json(answer);
342
407
  }
343
408
  case "/write": {
344
- await withSessionRecovery(sandbox, () =>
345
- sandbox.writeFile(abs(String(body.path ?? "")), String(body.content ?? "")),
346
- );
347
- return json({ ok: true });
409
+ const answer = await sandbox.write(abs(String(body.path ?? "")), String(body.content ?? ""));
410
+ return "status" in answer ? refused(answer) : json(answer);
348
411
  }
349
412
  default:
350
413
  return json({ error: "unknown route" }, 404);
351
414
  }
352
415
  } catch (err) {
353
- // Never an empty text (item 3): a message-less SDK error is named as
354
- // such, with the rollout hint.
416
+ // Never an empty text (item 3): a message-less SDK error is named as such.
355
417
  const msg = thrownText(thrownShape(err));
356
- // Nothing answered at the container's port (item 9): the file op never
357
- // reached a server, so a 503 the executor's transport retry re-sends —
358
- // by then the image's PID 1 has the server back — with the named reason
359
- // and the container in the text.
418
+ // Nothing answered at the container's control port (item 9): the file
419
+ // op never reached a runtime, so a 503 the executor's transport retry
420
+ // re-sends, with the named reason and the container in the text.
360
421
  if (isRuntimeUnreachableError(err)) return json(runtimeUnreachableAnswer(msg), 503);
361
- // A full fleet (docs/reference/specs/execution.md item 14): the SDK could not get a
362
- // container instance for this thread's Durable Object, so no session
363
- // exists and the file op never started — re-sending is safe by
364
- // construction. Named so the executor waits instead of reading it as a
365
- // dead sandbox; 503 because that is what it is.
422
+ // A full fleet (docs/reference/specs/execution.md item 14): no container
423
+ // instance for this thread's Durable Object, so the file op never
424
+ // started — re-sending is safe by construction. Named so the executor
425
+ // waits instead of reading it as a dead sandbox; 503 because that is
426
+ // what it is.
366
427
  if (isFleetBusyError(err)) return json(fleetBusyAnswer(msg), 503);
367
428
  return json({ error: msg }, 500);
368
429
  }
@@ -370,32 +431,26 @@ export default {
370
431
  } satisfies ExportedHandler<Env>;
371
432
 
372
433
  /** Execute a command and stream the response: immediate headers, a whitespace
373
- * heartbeat every 15s while the command runs, then one JSON document. All
374
- * outcomes arrive in-body with HTTP 200 (headers are long gone by the time
375
- * the result is known): a completed command as {stdout, stderr, exitCode},
376
- * a sandbox-enforced timeout as exit 124, and any other failure as {error}. */
377
- function streamExec(
378
- sandbox: {
379
- resetDefaultSession(): void | Promise<void>;
380
- exec(command: string, options?: ExecOptions): Promise<{ stdout?: string; stderr?: string; exitCode?: number }>;
381
- },
382
- command: string,
383
- options: ExecOptions,
384
- execTimeoutSecs: number,
385
- traceparent: string | undefined,
386
- ): Response {
434
+ * heartbeat every 15s while the command runs, then one JSON document. A long
435
+ * command otherwise holds a byteless HTTP request open for minutes, and some
436
+ * hop between the bot and this Worker silently drops idle connections
437
+ * (measured live: undici gives up 300s after sending a request that has
438
+ * received no headers). Heartbeats are pure whitespace, which is legal
439
+ * around a JSON document — the executor's res.json() on the full body parses
440
+ * unchanged. All outcomes arrive in-body with HTTP 200 (headers are long
441
+ * gone by the time the result is known): a completed command as {stdout,
442
+ * stderr, exitCode}, a sandbox-enforced timeout as exit 124, and any other
443
+ * failure as {error} (docs/reference/specs/execution.md item 3). */
444
+ function streamExec(run: () => Promise<ExecAnswer | ExecFailure>, traceparent: string | undefined): Response {
387
445
  const encoder = new TextEncoder();
388
- // Per ATTEMPT, not per request: withSessionRecovery may run the command a
389
- // second time after a stale-session reset, and a retry's own failure must be
390
- // judged on its own clock, not the first attempt's.
391
- let attemptStartedAt = systemClock();
446
+ const startedAt = systemClock();
392
447
  const stream = new ReadableStream<Uint8Array>({
393
448
  start(controller) {
394
449
  const beat = setInterval(() => {
395
450
  try {
396
451
  controller.enqueue(encoder.encode("\n"));
397
452
  } catch {
398
- clearInterval(beat); // client went away; the exec promise still settles
453
+ clearInterval(beat); // client went away; the promise still settles
399
454
  }
400
455
  }, 15_000);
401
456
  const finish = (payload: object) => {
@@ -407,81 +462,59 @@ function streamExec(
407
462
  // stream already errored/cancelled — nothing left to deliver to
408
463
  }
409
464
  };
410
- withSessionRecovery(sandbox, () => {
411
- attemptStartedAt = systemClock();
412
- return sandbox.exec(command, options);
413
- })
414
- .then((result) => {
415
- const exitCode = result.exitCode ?? 0;
416
- // coreutils `timeout` exits 124 when the deadline killed the
417
- // command (137 when the follow-up SIGKILL had to) — annotate so the
418
- // agent knows what happened and how to adapt.
419
- const timedOut = exitCode === 124 || exitCode === 137;
420
- // A job past the ceiling is detached with `setsid -f`: every /exec
421
- // runs under `timeout … bash -c`, whose process group is reaped when
422
- // the command returns, so a plain background job dies with it.
423
- const note = timedOut
424
- ? `command timed out in the sandbox after ${execTimeoutSecs}s (pass the bash tool's timeoutMs for longer commands, max ${BASH_TIMEOUT_MAX_MS} ms); ` +
425
- "re-run as smaller/faster steps, or start it detached with `setsid -f sh -c '<command> > /tmp/job.log 2>&1'` and poll the log on later calls"
426
- : "";
465
+ run()
466
+ .then((answer) => {
467
+ if ("error" in answer) {
468
+ // A command the sandbox never answered for, named by the Durable
469
+ // Object: an infra failure, classified, no message on the span.
470
+ const root = execRoot(startedAt, traceparent);
471
+ root.fail(classifyError(new Error("sandbox exec failed"), { kind: "infra" }));
472
+ root.end("error");
473
+ finish(answer);
474
+ return;
475
+ }
427
476
  // The command as the Worker's own root (docs/reference/specs/tracing.md item 22).
428
- execRoot(attemptStartedAt, traceparent).end(exitCode === 0 ? "ok" : "error", {
429
- exitCode: timedOut ? 124 : exitCode,
430
- ...(timedOut ? { timedOut: true } : {}),
431
- });
432
- finish({
433
- stdout: result.stdout ?? "",
434
- stderr: [result.stderr ?? "", note].filter(Boolean).join("\n"),
435
- exitCode: timedOut ? 124 : exitCode,
436
- // The command's wall time in the sandbox, for the bot's `exec.exec`
437
- // span (docs/reference/specs/tracing.md item 19); an older client ignores it.
438
- durationMs: systemClock() - attemptStartedAt,
477
+ execRoot(startedAt, traceparent).end(answer.exitCode === 0 ? "ok" : "error", {
478
+ exitCode: answer.exitCode,
479
+ ...(answer.exitCode === 124 ? { timedOut: true } : {}),
439
480
  });
481
+ finish(answer);
440
482
  })
441
483
  .catch((err: unknown) => {
442
- // A command the sandbox never answered for: an infra failure, classified, no message.
443
- const root = execRoot(attemptStartedAt, traceparent);
484
+ // The Durable Object threw: a failure its own classification did
485
+ // not name, seen here after the RPC boundary (name and message kept,
486
+ // prototype dropped), so the shared classifiers read the shape.
487
+ const root = execRoot(startedAt, traceparent);
444
488
  root.fail(classifyError(new Error("sandbox exec failed"), { kind: "infra" }));
445
489
  root.end("error");
446
490
  const shape = thrownShape(err);
447
- // The text that leaves the Worker is never empty (item 3): a
448
- // message-less SDK error is named, with the rollout hint. The
449
- // classifiers below still read the raw `shape`.
450
491
  const raw = thrownText(shape);
451
- // Nothing answered at the container's port (item 9): named, with the
452
- // container, in the dual shape — and never re-sent by anyone, since a
453
- // command in flight when the server died may have run.
454
492
  if (isRuntimeUnreachableError(err)) {
455
493
  finish(runtimeUnreachableExecAnswer(raw));
456
494
  return;
457
495
  }
458
- // A full fleet (docs/reference/specs/execution.md item 14): session creation
459
- // failed because no container instance was free, so the command
460
- // never started — re-sending it is safe by construction. The named
461
- // `reason` is what the executor waits on; the dual shape below is
462
- // kept so an older executor still renders it as exit 127.
463
496
  if (isFleetBusyError(err)) {
464
497
  finish(fleetBusyExecAnswer(raw));
465
498
  return;
466
499
  }
467
- // The container was replaced under the command (docs/reference/specs/execution.md
468
- // item 2): certain when the SDK says so with a typed error, inferred
469
- // when a recycle-shaped text arrives minutes into this attempt. Say
470
- // so, and that /workspace is gone. Still exit 127: the workspace
471
- // really is gone.
500
+ // A recycle the Durable Object did not catch by type: by name, or
501
+ // a recycle-shaped text minutes into the attempt (item 9).
472
502
  const certain = shape.name !== undefined && isRecycleError({ name: shape.name });
473
- const msg = recycledMidCommandMessage(systemClock() - attemptStartedAt, raw, certain);
474
- // Carry the failure in BOTH shapes so rollout order can't create
475
- // a silent-success window: a new executor throws on `error`, and
476
- // an executor that predates in-body errors (only checks exitCode)
477
- // still renders "exit 127: <message>" instead of "(no output)".
478
- finish({ error: msg, stdout: "", stderr: msg, exitCode: 127 });
503
+ const msg = recycledMidCommandMessage(systemClock() - startedAt, raw, certain);
504
+ finish({ error: msg, stdout: "", stderr: msg, exitCode: 127 } satisfies ExecFailure);
479
505
  });
480
506
  },
481
507
  });
482
508
  return new Response(stream, { headers: { "content-type": "application/json" } });
483
509
  }
484
510
 
511
+ /** A file route's refusal as the fetch handler answers it: the text, and the
512
+ * machine token when the Durable Object named one — the executor matches
513
+ * `reason`, not the text (docs/reference/specs/execution.md items 9 and 14). */
514
+ function refused(r: FileRefusal): Response {
515
+ return json(r.reason ? { error: r.error, reason: r.reason } : { error: r.error }, r.status);
516
+ }
517
+
485
518
  function abs(p: string): string {
486
519
  if (!p) throw new Error("missing path");
487
520
  const path = p.startsWith("/") ? p : `${WORKDIR}/${p}`;