@inflexa-ai/harness 0.42.2 → 0.43.0

package/dist/sandbox/create-sandbox.js

@@ -8,38 +8,28 @@
  *
  * Registry write/clear callbacks are wired here so the per-backend
  * `createSandbox` / `teardown` implementations stay free of state-layer
- * coupling (they just call the closures we hand them). `submitExec` and
- * `awaitExec` are backend-agnostic — they only need the SandboxRef.
+ * coupling (they just call the closures we hand them). `exec` is
+ * backend-agnostic — it only needs the SandboxRef.
  */
 import { chmod, mkdir } from "node:fs/promises";
 import { join } from "node:path";
+import { DBOS } from "@dbos-inc/dbos-sdk";
 import { ResultAsync, err, okAsync } from "neverthrow";
 import { passGate } from "../lib/hooks.js";
 import { recordSandboxExec } from "../lib/metrics.js";
+import { ATTR_INFLEXA_EXEC_ID, stableSpan } from "../lib/otel-spans.js";
 import { clampResources } from "../config/resource-limits.js";
 import { tailWritePrefix } from "../workspace/paths.js";
 import { tryMutation } from "../lib/db-result.js";
 import { unwrapOrThrow } from "../lib/result.js";
 import { clearSandboxRef, setSandboxRef } from "../state/index.js";
 import { capExecStreams, EXEC_STREAM_BYTE_CAP } from "../tools/workspace/result-bounds.js";
-import { awaitExec } from "./await-exec.js";
 import { createDockerSandboxOps } from "./docker-client.js";
+import { runExec, stepExecId } from "./exec.js";
 import { noteExecOutcome, sandboxExecOutcomeOf, summarizeExec } from "./exec-outcome.js";
 import { createK8sSandboxOps } from "./k8s-client.js";
 import { mountCoordsOf, sandboxWriteTail } from "./mount-plan.js";
 import { SandboxFailure } from "./sandbox-error.js";
-import { submitExec } from "./submit-exec.js";
-import { toPersistedRef, } from "./types.js";
-/**
- * Assemble `awaitExec`'s options: the liveness probe self-wires from the
- * backend ops (the poll loop's escalation always has its arbiter under the
- * client), explicit seam injections in `base` win over the self-wired probe,
- * and the transport is client-owned — never overridable through the seam bag.
- * Exported for tests.
- */
-export function composeAwaitOptions(base, transport, isAlive) {
-    return { isAlive, ...base, transport };
-}
 /**
  * Backend-agnostic pre-creation of the writable tree under the analysis's
  * resolved workspace root. On Docker that root is the host dir the container
@@ -77,7 +67,6 @@ export async function precreateStepTree(deps, coords) {
 }
 export function createSandboxClient(config) {
     const backend = config.backend ?? config.env.backend;
-    const transport = config.transport ?? "poll";
     // Normalized once, here, and handed to the backends and to the client
     // alike: the mount plan and the orient-core prompt then key on one value,
     // and an absent declaration means "store" on both sides.
@@ -112,13 +101,11 @@ export function createSandboxClient(config) {
      */
     const failing = (e) => new SandboxFailure(e);
     const registerSandbox = async (session, spec, ref) => {
-        unwrapOrThrow(await setSandboxRef(config.pool, session.runFrame.runId, session.runFrame.stepId, toPersistedRef(ref), spec.execId ?? null));
+        unwrapOrThrow(await setSandboxRef(config.pool, session.runFrame.runId, session.runFrame.stepId, ref));
     };
     const ops = backend === "k8s"
         ? createK8sSandboxOps({
             image: config.image,
-            cortexBaseUrl: config.cortexBaseUrl,
-            transport,
             namespace: config.namespace ?? config.env.namespace,
             sessionPvc: config.sessionPvc,
             sessionPvcRoot: config.sessionPvcRoot,
@@ -137,8 +124,6 @@ export function createSandboxClient(config) {
         })
         : createDockerSandboxOps({
             image: config.image,
-            cortexBaseUrl: config.cortexBaseUrl,
-            transport,
             resolveWorkspaceRoot: config.resolveWorkspaceRoot,
             libStorePath: config.libStorePath,
             farmSource: config.farmSource,
@@ -164,7 +149,7 @@ export function createSandboxClient(config) {
         unwrapOrThrow(await tryMutation("createSandbox.teardownClearSandboxRef", async () => {
             await config.pool.query({
                 text: `UPDATE cortex_step_executions
-            SET sandbox_ref = NULL, exec_id = NULL
+            SET sandbox_ref = NULL
             WHERE sandbox_ref->>'sandboxId' = $1`,
                 values: [ref.sandboxId],
             });
@@ -173,6 +158,8 @@ export function createSandboxClient(config) {
     // One arbiter for both surfaces: the client's `isAlive` method and the
     // poll loop's escalation probe are the same backend inspect.
     const isAlive = async (ref) => unwrapOrThrow((await ops.isAlive(ref)).mapErr(failing));
+    const streamCap = config.execStreamByteCap ?? EXEC_STREAM_BYTE_CAP;
+    const execDeps = { isAlive, ...config.execDeps, execStreamByteCap: streamCap };
     const hostLabelsFor = (session) => {
         const hook = config.resolveSandboxLabels;
         if (hook === undefined)
@@ -217,10 +204,6 @@ export function createSandboxClient(config) {
     return {
         toolchainSource,
         createSandbox: (session, spec, identity) => new ResultAsync(createSandbox(session, spec, identity)),
-        submitExec: async (ref, body) => submitExec(ref, body, {
-            ...config.submitDeps,
-            execStreamByteCap: config.submitDeps?.execStreamByteCap ?? config.execStreamByteCap,
-        }),
         // The one place an ExecResult crosses from the wire into the process, so
         // the cap lands here rather than at each consumer. Every downstream use —
         // tool results, workflow return values, durable step outputs — is bounded
@@ -232,11 +215,13 @@ export function createSandboxClient(config) {
         // and it sees the outcome the caller then folds into a tool result, so
         // one record here counts what a consumer would each have to count for
         // itself. `recordSandboxExec` is idempotent over the exec id, because a
-        // replayed body reaches this line again. `capExecStreams` cuts the
+        // recovered step reaches this line again. `capExecStreams` cuts the
         // streams and keeps every other field, thus the kernel accounting the
         // sandbox reported reaches the record intact.
-        awaitExec: async (ref, execId, emit, deadline) => {
-            const result = capExecStreams(await awaitExec(ref, execId, emit, deadline, composeAwaitOptions(config.awaitOptions, transport, isAlive)), config.execStreamByteCap ?? EXEC_STREAM_BYTE_CAP);
+        exec: (ref, request, emit, deadline) => DBOS.runStep(async () => {
+            const execId = stepExecId();
+            stableSpan("sandbox.exec", "sandbox.exec", { [ATTR_INFLEXA_EXEC_ID]: execId });
+            const result = capExecStreams(await runExec(ref, execId, request, emit, deadline, execDeps), streamCap);
             recordSandboxExec({
                 execId,
                 outcome: sandboxExecOutcomeOf(result),
@@ -246,7 +231,7 @@ export function createSandboxClient(config) {
             });
             noteExecOutcome(ref.sandboxId, summarizeExec(result));
             return result;
-        },
+        }, { name: "sandbox.exec" }),
         isAlive,
         isAliveById: async (sandboxId) => unwrapOrThrow((await ops.isAliveById(sandboxId)).mapErr(failing)),
         teardown,

package/dist/sandbox/docker-client.d.ts

@@ -17,26 +17,20 @@
  * that the sandbox sees. `teardown` removes the directory when this process
  * wrote it. A directory from an earlier process stays.
  *
- * ## Transport and confinement
+ * ## Confinement
  *
  * The container joins the default bridge and publishes sandbox-server's port to
- * `127.0.0.1` only, so the host can reach `/exec` but the LAN cannot. What the
- * sandbox may do *outbound* depends on the transport:
+ * `127.0.0.1` only, so the host can reach `/exec` but the LAN cannot. The
+ * sandbox initiates nothing, so it gets no egress. The container is created as
+ * root with `CAP_NET_ADMIN` and the `SANDBOX_EGRESS_FIREWALL` flag; the image's
+ * root entrypoint installs `iptables -P OUTPUT DROP` (allowing loopback and
+ * established return traffic) and then `setpriv`-drops to the uid-1000
+ * workload, which can neither reach the network nor flush the rules. The host
+ * polls `GET /exec/{execId}?since={cursor}` over the published port; the reply
+ * rides the established connection, so polling works with egress hard-blocked.
  *
- *   - **poll** (default): the sandbox initiates nothing, so it needs no egress.
- *     The container is created as root with `CAP_NET_ADMIN` and the
- *     `SANDBOX_EGRESS_FIREWALL` flag; the image's root entrypoint installs
- *     `iptables -P OUTPUT DROP` (allowing loopback and established return
- *     traffic) and then `setpriv`-drops to the uid-1000 workload, which can
- *     neither reach the network nor flush the rules. The host polls
- *     `GET /exec/{execId}?since={cursor}` over the published port; the reply
- *     rides the established connection, so polling works with egress hard-blocked.
- *   - **callback**: the sandbox is *allowed* egress and POSTs signed callbacks to
- *     `CORTEX_BASE_URL`. It runs as uid 1000 throughout with no `NET_ADMIN` and
- *     no firewall.
- *
- * Either way the workload ends as uid 1000 with `no-new-privileges` and no
- * effective capabilities.
+ * The workload ends as uid 1000 with `no-new-privileges` and no effective
+ * capabilities.
  */
 import Docker from "dockerode";
 import { ResultAsync } from "neverthrow";
@@ -44,13 +38,9 @@ import type { SpawnSession } from "../auth/types.js";
 import type { Logger } from "../lib/logger.js";
 import { type ResolveWorkspaceRoot } from "../workspace/paths.js";
 import { type SandboxError } from "./sandbox-error.js";
-import type { FarmSource, ManagedSandbox, SandboxIdentity, SandboxLabels, SandboxLiveness, SandboxRef, SandboxSpec, SandboxTransport, ToolchainSource } from "./types.js";
+import type { FarmSource, ManagedSandbox, SandboxIdentity, SandboxLabels, SandboxLiveness, SandboxRef, SandboxSpec, ToolchainSource } from "./types.js";
 export interface DockerClientConfig {
     image: string;
-    /** Cortex base URL injected into the sandbox env in callback mode so callbacks land here. Unused in poll mode. */
-    cortexBaseUrl: string;
-    /** Result transport. `poll` (default) confines the sandbox with the egress firewall; `callback` permits egress. */
-    transport?: SandboxTransport;
     /** Workspace-root resolution seam; each analysis's resolved root is the bind source for its tree mounts. */
     resolveWorkspaceRoot: ResolveWorkspaceRoot;
     /** Host lib store; bind-mounted read-only at `/mnt/libs` when set. */

package/dist/sandbox/docker-client.js

@@ -17,26 +17,20 @@
  * that the sandbox sees. `teardown` removes the directory when this process
  * wrote it. A directory from an earlier process stays.
  *
- * ## Transport and confinement
+ * ## Confinement
  *
  * The container joins the default bridge and publishes sandbox-server's port to
- * `127.0.0.1` only, so the host can reach `/exec` but the LAN cannot. What the
- * sandbox may do *outbound* depends on the transport:
+ * `127.0.0.1` only, so the host can reach `/exec` but the LAN cannot. The
+ * sandbox initiates nothing, so it gets no egress. The container is created as
+ * root with `CAP_NET_ADMIN` and the `SANDBOX_EGRESS_FIREWALL` flag; the image's
+ * root entrypoint installs `iptables -P OUTPUT DROP` (allowing loopback and
+ * established return traffic) and then `setpriv`-drops to the uid-1000
+ * workload, which can neither reach the network nor flush the rules. The host
+ * polls `GET /exec/{execId}?since={cursor}` over the published port; the reply
+ * rides the established connection, so polling works with egress hard-blocked.
  *
- *   - **poll** (default): the sandbox initiates nothing, so it needs no egress.
- *     The container is created as root with `CAP_NET_ADMIN` and the
- *     `SANDBOX_EGRESS_FIREWALL` flag; the image's root entrypoint installs
- *     `iptables -P OUTPUT DROP` (allowing loopback and established return
- *     traffic) and then `setpriv`-drops to the uid-1000 workload, which can
- *     neither reach the network nor flush the rules. The host polls
- *     `GET /exec/{execId}?since={cursor}` over the published port; the reply
- *     rides the established connection, so polling works with egress hard-blocked.
- *   - **callback**: the sandbox is *allowed* egress and POSTs signed callbacks to
- *     `CORTEX_BASE_URL`. It runs as uid 1000 throughout with no `NET_ADMIN` and
- *     no firewall.
- *
- * Either way the workload ends as uid 1000 with `no-new-privileges` and no
- * effective capabilities.
+ * The workload ends as uid 1000 with `no-new-privileges` and no effective
+ * capabilities.
  */
 import { lstatSync } from "node:fs";
 import { mkdir, rm, writeFile } from "node:fs/promises";
@@ -59,14 +53,12 @@ const SANDBOX_SERVER_PORT = 8765;
 const HEALTH_TIMEOUT_MS = 30_000;
 /** Only this process dials the published port; binding it on every host interface would expose `/exec` to the LAN. */
 const SANDBOX_PORT_HOST_IP = "127.0.0.1";
-/** Matches `USER sandbox` (uid/gid 1000) in the sandbox images — the callback-mode workload user. */
-const SANDBOX_USER = "1000:1000";
 /**
- * Poll mode starts the container as root so the entrypoint can install the
- * egress firewall; the entrypoint then `setpriv`-drops to uid 1000, so the
- * workload still runs unprivileged. sandbox-server refuses to start as root
- * when the firewall flag is set, so an image whose entrypoint skips the drop
- * fails at create time rather than running privileged and unconfined.
+ * The container starts as root so the entrypoint can install the egress
+ * firewall; the entrypoint then `setpriv`-drops to uid 1000, so the workload
+ * still runs unprivileged. sandbox-server refuses to start as root when the
+ * firewall flag is set, so an image whose entrypoint skips the drop fails at
+ * create time rather than running privileged and unconfined.
  */
 const SANDBOX_ROOT_USER = "0:0";
 const HEALTH_POLL_MS = 250;
@@ -223,15 +215,13 @@ export function createDockerSandboxOps(config) {
     const docker = config.docker ?? (connection ? new Docker(connection) : new Docker());
     const logger = (config.logger ?? createNoopLogger()).named("docker-client");
     const fetchImpl = config.fetch ?? fetch;
-    const transport = config.transport ?? "poll";
-    const pollMode = transport === "poll";
     const readHostCpuinfo = config.readHostCpuinfo ?? readHostCpuinfoOnce;
     /** The cpu directory of each sandbox that this process wrote, for `teardown`. */
     const cpuDirs = new Map();
     return {
         createSandbox(session, spec, identity, hostLabels) {
             return new ResultAsync((async () => {
-                const { sandboxId, callbackSecret } = identity;
+                const { sandboxId } = identity;
                 const analysisId = session.scope.analysisId;
                 const coords = mountCoordsOf(session, spec);
                 // Farm resolution comes before any container work: a resolver
@@ -337,17 +327,14 @@ export function createDockerSandboxOps(config) {
                 });
                 const image = spec.image ?? config.image;
                 const limits = spec.resources;
-                // Poll mode never dials out and carries no CORTEX_BASE_URL; it sets the
-                // firewall flag so the root entrypoint installs the egress block before
-                // dropping to uid 1000. Callback mode is the inverse.
+                // The firewall flag makes the root entrypoint install the egress block
+                // before it drops to uid 1000.
                 //
                 // Composed as one record, not as a list of `K=V`. A duplicate key in
                 // the Env array resolves at the libc that scans `environ`, thus the
                 // later spread must win here, not there.
                 const env = Object.entries({
-                    SANDBOX_TRANSPORT: transport,
-                    SANDBOX_CALLBACK_SECRET: callbackSecret,
-                    ...(pollMode ? { SANDBOX_EGRESS_FIREWALL: "1" } : { CORTEX_BASE_URL: config.cortexBaseUrl }),
+                    SANDBOX_EGRESS_FIREWALL: "1",
                     ...threadLimitEnv(limits),
                     ...plan.env,
                     ...(spec.extraEnv ?? {}),
@@ -357,7 +344,7 @@ export function createDockerSandboxOps(config) {
                     ...(config.platform !== undefined && { platform: config.platform }),
                     Image: image,
                     Env: env,
-                    User: pollMode ? SANDBOX_ROOT_USER : SANDBOX_USER,
+                    User: SANDBOX_ROOT_USER,
                     WorkingDir: plan.workingDir,
                     Labels: mergeLabels(hostLabels, { ...harnessLabels(session, sandboxId), [OWNER_WORKFLOW_LABEL]: spec.childWorkflowId }),
                     ExposedPorts: { [`${SANDBOX_SERVER_PORT}/tcp`]: {} },
@@ -368,14 +355,14 @@ export function createDockerSandboxOps(config) {
                             [`${SANDBOX_SERVER_PORT}/tcp`]: [{ HostIp: SANDBOX_PORT_HOST_IP, HostPort: "0" }],
                         },
                         CapDrop: ["ALL"],
-                        // Poll mode grants the ROOT entrypoint exactly what its privileged
-                        // setup needs: NET_ADMIN to install the egress iptables rules,
-                        // SETUID/SETGID for the setpriv uid/gid drop to the workload user
-                        // (setresuid/setresgid fail EPERM without them), and SETPCAP to apply
-                        // setpriv's `--bounding-set=-all`. The entrypoint drops all of them
-                        // before the workload runs — the workload's capability sets end empty
-                        // — and `no-new-privileges` prevents regaining any.
-                        ...(pollMode ? { CapAdd: ["NET_ADMIN", "SETUID", "SETGID", "SETPCAP"] } : {}),
+                        // The ROOT entrypoint gets exactly what its privileged setup needs:
+                        // NET_ADMIN to install the egress iptables rules, SETUID/SETGID for
+                        // the setpriv uid/gid drop to the workload user (setresuid/setresgid
+                        // fail EPERM without them), and SETPCAP to apply setpriv's
+                        // `--bounding-set=-all`. The entrypoint drops all of them before the
+                        // workload runs — the workload's capability sets end empty — and
+                        // `no-new-privileges` prevents regaining any.
+                        CapAdd: ["NET_ADMIN", "SETUID", "SETGID", "SETPCAP"],
                         SecurityOpt: ["no-new-privileges"],
                         NanoCpus: Math.round(limits.cpu * 1e9),
                         Memory: limits.memoryGb * 1024 ** 3,
@@ -415,7 +402,6 @@ export function createDockerSandboxOps(config) {
                     host,
                     port,
                     backend: "docker",
-                    callbackSecret,
                 };
                 const registered = await trySandbox(() => config.registerSandbox(session, spec, ref), createFailed);
                 if (registered.isErr())

package/dist/sandbox/exec.d.ts

@@ -0,0 +1,58 @@
+/**
+ * One sandbox exec: submit the command, then poll until it is terminal.
+ *
+ * The whole exec runs inside ONE durable step (`SandboxClient.exec` wraps it),
+ * thus nothing here touches DBOS but the cancellation read. The polls, the
+ * sleeps, and the liveness probes are plain calls. A replay returns the cached
+ * result of the step and never reaches this module.
+ *
+ * A recovered step runs this module again from the start. The sandbox makes
+ * that safe: the submit is idempotent on `execId`, and the sandbox keeps the
+ * record of each exec it ran, thus a second submit attaches to the running or
+ * finished exec instead of a second run. The poll then starts at cursor 0, and
+ * the caller gets the events of the exec again.
+ */
+import { type ExecEmit, type ExecRequest, type ExecResult, type SandboxLiveness, type SandboxRef } from "./types.js";
+export declare class ExecTimeoutError extends Error {
+    readonly execId: string;
+    constructor(execId: string);
+}
+/** The effects of one exec. Each member has a default, and a test replaces it. */
+export interface ExecDeps {
+    /** Defaults to `globalThis.fetch`. */
+    readonly fetch?: typeof fetch;
+    /** The clock of the deadline and of the cancel reads. Defaults to `Date.now`. */
+    readonly now?: () => number;
+    /** The pause between two polls. Defaults to a timer. */
+    readonly sleep?: (ms: number) => Promise<void>;
+    /**
+     * Sink for advisory warnings: events that the sandbox ring shed. Defaults to
+     * `DBOS.logger.warn`. The terminal result, not the event stream, is the
+     * outcome of the exec.
+     */
+    readonly warn?: (message: string) => void;
+    /**
+     * Backend inspect for the liveness escalation (`liveness.ts`). Absent, the
+     * exec never probes and only the deadline bounds it. A throw is an
+     * inconclusive probe, never a failed exec.
+     */
+    readonly isAlive?: (ref: SandboxRef) => Promise<SandboxLiveness>;
+    /** True once the owning workflow is cancelled. Defaults to a read of the DBOS workflow status. */
+    readonly isCancelled?: () => Promise<boolean>;
+    /** Per-stream retention budget sent with the submit. Defaults to `EXEC_STREAM_BYTE_CAP`. */
+    readonly execStreamByteCap?: number;
+}
+/**
+ * The id of the exec that runs in the current durable step: the workflow id
+ * and the function id of the step. Both are stable across a replay, and DBOS
+ * makes the pair unique, thus no caller mints an exec id.
+ */
+export declare function stepExecId(): string;
+/**
+ * Run one exec to its terminal result.
+ *
+ * `deadlineMs` is an absolute unix-ms timestamp. A machine that the liveness
+ * probe finds dead ends the exec with a synthetic-failure result. A cancel of
+ * the owning workflow throws `DBOSWorkflowCancelledError`.
+ */
+export declare function runExec(ref: SandboxRef, execId: string, request: ExecRequest, emit: ExecEmit, deadlineMs: number, deps?: ExecDeps): Promise<ExecResult>;

package/dist/sandbox/exec.js

@@ -0,0 +1,197 @@
+/**
+ * One sandbox exec: submit the command, then poll until it is terminal.
+ *
+ * The whole exec runs inside ONE durable step (`SandboxClient.exec` wraps it),
+ * thus nothing here touches DBOS but the cancellation read. The polls, the
+ * sleeps, and the liveness probes are plain calls. A replay returns the cached
+ * result of the step and never reaches this module.
+ *
+ * A recovered step runs this module again from the start. The sandbox makes
+ * that safe: the submit is idempotent on `execId`, and the sandbox keeps the
+ * record of each exec it ran, thus a second submit attaches to the running or
+ * finished exec instead of a second run. The poll then starts at cursor 0, and
+ * the caller gets the events of the exec again.
+ */
+import { DBOS, Error as DBOSErrors, StatusString } from "@dbos-inc/dbos-sdk";
+import { untracedFetch } from "../lib/otel-spans.js";
+import { EXEC_STREAM_BYTE_CAP } from "../tools/workspace/result-bounds.js";
+import { createEscalationPolicy, probeLiveness, syntheticFailureReason, syntheticFailureResult } from "./liveness.js";
+import { PollResponseSchema } from "./types.js";
+const SUBMIT_HTTP_TIMEOUT_MS = 30_000;
+const POLL_HTTP_TIMEOUT_MS = 10_000;
+/**
+ * Two-phase poll cadence: 1.5 s for the first 40 polls (about the first minute
+ * of the exec) keeps a short command prompt, and 10 s after that bounds the
+ * request rate of an exec that runs for hours.
+ */
+const FAST_POLL_INTERVAL_MS = 1_500;
+const FAST_POLL_ATTEMPTS = 40;
+const SLOW_POLL_INTERVAL_MS = 10_000;
+/**
+ * The minimum gap between two reads of the workflow status. A step does not see
+ * a cancel on its own, thus the exec reads the status of its workflow and stops
+ * when the workflow is cancelled.
+ */
+const CANCEL_CHECK_INTERVAL_MS = 10_000;
+export class ExecTimeoutError extends Error {
+    execId;
+    constructor(execId) {
+        super(`exec[${execId}]: deadline exceeded before a terminal result`);
+        this.execId = execId;
+    }
+}
+/**
+ * The id of the exec that runs in the current durable step: the workflow id
+ * and the function id of the step. Both are stable across a replay, and DBOS
+ * makes the pair unique, thus no caller mints an exec id.
+ */
+export function stepExecId() {
+    const workflowId = DBOS.workflowID;
+    const stepId = DBOS.stepID;
+    if (workflowId === undefined || stepId === undefined) {
+        throw new Error("a sandbox exec runs inside a DBOS workflow, and none is active");
+    }
+    return `${workflowId}:${stepId}`;
+}
+/**
+ * Run one exec to its terminal result.
+ *
+ * `deadlineMs` is an absolute unix-ms timestamp. A machine that the liveness
+ * probe finds dead ends the exec with a synthetic-failure result. A cancel of
+ * the owning workflow throws `DBOSWorkflowCancelledError`.
+ */
+export async function runExec(ref, execId, request, emit, deadlineMs, deps = {}) {
+    const fetchImpl = deps.fetch ?? fetch;
+    const cap = deps.execStreamByteCap ?? EXEC_STREAM_BYTE_CAP;
+    await submitExec(fetchImpl, ref, {
+        command: [...request.command],
+        execId,
+        ...(request.cwd === undefined ? {} : { cwd: request.cwd }),
+        ...(request.env === undefined ? {} : { env: { ...request.env } }),
+        ...(request.timeoutSeconds === undefined ? {} : { timeoutSeconds: request.timeoutSeconds }),
+        stdoutByteCap: cap,
+        stderrByteCap: cap,
+    });
+    return pollExec(ref, execId, emit, deadlineMs, deps);
+}
+/**
+ * POST the command to `/exec` and return after the 202 ack. A non-202 throws.
+ * The server dedups on `execId`: a submit of an exec that it already holds
+ * returns 202 with the stored status and runs nothing.
+ */
+async function submitExec(fetchImpl, ref, body) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), SUBMIT_HTTP_TIMEOUT_MS);
+    try {
+        const res = await untracedFetch(fetchImpl, `http://${ref.host}:${ref.port}/exec`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(body),
+            signal: controller.signal,
+        });
+        if (res.status !== 202) {
+            const text = await res.text().catch(() => "");
+            throw Object.assign(new Error(`submitExec: sandbox-server returned ${res.status} for execId=${body.execId}: ${text}`), { statusCode: res.status });
+        }
+        // Read and discard the ack, so the socket is released.
+        await res.text().catch(() => "");
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+async function pollOnce(fetchImpl, ref, execId, cursor) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), POLL_HTTP_TIMEOUT_MS);
+    try {
+        const res = await untracedFetch(fetchImpl, `http://${ref.host}:${ref.port}/exec/${encodeURIComponent(execId)}?since=${cursor}`, {
+            method: "GET",
+            signal: controller.signal,
+        });
+        if (!res.ok) {
+            await res.text().catch(() => "");
+            return { kind: "unavailable", detail: `status ${res.status}` };
+        }
+        return { kind: "ok", raw: await res.text() };
+    }
+    catch (cause) {
+        return { kind: "unavailable", detail: cause instanceof Error ? cause.message : String(cause) };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * Poll until the exec is terminal. Each poll reads
+ * `{ status, events, cursor, result? }`, forwards the events past the local
+ * cursor through `emit`, and returns `result` once it is there.
+ *
+ * Consecutive `unavailable` polls arm a backend probe (`liveness.ts`). Only an
+ * observably dead machine fails the exec, because the result lives in that
+ * machine and nowhere else.
+ */
+async function pollExec(ref, execId, emit, deadlineMs, deps) {
+    const fetchImpl = deps.fetch ?? fetch;
+    const now = deps.now ?? Date.now;
+    const sleep = deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const warn = deps.warn ?? ((message) => DBOS.logger.warn(message));
+    const isCancelled = deps.isCancelled ?? workflowCancelled;
+    let cursor = 0;
+    let pollAttempt = 0;
+    let lastCancelCheckMs = now();
+    const escalation = createEscalationPolicy();
+    while (true) {
+        pollAttempt += 1;
+        const polled = await pollOnce(fetchImpl, ref, execId, cursor);
+        if (polled.kind === "ok") {
+            const resp = PollResponseSchema.parse(JSON.parse(polled.raw));
+            // Each event is emitted at most once, whatever `since` the server applied.
+            const newEvents = resp.events.filter((ev) => ev.seq > cursor);
+            // Sequence numbers are contiguous per exec, thus a hole between the local
+            // cursor and the next delivered (or high-water) seq proves that the ring
+            // shed events before the host saw them.
+            const nextSeq = newEvents.length > 0 ? newEvents[0].seq : resp.cursor > cursor ? resp.cursor + 1 : cursor + 1;
+            const lost = nextSeq - cursor - 1;
+            if (lost > 0) {
+                warn(`exec[${execId}]: ${lost} progress event(s) (seq ${cursor + 1}-${cursor + lost}) shed by the sandbox ring before delivery; the terminal result is unaffected`);
+            }
+            // Forwarded before the return, thus trailing progress lands ahead of the result.
+            for (const ev of newEvents) {
+                await emit(ev.payload);
+            }
+            if (resp.cursor > cursor)
+                cursor = resp.cursor;
+            if (resp.result !== undefined)
+                return resp.result;
+        }
+        if (deps.isAlive && escalation.onPoll(polled.kind === "ok" ? "ok" : "unavailable")) {
+            const verdict = await probeLiveness(deps.isAlive, ref);
+            // `alive` (a slow exec) and `inconclusive` (a backend
+            // API error) both resume the poll, bounded by the deadline.
+            if (verdict.kind === "dead") {
+                return syntheticFailureResult(execId, syntheticFailureReason({ oomKilled: verdict.oomKilled }));
+            }
+        }
+        // The deadline gate sits AFTER the poll, thus the loop always asks once more
+        // before it declares a timeout: a missed poll window is not a slow command.
+        const nowMs = now();
+        const remainingMs = deadlineMs - nowMs;
+        if (remainingMs <= 0)
+            throw new ExecTimeoutError(execId);
+        if (nowMs - lastCancelCheckMs >= CANCEL_CHECK_INTERVAL_MS) {
+            lastCancelCheckMs = nowMs;
+            if (await isCancelled())
+                throw new DBOSErrors.DBOSWorkflowCancelledError(DBOS.workflowID ?? execId);
+        }
+        const intervalMs = pollAttempt <= FAST_POLL_ATTEMPTS ? FAST_POLL_INTERVAL_MS : SLOW_POLL_INTERVAL_MS;
+        await sleep(Math.min(intervalMs, remainingMs));
+    }
+}
+/** Read the status of the workflow that runs this exec. Outside a workflow, nothing can cancel it. */
+async function workflowCancelled() {
+    const workflowId = DBOS.workflowID;
+    if (workflowId === undefined)
+        return false;
+    const status = await DBOS.getWorkflowStatus(workflowId);
+    return status?.status === StatusString.CANCELLED;
+}

package/dist/sandbox/identity.d.ts

@@ -1,10 +1,10 @@
 /**
  * Sandbox identity minting — the durable half of the two-step create (ADR
- * 0016). Backend-agnostic: the `sandboxId` and HMAC `callbackSecret` are minted
- * here, checkpointed by the `sandbox.mint` DBOS step, then handed to the
- * backend's spawn step. Because the identity is durable before any machine
- * exists, a crash between spawn and the spawn-step checkpoint is recoverable —
- * the re-run adopts the existing machine under the same name + secret.
+ * 0016). Backend-agnostic: the `sandboxId` is minted here, checkpointed by the
+ * `sandbox.mint` DBOS step, then handed to the backend's spawn step. Because the
+ * identity is durable before any machine exists, a crash between spawn and the
+ * spawn-step checkpoint is recoverable — the re-run adopts the existing machine
+ * under the same name.
  */
 import type { SandboxIdentity } from "./types.js";
 /** `sbx-{run8}-{rand8}` — run prefix aids `kubectl` triage; the 8-hex random

package/dist/sandbox/identity.js

@@ -1,12 +1,12 @@
 /**
  * Sandbox identity minting — the durable half of the two-step create (ADR
- * 0016). Backend-agnostic: the `sandboxId` and HMAC `callbackSecret` are minted
- * here, checkpointed by the `sandbox.mint` DBOS step, then handed to the
- * backend's spawn step. Because the identity is durable before any machine
- * exists, a crash between spawn and the spawn-step checkpoint is recoverable —
- * the re-run adopts the existing machine under the same name + secret.
+ * 0016). Backend-agnostic: the `sandboxId` is minted here, checkpointed by the
+ * `sandbox.mint` DBOS step, then handed to the backend's spawn step. Because the
+ * identity is durable before any machine exists, a crash between spawn and the
+ * spawn-step checkpoint is recoverable — the re-run adopts the existing machine
+ * under the same name.
  */
-import { randomBytes, randomUUID } from "node:crypto";
+import { randomUUID } from "node:crypto";
 /** `sbx-{run8}-{rand8}` — run prefix aids `kubectl` triage; the 8-hex random
  *  suffix makes within-run collisions across a run's concurrent steps
  *  negligible. Names are NOT the idempotency mechanism (the checkpoint is), and
@@ -15,8 +15,5 @@ import { randomBytes, randomUUID } from "node:crypto";
 export function mintSandboxIdentity(runId) {
     const run8 = runId.replace(/-/g, "").slice(0, 8);
     const suffix = randomUUID().replace(/-/g, "").slice(0, 8);
-    return {
-        sandboxId: `sbx-${run8}-${suffix}`,
-        callbackSecret: "base64:" + randomBytes(32).toString("base64"),
-    };
+    return { sandboxId: `sbx-${run8}-${suffix}` };
 }