@prized/sdk 0.0.0-stage → 0.1.1

package/dist/esm/commands.js

@@ -0,0 +1,590 @@
+// sandbox.exec: one call for short commands, a detached run for background
+// work, long timeouts and streamed output.
+import { CommandTimeoutError, PrizedError } from './errors.js';
+import { backoffMs, cutByDeadline, LAST_LOOK_MAX_MS, MAX_RETRY_AFTER_MS, Polls, validOutage } from './http.js';
+import { abandonScript, cleanupScript, commandText, killScript, NO_PID_EXIT, pollScript, RUN_ID_RE, sandboxPath, SIGNALS, startScript, } from './shell.js';
+import { mayHaveActed, pollFailed, runAborted, signalEffect, startAbandoned, startEffect, withOutcome } from './outcome.js';
+import { execFullOutput } from './output.js';
+import { helperExec, rawExec } from './toolbox.js';
+/** The timeout a command gets when the call names none. */
+export const DEFAULT_EXEC_TIMEOUT_MS = 60_000;
+/**
+ * The longest command one blocking call carries. Longer timeouts (and
+ * `timeoutMs: 0`) run detached and are polled: Node's fetch gives up on an
+ * answer after 300 s, and load balancers drop idle connections not long after.
+ */
+export const MAX_BLOCKING_EXEC_MS = 240_000;
+/** Bytes of each stream one poll brings back. */
+const POLL_CHUNK_BYTES = 1024 * 1024;
+const POLL_MIN_MS = 200;
+const POLL_MAX_MS = 1_000;
+/**
+ * The bound on each teardown request the SDK makes on its own (stopping or
+ * abandoning a run, removing its files), so it never holds up the caller's
+ * result or error for long. The Python SDK's TEARDOWN_SECONDS.
+ */
+const TEARDOWN_MS = 10_000;
+/**
+ * The bound on abandoning a start (ABANDON_SECONDS): the abandon script
+ * waits up to 10 s for a slow start's pid before it KILLs the run, and a
+ * request cut off sooner would stop it before the KILL.
+ */
+const ABANDON_MS = 20_000;
+/** How long a stopped run's cleanup keeps trying while its group goes (CLEANUP_SECONDS). */
+const CLEANUP_MS = 3_000;
+const CLEANUP_RETRY_MS = 200;
+/**
+ * The errors a wait's poll threw (waitRaw) that leave open what became of
+ * the run: the run polled, and whether the wait had already seen it end.
+ * exec tells them from a callback's own and says what became of the command.
+ */
+const failedPolls = new WeakMap();
+/**
+ * The poll errors that settle what became of the run: it is gone with its
+ * sandbox (deleted, or not running), or there is no such run. Any other (no
+ * answer, an answer that could not be read, a 5xx, a rate limit, a refused
+ * key) leaves the command perhaps still running. The Python SDK's
+ * _RUN_SETTLED.
+ */
+const RUN_SETTLED = new Set(['not_found', 'sandbox_deleted', 'sandbox_not_running']);
+/** The poll errors with no answer behind them that an outage window rides out. */
+const NO_ANSWER = new Set(['network_error', 'timeout', 'bad_response']);
+/**
+ * Whether a failed poll is one an outage window rides out (sdk/retry-spec.json
+ * `pollOutage`): no answer, an answer that could not be read, a rate limit or
+ * any 5xx. Any other (a refused key, any other refusal, an answer that
+ * settles the run, an abort) ends the wait at once. The Python SDK's
+ * rides_out.
+ */
+export function ridesOut(error) {
+    if (RUN_SETTLED.has(error.code))
+        return false;
+    if (error.status === null)
+        return NO_ANSWER.has(error.code);
+    return error.status >= 500 || error.status === 429 || error.code === 'bad_response';
+}
+/**
+ * The failed polls of a wait that rides out an outage of up to `windowMs`
+ * (pollOutageMs), from when the first poll's retries ran out, and, while the
+ * command runs, never past the wait's own deadline (`waitDeadline`, epoch
+ * ms; null for none): polled again with backoff, each poll and its retries
+ * ending before the last look, which goes out LAST_LOOK_MAX_MS before the
+ * window's end (half the window, for one shorter than twice that) with room
+ * for its own retries. So an outage that ends inside the window is always
+ * seen to end. The Python SDK's Outage.
+ */
+export class Outage {
+    windowMs;
+    waitDeadline;
+    end = null;
+    last = null;
+    tries = 0;
+    room = 0;
+    finished = false;
+    constructor(windowMs, waitDeadline = null) {
+        this.windowMs = windowMs;
+        this.waitDeadline = waitDeadline;
+    }
+    /** The command has ended: draining its output, and an outage meanwhile, are not held to the deadline. */
+    finish() {
+        this.finished = true;
+    }
+    /**
+     * Whether a wait's failed poll (`error`, at `now`; `cut`: its bound cut it
+     * short) means the wait's own time ran out: its deadline cut the poll
+     * short while the command ran, with no outage being ridden out, so the
+     * command was still running when last seen. With no window, any timeout
+     * while it runs says so, as it always has. Never once the command has
+     * ended. The Python SDK's Outage.ran_out.
+     */
+    ranOut(error, now, cut) {
+        if (this.waitDeadline === null || this.finished || error.code !== 'timeout')
+            return false;
+        return this.windowMs <= 0 || (this.end === null && (cut || now >= this.waitDeadline));
+    }
+    /** The bound on a poll sent `now`: undefined unless polls are failing; else the last look's start, or the window's end for the last look. */
+    deadline(now) {
+        if (this.end === null)
+            return undefined;
+        // (A millisecond's slack, as Polls has.)
+        return now < this.end - this.room - 1 ? this.end - this.room : this.end;
+    }
+    /** A poll was answered: the outage, if any, is over. */
+    answered() {
+        this.end = null;
+        this.last = null;
+        this.tries = 0;
+    }
+    /**
+     * A poll sent at `sentAt` failed `now` with `error` (`cut`: its bound cut
+     * it short). Returns the pause before polling again, or null to give up,
+     * with the failure to give up with: the last one a poll met on its own,
+     * never the cut.
+     */
+    failed(error, sentAt, now, cut, backoff) {
+        if (this.windowMs <= 0 || !ridesOut(error))
+            return { pause: null, failure: error };
+        if (this.end === null) {
+            this.end = Math.min(now + this.windowMs, this.finished || this.waitDeadline === null ? Infinity : this.waitDeadline);
+            this.room = Math.max(0, Math.min(LAST_LOOK_MAX_MS, (this.end - now) / 2));
+        }
+        const met = error.cause instanceof PrizedError ? error.cause : null;
+        if (!cut) {
+            this.last = error;
+            this.tries += 1;
+        }
+        else if (met !== null && ridesOut(met) && met.code !== this.last?.code) {
+            // What the cut poll met, when it is news (a 503 after lost connections).
+            this.last = met;
+        }
+        const failure = this.last ?? error;
+        // The last look has been taken: this poll went out with no more than its room left.
+        if (this.end - sentAt <= this.room + 1)
+            return { pause: null, failure };
+        // 1 s, 2 s, 4 s, then 8 s apart (jittered as retries are), each poll with
+        // its own retries, none past the last look's start; straight on to it
+        // when the bound cut this one short.
+        const toLast = this.end - this.room - now;
+        if (cut)
+            return { pause: Math.max(0, toLast), failure };
+        // A Retry-After the last answer asked for is waited out too (up to MAX_RETRY_AFTER_MS).
+        const pause = Math.max(backoff(Math.min(this.tries, 4)), Math.min((error.retryAfter ?? 0) * 1000, MAX_RETRY_AFTER_MS));
+        return { pause: Math.max(0, Math.min(pause, toLast)), failure };
+    }
+}
+function randomHex(bytes) {
+    const out = new Uint8Array(bytes);
+    const webcrypto = globalThis.crypto;
+    if (webcrypto?.getRandomValues) {
+        webcrypto.getRandomValues(out);
+    }
+    else {
+        for (let i = 0; i < bytes; i += 1)
+            out[i] = Math.floor(Math.random() * 256);
+    }
+    return Array.from(out, (b) => b.toString(16).padStart(2, '0')).join('');
+}
+function base64Bytes(text) {
+    if (text === '')
+        return new Uint8Array(0);
+    const binary = atob(text);
+    const out = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i += 1)
+        out[i] = binary.charCodeAt(i);
+    return out;
+}
+const REPORT_RE = /^prized-run (running|done|gone|missing) (-|-?[0-9]+) ([0-9]+) ([0-9]+) ([01]) (-|-?[0-9]+)$/;
+function parseReport(result, toolbox, id) {
+    const lines = result.stdout.split('\n');
+    const head = REPORT_RE.exec(lines[0] ?? '');
+    if (result.exitCode !== 0 || head === null) {
+        throw new PrizedError('bad_response', `Could not read a background command's state: ${(result.stderr || lines[0] || '').slice(0, 200)}`, {
+            sandboxId: toolbox.sandboxId,
+        });
+    }
+    let stdout = new Uint8Array(0);
+    let stderr = new Uint8Array(0);
+    try {
+        // A missing run is not_found (poll()), whatever follows its line.
+        if (head[1] !== 'missing') {
+            stdout = base64Bytes((lines[1] ?? '').trim());
+            stderr = base64Bytes((lines[2] ?? '').trim());
+        }
+    }
+    catch (error) {
+        // As the Python SDK reads it: an answer that could not be read.
+        throw new PrizedError('bad_response', `Unreadable report for ${id}: ${error instanceof Error ? error.message : String(error)}`, {
+            sandboxId: toolbox.sandboxId,
+            cause: error,
+        });
+    }
+    return {
+        state: head[1],
+        exitCode: head[2] === '-' ? null : Number(head[2]),
+        outSize: Number(head[3]),
+        errSize: Number(head[4]),
+        timedOut: head[5] === '1',
+        durationMs: head[6] === '-' ? null : Number(head[6]),
+        stdout,
+        stderr,
+    };
+}
+/** One output stream: bytes read so far, decoded as UTF-8 across chunk boundaries. */
+class StreamReader {
+    onChunk;
+    offset = 0;
+    text = '';
+    decoder = new TextDecoder('utf-8');
+    constructor(onChunk) {
+        this.onChunk = onChunk;
+    }
+    push(bytes) {
+        if (bytes.length === 0)
+            return;
+        this.offset += bytes.length;
+        this.emit(this.decoder.decode(bytes, { stream: true }));
+    }
+    finish() {
+        this.emit(this.decoder.decode());
+        return this.text;
+    }
+    emit(chunk) {
+        if (chunk === '')
+            return;
+        this.text += chunk;
+        this.onChunk?.(chunk);
+    }
+}
+/**
+ * A command running in the background (`sandbox.exec(cmd, { background: true })`).
+ * It runs in its own session: it keeps running after this process exits, and
+ * while it runs, kill() reaches every process it started in its process group.
+ */
+export class CommandHandle {
+    /** The run's id (`run_…`). */
+    id;
+    /** The process group id of the command inside the sandbox. */
+    pid;
+    sandboxId;
+    toolbox;
+    startedAt;
+    /** @internal The time limit exec gave the command, in ms; null for none. */
+    timeoutMs = null;
+    /** @internal */
+    constructor(toolbox, id, pid, startedAt) {
+        this.toolbox = toolbox;
+        this.id = id;
+        this.pid = pid;
+        this.sandboxId = toolbox.sandboxId;
+        this.startedAt = startedAt;
+    }
+    async poll(outOffset, errOffset, maxChunk, signal, deadline) {
+        const result = await helperExec(this.toolbox, pollScript(this.id, outOffset, errOffset, maxChunk), { signal, deadline });
+        const report = parseReport(result, this.toolbox, this.id);
+        if (report.state === 'missing') {
+            throw new PrizedError('not_found', `No background command ${this.id} in this sandbox.`, {
+                sandboxId: this.sandboxId,
+            });
+        }
+        return report;
+    }
+    /** Whether the command is still running, and its exit code once it is not. */
+    async status() {
+        const report = await this.poll(0, 0, 0);
+        if (report.state === 'running')
+            return { running: true, exitCode: null };
+        const exitCode = report.timedOut || report.state !== 'done' || report.exitCode === null ? -1 : report.exitCode;
+        return { running: false, exitCode };
+    }
+    /** Everything the command has printed so far. */
+    async logs() {
+        const out = new StreamReader();
+        const err = new StreamReader();
+        for (;;) {
+            const report = await this.poll(out.offset, err.offset, POLL_CHUNK_BYTES);
+            out.push(report.stdout);
+            err.push(report.stderr);
+            const caughtUp = (report.stdout.length === 0 || out.offset >= report.outSize) &&
+                (report.stderr.length === 0 || err.offset >= report.errSize);
+            if (caughtUp)
+                return { stdout: out.finish(), stderr: err.finish() };
+        }
+    }
+    /**
+     * Wait for the command to finish and return everything it printed.
+     * `onStdout`/`onStderr` receive the output as it arrives, from the start.
+     * A `timeoutMs` here bounds the wait only; the command keeps running. If
+     * the command ran past its own timeout (the one given to exec), it was
+     * killed and this throws CommandTimeoutError with what it printed. While
+     * polls fail (a dropped connection, a 5xx), it keeps polling for up to
+     * `pollOutageMs` (the client's, 120 000 by default) before it throws.
+     */
+    async wait(options = {}) {
+        const outageMs = validOutage(options.pollOutageMs) ?? this.toolbox.config.pollOutageMs;
+        return finished(await this.waitRaw(options, outageMs), this.timeoutMs, this.sandboxId);
+    }
+    /**
+     * @internal wait() without the timeout check: exec cleans up before it
+     * throws. `outageMs` (pollOutageMs): how long polls may fail before the
+     * wait gives up (Outage), never past the wait's own deadline.
+     */
+    async waitRaw(options, outageMs) {
+        const config = this.toolbox.config;
+        const deadline = options.timeoutMs ? config.now() + options.timeoutMs : null;
+        const out = new StreamReader(options.onStdout);
+        const err = new StreamReader(options.onStderr);
+        let delay = POLL_MIN_MS;
+        let finished = false;
+        const polls = deadline === null ? null : new Polls(config, deadline);
+        const outage = new Outage(outageMs, deadline);
+        for (;;) {
+            let report;
+            const sentAt = config.now();
+            polls?.sent();
+            try {
+                // The wait's deadline bounds each poll and its retries too, so a
+                // stalled or rate-limited edge cannot hold a short wait for minutes.
+                // Once the command has finished, draining the rest of its output is
+                // not held to it. While polls fail, the outage window's end bounds them.
+                const until = outage.deadline(sentAt) ?? (finished ? undefined : deadline ?? undefined);
+                report = await this.poll(out.offset, err.offset, POLL_CHUNK_BYTES, options.signal, until);
+                polls?.answered();
+            }
+            catch (error) {
+                const now = config.now();
+                if (error instanceof PrizedError && outage.ranOut(error, now, cutByDeadline(error))) {
+                    throw this.waitTimeout(options.timeoutMs);
+                }
+                let failure = error;
+                if (error instanceof PrizedError && !options.signal?.aborted) {
+                    // A dropped connection, an edge restarting: the command may be
+                    // fine. Poll again until the window runs out, then give up.
+                    const step = outage.failed(error, sentAt, now, cutByDeadline(error), (tries) => backoffMs(config, tries));
+                    if (step.pause !== null) {
+                        await config.sleep(step.pause, options.signal);
+                        continue;
+                    }
+                    failure = step.failure;
+                }
+                if (failure instanceof PrizedError && !RUN_SETTLED.has(failure.code))
+                    failedPolls.set(failure, { runId: this.id, ended: finished });
+                throw failure;
+            }
+            outage.answered();
+            if (report.state !== 'running') {
+                finished = true;
+                outage.finish();
+            }
+            out.push(report.stdout);
+            err.push(report.stderr);
+            const more = (report.stdout.length > 0 && out.offset < report.outSize) ||
+                (report.stderr.length > 0 && err.offset < report.errSize);
+            if (report.state !== 'running' && !more)
+                return this.result(report, out.finish(), err.finish());
+            // Checked before draining more output, so a chatty command cannot hold
+            // the wait past its limit.
+            if (report.state === 'running' && deadline !== null && config.now() >= deadline) {
+                throw this.waitTimeout(options.timeoutMs);
+            }
+            if (more)
+                continue;
+            // Back off while it is quiet; poll briskly again once output flows.
+            if (report.stdout.length > 0 || report.stderr.length > 0)
+                delay = POLL_MIN_MS;
+            // With a limit, the last poll goes out just before it (Polls).
+            const pause = polls === null ? delay : polls.pause(delay);
+            if (pause === null)
+                throw this.waitTimeout(options.timeoutMs);
+            if (pause > 0)
+                await config.sleep(pause, options.signal);
+            delay = Math.min(POLL_MAX_MS, delay * 2);
+        }
+    }
+    waitTimeout(timeoutMs) {
+        return new PrizedError('timeout', `${this.id} was still running after ${timeoutMs} ms`, { sandboxId: this.sandboxId });
+    }
+    result(report, stdout, stderr) {
+        // Both come from the run's own files, not from when the caller looked.
+        const durationMs = report.durationMs ?? Math.max(0, this.toolbox.config.now() - this.startedAt);
+        const exitCode = report.timedOut || report.state !== 'done' || report.exitCode === null ? -1 : report.exitCode;
+        return { stdout, stderr, exitCode, durationMs, timedOut: report.timedOut, truncated: false };
+    }
+    /**
+     * Send a signal (default SIGTERM) to the command and everything it
+     * started in its process group. Resolves true if it was sent, false if
+     * the run had already ended. Processes that left the group (`setsid`, a
+     * daemon) are not reached, nor anything after the run itself has ended.
+     */
+    async kill(signal = 'SIGTERM') {
+        const name = signal.toUpperCase().replace(/^SIG/, '');
+        if (!SIGNALS.has(name)) {
+            throw new PrizedError('invalid_argument', `Unsupported signal ${signal}; use one of ${[...SIGNALS].map((s) => `SIG${s}`).join(', ')}`);
+        }
+        // Like a command: a signal whose answer was lost is not sent again (the
+        // second would report false for a kill that landed).
+        const result = await helperExec(this.toolbox, killScript(this.id, name), { retry: 'exec', effect: signalEffect(this.id, name) });
+        if (result.exitCode === 0)
+            return true;
+        if (result.exitCode === 3) {
+            throw new PrizedError('not_found', `No background command ${this.id} in this sandbox.`, { sandboxId: this.sandboxId });
+        }
+        return false;
+    }
+    /**
+     * Remove the command's files (its output under `~/.prized/runs/<id>/`)
+     * from the sandbox once it has ended: call it after wait() resolves or
+     * throws CommandTimeoutError (after a kill(), await wait() first). On a
+     * command still running it does nothing, so status(), wait() and kill()
+     * keep working. Once its files are gone, status(), logs(), wait() and
+     * kill() reject with `not_found`. Best effort: it never rejects.
+     */
+    async cleanup() {
+        try {
+            // Exit 5 (still running) is not an error: the files stay on purpose.
+            await helperExec(this.toolbox, cleanupScript(this.id));
+        }
+        catch {
+            // Best effort: a leftover run directory costs a few bytes of disk.
+        }
+    }
+}
+/**
+ * One teardown helper the SDK sends on its own (stopping or abandoning a
+ * run, removing its files): best effort, and within `deadline` (TEARDOWN_MS
+ * from now by default), so it never holds up the caller's result or error
+ * for long. Its exit code, or null when it got no answer.
+ */
+async function teardown(toolbox, script, deadline = toolbox.config.now() + TEARDOWN_MS) {
+    try {
+        return (await helperExec(toolbox, script, { deadline })).exitCode;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * SIGKILL a run for a caller that is leaving, sent again like a read (a
+ * second KILL changes nothing). True once nothing of it is running: sent,
+ * or there was nothing left to send it to.
+ */
+async function stopRun(toolbox, id) {
+    const code = await teardown(toolbox, killScript(id, 'KILL'));
+    return code === 0 || code === 3 || code === 4;
+}
+/**
+ * Remove the files of a run just stopped: its group may take a moment to
+ * go, and cleanup keeps a running run's files (exit 5), so it is tried
+ * again for CLEANUP_MS.
+ */
+async function cleanupOnceGone(toolbox, id) {
+    const config = toolbox.config;
+    const deadline = config.now() + CLEANUP_MS;
+    for (;;) {
+        if ((await teardown(toolbox, cleanupScript(id), deadline)) !== 5 || config.now() + CLEANUP_RETRY_MS >= deadline)
+            return;
+        try {
+            await config.sleep(CLEANUP_RETRY_MS);
+        }
+        catch {
+            return; // never in place of the error the caller is leaving with
+        }
+    }
+}
+async function startBackground(toolbox, command, cwd, env, timeoutMs, signal, unbuffered) {
+    // Nothing sent yet: nothing to stop.
+    if (signal?.aborted)
+        throw withOutcome(new PrizedError('aborted', null, { sandboxId: toolbox.sandboxId }), startEffect(), 'never');
+    const id = `run_${randomHex(8)}`;
+    if (!RUN_ID_RE.test(id))
+        throw new Error('unreachable: bad run id');
+    const startedAt = toolbox.config.now();
+    let pid;
+    // Launched, but no pid came within the start's own wait: it may yet run.
+    let unsettled = false;
+    try {
+        const result = await rawExec(toolbox, { command: startScript(id, command, timeoutMs === null ? null : timeoutMs / 1000, unbuffered), cwd, env, timeoutMs: 60_000 }, 
+        // Keyed by a run id this call chose, so a resend finds the first start
+        // instead of starting the command twice.
+        { retry: 'safe', signal, effect: startEffect() });
+        pid = Number(result.stdout.trim());
+        if (result.exitCode !== 0 || !Number.isInteger(pid) || pid <= 0) {
+            unsettled = result.exitCode === NO_PID_EXIT;
+            throw new PrizedError('command_failed', `Could not start the background command: ${(result.stderr || result.stdout).trim().slice(0, 300)}`, {
+                sandboxId: toolbox.sandboxId,
+            });
+        }
+    }
+    catch (error) {
+        // The start may have run even though its answer never came (or the
+        // caller left mid-start). The caller gets no handle, so the run is
+        // stopped by its id rather than left going in its own session, and the
+        // error says so when it does not settle whether the command started.
+        const stopped = (await teardown(toolbox, abandonScript(id), toolbox.config.now() + ABANDON_MS)) === 0;
+        if (mayHaveActed(error) || (unsettled && error instanceof PrizedError))
+            throw startAbandoned(error, id, stopped);
+        throw error;
+    }
+    return new CommandHandle(toolbox, id, pid, startedAt);
+}
+function validTimeout(value) {
+    if (value !== undefined && (!Number.isFinite(value) || value < 0)) {
+        throw new PrizedError('invalid_argument', 'timeoutMs must be a number of ms ≥ 0');
+    }
+}
+/**
+ * A command's result for the caller: throw CommandTimeoutError when its
+ * timeout killed it, so a cut-off run never reads as a finished one. The
+ * SDK's own helpers read their results raw.
+ */
+function finished(result, timeoutMs, sandboxId) {
+    if (result.timedOut)
+        throw new CommandTimeoutError(result, timeoutMs, sandboxId);
+    return result;
+}
+/** sandbox.exec's body. */
+export async function execCommand(toolbox, command, options = {}) {
+    const text = commandText(command);
+    if (text.trim() === '')
+        throw new PrizedError('invalid_argument', 'The command is empty.');
+    validTimeout(options.timeoutMs);
+    const outageMs = validOutage(options.pollOutageMs) ?? toolbox.config.pollOutageMs;
+    const cwd = options.cwd === undefined ? undefined : sandboxPath(options.cwd);
+    if (options.background) {
+        const limit = options.timeoutMs ? options.timeoutMs : null;
+        const handle = await startBackground(toolbox, text, cwd, options.env, limit, options.signal, true);
+        handle.timeoutMs = limit;
+        return handle;
+    }
+    const timeoutMs = options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS;
+    const streamed = options.onStdout !== undefined || options.onStderr !== undefined;
+    if (streamed || timeoutMs === 0 || timeoutMs > MAX_BLOCKING_EXEC_MS) {
+        const limit = timeoutMs === 0 ? null : timeoutMs;
+        const handle = await startBackground(toolbox, text, cwd, options.env, limit, options.signal, streamed);
+        const waitOptions = {};
+        if (options.onStdout)
+            waitOptions.onStdout = options.onStdout;
+        if (options.onStderr)
+            waitOptions.onStderr = options.onStderr;
+        if (options.signal)
+            waitOptions.signal = options.signal;
+        let result;
+        try {
+            result = await handle.waitRaw(waitOptions, outageMs);
+        }
+        catch (error) {
+            // Leaving the call (an abort, a callback that threw, polls that kept
+            // failing past pollOutageMs, or failed in a way waiting cannot mend)
+            // stops the command, as a blocking call would: the caller has
+            // no handle to it. Its files go only once the stop is confirmed: until
+            // then they are how the run is found and stopped by its id.
+            const stopped = await stopRun(toolbox, handle.id);
+            if (stopped)
+                await cleanupOnceGone(toolbox, handle.id);
+            // The caller's abort says what became of the command (the Python SDK
+            // has no signal), as does a poll that failed: with the network down,
+            // the stop may not have got through either. A callback's own error
+            // stays as it was thrown.
+            if (options.signal?.aborted && error instanceof PrizedError && error.code === 'aborted') {
+                throw runAborted(error, handle.id, stopped, toolbox.sandboxId);
+            }
+            if (error instanceof PrizedError) {
+                const poll = failedPolls.get(error);
+                // A run the wait saw end is not running, whatever became of the stop.
+                if (poll?.runId === handle.id)
+                    throw pollFailed(error, handle.id, stopped || poll.ended, toolbox.sandboxId);
+            }
+            throw error;
+        }
+        await teardown(toolbox, cleanupScript(handle.id));
+        return finished(result, limit, toolbox.sandboxId);
+    }
+    if (options.fullOutput) {
+        const result = await execFullOutput(toolbox, `exec_${randomHex(8)}`, { command: text, cwd, env: options.env, timeoutMs }, options.signal);
+        return finished(result, timeoutMs, toolbox.sandboxId);
+    }
+    // A command that reached the sandbox may have run even if its answer was
+    // lost, so a blocking exec is retried only when the edge proves it never
+    // delivered it.
+    const result = await rawExec(toolbox, { command: text, cwd, env: options.env, timeoutMs }, { retry: 'exec', signal: options.signal });
+    return finished(result, timeoutMs, toolbox.sandboxId);
+}