@prized/sdk 0.1.0 → 0.1.2

package/dist/esm/output.js

@@ -1,22 +1,24 @@
 // exec(..., { fullOutput: true }): one call that keeps a command's whole
 // output, not just the first 4 MiB of each stream the sandbox answers with.
 //
-// The command's stdout and stderr stay pipes, as in a plain exec. A reader on
-// each copies what comes through to a file in the exec's own directory, at a
-// fixed path under the sandbox user's home, and drops what passes 100 MiB, so
-// a runaway command cannot fill the disk:
+// The command's stdout and stderr stay pipes, as in a plain exec. A relay on
+// each passes what comes through to a reader that copies it to a file in the
+// exec's own directory, at a fixed path under the sandbox user's home, and
+// drops what passes 100 MiB, so a runaway command cannot fill the disk:
 //
 //   ~/.prized/exec/<exec id>/stdout
 //   ~/.prized/exec/<exec id>/stderr
 //
 // Both under 4 MiB: the script prints them back and removes the directory,
-// so the answer carries the same output a plain exec's does. Bigger: it
-// prints one marker line instead, and the SDK downloads both files through
-// the files API and removes the directory. The scripts are the Python SDK's
+// so the answer carries the same output a plain exec's does. Bigger, or a
+// reader that could not write its file (a full disk): it prints one marker
+// line instead, and the SDK downloads both files through the files API and
+// removes the directory. A stream whose write failed comes back as far as it
+// got, `truncated` with `truncatedReason: 'write_failed'`. The scripts are the Python SDK's
 // (sdk/python/src/prized/_output.py), rendered the same; both SDKs check
 // them against sdk/output-scripts/.
-import { PrizedError } from './errors.js';
-import { request } from './http.js';
+import { OutputReadError, PrizedError } from './errors.js';
+import { request, timing } from './http.js';
 import { restate } from './outcome.js';
 import { LAUNCH_PATH } from './shell.js';
 import { helperExec, MAX_FILE_BYTES, MAX_OUTPUT_BYTES, rawExec } from './toolbox.js';
@@ -49,45 +51,102 @@ function checkId(id) {
         throw new Error(`unreachable: bad exec id ${id}`);
 }
 /**
- * The reader on one stream: tee copies it to the file as it comes (each read
- * written at once) until the file reaches FULL_OUTPUT_BYTES and one 512-byte
- * block more, which tells a cut stream from a full one. The file size limit
- * (ulimit -f, in 512-byte blocks in POSIX mode) stops the file there with
- * EFBIG, not SIGXFSZ, and tee goes on draining into /dev/null, so the command
- * never meets a closed pipe and a runaway one cannot fill the disk. One
- * process per stream (exec), in a session of its own (setsid, same pid): a
- * time limit or a `kill 0` that ends the command's process group leaves it to
+ * The reader's own shell (`$1` the file, `$2` its flag): cat writes the file
+ * until a write fails; then the flag, an empty file (it takes no free block),
+ * then the rest drained into /dev/null, and exit 1. `true`, not `:`: a
+ * redirection that fails on a special builtin ends the shell, and with it
+ * the draining.
+ */
+const READER_SHELL = `'cat > "$1" || { true > "$2" || :; cat > /dev/null; exit 1; }'`;
+/**
+ * The reader on one stream: cat copies what the relay passes it to the file
+ * as it comes (each read written at once) until the file reaches
+ * FULL_OUTPUT_BYTES and one 512-byte block more, which tells a cut stream
+ * from a full one. The file size limit (ulimit -f, in 512-byte blocks in
+ * POSIX mode) stops the file there with EFBIG, not SIGXFSZ, and the reader
+ * goes on draining into /dev/null, so the command never meets a closed pipe
+ * and a runaway one cannot fill the disk. A write that fails short of that (a
+ * full disk) stops the file the same way. Either leaves `<name>.failed` at
+ * once, which collect reads when a time limit ended the script before it
+ * could wait for the reader, and the reader exits 1 once its input ends. A
+ * shell per stream (exec), in a session of its own (setsid, same pid): a time
+ * limit or a `kill 0` that ends the command's process group leaves it to
  * drain what the pipe still holds.
  */
 function reader(d, name, fifo) {
     const blocks = FULL_OUTPUT_BYTES / 512 + 1;
     return (`( exec > /dev/null 2>&1; PATH=${LAUNCH_PATH}; trap '' XFSZ; ( set -o posix ) && set -o posix; ` +
-        `ulimit -f ${blocks} || :; exec setsid tee ${d}/${name} < ${d}/${fifo} ) &`);
+        `ulimit -f ${blocks} || :; exec setsid /bin/sh -c ${READER_SHELL} sh ${d}/${name} ${d}/${name}.failed < ${d}/${fifo} ) &`);
+}
+/**
+ * The relay on one stream: cat reads the command's pipe and passes it on to
+ * the reader's. It is what the watchdog stops, never the reader, so the
+ * reader always ends on the end of its input, and its exit status says
+ * whether every write to its file went through. In a session of its own
+ * (setsid, same pid), as the reader is.
+ */
+function relay(d, from, to) {
+    return `( exec > ${d}/${to} 2> /dev/null < /dev/null; PATH=${LAUNCH_PATH}; exec setsid cat ${d}/${from} ) &`;
+}
+/**
+ * A shell test: process `pid` (shell text) is this exec's relay on `fifo`,
+ * not another process that reused its pid. Its arguments must be exactly
+ * `cat <fifo>` (as setsid execs it), not merely name the exec's directory
+ * somewhere in the command line, as a process the command started may
+ * (`tail -f ~/.prized/exec/<id>/stdout`). /proc ends each argument with a
+ * NUL; here each is one line (a NUL becomes a newline, a newline a space),
+ * as the run scripts' isRun (shell.ts) reads them.
+ */
+function isRelay(pid, fifo) {
+    return `[ "$(tr "\\0\\n" "\\n " < "/proc/${pid}/cmdline" 2> /dev/null | tr "\\n" " ")" = "cat ${fifo} " ]`;
 }
 /**
  * The bound on waiting for the readers once the command has ended: GRACE_S
- * seconds, as the sandbox waits for a plain exec's pipes, then the readers
- * still running stop (a pid is checked to be this exec's tee first, never
- * one reused since), and a process the command left holding its output
- * meets a closed pipe, as it would there. Stopped early (TERM, once the
- * readers are done), it takes its sleep with it.
+ * seconds, as the sandbox waits for a plain exec's pipes, then the relays
+ * still running stop (each pid checked to be this exec's relay on its own
+ * pipe first, never one reused since): its reader then ends with what came
+ * through, and a process the command left holding its output meets a closed
+ * pipe, as it would there. Stopped early (TERM, once the readers are done),
+ * it takes its sleep with it.
  */
 function watchdog(d) {
     return (`( s=; trap '[ -z "$s" ] || kill "$s" 2> /dev/null; exit 0' TERM; sleep ${GRACE_S} & s=$!; wait "$s" || :; ` +
-        `for p in "$2" "$3"; do if tr "\\0" " " < "/proc/$p/cmdline" 2> /dev/null | grep -qF -- ${d}/; then kill "$p" 2> /dev/null || :; fi; done ) > /dev/null 2>&1 &`);
+        `if ${isRelay('$4', `${d}/o`)}; then kill "$4" 2> /dev/null || :; fi; ` +
+        `if ${isRelay('$5', `${d}/e`)}; then kill "$5" 2> /dev/null || :; fi ) > /dev/null 2>&1 &`);
 }
-/** Sizes, the cut past FULL_OUTPUT_BYTES and the marker line, with `d` set to the directory. */
+/**
+ * Sizes, the cut past FULL_OUTPUT_BYTES and the marker line, with `d` set to
+ * the directory and each reader's write failure in `x` and `y` (1 or 0,
+ * failedLines). A cut leaves `<name>.cut`, so collect sent again still reads
+ * the stream as past the bound (its flag is still there, its size no longer
+ * says so). `true`, not `:`, for the same reason as in READER_SHELL.
+ */
 function reportLines(id) {
     const max = FULL_OUTPUT_BYTES;
     return {
         sizes: ['o=$(stat -c %s "$d/stdout" 2> /dev/null || echo 0)', 'e=$(stat -c %s "$d/stderr" 2> /dev/null || echo 0)'],
         cut: [
-            `if [ "$o" -gt ${max} ]; then truncate -s ${max} "$d/stdout" || :; fi`,
-            `if [ "$e" -gt ${max} ]; then truncate -s ${max} "$d/stderr" || :; fi`,
-            `echo "prized-output ${id} $o $e"`,
+            `if [ "$o" -gt ${max} ]; then truncate -s ${max} "$d/stdout" && true > "$d/stdout.cut" || :; fi`,
+            `if [ "$e" -gt ${max} ]; then truncate -s ${max} "$d/stderr" && true > "$d/stderr.cut" || :; fi`,
+            `echo "prized-output ${id} $o $e $x $y"`,
         ],
     };
 }
+/**
+ * Whether each reader failed to write its file (`x` for stdout, `y` for
+ * stderr: nonzero from its exit status, or its flag, after `sizes`): 1 when
+ * it ended in error short of the bound (a full disk or a quota, a file it
+ * could not create, a reader killed), else 0. Past the bound its error is the
+ * cut (EFBIG). The watchdog stops relays, never readers, so a reader's status
+ * is its own.
+ */
+function failedLines() {
+    const max = FULL_OUTPUT_BYTES;
+    return [
+        `if [ "$x" -eq 0 ] || [ "$o" -gt ${max} ]; then x=0; else x=1; fi`,
+        `if [ "$y" -eq 0 ] || [ "$e" -gt ${max} ]; then y=0; else y=1; fi`,
+    ];
+}
 /**
  * The script a fullOutput exec runs (RUN_SCRIPT, from SCRIPT_VAR). The
  * caller's command, from COMMAND_VAR, runs as a plain exec runs it (`bash
@@ -97,10 +156,12 @@ function reportLines(id) {
  * Nothing is assigned before it runs but the positional parameters, so every
  * variable the caller's env sets reaches it unchanged, PATH included; and
  * every step holds under an errexit the env may switch on (SHELLOPTS). As in
- * a plain exec, the readers stop 5 s after the command ends if a process it
+ * a plain exec, the relays stop 5 s after the command ends if a process it
  * left running still holds its output, keeping what came through. A command
- * a signal ends exits 128+n, as a background command does. Without tee,
- * setsid or a directory to write to, the command runs as a plain exec.
+ * a signal ends exits 128+n, as a background command does. A reader that
+ * could not write its file (failedLines) sends the output to the files API
+ * whatever its size, with the marker saying which. Without /bin/sh, setsid,
+ * cat or a directory to write to, the command runs as a plain exec.
  */
 export function captureScript(id) {
     checkId(id);
@@ -109,21 +170,28 @@ export function captureScript(id) {
     return [
         `set -- "\${${COMMAND_VAR}:?}" /bin/bash; unset ${COMMAND_VAR} ${SCRIPT_VAR}; [ -x "$2" ] || set -- "$1" /bin/sh`,
         `if ( PATH=${LAUNCH_PATH}; mkdir -p ${OUTPUT_DIR} && mkdir ${d} ) 2> /dev/null; then`,
-        `if ( PATH=${LAUNCH_PATH}; command -v tee && command -v setsid && mkfifo ${d}/o ${d}/e ) > /dev/null 2>&1; then`,
-        reader(d, 'stdout', 'o'),
+        `if ( PATH=${LAUNCH_PATH}; [ -x /bin/sh ] && command -v setsid && command -v cat && mkfifo ${d}/o ${d}/e ${d}/p ${d}/q ) > /dev/null 2>&1; then`,
+        // The command writes to o and e; a relay passes each on to p and q, where its reader is.
+        reader(d, 'stdout', 'p'),
         'set -- "$1" "$2" "$!"',
-        reader(d, 'stderr', 'e'),
+        reader(d, 'stderr', 'q'),
         'set -- "$1" "$2" "$3" "$!"',
+        relay(d, 'o', 'p'),
+        'set -- "$1" "$2" "$3" "$4" "$!"',
+        relay(d, 'e', 'q'),
+        'set -- "$1" "$2" "$3" "$4" "$5" "$!"',
         // bash's "Killed" line for a command a signal ended goes to the braces' /dev/null.
-        `if { "$2" -c "$1" > ${d}/o 2> ${d}/e < /dev/null; } 2> /dev/null; then set -- 0 "$3" "$4"; else set -- "$?" "$3" "$4"; fi`,
+        `if { "$2" -c "$1" > ${d}/o 2> ${d}/e < /dev/null; } 2> /dev/null; then set -- 0 "$3" "$4" "$5" "$6"; else set -- "$?" "$3" "$4" "$5" "$6"; fi`,
         `PATH=${LAUNCH_PATH}`,
         `c=$1; d=${d}`,
         watchdog(d),
         'w=$!',
-        'wait "$2" "$3" 2> /dev/null || :',
+        'if wait "$2" 2> /dev/null; then x=0; else x=$?; fi',
+        'if wait "$3" 2> /dev/null; then y=0; else y=$?; fi',
         'kill "$w" 2> /dev/null || :',
         ...sizes,
-        `if [ "$o" -lt ${MAX_OUTPUT_BYTES} ] && [ "$e" -lt ${MAX_OUTPUT_BYTES} ]; then ` +
+        ...failedLines(),
+        `if [ "$o" -lt ${MAX_OUTPUT_BYTES} ] && [ "$e" -lt ${MAX_OUTPUT_BYTES} ] && [ "$x$y" = 00 ]; then ` +
             'cat "$d/stdout" 2> /dev/null || :; cat "$d/stderr" >&2 2> /dev/null || :; rm -rf "$d" || :; exit "$c"; fi',
         ...cut,
         'exit "$c"',
@@ -137,7 +205,10 @@ export function captureScript(id) {
  * What a fullOutput command left when the script could not print it (its
  * time limit, or a signal, ended the script): the marker line, or `gone`,
  * with the directory removed, when there is nothing to read (it was never
- * made, or its output was already printed back).
+ * made, or its output was already printed back). A failed write is read from
+ * the reader's flag, before the size, which is final once the flag is there
+ * (the script's wait for the reader's exit status went with the script); a
+ * reader killed, or one that could not make its flag, is not seen here.
  */
 export function collectScript(id) {
     checkId(id);
@@ -146,7 +217,13 @@ export function collectScript(id) {
         `d=${OUTPUT_DIR}/${id}`,
         `PATH=${LAUNCH_PATH}`,
         `if [ ! -f "$d/stdout" ] || [ ! -f "$d/stderr" ]; then rm -rf "$d" || :; echo "prized-output ${id} gone"; exit 0; fi`,
+        'if [ -e "$d/stdout.failed" ]; then x=1; else x=0; fi',
+        'if [ -e "$d/stderr.failed" ]; then y=1; else y=0; fi',
         ...sizes,
+        // Cut by an earlier collect whose answer was lost: as long as a cut stream (the bound and a block).
+        `if [ -e "$d/stdout.cut" ]; then o=$((${FULL_OUTPUT_BYTES} + 512)); fi`,
+        `if [ -e "$d/stderr.cut" ]; then e=$((${FULL_OUTPUT_BYTES} + 512)); fi`,
+        ...failedLines(),
         ...cut,
     ].join('\n');
 }
@@ -160,14 +237,22 @@ export function removeOutputScript(id) {
     return (`PATH=${LAUNCH_PATH}; rm -rf ${OUTPUT_DIR}/${id}; ` +
         `find ${OUTPUT_DIR} -mindepth 1 -maxdepth 1 -name 'exec_*' -mmin +${STALE_MIN} -exec rm -rf -- {} + 2> /dev/null || :`);
 }
-/** The marker's byte counts (a stream cut at FULL_OUTPUT_BYTES reads more), `gone`, or null for no marker. */
+/** The marker (one without failure flags, as an earlier SDK's collect printed, has none), `gone`, or null for no marker. */
 export function parseMarker(stdout, id) {
-    const m = new RegExp(`(?:^|\\n)prized-output ${id} (?:([0-9]+) ([0-9]+)|(gone))\\n$`).exec(stdout);
+    const m = new RegExp(`(?:^|\\n)prized-output ${id} (?:([0-9]+) ([0-9]+)(?: ([01]) ([01]))?|(gone))\\n$`).exec(stdout);
     if (m === null)
         return null;
-    if (m[3] === 'gone')
+    if (m[5] === 'gone')
         return 'gone';
-    return { out: Number(m[1]), err: Number(m[2]) };
+    return { out: Number(m[1]), err: Number(m[2]), outFailed: m[3] === '1', errFailed: m[4] === '1' };
+}
+/** Why the read-back output is cut, if it is: a failed write first, since it is the one the caller cannot foresee. */
+export function markerTruncation(marker) {
+    if (marker.outFailed || marker.errFailed)
+        return 'write_failed';
+    if (marker.out > FULL_OUTPUT_BYTES || marker.err > FULL_OUTPUT_BYTES)
+        return 'output_limit';
+    return null;
 }
 /** A request the SDK sends on its own to collect or remove the files: best effort, within TEARDOWN_MS. */
 async function teardown(toolbox, script) {
@@ -185,6 +270,7 @@ async function download(toolbox, path, signal) {
         retry: 'safe',
         read: 'bytes',
         timeoutMs: READ_MS,
+        stallMs: timing.transferStallMs,
         deadline: toolbox.config.now() + READ_MS,
         signal,
         sandboxId: toolbox.sandboxId,
@@ -193,7 +279,7 @@ async function download(toolbox, path, signal) {
     return new TextDecoder('utf-8', { ignoreBOM: true }).decode(result.bytes ?? new Uint8Array(0));
 }
 /** The files' contents in place of the answer's output. */
-async function readBack(toolbox, id, sizes, result, signal) {
+async function readBack(toolbox, id, marker, result, signal) {
     const dir = `${OUTPUT_DIR}/${id}`;
     // Both at once; one that fails stops the other, as the caller's abort stops both.
     const both = new AbortController();
@@ -210,17 +296,17 @@ async function readBack(toolbox, id, sizes, result, signal) {
     let stdout;
     let stderr;
     try {
-        [stdout, stderr] = await Promise.all([read('stdout', sizes.out), read('stderr', sizes.err)]);
+        [stdout, stderr] = await Promise.all([read('stdout', marker.out), read('stderr', marker.err)]);
     }
     finally {
         signal?.removeEventListener('abort', stop);
     }
-    return {
-        ...result,
-        stdout,
-        stderr,
-        truncated: sizes.out > FULL_OUTPUT_BYTES || sizes.err > FULL_OUTPUT_BYTES,
-    };
+    const reason = markerTruncation(marker);
+    const { exitCode, durationMs, timedOut } = result;
+    const full = { stdout, stderr, exitCode, durationMs, timedOut, truncated: reason !== null };
+    if (reason !== null)
+        full.truncatedReason = reason;
+    return full;
 }
 /**
  * A blocking exec that keeps the whole output (up to FULL_OUTPUT_BYTES of
@@ -245,22 +331,22 @@ export async function execFullOutput(toolbox, id, req, signal) {
         await teardown(toolbox, removeOutputScript(id));
         throw error;
     }
-    let sizes = parseMarker(result.stdout, id);
-    if (sizes === null && (result.timedOut || result.exitCode === -1)) {
+    let marker = parseMarker(result.stdout, id);
+    if (marker === null && (result.timedOut || result.exitCode === -1)) {
         // Its time limit (or a signal) ended the script with the command, before
         // it printed anything back: what the command printed is in its files.
         const collected = await teardown(toolbox, collectScript(id));
-        sizes = collected === null ? null : parseMarker(collected.stdout, id);
-        if (sizes === null)
+        marker = collected === null ? null : parseMarker(collected.stdout, id);
+        if (marker === null)
             return result; // no answer: nothing more to be done
     }
     // No marker: the output came back in the answer (or the command ran as a
     // plain exec), and the directory is already gone.
-    if (sizes === null || sizes === 'gone')
+    if (marker === null || marker === 'gone')
         return result;
     let full;
     try {
-        full = await readBack(toolbox, id, sizes, result, signal);
+        full = await readBack(toolbox, id, marker, result, signal);
     }
     catch (error) {
         const aborted = !(error instanceof PrizedError) || error.code === 'aborted';
@@ -271,9 +357,18 @@ export async function execFullOutput(toolbox, id, req, signal) {
                 throw error;
             return result;
         }
+        if (error.code === 'sandbox_deleted') {
+            // Nothing is kept: the output went with the sandbox, as its code says for any call on it.
+            // The command ran: its exit code goes on the error too, so a caller
+            // that replaces the sandbox can tell not to run it again.
+            throw restate(error, `The command ran (exit ${result.exitCode}), but the sandbox was deleted before its output could be read back.`, {
+                cause: error,
+                exitCode: result.exitCode,
+            });
+        }
         const dir = `${OUTPUT_DIR}/${id}`;
-        throw restate(error, `The command ran (exit ${result.exitCode}), but its output could not be read back (${error.message.replace(/[.\s]+$/, '')}). ` +
-            `It is kept in ${dir}/stdout and ${dir}/stderr for an hour: read them with files.read(), then remove ${dir}.`);
+        throw new OutputReadError(`The command ran (exit ${result.exitCode}), but its output could not be read back (${error.message.replace(/[.\s]+$/, '')}). ` +
+            `It is kept in ${dir}/stdout and ${dir}/stderr for an hour: read them with files.read(), then remove ${dir}.`, { cause: error, exitCode: result.exitCode, stdoutPath: `${dir}/stdout`, stderrPath: `${dir}/stderr` });
     }
     await teardown(toolbox, removeOutputScript(id));
     return full;

package/dist/esm/sandbox.d.ts

@@ -128,8 +128,9 @@ export declare class Sandbox {
     /**
      * Delete every sandbox carrying all of `labels` (at least one) and resolve
      * with their ids (the API takes up to 250 a request; this repeats it until
-     * none are left). Safe to repeat: sandboxes already being
-     * deleted are in the answer again, and none matching resolves `[]`. Each
+     * none are left). Each request selects the current matches, so a repeat
+     * can delete sandboxes created since the first request. An uncertain
+     * answer is not retried; list the matches before deleting again. Each
      * counts toward the workspace's quotas until it is gone;
      * `Sandbox.list({ labels })` coming back empty means they all are.
      */

package/dist/esm/sandbox.js

@@ -225,6 +225,32 @@ function failedState(info) {
     }
     return null;
 }
+/** How the API's errorReason starts on each copy it deletes because another
+ *  copy of the same fork could not start. */
+const FORK_UNDONE_PREFIX = 'fork undone';
+/**
+ * The error a fork throws when not every copy runs, or null. The API undoes
+ * a fork one of whose copies could not start, deleting every copy, so the
+ * copy that failed may already read `deleting`, with its errorReason, by the
+ * time it is read: the error is that copy's reason (one that is not the
+ * undo's own), else a copy in error, else any non-running copy's state.
+ */
+function forkFailure(copies) {
+    const failed = copies.filter((copy) => copy.state !== 'running');
+    if (failed.length === 0)
+        return null;
+    const cause = failed.find((copy) => copy.errorReason && !copy.errorReason.startsWith(FORK_UNDONE_PREFIX)) ??
+        failed.find((copy) => copy.state === 'error') ??
+        failed.find((copy) => copy.errorReason) ??
+        failed[0];
+    if (cause.errorReason) {
+        return new PrizedError('sandbox_error', `The sandbox failed: ${cause.errorReason}`, {
+            sandboxId: cause.id,
+            reason: cause.errorReason,
+        });
+    }
+    return failedState(cause) ?? new PrizedError('sandbox_error', null, { sandboxId: cause.id });
+}
 /**
  * A wait's timeout. One ended by a read its deadline cut short (the retry it
  * was due would have ended past it: a 503's Retry-After, say) may come
@@ -411,8 +437,9 @@ export class Sandbox {
     /**
      * Delete every sandbox carrying all of `labels` (at least one) and resolve
      * with their ids (the API takes up to 250 a request; this repeats it until
-     * none are left). Safe to repeat: sandboxes already being
-     * deleted are in the answer again, and none matching resolves `[]`. Each
+     * none are left). Each request selects the current matches, so a repeat
+     * can delete sandboxes created since the first request. An uncertain
+     * answer is not retried; list the matches before deleting again. Each
      * counts toward the workspace's quotas until it is gone;
      * `Sandbox.list({ labels })` coming back empty means they all are.
      */
@@ -426,7 +453,15 @@ export class Sandbox {
         const deleted = new Set();
         const stuck = (calls) => new PrizedError('delete_incomplete', `Deleted ${deleted.size} sandboxes in ${calls} requests and some are left; call again to delete the rest.`);
         for (let call = 1; call <= MAX_BULK_DELETE_CALLS; call += 1) {
-            const result = await request(config, { method: 'DELETE', url: apiUrl(config, `/sandboxes${query}`), retry: 'safe', deadline });
+            // A label is a live selector, not a stable resource or idempotency key.
+            const result = await request(config, {
+                method: 'DELETE', url: apiUrl(config, `/sandboxes${query}`), retry: 'exec', deadline,
+                effect: {
+                    maybe: 'some matching sandboxes may already be deleting',
+                    did: 'The matching sandboxes were marked for deletion',
+                    check: 'It was not sent again: list the matching sandboxes before deleting again, because the matches may have changed.',
+                },
+            });
             const answer = result.json;
             const ids = answer?.ids;
             if (!Array.isArray(ids) || !ids.every((id) => typeof id === 'string')) {
@@ -560,9 +595,9 @@ export class Sandbox {
             return children;
         try {
             await Sandbox.settle(this.config, children, deadline, timeoutMs, options.signal, this.id);
-            const failed = children.find((child) => child.state !== 'running');
+            const failed = forkFailure(children.map((child) => child.current));
             if (failed)
-                throw failedState(failed.current) ?? new PrizedError('sandbox_error', null, { sandboxId: failed.id });
+                throw failed;
         }
         catch (error) {
             // All or nothing: copies the caller will never get are not left billing.

package/dist/esm/shell.d.ts

@@ -11,7 +11,8 @@ export declare const RUN_ID_RE: RegExp;
 /**
  * Minutes (by the sandbox's clock) a cleaned or abandoned run's directory
  * stays: what keeps a resend of its start from running the command again.
- * The SDK stops resending a start within minutes, so an hour is ample; each
+ * The SDK stops resending a start once pollOutageMs (2 minutes by default)
+ * has run out after its first copy's retries, so an hour is ample; each
  * cleanup then removes the ones older than that, so they never pile up.
  */
 export declare const CLEANED_KEEP_MIN = 60;

package/dist/esm/shell.js

@@ -50,7 +50,8 @@ export const RUN_ID_RE = /^run_[0-9a-f]{16}$/;
 /**
  * Minutes (by the sandbox's clock) a cleaned or abandoned run's directory
  * stays: what keeps a resend of its start from running the command again.
- * The SDK stops resending a start within minutes, so an hour is ample; each
+ * The SDK stops resending a start once pollOutageMs (2 minutes by default)
+ * has run out after its first copy's retries, so an hour is ample; each
  * cleanup then removes the ones older than that, so they never pile up.
  */
 export const CLEANED_KEEP_MIN = 60;

package/dist/esm/toolbox.d.ts

@@ -30,6 +30,8 @@ export declare function rawExec(toolbox: Toolbox, req: RawExecRequest, options:
     deadline?: number | undefined;
     /** What the command does, for an error that says whether it happened (`exec`: a command unless given). */
     effect?: Effect | undefined;
+    /** An earlier copy of a keyed start may have reached the sandbox. */
+    acted?: boolean | undefined;
 }): Promise<ExecResult>;
 /** A helper command the SDK sends itself; a non-zero exit is the caller's to judge. */
 export declare function helperExec(toolbox: Toolbox, command: string, options?: {

package/dist/esm/toolbox.js

@@ -46,6 +46,7 @@ export async function rawExec(toolbox, req, options) {
         deadline: options.deadline,
         sandboxId: toolbox.sandboxId,
         ...(effect === undefined ? {} : { effect }),
+        ...(options.acted ? { acted: true } : {}),
     });
     const doc = result.json;
     if (doc === null || typeof doc !== 'object' || typeof doc.exitCode !== 'number') {
@@ -58,13 +59,15 @@ export async function rawExec(toolbox, req, options) {
     }
     const stdout = typeof doc.stdout === 'string' ? doc.stdout : '';
     const stderr = typeof doc.stderr === 'string' ? doc.stderr : '';
+    const truncated = keptBytes(stdout) >= MAX_OUTPUT_BYTES || keptBytes(stderr) >= MAX_OUTPUT_BYTES;
     return {
         stdout,
         stderr,
         exitCode: doc.exitCode,
         durationMs: typeof doc.durationMs === 'number' ? doc.durationMs : 0,
         timedOut: doc.timedOut === true,
-        truncated: keptBytes(stdout) >= MAX_OUTPUT_BYTES || keptBytes(stderr) >= MAX_OUTPUT_BYTES,
+        truncated,
+        ...(truncated ? { truncatedReason: 'output_limit' } : {}),
     };
 }
 /** A helper command the SDK sends itself; a non-zero exit is the caller's to judge. */

package/dist/esm/types.d.ts

@@ -70,10 +70,22 @@ export interface ExecResult {
      * True when stdout or stderr reached the 4 MiB per-stream cap of a single
      * call: the command kept running, the rest of that stream was dropped.
      * Pass `fullOutput: true` for commands that may print more (it keeps up
-     * to 100 MiB of each stream, and sets this past that).
+     * to 100 MiB of each stream, and sets this past that). `truncatedReason`
+     * says why.
      */
     truncated: boolean;
+    /**
+     * Why `truncated` is true; absent when it is false. `output_limit`: a
+     * stream reached its cap (4 MiB in one call, 100 MiB with `fullOutput`).
+     * `write_failed` (`fullOutput` only): the sandbox could not save a stream
+     * as the command printed it, most often because its disk is full, so that
+     * stream stops where the write failed, however short; free some space and
+     * run it again.
+     */
+    truncatedReason?: TruncatedReason;
 }
+/** Why a command's output was cut: ExecResult.truncatedReason. */
+export type TruncatedReason = 'output_limit' | 'write_failed';
 /** One directory entry, as `sandbox.files.list()` returns it. */
 export interface FileInfo {
     name: string;

package/dist/esm/version.d.ts

@@ -1,2 +1,2 @@
 /** This package's version; test/version.test.ts holds it equal to package.json. */
-export declare const VERSION = "0.1.0";
+export declare const VERSION = "0.1.2";

package/dist/esm/version.js

@@ -1,2 +1,2 @@
 /** This package's version; test/version.test.ts holds it equal to package.json. */
-export const VERSION = '0.1.0';
+export const VERSION = '0.1.2';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prized/sdk",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Prized Sandboxes (beta): create microVM sandboxes, run commands, move files, pause, resume and fork them from TypeScript.",
   "homepage": "https://prized.dev/docs/sandboxes",
   "license": "Apache-2.0",