@prized/sdk 0.1.1 → 0.1.2

package/dist/cjs/outcome.js

@@ -10,6 +10,7 @@ exports.mayHaveActed = mayHaveActed;
 exports.restate = restate;
 exports.withOutcome = withOutcome;
 exports.withAbort = withAbort;
+exports.startGaveUp = startGaveUp;
 exports.startAbandoned = startAbandoned;
 exports.runAborted = runAborted;
 exports.pollFailed = pollFailed;
@@ -105,12 +106,14 @@ function maybeTail(effect, closed) {
         tail += ` ${effect.check}`;
     return tail;
 }
+/** How an error says its request never reached the sandbox (withOutcome). */
+const NEVER_TAIL = ' It never reached the sandbox, so it had no effect.';
 /** `error` (same code, status and body), its message saying what it means for `effect`. */
 function withOutcome(error, effect, outcome) {
     if (outcome === null)
         return error;
     if (outcome === 'never')
-        return restate(error, `${trimmed(error.message)}. It never reached the sandbox, so it had no effect.`);
+        return restate(error, `${trimmed(error.message)}.${NEVER_TAIL}`);
     if (outcome === 'did')
         return acting(error, `${effect.did}, but the answer could not be read (${detail(error)}).`);
     return acting(error, `No clear answer came back (${detail(error)}), so ${maybeTail(effect, false)}`);
@@ -134,6 +137,23 @@ function stopTail(runId, stopped) {
         ? ` The SDK stopped ${runId}, or it had already ended.`
         : ` The SDK could not confirm it stopped: it may still be running as ${runId} (exec('ps -ef | grep ${runId}') finds it).`;
 }
+/**
+ * What a start that gave up throws when the failure it gives up with
+ * (`failure`, Outage's pick) is not the last copy's own: an earlier copy's,
+ * worded before a later copy may have reached the sandbox, or what a copy
+ * cut short by the window met, not yet worded. Worded for the start: as one
+ * that may have started when any copy may have reached the sandbox
+ * (`anyActed`), else as one that never did. The Python SDK's start_gave_up.
+ */
+function startGaveUp(failure, anyActed) {
+    if (acted.has(failure))
+        return failure;
+    const never = failure.message.endsWith(NEVER_TAIL);
+    if (never && !anyActed)
+        return failure;
+    const plain = never ? restate(failure, failure.message.slice(0, -NEVER_TAIL.length)) : failure;
+    return withOutcome(plain, startEffect(), anyActed ? 'maybe' : 'never');
+}
 /**
  * A background start that may have started (`error`, from withOutcome or
  * withAbort) and that the SDK then stopped by its run id (`stopped`: the

package/dist/cjs/output.js

@@ -279,6 +279,7 @@ async function download(toolbox, path, signal) {
         retry: 'safe',
         read: 'bytes',
         timeoutMs: READ_MS,
+        stallMs: http_js_1.timing.transferStallMs,
         deadline: toolbox.config.now() + READ_MS,
         signal,
         sandboxId: toolbox.sandboxId,

package/dist/cjs/sandbox.d.ts

@@ -128,8 +128,9 @@ export declare class Sandbox {
     /**
      * Delete every sandbox carrying all of `labels` (at least one) and resolve
      * with their ids (the API takes up to 250 a request; this repeats it until
-     * none are left). Safe to repeat: sandboxes already being
-     * deleted are in the answer again, and none matching resolves `[]`. Each
+     * none are left). Each request selects the current matches, so a repeat
+     * can delete sandboxes created since the first request. An uncertain
+     * answer is not retried; list the matches before deleting again. Each
      * counts toward the workspace's quotas until it is gone;
      * `Sandbox.list({ labels })` coming back empty means they all are.
      */

package/dist/cjs/sandbox.js

@@ -440,8 +440,9 @@ class Sandbox {
     /**
      * Delete every sandbox carrying all of `labels` (at least one) and resolve
      * with their ids (the API takes up to 250 a request; this repeats it until
-     * none are left). Safe to repeat: sandboxes already being
-     * deleted are in the answer again, and none matching resolves `[]`. Each
+     * none are left). Each request selects the current matches, so a repeat
+     * can delete sandboxes created since the first request. An uncertain
+     * answer is not retried; list the matches before deleting again. Each
      * counts toward the workspace's quotas until it is gone;
      * `Sandbox.list({ labels })` coming back empty means they all are.
      */
@@ -455,7 +456,15 @@ class Sandbox {
         const deleted = new Set();
         const stuck = (calls) => new errors_js_1.PrizedError('delete_incomplete', `Deleted ${deleted.size} sandboxes in ${calls} requests and some are left; call again to delete the rest.`);
         for (let call = 1; call <= MAX_BULK_DELETE_CALLS; call += 1) {
-            const result = await (0, http_js_1.request)(config, { method: 'DELETE', url: apiUrl(config, `/sandboxes${query}`), retry: 'safe', deadline });
+            // A label is a live selector, not a stable resource or idempotency key.
+            const result = await (0, http_js_1.request)(config, {
+                method: 'DELETE', url: apiUrl(config, `/sandboxes${query}`), retry: 'exec', deadline,
+                effect: {
+                    maybe: 'some matching sandboxes may already be deleting',
+                    did: 'The matching sandboxes were marked for deletion',
+                    check: 'It was not sent again: list the matching sandboxes before deleting again, because the matches may have changed.',
+                },
+            });
             const answer = result.json;
             const ids = answer?.ids;
             if (!Array.isArray(ids) || !ids.every((id) => typeof id === 'string')) {

package/dist/cjs/shell.d.ts

@@ -11,7 +11,8 @@ export declare const RUN_ID_RE: RegExp;
 /**
  * Minutes (by the sandbox's clock) a cleaned or abandoned run's directory
  * stays: what keeps a resend of its start from running the command again.
- * The SDK stops resending a start within minutes, so an hour is ample; each
+ * The SDK stops resending a start once pollOutageMs (2 minutes by default)
+ * has run out after its first copy's retries, so an hour is ample; each
  * cleanup then removes the ones older than that, so they never pile up.
  */
 export declare const CLEANED_KEEP_MIN = 60;

package/dist/cjs/shell.js

@@ -64,7 +64,8 @@ exports.RUN_ID_RE = /^run_[0-9a-f]{16}$/;
 /**
  * Minutes (by the sandbox's clock) a cleaned or abandoned run's directory
  * stays: what keeps a resend of its start from running the command again.
- * The SDK stops resending a start within minutes, so an hour is ample; each
+ * The SDK stops resending a start once pollOutageMs (2 minutes by default)
+ * has run out after its first copy's retries, so an hour is ample; each
  * cleanup then removes the ones older than that, so they never pile up.
  */
 exports.CLEANED_KEEP_MIN = 60;

package/dist/cjs/toolbox.d.ts

@@ -30,6 +30,8 @@ export declare function rawExec(toolbox: Toolbox, req: RawExecRequest, options:
     deadline?: number | undefined;
     /** What the command does, for an error that says whether it happened (`exec`: a command unless given). */
     effect?: Effect | undefined;
+    /** An earlier copy of a keyed start may have reached the sandbox. */
+    acted?: boolean | undefined;
 }): Promise<ExecResult>;
 /** A helper command the SDK sends itself; a non-zero exit is the caller's to judge. */
 export declare function helperExec(toolbox: Toolbox, command: string, options?: {

package/dist/cjs/toolbox.js

@@ -51,6 +51,7 @@ async function rawExec(toolbox, req, options) {
         deadline: options.deadline,
         sandboxId: toolbox.sandboxId,
         ...(effect === undefined ? {} : { effect }),
+        ...(options.acted ? { acted: true } : {}),
     });
     const doc = result.json;
     if (doc === null || typeof doc !== 'object' || typeof doc.exitCode !== 'number') {

package/dist/cjs/version.d.ts

@@ -1,2 +1,2 @@
 /** This package's version; test/version.test.ts holds it equal to package.json. */
-export declare const VERSION = "0.1.1";
+export declare const VERSION = "0.1.2";

package/dist/cjs/version.js

@@ -2,4 +2,4 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
 /** This package's version; test/version.test.ts holds it equal to package.json. */
-exports.VERSION = '0.1.1';
+exports.VERSION = '0.1.2';

package/dist/esm/commands.d.ts

@@ -104,7 +104,9 @@ export interface ExecOptions {
      * its polls fail (no answer, a 5xx, a rate limit), in ms, before it gives
      * up and stops the command. Default: the client's `pollOutageMs`
      * (120 000). A refused key, or an answer that settles the run (the
-     * sandbox deleted or not running, the run gone), stops it at once.
+     * sandbox deleted or not running, the run gone), stops it at once. The
+     * start of a background, streamed or long command is sent again for as
+     * long (a resend finds the first start, so the command runs once).
      */
     pollOutageMs?: number;
 }

package/dist/esm/commands.js

@@ -3,7 +3,7 @@
 import { CommandTimeoutError, PrizedError } from './errors.js';
 import { backoffMs, cutByDeadline, LAST_LOOK_MAX_MS, MAX_RETRY_AFTER_MS, Polls, validOutage } from './http.js';
 import { abandonScript, cleanupScript, commandText, killScript, NO_PID_EXIT, pollScript, RUN_ID_RE, sandboxPath, SIGNALS, startScript, } from './shell.js';
-import { mayHaveActed, pollFailed, runAborted, signalEffect, startAbandoned, startEffect, withOutcome } from './outcome.js';
+import { mayHaveActed, pollFailed, runAborted, signalEffect, startAbandoned, startEffect, startGaveUp, withAbort, withOutcome } from './outcome.js';
 import { execFullOutput } from './output.js';
 import { helperExec, rawExec } from './toolbox.js';
 /** The timeout a command gets when the call names none. */
@@ -470,7 +470,48 @@ async function cleanupOnceGone(toolbox, id) {
         }
     }
 }
-async function startBackground(toolbox, command, cwd, env, timeoutMs, signal, unbuffered) {
+/**
+ * Send a start script. It is keyed by a run id the SDK chose, so a resend
+ * finds the first start instead of starting the command twice. Once one
+ * copy's retries run out, it is sent again through an outage of up to
+ * `outageMs` (pollOutageMs), paced as a wait's polls are (Outage): an edge
+ * that restarts mid-start does not cost the command, as it does not cost a
+ * running one its wait. Each copy's error counts the copies before it
+ * (`acted`), so one that gives up says the command may have started when any
+ * copy may have reached the sandbox. The Python SDK's _send_start.
+ */
+async function sendStart(toolbox, req, signal, outageMs) {
+    const config = toolbox.config;
+    const outage = new Outage(outageMs);
+    // Whether an earlier copy may have reached the sandbox: an abort between
+    // copies then says the command may have started, else that it never did.
+    let acted = false;
+    const aborted = (cause) => withAbort(new PrizedError('aborted', null, { sandboxId: toolbox.sandboxId, cause }), startEffect(), acted ? 'maybe' : 'never');
+    for (;;) {
+        const sentAt = config.now();
+        try {
+            return await rawExec(toolbox, req, { retry: 'safe', signal, deadline: outage.deadline(sentAt), effect: startEffect(), acted });
+        }
+        catch (error) {
+            if (!(error instanceof PrizedError))
+                throw error;
+            acted = acted || mayHaveActed(error);
+            if (signal?.aborted)
+                throw error;
+            const step = outage.failed(error, sentAt, config.now(), cutByDeadline(error), (tries) => backoffMs(config, tries));
+            // What a copy met, not the window's cut.
+            if (step.pause === null)
+                throw step.failure === error ? error : startGaveUp(step.failure, acted);
+            try {
+                await config.sleep(step.pause, signal);
+            }
+            catch (abort) {
+                throw abort instanceof PrizedError && abort.code === 'aborted' ? aborted(step.failure) : abort;
+            }
+        }
+    }
+}
+async function startBackground(toolbox, command, cwd, env, timeoutMs, signal, unbuffered, outageMs) {
     // Nothing sent yet: nothing to stop.
     if (signal?.aborted)
         throw withOutcome(new PrizedError('aborted', null, { sandboxId: toolbox.sandboxId }), startEffect(), 'never');
@@ -482,10 +523,7 @@ async function startBackground(toolbox, command, cwd, env, timeoutMs, signal, un
     // Launched, but no pid came within the start's own wait: it may yet run.
     let unsettled = false;
     try {
-        const result = await rawExec(toolbox, { command: startScript(id, command, timeoutMs === null ? null : timeoutMs / 1000, unbuffered), cwd, env, timeoutMs: 60_000 }, 
-        // Keyed by a run id this call chose, so a resend finds the first start
-        // instead of starting the command twice.
-        { retry: 'safe', signal, effect: startEffect() });
+        const result = await sendStart(toolbox, { command: startScript(id, command, timeoutMs === null ? null : timeoutMs / 1000, unbuffered), cwd, env, timeoutMs: 60_000 }, signal, outageMs);
         pid = Number(result.stdout.trim());
         if (result.exitCode !== 0 || !Number.isInteger(pid) || pid <= 0) {
             unsettled = result.exitCode === NO_PID_EXIT;
@@ -531,7 +569,7 @@ export async function execCommand(toolbox, command, options = {}) {
     const cwd = options.cwd === undefined ? undefined : sandboxPath(options.cwd);
     if (options.background) {
         const limit = options.timeoutMs ? options.timeoutMs : null;
-        const handle = await startBackground(toolbox, text, cwd, options.env, limit, options.signal, true);
+        const handle = await startBackground(toolbox, text, cwd, options.env, limit, options.signal, true, outageMs);
         handle.timeoutMs = limit;
         return handle;
     }
@@ -539,7 +577,7 @@ export async function execCommand(toolbox, command, options = {}) {
     const streamed = options.onStdout !== undefined || options.onStderr !== undefined;
     if (streamed || timeoutMs === 0 || timeoutMs > MAX_BLOCKING_EXEC_MS) {
         const limit = timeoutMs === 0 ? null : timeoutMs;
-        const handle = await startBackground(toolbox, text, cwd, options.env, limit, options.signal, streamed);
+        const handle = await startBackground(toolbox, text, cwd, options.env, limit, options.signal, streamed, outageMs);
         const waitOptions = {};
         if (options.onStdout)
             waitOptions.onStdout = options.onStdout;

package/dist/esm/files.d.ts

@@ -13,9 +13,10 @@ export interface WriteOptions {
 }
 /**
  * The sandbox's file transfers take at most this long, retries included:
- * the edge's own limit on one transfer. A write's upload sent again (a
- * staged one is, after a lost answer or a 502, 503 or 504) is sent only
- * while time is left, so a transfer that timed out is not.
+ * the edge's own limit on one transfer. An attempt that moves no bytes for
+ * timing.transferStallMs (2 minutes) is given up; a download, or a write's
+ * staged upload, is then sent again (as after a lost answer or a 502, 503 or
+ * 504) while time is left.
  */
 export declare const FILE_TIMEOUT_MS: number;
 /** The publish script's own exits (the Python SDK's too). */
@@ -60,8 +61,8 @@ export declare class Files {
      * stays executable; a new file gets the sandbox's default (0600 in
      * prized-base-v1 and prized-agent-v1). `mode` sets the permission bits either way.
      * The bytes go up under a temporary name beside the file, sent again if
-     * need be (a lost answer, a 502, 503 or 504; all copies within the
-     * transfer's 10 minutes),
+     * need be (a lost answer, a 502, 503 or 504, 2 minutes with no bytes
+     * moving; all copies within the transfer's 10 minutes),
      * and one command that is never sent again puts them in place: a write
      * whose answer to that command was lost throws, since it may have landed
      * and the sandbox written the file since (read it back, or write again,

package/dist/esm/files.js

@@ -1,14 +1,15 @@
 // sandbox.files: read, write, list, remove, upload, download.
 import { PrizedError } from './errors.js';
-import { request } from './http.js';
+import { request, timing } from './http.js';
 import { mayHaveActed, removeEffect, withOutcome, writeEffect } from './outcome.js';
 import { existsScript, listScript, removeScript, sandboxPath, shellQuote, WORKSPACE_DIR } from './shell.js';
 import { helperExec, MAX_FILE_BYTES, rawExec } from './toolbox.js';
 /**
  * The sandbox's file transfers take at most this long, retries included:
- * the edge's own limit on one transfer. A write's upload sent again (a
- * staged one is, after a lost answer or a 502, 503 or 504) is sent only
- * while time is left, so a transfer that timed out is not.
+ * the edge's own limit on one transfer. An attempt that moves no bytes for
+ * timing.transferStallMs (2 minutes) is given up; a download, or a write's
+ * staged upload, is then sent again (as after a lost answer or a 502, 503 or
+ * 504) while time is left.
  */
 export const FILE_TIMEOUT_MS = 10 * 60_000;
 function byteSize(data) {
@@ -163,6 +164,7 @@ export class Files {
             retry: 'safe',
             read: 'bytes',
             timeoutMs: FILE_TIMEOUT_MS,
+            stallMs: timing.transferStallMs,
             deadline: this.toolbox.config.now() + FILE_TIMEOUT_MS,
             sandboxId: this.toolbox.sandboxId,
         });
@@ -175,8 +177,8 @@ export class Files {
      * stays executable; a new file gets the sandbox's default (0600 in
      * prized-base-v1 and prized-agent-v1). `mode` sets the permission bits either way.
      * The bytes go up under a temporary name beside the file, sent again if
-     * need be (a lost answer, a 502, 503 or 504; all copies within the
-     * transfer's 10 minutes),
+     * need be (a lost answer, a 502, 503 or 504, 2 minutes with no bytes
+     * moving; all copies within the transfer's 10 minutes),
      * and one command that is never sent again puts them in place: a write
      * whose answer to that command was lost throws, since it may have landed
      * and the sandbox written the file since (read it back, or write again,
@@ -256,6 +258,7 @@ export class Files {
             retry,
             effect,
             timeoutMs: FILE_TIMEOUT_MS,
+            stallMs: timing.transferStallMs,
             deadline: this.toolbox.config.now() + FILE_TIMEOUT_MS,
             sandboxId: this.toolbox.sandboxId,
         });

package/dist/esm/http.d.ts

@@ -32,7 +32,9 @@ export interface ClientOptions {
      * rate limit), in ms, before it gives up: a streamed or long `exec` then
      * stops the command, a handle's `wait()` throws. Default 120 000. `0`
      * gives up once one poll's retries run out. `exec` and `wait()` take it
-     * per call too.
+     * per call too. The start of a background, streamed or long `exec` is
+     * sent again for as long, so an edge that restarts mid-start does not
+     * cost the command.
      */
     pollOutageMs?: number;
     /** Sleep seam (tests inject one that never really waits). */
@@ -54,13 +56,21 @@ export interface Config {
     sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
     now: () => number;
     random: () => number;
+    /**
+     * Whether an upload's body goes out as a stream the SDK feeds piece by
+     * piece, so a stalled one is seen (RequestSpec.stallMs): Node's own fetch
+     * only, which asks for each piece once the last has gone into the socket.
+     * Elsewhere (another runtime, a fetch you pass) the body goes as it is.
+     */
+    streamBodies: boolean;
 }
 export declare function abortableSleep(ms: number, signal?: AbortSignal): Promise<void>;
 /**
  * How long a wait (a streamed or long exec, a handle's wait()) keeps polling
  * while its polls fail before it gives up (pollOutageMs), and a streamed or
- * long exec stops its command: long enough to ride out a dropped connection
- * or an edge that restarts. The Python SDK's DEFAULT_POLL_OUTAGE.
+ * long exec stops its command, and how long a start is sent again: long
+ * enough to ride out a dropped connection or an edge that restarts (about
+ * 70 s, the last time it crashed). The Python SDK's DEFAULT_POLL_OUTAGE.
  */
 export declare const DEFAULT_POLL_OUTAGE_MS = 120000;
 /** A pollOutageMs given (a number of ms ≥ 0), or undefined. */
@@ -97,8 +107,8 @@ export declare function warmEdge(config: Config): Promise<void>;
  *   Retries network failures, 429/502/503/504, and everything `exec`
  *   retries (the edge's 500 control_plane_unavailable among them). A GET
  *   retries any 500 too (the API's `internal`).
- * - `exec`: a command, an upload or a remove: what the sandbox does
- *   meanwhile could be undone by a second copy. Retried only when it
+ * - `exec`: a command, an upload, a remove or a label-selected delete:
+ *   a second copy could affect later work. Retried only when it
  *   provably never arrived: a connection that never opened, or an answer
  *   saying so (the edge's rate limit, its lookup failing). A lost answer
  *   is never retried: it may already have run.
@@ -139,6 +149,14 @@ export interface RequestSpec {
     sandboxId?: string | null;
     /** How to read a 2xx body. Default `json`. */
     read?: 'json' | 'bytes' | 'none';
+    /**
+     * A file transfer's no-progress bound: an attempt that moves no bytes
+     * either way for this long (a connection dropped without a reset) ends as
+     * a timeout, sent again by `retry`'s rules while `deadline` allows. An
+     * upload's sending is watched only where its body is streamed
+     * (Config.streamBodies); its answer always is.
+     */
+    stallMs?: number;
     /**
      * Told of every attempt that failed (no answer, an error answer, or a 2xx
      * that could not be read), before it is retried or thrown. A create or
@@ -153,6 +171,12 @@ export interface RequestSpec {
      * its error says so too.
      */
     effect?: Effect;
+    /**
+     * An earlier copy of this keyed request may have reached the sandbox (the
+     * background start, sent again through an outage), so its error says so
+     * too.
+     */
+    acted?: boolean;
 }
 export interface HttpResult {
     status: number;
@@ -215,6 +239,25 @@ export declare class Polls {
 }
 /** Delay before retry number `attempt` (0-based): exponential with jitter. */
 export declare function backoffMs(config: Config, attempt: number): number;
+/**
+ * How long a file transfer may move no bytes before the attempt is given up
+ * (sdk/retry-spec.json transferStallMs; the Python SDK's TRANSFER_STALL).
+ * The edge holds a request about 20 s at most before it reads the body (its
+ * admission queue, then waking a paused sandbox), and ends a body that brings
+ * nothing for 60 s itself: kept above that, a stalled copy is over at the
+ * edge before the next one goes to the same staged name, so the two never
+ * race. Once an upload's last byte is handed over, its answer is waited for
+ * that and a second more for every uploadAnswerBytesPerSecond it sent: what
+ * the socket's buffers, or a proxy that takes a whole body before it sends it
+ * on, still hold moves unseen, and must not read as a stall (the Python SDK's
+ * UPLOAD_ANSWER_RATE). Mutable for tests only.
+ */
+export declare const timing: {
+    transferStallMs: number;
+    uploadAnswerBytesPerSecond: number;
+};
+/** A streamed upload goes out this many bytes at a time. */
+export declare const UPLOAD_CHUNK: number;
 /**
  * What an error answer (not a 2xx: request() judges those) says about
  * whether the sandbox acted: `never` for the codes that prove the request