@promigence/sdk 0.1.0-beta.15 → 0.1.0-beta.17

package/dist/client.js

@@ -2,7 +2,7 @@ import { readFileSync, statSync } from "node:fs";
 import { homedir } from "node:os";
 import { inspect } from "node:util";
 import { dirname, join } from "node:path";
-import { ApiError, AuthError, EnvFailedError, PromigenceError, InvalidArgumentError, NotFoundError, NotReleasedError, NotVerifiedError, RateLimitError, ServiceRestartError, SpendCapError, TimeoutError, UnreachableError, DEFAULT_API_URL, ERRORS_DOCS_URL, FILE_PUT_CHUNK_BYTES, MAX_OUTPUT_BYTES_DEFAULT, VERSION, WAIT_TIMEOUT_MS, PreviewsUnavailableError, exitForCode, previewsOff, withPreviewsFix, } from "./core.js";
+import { ApiError, AuthError, EnvFailedError, ExecLostError, PromigenceError, InvalidArgumentError, NotFoundError, NotReleasedError, NotVerifiedError, RateLimitError, ServiceRestartError, SpendCapError, TimeoutError, UnreachableError, DEFAULT_API_URL, FILE_PUT_CHUNK_BYTES, MAX_OUTPUT_BYTES_DEFAULT, VERSION, WAIT_TIMEOUT_MS, PreviewsUnavailableError, SANDBOX_ENDED_FIX, SandboxEndedError, errorsDocsFor, exitForCode, previewsOff, sandboxEndedIn, withPreviewsFix, } from "./core.js";
 import { buildBundle } from "./bundle.js";
 import { keychainReadCached, keychainStore, storeLabel } from "./credstore.js";
 import { CALLER_HEADER, resolveCaller, userAgentWithCaller } from "./caller.js";
@@ -16,6 +16,7 @@ import { isKeyedCreate } from "./keyed.js";
 import { customDispatcher, longRequestDispatcher } from "./dispatcher.js";
 import { compromisedKeyCause, guardedFetchDispatcher } from "./ca-guard.js";
 import { hintArg } from "./hint.js";
+import { answerTimedOut } from "./command-errors.js";
 import { GITHUB_REPO_PREFIX, githubCreateBody, } from "./github.js";
 import { GITLAB_REPO_PREFIX, } from "./gitlab.js";
 export const FILE_TIMEOUT_MS = 15 * 60_000;
@@ -93,27 +94,46 @@ function pageQuery(limit, max, cursor) {
 }
 export const OUTPUT_RETENTION_MAX_DAYS = 30;
 function projectUpdateBody(u) {
+    return settingsBody(u, "projects.update", "the organization's");
+}
+function orgUpdateBody(u) {
+    return settingsBody(u, "orgs.update", "the default");
+}
+function settingsBody(u, what, fallback) {
     const body = {};
     if (u.name !== undefined)
         body["name"] = u.name;
     if (u.outputRetentionDays !== undefined) {
         const d = u.outputRetentionDays;
         if (d !== null && (!Number.isInteger(d) || d < 1 || d > OUTPUT_RETENTION_MAX_DAYS)) {
-            throw new InvalidArgumentError(`outputRetentionDays must be a whole number from 1 to ${OUTPUT_RETENTION_MAX_DAYS}, or null, not ${String(d)}`, "Pass null for the organization's setting.");
+            throw new InvalidArgumentError(`outputRetentionDays must be a whole number from 1 to ${OUTPUT_RETENTION_MAX_DAYS}, or null, not ${String(d)}`, `Pass null for ${fallback} setting.`);
         }
         body["output_retention_days"] = d;
     }
     if (u.failureCaptures !== undefined) {
         if (u.failureCaptures !== null && typeof u.failureCaptures !== "boolean") {
-            throw new InvalidArgumentError("failureCaptures must be true, false or null", "Pass null for the organization's setting.");
+            throw new InvalidArgumentError("failureCaptures must be true, false or null", `Pass null for ${fallback} setting.`);
         }
         if (Object.keys(body).length) {
             throw new InvalidArgumentError("failureCaptures is set on its own", "Change it in a call of its own, without name or outputRetentionDays.");
         }
         body["failure_captures"] = u.failureCaptures;
     }
+    if (u.previewLinkLength !== undefined) {
+        const l = u.previewLinkLength;
+        if (l === "custom") {
+            throw new InvalidArgumentError("custom domains for preview links are coming soon", `Pass "short", "long" or null (${fallback}) for now.`);
+        }
+        if (l !== null && l !== "short" && l !== "long") {
+            throw new InvalidArgumentError(`previewLinkLength must be "short", "long" or null, not ${JSON.stringify(l)}`, `Pass null for ${fallback} setting.`);
+        }
+        if (Object.keys(body).length) {
+            throw new InvalidArgumentError("previewLinkLength is set on its own", "Change it in a call of its own, without the other settings.");
+        }
+        body["preview_link_length"] = l;
+    }
     if (!Object.keys(body).length) {
-        throw new InvalidArgumentError("projects.update needs something to change", "Pass name, outputRetentionDays or failureCaptures.");
+        throw new InvalidArgumentError(`${what} needs something to change`, "Pass name, outputRetentionDays, failureCaptures or previewLinkLength.");
     }
     return body;
 }
@@ -125,6 +145,14 @@ function expiryFields(o) {
         body["expires_at"] = o.expiresAt;
     return body;
 }
+function checkPreviewLength(length) {
+    if (length === undefined || length === "short" || length === "long")
+        return;
+    if (length === "custom") {
+        throw new InvalidArgumentError("custom domains for preview links are coming soon", "Pass length: \"short\" (the default) or \"long\" for now.");
+    }
+    throw new InvalidArgumentError(`length must be "short" or "long", got ${JSON.stringify(length)}`, "Omit it for the length your project or organization chose.");
+}
 export const NOTICE_HEADER = "promigence-notice";
 export const MAX_SOURCES = 8;
 export function configDir(env = process.env) {
@@ -276,30 +304,6 @@ export function readCredentials(env = process.env) {
 function storedName(store, env) {
     return store === undefined || store === "file" ? `the credentials file (${credentialsPath(env)})` : storeLabel(store);
 }
-const COMPAT_REFUSED_HOSTS = /(^|\.)e2b(-staging)?\.(app|dev|pro)$/i;
-export function apiUrlFromCompatUrl(url) {
-    const u = (url ?? "").trim();
-    if (!u)
-        return undefined;
-    let host = "";
-    try {
-        host = new URL(u.includes("://") ? u : `https://${u}`).hostname;
-    }
-    catch {
-        return undefined;
-    }
-    return COMPAT_REFUSED_HOSTS.test(host) ? undefined : u;
-}
-export function apiUrlFromCompatDomain(domain) {
-    const d = (domain ?? "").trim().replace(/^https?:\/\//, "").replace(/\/+$/, "");
-    if (!d || COMPAT_REFUSED_HOSTS.test(d))
-        return undefined;
-    return `https://api.${d}/v1`;
-}
-export function compatApiKey(key) {
-    const k = key?.trim();
-    return k && !k.startsWith("e2b_") ? k : undefined;
-}
 export function keyOrigin(env = process.env) {
     if (env["PROMIGENCE_API_KEY"]?.trim())
         return "PROMIGENCE_API_KEY";
@@ -326,12 +330,13 @@ function resolveCredentials(opts, env = process.env) {
     const inputs = optKey === undefined && envKey === undefined
         ? undefined
         : `${optKey ?? "\0"}\n${envKey ?? "\0"}\n${envUrl ?? "\0"}\n${opts.apiUrl ?? "\0"}`;
-    if (inputs !== undefined && inputs === credsLastInputs && credsLast !== undefined)
-        return { apiKey: credsLast.apiKey, apiUrl: credsLast.apiUrl };
+    if (inputs !== undefined && inputs === credsLastInputs && credsLast !== undefined) {
+        return { apiKey: credsLast.apiKey, apiUrl: credsLast.apiUrl, ...(credsLast.origin ? { origin: credsLast.origin } : {}) };
+    }
     const worked = resolveCredentialsFrom(opts, env, set, trim, optKey, envKey, envUrl);
     if (inputs !== undefined && worked.refusal === undefined) {
         credsLastInputs = inputs;
-        credsLast = { apiKey: worked.apiKey, apiUrl: worked.apiUrl };
+        credsLast = { apiKey: worked.apiKey, apiUrl: worked.apiUrl, ...(worked.origin ? { origin: worked.origin } : {}) };
     }
     return worked;
 }
@@ -365,8 +370,15 @@ function resolveCredentialsFrom(opts, env, set, trim, optKey, envKey, envUrl) {
         if (refusal)
             return { apiKey: undefined, apiUrl, refusal };
     }
-    return { apiKey: key, apiUrl };
+    const origin = !key ? undefined
+        : source === "opts" ? { source: "option" }
+            : source === "env" ? { source: "env" }
+                : source === "file" ? { source: "stored", ...(store === undefined ? {} : { store }) }
+                    : { source: "fallback" };
+    return { apiKey: key, apiUrl, ...(origin ? { origin } : {}) };
 }
+export const NO_KEY_MESSAGE = "no API key: none was given, PROMIGENCE_API_KEY is not set, and none is stored";
+export const NO_KEY_FIX = "Pass { apiKey } or set PROMIGENCE_API_KEY. No key yet: `promigence login` signs you up or in and stores one.";
 function keyRefusal(key, from) {
     if (/^[\x21-\x7E]+$/.test(key))
         return undefined;
@@ -646,6 +658,8 @@ export function snapshotCreateBody(opts) {
     }
     if (o.alias)
         body["alias"] = o.alias;
+    if (o.alias && o.moveAlias)
+        body["move_alias"] = true;
     if (o.grade) {
         if (!o.grade.cmd)
             throw new InvalidArgumentError("snapshots.create grade needs a cmd", "Pass { grade: { cmd: 'pytest -q tests/' } }.");
@@ -680,6 +694,9 @@ const REFUSED_503_CODES = new Set([
     "builder_unavailable", "builder_busy", "overloaded", "tier_unavailable", "registry_unavailable",
     "secrets_unavailable",
 ]);
+const SDK_FIXES = {
+    cap_exceeded: "Raise `spendCap`, lower `timeoutMs` or `count`, or pass `allowPartial` to launch what fits.",
+};
 function withoutCliHelp(fix) {
     const out = fix.replace(/\s*See `promigence [^`]*--help`\.?\s*$/, "").trim();
     if (out === fix.trim())
@@ -689,7 +706,7 @@ function withoutCliHelp(fix) {
 export function errorFor(code, message, fix, extra = {}) {
     const attach = (e) => {
         const w = e;
-        w.docs = extra.docs ?? `${ERRORS_DOCS_URL}#${code}`;
+        w.docs = extra.docs !== undefined && e.code === code ? extra.docs : errorsDocsFor(e.code);
         if (extra.status !== undefined)
             w.status = extra.status;
         if (extra.requestId !== undefined)
@@ -705,6 +722,8 @@ export function errorFor(code, message, fix, extra = {}) {
     if ((code === "usage" || code === "invalid_argument" || code === "unsupported") && previewsOff(message)) {
         return attach(new PreviewsUnavailableError());
     }
+    if (!extra.cli && Object.prototype.hasOwnProperty.call(SDK_FIXES, code))
+        fix = SDK_FIXES[code];
     switch (code) {
         case "auth": return attach(new AuthError(message, fix));
         case "not_found": return attach(new NotFoundError(message, fix));
@@ -718,8 +737,13 @@ export function errorFor(code, message, fix, extra = {}) {
         case "not_verified": return attach(new NotVerifiedError(message, fix));
         case "timeout": return attach(new TimeoutError(message, fix));
         case "usage":
-        case "invalid_argument":
+        case "invalid_argument": {
+            const ended = code === "usage" ? sandboxEndedIn(message) : undefined;
+            if (ended !== undefined) {
+                return attach(new SandboxEndedError(message, ended.sandboxId, ended.state, extra.cli ? fix : SANDBOX_ENDED_FIX));
+            }
             return attach(new InvalidArgumentError(message, extra.cli ? fix : withoutCliHelp(fix), code));
+        }
         case "not_released": return attach(new NotReleasedError(message));
         case "service_restart": return attach(new ServiceRestartError(message, fix));
         default:
@@ -772,6 +796,76 @@ function parseRetryAfter(header) {
 const MAX_RETRY_SLEEP_MS = 10_000;
 const STREAM_IDLE_MS = 90_000;
 const sleep = (ms) => new Promise((r) => { setTimeout(r, ms); });
+const EXEC_KEY_HEADER = "promigence-exec-key";
+const EXEC_HOLD_DEFAULT_S = 120;
+const REATTACH_SLACK_MS = 30_000;
+const REATTACH_BACKOFF_MS = 250;
+const REATTACH_BACKOFF_MAX_MS = 2_000;
+const REATTACH_RETRY_AFTER_MAX_MS = 5_000;
+const REATTACH_TAKEN_MS = 10_000;
+const EXEC_CANCEL_TIMEOUT_MS = 3_000;
+const REATTACH_AGAIN = new Set([429, 502, 503, 504]);
+function newExecKey() {
+    const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16));
+    let s = "";
+    for (const b of bytes)
+        s += b.toString(16).padStart(2, "0");
+    return s;
+}
+const UNSENT_CODES = /^(?:ECONNREFUSED|ENOTFOUND|EAI_[A-Z]+|ENETUNREACH|EHOSTUNREACH|UND_ERR_CONNECT_TIMEOUT|ERR_TLS_[A-Z_]+|ERR_SSL_[A-Z_]+|CERT_[A-Z_]+|UNABLE_TO_[A-Z_]+|DEPTH_ZERO_SELF_SIGNED_CERT|SELF_SIGNED_CERT_IN_CHAIN|HOSTNAME_MISMATCH|EPROTO)$/;
+function provedUnsent(err) {
+    if (compromisedKeyCause(err) !== undefined)
+        return true;
+    const top = err;
+    if (top?.name === "TypeError" && /header/i.test(top.message ?? ""))
+        return true;
+    const unsent = (x) => {
+        const code = x?.code;
+        return typeof code === "string" && UNSENT_CODES.test(code);
+    };
+    let c = err;
+    for (let depth = 0; c !== null && typeof c === "object" && depth < 6; depth++) {
+        if (unsent(c))
+            return true;
+        const each = c.errors;
+        if (Array.isArray(each) && each.length > 0 && each.every(unsent))
+            return true;
+        c = c.cause;
+    }
+    return false;
+}
+class HeldDrop extends Error {
+    key;
+    constructor(key, why) {
+        super(why);
+        this.key = key;
+        this.name = "HeldDrop";
+    }
+}
+function sleepUnless(ms, signal) {
+    if (signal === undefined)
+        return sleep(ms);
+    return new Promise((resolve) => {
+        if (signal.aborted) {
+            resolve();
+            return;
+        }
+        const done = () => { clearTimeout(t); signal.removeEventListener("abort", done); resolve(); };
+        const t = setTimeout(done, ms);
+        signal.addEventListener("abort", done, { once: true });
+    });
+}
+function reattachWait(attempt, retryAfterMs) {
+    if (retryAfterMs !== undefined)
+        return Math.min(retryAfterMs, REATTACH_RETRY_AFTER_MAX_MS);
+    return Math.min(REATTACH_BACKOFF_MAX_MS, REATTACH_BACKOFF_MS * 2 ** Math.min(attempt, 8)) * (0.5 + Math.random() * 0.5);
+}
+function stoppedUnattended(answer) {
+    return answer?.reason === "not_reattached";
+}
+function abortedByCaller(method, path) {
+    return new TimeoutError(`request ${method} ${path} was aborted by the caller's signal`, "Nothing to fix: the caller aborted it.");
+}
 const POLLING_TICK_MS = 5;
 const PARKED_TICK_MS = 50;
 async function keepingLoopAwake(everyMs, fn) {
@@ -790,6 +884,21 @@ const WAIT_MS_CAP = 30_000;
 export const LONG_POLL_MS = 5_000;
 const BUILD_POLL_MS = 10_000;
 const PROGRESS_POLL_MS = 200;
+export const RUN_SETTLE_MS = 5_000;
+export function runWorkDone(run, endedSince, now = Date.now()) {
+    if (run.state === "completed" || run.state === "cap_killed" || run.state === "aborted")
+        return true;
+    const sbs = Array.isArray(run.sandboxes) ? run.sandboxes : [];
+    const allDone = sbs.length > 0 && sbs.every((s) => s.state === "ended" || s.command_outcome != null);
+    if (!allDone) {
+        delete endedSince.at;
+        return false;
+    }
+    if (sbs.some((s) => s.state !== "ended"))
+        return true;
+    endedSince.at ??= now;
+    return now - endedSince.at >= RUN_SETTLE_MS;
+}
 const WAITED_HEADER = "promigence-waited-ms";
 const seg = (s) => encodeURIComponent(s).replace(/%40/g, "@");
 function waitQuery(waitMs) {
@@ -827,6 +936,9 @@ export class PromigenceClient {
     apiUrl;
     #apiKey;
     #keyRefusal;
+    #keyOrigin;
+    #onKeyUse;
+    #keyUseSaid = false;
     #bearer;
     fetchImpl;
     ownFetch;
@@ -848,6 +960,8 @@ export class PromigenceClient {
         const resolved = resolveCredentials(opts);
         this.#apiKey = resolved.apiKey;
         this.#keyRefusal = resolved.refusal;
+        this.#keyOrigin = resolved.origin;
+        this.#onKeyUse = opts.onKeyUse;
         this.apiUrl = resolved.apiUrl;
         const env = opts.fetch === undefined ? transportEnv() : undefined;
         const proxy = env !== undefined ? proxyFor(this.apiUrl, env) : undefined;
@@ -913,11 +1027,18 @@ export class PromigenceClient {
         catch {
         }
     }
+    get keyOrigin() {
+        return this.#keyOrigin;
+    }
     requireKey() {
         if (this.#keyRefusal)
             throw this.#keyRefusal;
         if (!this.#apiKey)
-            throw new AuthError();
+            throw this.cliErrors ? new AuthError(NO_KEY_MESSAGE) : new AuthError(NO_KEY_MESSAGE, NO_KEY_FIX);
+        if (!this.#keyUseSaid && this.#onKeyUse !== undefined && this.#keyOrigin !== undefined) {
+            this.#keyUseSaid = true;
+            this.#onKeyUse(this.#keyOrigin);
+        }
         return this.#apiKey;
     }
     bearer() {
@@ -965,6 +1086,82 @@ export class PromigenceClient {
         await sleep(backoff * (0.5 + Math.random() * 0.5));
         return true;
     }
+    async heldExec(sandboxId, req) {
+        const signal = req.signal;
+        const abortedBefore = signal?.aborted === true;
+        let key;
+        try {
+            let res;
+            try {
+                res = await this.request({ ...req, held: { onKey: (k) => { key = k; } } });
+            }
+            catch (err) {
+                if (!(err instanceof HeldDrop))
+                    throw err;
+                res = await this.reattachBuffered(sandboxId, err.key, req);
+            }
+            if (stoppedUnattended(res)) {
+                throw new ExecLostError(`the command on sandbox ${sandboxId} was stopped: its connection to the service dropped and no client came back for it in time`, { sandboxId }, "not_reattached");
+            }
+            return res;
+        }
+        catch (err) {
+            if (key !== undefined && signal?.aborted && !abortedBefore)
+                await this.cancelHeldExec(sandboxId, key);
+            throw err;
+        }
+    }
+    async reattachBuffered(sandboxId, key, req) {
+        const budgetMs = EXEC_HOLD_DEFAULT_S * 1000 + REATTACH_SLACK_MS;
+        let droppedAt = Date.now();
+        for (let attempt = 0;; attempt++) {
+            if (req.signal?.aborted)
+                throw abortedByCaller(req.method, req.path);
+            this.stats.retries++;
+            const sentAt = Date.now();
+            try {
+                return await this.request({
+                    method: "POST",
+                    path: `/sandboxes/${encodeURIComponent(sandboxId)}/execs/attach`,
+                    body: { key },
+                    once: true,
+                    ...(req.timeoutMs === undefined ? {} : { timeoutMs: req.timeoutMs }),
+                    ...(req.signal === undefined ? {} : { signal: req.signal }),
+                });
+            }
+            catch (err) {
+                if (req.signal?.aborted)
+                    throw abortedByCaller(req.method, req.path);
+                const status = err instanceof PromigenceError ? err.status : undefined;
+                if (status === 404) {
+                    throw new ExecLostError(`the connection to the service dropped before the command on sandbox ${sandboxId} answered, and it could not be picked up again: nothing is kept for it (it ended, it never started, or this service cannot keep it)`, { sandboxId }, "gone");
+                }
+                const lostConnection = status === undefined && (err instanceof UnreachableError || err instanceof TimeoutError);
+                if (!lostConnection && !(status !== undefined && REATTACH_AGAIN.has(status)))
+                    throw err;
+                if (Date.now() - sentAt >= REATTACH_TAKEN_MS)
+                    droppedAt = Date.now();
+                const left = budgetMs - (Date.now() - droppedAt);
+                if (left <= 0) {
+                    throw new ExecLostError(`the connection to the service dropped while the command on sandbox ${sandboxId} was running, and the service could not be reached again in time: the command is stopped ${EXEC_HOLD_DEFAULT_S} s after the drop`, { sandboxId }, "unreachable");
+                }
+                await sleepUnless(Math.min(left, reattachWait(attempt, err instanceof PromigenceError ? err.retryAfterMs : undefined)), req.signal);
+            }
+        }
+    }
+    async cancelHeldExec(sandboxId, key) {
+        try {
+            await this.request({
+                method: "POST",
+                path: `/sandboxes/${encodeURIComponent(sandboxId)}/execs/cancel`,
+                body: { key },
+                timeoutMs: EXEC_CANCEL_TIMEOUT_MS,
+                once: true,
+            });
+        }
+        catch {
+        }
+    }
     async request(opts) {
         const tr = traceBegin(opts.method, opts.path);
         const url = `${this.apiUrl}${opts.path}`;
@@ -995,9 +1192,16 @@ export class PromigenceClient {
         const timeoutMs = opts.timeoutMs ?? this.requestTimeoutMs;
         let dispatcher;
         let resentDelete = false;
+        const maxRetries = opts.once === true ? 0 : this.maxRetries;
+        let execKey;
         for (let attempt = 0;; attempt++) {
             let res;
             this.stats.requests++;
+            if (opts.held !== undefined) {
+                execKey = newExecKey();
+                headers[EXEC_KEY_HEADER] = execKey;
+                opts.held.onKey(execKey);
+            }
             const sentAt = performance.now();
             try {
                 if (this.h2) {
@@ -1040,11 +1244,14 @@ export class PromigenceClient {
             }
             catch (err) {
                 const e = err;
+                if (execKey !== undefined && !opts.signal?.aborted && !provedUnsent(err)) {
+                    throw new HeldDrop(execKey, transportDetail(e));
+                }
                 if (keyedCreate && idempotencyKey && await this.againKeyed(attempt, opts.signal))
                     continue;
                 const cutOff = e?.cause?.code === "UND_ERR_HEADERS_TIMEOUT" || e?.cause?.code === "UND_ERR_BODY_TIMEOUT";
                 const timedOut = e?.name === "TimeoutError" || e?.name === "AbortError" || cutOff;
-                if (!timedOut && replayable && attempt < this.maxRetries && !opts.signal?.aborted) {
+                if (!timedOut && replayable && attempt < maxRetries && !opts.signal?.aborted) {
                     this.stats.retries++;
                     if (opts.method === "DELETE")
                         resentDelete = true;
@@ -1052,9 +1259,8 @@ export class PromigenceClient {
                     await sleep(backoff * (0.5 + Math.random() * 0.5));
                     continue;
                 }
-                if (opts.signal?.aborted) {
-                    throw new TimeoutError(`request ${opts.method} ${opts.path} was aborted by the caller's signal`, "Nothing to fix: the caller aborted it.");
-                }
+                if (opts.signal?.aborted)
+                    throw abortedByCaller(opts.method, opts.path);
                 if (e?.name === "TimeoutError" || e?.name === "AbortError") {
                     throw new TimeoutError(`request ${opts.method} ${opts.path} timed out after ${timeoutMs} ms`, "Retry. To allow longer: promigence --request-timeout 120s <command>, " +
                         "PROMIGENCE_REQUEST_TIMEOUT=120s, or requestTimeoutMs in the SDK. A --wait is bounded " +
@@ -1071,6 +1277,11 @@ export class PromigenceClient {
             const gateway = (res.status === 502 || res.status === 504) && replayable;
             let retryable = (res.status === 429 || res.status === 503 || gateway) && (retryAfterMs ?? 0) <= MAX_RETRY_SLEEP_MS;
             let body;
+            if (execKey !== undefined && (res.status === 502 || res.status === 504)) {
+                body = await res.text().catch(() => "");
+                if (errorCodeOf(body) === undefined)
+                    throw new HeldDrop(execKey, `HTTP ${res.status}`);
+            }
             if (retryable && res.status === 503 && !replayable && retryAfterMs === undefined) {
                 body = await res.text().catch(() => "");
                 retryable = REFUSED_503_CODES.has(errorCodeOf(body) ?? "");
@@ -1079,7 +1290,7 @@ export class PromigenceClient {
                 body = await res.text().catch(() => "");
                 retryable = errorCodeOf(body) === undefined;
             }
-            if (retryable && attempt < this.maxRetries) {
+            if (retryable && attempt < maxRetries) {
                 const backoff = Math.min(MAX_RETRY_SLEEP_MS, this.retryBaseMs * 2 ** attempt);
                 const jitter = Math.random();
                 const waitMs = retryAfterMs !== undefined
@@ -1112,8 +1323,10 @@ export class PromigenceClient {
                         text = "";
                     else {
                         const e = err;
+                        if (execKey !== undefined && !opts.signal?.aborted)
+                            throw new HeldDrop(execKey, transportDetail(e));
                         const timedOut = e?.name === "TimeoutError" || e?.name === "AbortError";
-                        if (!timedOut && replayable && attempt < this.maxRetries && !opts.signal?.aborted) {
+                        if (!timedOut && replayable && attempt < maxRetries && !opts.signal?.aborted) {
                             this.stats.retries++;
                             if (opts.method === "DELETE")
                                 resentDelete = true;
@@ -1239,10 +1452,12 @@ export class PromigenceClient {
                 continue;
             }
             const code = payload.error?.code ?? codeForStatus(res.status);
+            const rid = headerOf(res, "request-id") ?? headerOf(res, "x-request-id");
             throw errorFor(code, payload.error?.message ?? `HTTP ${res.status} from ${method} ${path}`, payload.error?.fix ?? "See the API response for details.", {
                 status: res.status,
                 ...bodyExtra(payload),
                 ...(retryAfterMs === undefined ? {} : { retryAfterMs }),
+                ...(rid ? { requestId: rid } : {}),
             });
         }
     }
@@ -1322,10 +1537,12 @@ export class PromigenceClient {
                 continue;
             }
             const code = payload.error?.code ?? codeForStatus(res.status);
+            const rid = headerOf(res, "request-id") ?? headerOf(res, "x-request-id");
             throw errorFor(code, payload.error?.message ?? `HTTP ${res.status} from ${method} ${path}`, payload.error?.fix ?? "See the API response for details.", {
                 status: res.status,
                 ...bodyExtra(payload),
                 ...(retryAfterMs === undefined ? {} : { retryAfterMs }),
+                ...(rid ? { requestId: rid } : {}),
             });
         }
     }
@@ -1486,6 +1703,7 @@ export class PromigenceClient {
                     method: "PATCH", path: `/orgs/${orgSeg(org)}/projects/${encodeURIComponent(project)}`, body: projectUpdateBody(update),
                 }),
             },
+            update: async (org, update) => this.request({ method: "PATCH", path: `/orgs/${orgSeg(org)}`, body: orgUpdateBody(update) }),
         };
     }
     group_credits() {
@@ -1565,6 +1783,8 @@ export class PromigenceClient {
                     body["network"] = opts.network;
                 if (opts.optimizeFor !== undefined)
                     body["optimize_for"] = opts.optimizeFor;
+                if (opts.capture !== undefined)
+                    body["capture"] = opts.capture;
                 if (opts.secrets?.length)
                     body["secrets"] = opts.secrets;
                 if (opts.labels && Object.keys(opts.labels).length)
@@ -1701,12 +1921,8 @@ export class PromigenceClient {
                 return { sandboxes: r.value, timedOut: r.timedOut, waitedMs: r.waitedMs };
             },
             waitDone: async (runId, opts = {}) => {
-                const r = await this.waitLoop(opts, LONG_POLL_MS, { run_id: runId, state: "running" }, (read) => this.runs.get(runId, read), (run) => {
-                    const terminal = run.state === "completed" || run.state === "cap_killed" || run.state === "aborted";
-                    const allDone = Array.isArray(run.sandboxes) && run.sandboxes.length > 0 &&
-                        run.sandboxes.every((s) => s.state === "ended" || s.command_outcome != null);
-                    return terminal || allDone;
-                });
+                const endedSince = {};
+                const r = await this.waitLoop(opts, LONG_POLL_MS, { run_id: runId, state: "running" }, (read) => this.runs.get(runId, read), (run) => runWorkDone(run, endedSince));
                 return { run: r.value, timedOut: r.timedOut, waitedMs: r.waitedMs };
             },
         };
@@ -1802,7 +2018,9 @@ export class PromigenceClient {
                     req.idempotencyKey = opts.idempotencyKey;
                 if (opts.signal)
                     req.signal = opts.signal;
-                return this.request(req);
+                if (opts.terminal)
+                    return this.request(req);
+                return this.heldExec(sandboxId, req);
             },
             execStream: async (sandboxId, opts, on = {}) => {
                 if (!opts.argv?.length && !opts.sh) {
@@ -1831,7 +2049,9 @@ export class PromigenceClient {
                 body["timeout_ms"] = commandTimeoutMs(opts.timeoutMs) ?? 15 * 60_000;
                 body["max_output_bytes"] = opts.maxOutputBytes ?? MAX_OUTPUT_BYTES_DEFAULT;
                 const url = `${this.apiUrl}/sandboxes/${encodeURIComponent(sandboxId)}/execs`;
-                const idle = new AbortController();
+                const keyed = opts.terminal !== true;
+                let key;
+                let idle = new AbortController();
                 let silent = false;
                 let reader;
                 let timer;
@@ -1843,23 +2063,33 @@ export class PromigenceClient {
                         reader?.cancel().catch(() => { });
                     }, STREAM_IDLE_MS);
                 };
+                const fresh = () => { idle = new AbortController(); silent = false; reader = undefined; };
+                const abortedBefore = opts.signal?.aborted === true;
+                const abandon = async (err) => {
+                    if (key !== undefined && !abortedBefore)
+                        await this.cancelHeldExec(sandboxId, key);
+                    throw err;
+                };
                 let stream;
                 const bearer = `Bearer ${this.requireKey()}`;
                 const dispatcher = this.guardDispatcher();
+                const headers = () => ({
+                    accept: "text/event-stream",
+                    "content-type": "application/json",
+                    "user-agent": this.userAgent,
+                    authorization: bearer,
+                    ...(this.caller ? { [CALLER_HEADER]: this.caller } : {}),
+                });
                 for (let attempt = 0; stream === undefined; attempt++) {
                     let r;
                     this.stats.requests++;
+                    if (keyed)
+                        key = newExecKey();
                     arm();
                     try {
                         r = await this.fetchImpl(url, {
                             method: "POST",
-                            headers: {
-                                accept: "text/event-stream",
-                                "content-type": "application/json",
-                                "user-agent": this.userAgent,
-                                authorization: bearer,
-                                ...(this.caller ? { [CALLER_HEADER]: this.caller } : {}),
-                            },
+                            headers: { ...headers(), ...(key === undefined ? {} : { [EXEC_KEY_HEADER]: key }) },
                             body: JSON.stringify(body),
                             signal: opts.signal ? AbortSignal.any([opts.signal, idle.signal]) : idle.signal,
                             ...(dispatcher === undefined ? {} : { dispatcher }),
@@ -1869,7 +2099,10 @@ export class PromigenceClient {
                         clearTimeout(timer);
                         const e = err;
                         const why = silent ? `no answer in ${STREAM_IDLE_MS / 1000} s` : transportDetail(e);
-                        throw new UnreachableError(`cannot reach ${url}: ${why}`);
+                        const failed = new UnreachableError(`cannot reach ${url}: ${why}`);
+                        if (opts.signal?.aborted)
+                            return abandon(failed);
+                        throw failed;
                     }
                     if (r.ok && r.body) {
                         stream = r.body;
@@ -1900,87 +2133,166 @@ export class PromigenceClient {
                 const errOut = [];
                 let exit;
                 let failure;
-                const dec = new TextDecoder();
-                let buf = "";
+                let holdS;
+                const received = { stdout: 0, stderr: 0 };
                 const lost = (how) => new UnreachableError(`the exec stream for sandbox ${sandboxId} ${how}`, "The connection dropped before the command finished; its result is unknown. " +
                     `Read it by id (\`promigence exec output <exec_id> --sandbox ${hintArg(sandboxId)}\`) or re-run.`);
-                reader = stream.getReader();
-                try {
-                    for (;;) {
+                const consume = async (from) => {
+                    const dec = new TextDecoder();
+                    let buf = "";
+                    const rd = from.getReader();
+                    reader = rd;
+                    try {
+                        for (;;) {
+                            arm();
+                            let chunk;
+                            try {
+                                chunk = await rd.read();
+                            }
+                            catch (err) {
+                                if (opts.signal?.aborted)
+                                    return abandon(err);
+                                if (holdS !== undefined && (exit || failure))
+                                    return { done: true };
+                                if (silent)
+                                    return { done: false, why: `was silent for ${STREAM_IDLE_MS / 1000} s`, restart: false };
+                                const e = err;
+                                return { done: false, why: `was cut off (${e?.cause?.code ?? e?.message ?? "connection lost"})`, restart: false };
+                            }
+                            if (chunk.done)
+                                break;
+                            const value = chunk.value;
+                            buf += dec.decode(value, { stream: true });
+                            let sep;
+                            while ((sep = buf.indexOf("\n\n")) >= 0) {
+                                const block = buf.slice(0, sep);
+                                buf = buf.slice(sep + 2);
+                                let event = "";
+                                let data = "";
+                                for (const line of block.split("\n")) {
+                                    if (line.startsWith("event:"))
+                                        event = line.slice(6).trim();
+                                    else if (line.startsWith("data:"))
+                                        data = line.slice(5).replace(/^ /, "");
+                                }
+                                if (!event)
+                                    continue;
+                                if (event === "heartbeat") {
+                                    on.heartbeat?.();
+                                    continue;
+                                }
+                                if (event === "reconnect") {
+                                    rd.cancel().catch(() => { });
+                                    return { done: false, why: "was ended by a service restart", restart: true };
+                                }
+                                let parsed = {};
+                                try {
+                                    parsed = JSON.parse(data);
+                                }
+                                catch { }
+                                if (event === "held") {
+                                    const h = parsed["hold_s"];
+                                    holdS = typeof h === "number" && Number.isFinite(h) && h >= 0 ? h : EXEC_HOLD_DEFAULT_S;
+                                }
+                                else if (event === "stdout" || event === "stderr") {
+                                    const bytes = b64ToBytes(String(parsed["data"] ?? ""));
+                                    if (event === "stdout") {
+                                        received.stdout += bytes.length;
+                                        out.push(bytes);
+                                        on.stdout?.(bytes);
+                                    }
+                                    else {
+                                        received.stderr += bytes.length;
+                                        errOut.push(bytes);
+                                        on.stderr?.(bytes);
+                                    }
+                                }
+                                else if (event === "exit") {
+                                    exit = { ...(exit ?? {}), ...parsed };
+                                }
+                                else if (event === "error") {
+                                    failure = parsed;
+                                }
+                            }
+                        }
+                    }
+                    finally {
+                        clearTimeout(timer);
+                    }
+                    if (exit || failure)
+                        return { done: true };
+                    if (silent)
+                        return { done: false, why: `was silent for ${STREAM_IDLE_MS / 1000} s`, restart: false };
+                    return { done: false, why: "ended without an exit event", restart: false };
+                };
+                const reattach = async () => {
+                    const droppedAt = Date.now();
+                    const hold = holdS ?? EXEC_HOLD_DEFAULT_S;
+                    const budgetMs = hold * 1000 + REATTACH_SLACK_MS;
+                    for (let attempt = 0;; attempt++) {
+                        if (opts.signal?.aborted)
+                            return abandon(opts.signal.reason);
+                        fresh();
+                        this.stats.requests++;
+                        this.stats.retries++;
                         arm();
-                        let chunk;
+                        let retryAfterMs;
+                        let r;
                         try {
-                            chunk = await reader.read();
+                            r = await this.fetchImpl(`${url}/attach`, {
+                                method: "POST",
+                                headers: headers(),
+                                body: JSON.stringify({ key, stdout_offset: received.stdout, stderr_offset: received.stderr }),
+                                signal: opts.signal ? AbortSignal.any([opts.signal, idle.signal]) : idle.signal,
+                                ...(dispatcher === undefined ? {} : { dispatcher }),
+                            });
                         }
                         catch (err) {
+                            clearTimeout(timer);
                             if (opts.signal?.aborted)
-                                throw err;
-                            if (silent)
-                                throw lost(`was silent for ${STREAM_IDLE_MS / 1000} s`);
-                            const e = err;
-                            throw lost(`was cut off (${e?.cause?.code ?? e?.message ?? "connection lost"})`);
+                                return abandon(err);
                         }
-                        if (chunk.done)
-                            break;
-                        const value = chunk.value;
-                        buf += dec.decode(value, { stream: true });
-                        let sep;
-                        while ((sep = buf.indexOf("\n\n")) >= 0) {
-                            const block = buf.slice(0, sep);
-                            buf = buf.slice(sep + 2);
-                            let event = "";
-                            let data = "";
-                            for (const line of block.split("\n")) {
-                                if (line.startsWith("event:"))
-                                    event = line.slice(6).trim();
-                                else if (line.startsWith("data:"))
-                                    data = line.slice(5).replace(/^ /, "");
-                            }
-                            if (!event)
-                                continue;
-                            if (event === "heartbeat") {
-                                on.heartbeat?.();
-                                continue;
-                            }
-                            if (event === "reconnect") {
-                                reader.cancel().catch(() => { });
-                                throw new ServiceRestartError(`a service restart stopped the command on sandbox ${sandboxId} while its output was streaming`, "Not billed: the time the command ran is credited back. The sandbox kept running; the command may have done part of its work before it stopped: run it again if that is safe.");
-                            }
-                            let parsed = {};
+                        if (r !== undefined) {
+                            if (r.ok && r.body)
+                                return r.body;
+                            clearTimeout(timer);
+                            const text = await r.text().catch(() => "");
+                            let payload = {};
                             try {
-                                parsed = JSON.parse(data);
+                                payload = text ? JSON.parse(text) : {};
                             }
                             catch { }
-                            if (event === "stdout" || event === "stderr") {
-                                const bytes = b64ToBytes(String(parsed["data"] ?? ""));
-                                if (event === "stdout") {
-                                    out.push(bytes);
-                                    on.stdout?.(bytes);
-                                }
-                                else {
-                                    errOut.push(bytes);
-                                    on.stderr?.(bytes);
-                                }
+                            retryAfterMs = parseRetryAfter(r.headers.get("retry-after"));
+                            if (r.status === 404) {
+                                throw new ExecLostError(`the connection to the service dropped while the command on sandbox ${sandboxId} was running, and it could not be picked up again: it is no longer kept (it ended, or this service cannot keep it)`, { sandboxId }, "gone");
                             }
-                            else if (event === "exit") {
-                                exit = { ...(exit ?? {}), ...parsed };
-                            }
-                            else if (event === "error") {
-                                failure = parsed;
+                            if (!REATTACH_AGAIN.has(r.status)) {
+                                throw errorFor(payload.error?.code ?? codeForStatus(r.status), payload.error?.message ?? `HTTP ${r.status} from POST /sandboxes/${sandboxId}/execs/attach`, payload.error?.fix ?? "See the API response for details.", { status: r.status, ...bodyExtra(payload), ...(retryAfterMs === undefined ? {} : { retryAfterMs }) });
                             }
                         }
+                        const left = budgetMs - (Date.now() - droppedAt);
+                        if (left <= 0) {
+                            throw new ExecLostError(`the connection to the service dropped while the command on sandbox ${sandboxId} was running, and the service could not be reached again in time: the command is stopped ${hold} s after the drop`, { sandboxId }, "unreachable");
+                        }
+                        await sleepUnless(Math.min(left, reattachWait(attempt, retryAfterMs)), opts.signal);
                     }
-                }
-                finally {
-                    clearTimeout(timer);
+                };
+                for (let ended = await consume(stream); !ended.done; ended = await consume(stream)) {
+                    if (holdS === undefined || key === undefined) {
+                        if (ended.restart) {
+                            throw new ServiceRestartError(`a service restart stopped the command on sandbox ${sandboxId} while its output was streaming`, "Not billed: the time the command ran is credited back. The sandbox kept running; the command may have done part of its work before it stopped: run it again if that is safe.");
+                        }
+                        throw lost(ended.why);
+                    }
+                    stream = await reattach();
                 }
                 if (failure) {
                     throw errorFor(failure.code ?? "internal", failure.message ?? "the exec stream failed", failure.fix ?? "Re-run; if it repeats, include the exec_id.", { status: 200, ...bodyExtra({ error: failure }) });
                 }
-                if (!exit && silent)
-                    throw lost(`was silent for ${STREAM_IDLE_MS / 1000} s`);
-                if (!exit) {
+                if (!exit)
                     throw lost("ended without an exit event");
+                if (stoppedUnattended(exit)) {
+                    throw new ExecLostError(`the command on sandbox ${sandboxId} was stopped: its connection to the service dropped and no client came back for it ${holdS === undefined ? "in time" : `within ${holdS} s`}`, { sandboxId }, "not_reattached");
                 }
                 const rawCode = exit["exit_code"];
                 const signal = exit["signal"];
@@ -2101,12 +2413,14 @@ export class PromigenceClient {
                 }),
             },
             ports: {
-                expose: (sandboxId, port) => {
+                expose: (sandboxId, port, opts = {}) => {
                     if (!Number.isInteger(port) || port < 1 || port > 65535) {
                         throw new InvalidArgumentError(`port must be an integer 1..65535, got ${port}`, "Pass the port your process listens on inside the sandbox, e.g. 3000.");
                     }
+                    checkPreviewLength(opts.length);
                     return withPreviewsFix(this.request({
-                        method: "POST", path: `/sandboxes/${encodeURIComponent(sandboxId)}/ports`, body: { port },
+                        method: "POST", path: `/sandboxes/${encodeURIComponent(sandboxId)}/ports`,
+                        body: { port, ...(opts.length === undefined ? {} : { length: opts.length }) },
                     }), sandboxId, port);
                 },
                 list: async (sandboxId) => {
@@ -2115,10 +2429,19 @@ export class PromigenceClient {
                     }), sandboxId);
                     return r.ports ?? [];
                 },
-                revoke: (sandboxId, tokenOrPort) => this.request({
-                    method: "DELETE",
-                    path: `/sandboxes/${encodeURIComponent(sandboxId)}/ports/${encodeURIComponent(String(tokenOrPort))}`,
-                }),
+                revoke: (sandboxId, tokenOrPort) => {
+                    const which = String(tokenOrPort).trim();
+                    return /^[0-9]+$/.test(which)
+                        ? this.request({
+                            method: "DELETE",
+                            path: `/sandboxes/${encodeURIComponent(sandboxId)}/ports/${which}`,
+                        })
+                        : this.request({
+                            method: "POST",
+                            path: `/sandboxes/${encodeURIComponent(sandboxId)}/ports/revoke`,
+                            body: { token: which },
+                        });
+                },
             },
             ssh: {
                 open: (sandboxId, opts) => {
@@ -2615,10 +2938,10 @@ export class PromigenceClient {
                     throw new InvalidArgumentError("snapshots.delete needs a snapshot", "Pass a snap_ id, an alias, or alias@commit7.");
                 return this.request({ method: "DELETE", path: `/snapshots/${seg(idOrAlias)}` });
             },
-            alias: (idOrAlias, alias) => this.request({
+            alias: (idOrAlias, alias, opts = {}) => this.request({
                 method: "PUT",
                 path: `/snapshots/${seg(idOrAlias)}/alias`,
-                body: { alias },
+                body: { alias, ...(alias !== null && opts.moveAlias ? { move_alias: true } : {}) },
             }),
             waitBuild: async (buildId, opts = {}) => {
                 const r = await this.waitLoop(opts, BUILD_POLL_MS, { build_id: buildId, state: "running" }, (read) => this.snapshots.status(buildId, read), (b) => b.state !== "running" && b.state !== "queued");
@@ -2723,5 +3046,6 @@ export function toCommandResult(r) {
         stderr: r.stderr ?? "",
         ...(r.truncated === undefined ? {} : { truncated: r.truncated }),
         ...(r.exec_id === undefined ? {} : { execId: r.exec_id }),
+        timedOut: answerTimedOut(r),
     };
 }