@promigence/sdk 0.1.0-beta.16 → 0.1.0-beta.17

package/dist/client.js

@@ -2,7 +2,7 @@ import { readFileSync, statSync } from "node:fs";
 import { homedir } from "node:os";
 import { inspect } from "node:util";
 import { dirname, join } from "node:path";
-import { ApiError, AuthError, EnvFailedError, PromigenceError, InvalidArgumentError, NotFoundError, NotReleasedError, NotVerifiedError, RateLimitError, ServiceRestartError, SpendCapError, TimeoutError, UnreachableError, DEFAULT_API_URL, FILE_PUT_CHUNK_BYTES, MAX_OUTPUT_BYTES_DEFAULT, VERSION, WAIT_TIMEOUT_MS, PreviewsUnavailableError, SANDBOX_ENDED_FIX, SandboxEndedError, errorsDocsFor, exitForCode, previewsOff, sandboxEndedIn, withPreviewsFix, } from "./core.js";
+import { ApiError, AuthError, EnvFailedError, ExecLostError, PromigenceError, InvalidArgumentError, NotFoundError, NotReleasedError, NotVerifiedError, RateLimitError, ServiceRestartError, SpendCapError, TimeoutError, UnreachableError, DEFAULT_API_URL, FILE_PUT_CHUNK_BYTES, MAX_OUTPUT_BYTES_DEFAULT, VERSION, WAIT_TIMEOUT_MS, PreviewsUnavailableError, SANDBOX_ENDED_FIX, SandboxEndedError, errorsDocsFor, exitForCode, previewsOff, sandboxEndedIn, withPreviewsFix, } from "./core.js";
 import { buildBundle } from "./bundle.js";
 import { keychainReadCached, keychainStore, storeLabel } from "./credstore.js";
 import { CALLER_HEADER, resolveCaller, userAgentWithCaller } from "./caller.js";
@@ -16,6 +16,7 @@ import { isKeyedCreate } from "./keyed.js";
 import { customDispatcher, longRequestDispatcher } from "./dispatcher.js";
 import { compromisedKeyCause, guardedFetchDispatcher } from "./ca-guard.js";
 import { hintArg } from "./hint.js";
+import { answerTimedOut } from "./command-errors.js";
 import { GITHUB_REPO_PREFIX, githubCreateBody, } from "./github.js";
 import { GITLAB_REPO_PREFIX, } from "./gitlab.js";
 export const FILE_TIMEOUT_MS = 15 * 60_000;
@@ -93,27 +94,46 @@ function pageQuery(limit, max, cursor) {
 }
 export const OUTPUT_RETENTION_MAX_DAYS = 30;
 function projectUpdateBody(u) {
+    return settingsBody(u, "projects.update", "the organization's");
+}
+function orgUpdateBody(u) {
+    return settingsBody(u, "orgs.update", "the default");
+}
+function settingsBody(u, what, fallback) {
     const body = {};
     if (u.name !== undefined)
         body["name"] = u.name;
     if (u.outputRetentionDays !== undefined) {
         const d = u.outputRetentionDays;
         if (d !== null && (!Number.isInteger(d) || d < 1 || d > OUTPUT_RETENTION_MAX_DAYS)) {
-            throw new InvalidArgumentError(`outputRetentionDays must be a whole number from 1 to ${OUTPUT_RETENTION_MAX_DAYS}, or null, not ${String(d)}`, "Pass null for the organization's setting.");
+            throw new InvalidArgumentError(`outputRetentionDays must be a whole number from 1 to ${OUTPUT_RETENTION_MAX_DAYS}, or null, not ${String(d)}`, `Pass null for ${fallback} setting.`);
         }
         body["output_retention_days"] = d;
     }
     if (u.failureCaptures !== undefined) {
         if (u.failureCaptures !== null && typeof u.failureCaptures !== "boolean") {
-            throw new InvalidArgumentError("failureCaptures must be true, false or null", "Pass null for the organization's setting.");
+            throw new InvalidArgumentError("failureCaptures must be true, false or null", `Pass null for ${fallback} setting.`);
         }
         if (Object.keys(body).length) {
             throw new InvalidArgumentError("failureCaptures is set on its own", "Change it in a call of its own, without name or outputRetentionDays.");
         }
         body["failure_captures"] = u.failureCaptures;
     }
+    if (u.previewLinkLength !== undefined) {
+        const l = u.previewLinkLength;
+        if (l === "custom") {
+            throw new InvalidArgumentError("custom domains for preview links are coming soon", `Pass "short", "long" or null (${fallback}) for now.`);
+        }
+        if (l !== null && l !== "short" && l !== "long") {
+            throw new InvalidArgumentError(`previewLinkLength must be "short", "long" or null, not ${JSON.stringify(l)}`, `Pass null for ${fallback} setting.`);
+        }
+        if (Object.keys(body).length) {
+            throw new InvalidArgumentError("previewLinkLength is set on its own", "Change it in a call of its own, without the other settings.");
+        }
+        body["preview_link_length"] = l;
+    }
     if (!Object.keys(body).length) {
-        throw new InvalidArgumentError("projects.update needs something to change", "Pass name, outputRetentionDays or failureCaptures.");
+        throw new InvalidArgumentError(`${what} needs something to change`, "Pass name, outputRetentionDays, failureCaptures or previewLinkLength.");
     }
     return body;
 }
@@ -125,6 +145,14 @@ function expiryFields(o) {
         body["expires_at"] = o.expiresAt;
     return body;
 }
+function checkPreviewLength(length) {
+    if (length === undefined || length === "short" || length === "long")
+        return;
+    if (length === "custom") {
+        throw new InvalidArgumentError("custom domains for preview links are coming soon", "Pass length: \"short\" (the default) or \"long\" for now.");
+    }
+    throw new InvalidArgumentError(`length must be "short" or "long", got ${JSON.stringify(length)}`, "Omit it for the length your project or organization chose.");
+}
 export const NOTICE_HEADER = "promigence-notice";
 export const MAX_SOURCES = 8;
 export function configDir(env = process.env) {
@@ -768,6 +796,76 @@ function parseRetryAfter(header) {
 const MAX_RETRY_SLEEP_MS = 10_000;
 const STREAM_IDLE_MS = 90_000;
 const sleep = (ms) => new Promise((r) => { setTimeout(r, ms); });
+const EXEC_KEY_HEADER = "promigence-exec-key";
+const EXEC_HOLD_DEFAULT_S = 120;
+const REATTACH_SLACK_MS = 30_000;
+const REATTACH_BACKOFF_MS = 250;
+const REATTACH_BACKOFF_MAX_MS = 2_000;
+const REATTACH_RETRY_AFTER_MAX_MS = 5_000;
+const REATTACH_TAKEN_MS = 10_000;
+const EXEC_CANCEL_TIMEOUT_MS = 3_000;
+const REATTACH_AGAIN = new Set([429, 502, 503, 504]);
+function newExecKey() {
+    const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16));
+    let s = "";
+    for (const b of bytes)
+        s += b.toString(16).padStart(2, "0");
+    return s;
+}
+const UNSENT_CODES = /^(?:ECONNREFUSED|ENOTFOUND|EAI_[A-Z]+|ENETUNREACH|EHOSTUNREACH|UND_ERR_CONNECT_TIMEOUT|ERR_TLS_[A-Z_]+|ERR_SSL_[A-Z_]+|CERT_[A-Z_]+|UNABLE_TO_[A-Z_]+|DEPTH_ZERO_SELF_SIGNED_CERT|SELF_SIGNED_CERT_IN_CHAIN|HOSTNAME_MISMATCH|EPROTO)$/;
+function provedUnsent(err) {
+    if (compromisedKeyCause(err) !== undefined)
+        return true;
+    const top = err;
+    if (top?.name === "TypeError" && /header/i.test(top.message ?? ""))
+        return true;
+    const unsent = (x) => {
+        const code = x?.code;
+        return typeof code === "string" && UNSENT_CODES.test(code);
+    };
+    let c = err;
+    for (let depth = 0; c !== null && typeof c === "object" && depth < 6; depth++) {
+        if (unsent(c))
+            return true;
+        const each = c.errors;
+        if (Array.isArray(each) && each.length > 0 && each.every(unsent))
+            return true;
+        c = c.cause;
+    }
+    return false;
+}
+class HeldDrop extends Error {
+    key;
+    constructor(key, why) {
+        super(why);
+        this.key = key;
+        this.name = "HeldDrop";
+    }
+}
+function sleepUnless(ms, signal) {
+    if (signal === undefined)
+        return sleep(ms);
+    return new Promise((resolve) => {
+        if (signal.aborted) {
+            resolve();
+            return;
+        }
+        const done = () => { clearTimeout(t); signal.removeEventListener("abort", done); resolve(); };
+        const t = setTimeout(done, ms);
+        signal.addEventListener("abort", done, { once: true });
+    });
+}
+function reattachWait(attempt, retryAfterMs) {
+    if (retryAfterMs !== undefined)
+        return Math.min(retryAfterMs, REATTACH_RETRY_AFTER_MAX_MS);
+    return Math.min(REATTACH_BACKOFF_MAX_MS, REATTACH_BACKOFF_MS * 2 ** Math.min(attempt, 8)) * (0.5 + Math.random() * 0.5);
+}
+function stoppedUnattended(answer) {
+    return answer?.reason === "not_reattached";
+}
+function abortedByCaller(method, path) {
+    return new TimeoutError(`request ${method} ${path} was aborted by the caller's signal`, "Nothing to fix: the caller aborted it.");
+}
 const POLLING_TICK_MS = 5;
 const PARKED_TICK_MS = 50;
 async function keepingLoopAwake(everyMs, fn) {
@@ -988,6 +1086,82 @@ export class PromigenceClient {
         await sleep(backoff * (0.5 + Math.random() * 0.5));
         return true;
     }
+    async heldExec(sandboxId, req) {
+        const signal = req.signal;
+        const abortedBefore = signal?.aborted === true;
+        let key;
+        try {
+            let res;
+            try {
+                res = await this.request({ ...req, held: { onKey: (k) => { key = k; } } });
+            }
+            catch (err) {
+                if (!(err instanceof HeldDrop))
+                    throw err;
+                res = await this.reattachBuffered(sandboxId, err.key, req);
+            }
+            if (stoppedUnattended(res)) {
+                throw new ExecLostError(`the command on sandbox ${sandboxId} was stopped: its connection to the service dropped and no client came back for it in time`, { sandboxId }, "not_reattached");
+            }
+            return res;
+        }
+        catch (err) {
+            if (key !== undefined && signal?.aborted && !abortedBefore)
+                await this.cancelHeldExec(sandboxId, key);
+            throw err;
+        }
+    }
+    async reattachBuffered(sandboxId, key, req) {
+        const budgetMs = EXEC_HOLD_DEFAULT_S * 1000 + REATTACH_SLACK_MS;
+        let droppedAt = Date.now();
+        for (let attempt = 0;; attempt++) {
+            if (req.signal?.aborted)
+                throw abortedByCaller(req.method, req.path);
+            this.stats.retries++;
+            const sentAt = Date.now();
+            try {
+                return await this.request({
+                    method: "POST",
+                    path: `/sandboxes/${encodeURIComponent(sandboxId)}/execs/attach`,
+                    body: { key },
+                    once: true,
+                    ...(req.timeoutMs === undefined ? {} : { timeoutMs: req.timeoutMs }),
+                    ...(req.signal === undefined ? {} : { signal: req.signal }),
+                });
+            }
+            catch (err) {
+                if (req.signal?.aborted)
+                    throw abortedByCaller(req.method, req.path);
+                const status = err instanceof PromigenceError ? err.status : undefined;
+                if (status === 404) {
+                    throw new ExecLostError(`the connection to the service dropped before the command on sandbox ${sandboxId} answered, and it could not be picked up again: nothing is kept for it (it ended, it never started, or this service cannot keep it)`, { sandboxId }, "gone");
+                }
+                const lostConnection = status === undefined && (err instanceof UnreachableError || err instanceof TimeoutError);
+                if (!lostConnection && !(status !== undefined && REATTACH_AGAIN.has(status)))
+                    throw err;
+                if (Date.now() - sentAt >= REATTACH_TAKEN_MS)
+                    droppedAt = Date.now();
+                const left = budgetMs - (Date.now() - droppedAt);
+                if (left <= 0) {
+                    throw new ExecLostError(`the connection to the service dropped while the command on sandbox ${sandboxId} was running, and the service could not be reached again in time: the command is stopped ${EXEC_HOLD_DEFAULT_S} s after the drop`, { sandboxId }, "unreachable");
+                }
+                await sleepUnless(Math.min(left, reattachWait(attempt, err instanceof PromigenceError ? err.retryAfterMs : undefined)), req.signal);
+            }
+        }
+    }
+    async cancelHeldExec(sandboxId, key) {
+        try {
+            await this.request({
+                method: "POST",
+                path: `/sandboxes/${encodeURIComponent(sandboxId)}/execs/cancel`,
+                body: { key },
+                timeoutMs: EXEC_CANCEL_TIMEOUT_MS,
+                once: true,
+            });
+        }
+        catch {
+        }
+    }
     async request(opts) {
         const tr = traceBegin(opts.method, opts.path);
         const url = `${this.apiUrl}${opts.path}`;
@@ -1018,9 +1192,16 @@ export class PromigenceClient {
         const timeoutMs = opts.timeoutMs ?? this.requestTimeoutMs;
         let dispatcher;
         let resentDelete = false;
+        const maxRetries = opts.once === true ? 0 : this.maxRetries;
+        let execKey;
         for (let attempt = 0;; attempt++) {
             let res;
             this.stats.requests++;
+            if (opts.held !== undefined) {
+                execKey = newExecKey();
+                headers[EXEC_KEY_HEADER] = execKey;
+                opts.held.onKey(execKey);
+            }
             const sentAt = performance.now();
             try {
                 if (this.h2) {
@@ -1063,11 +1244,14 @@ export class PromigenceClient {
             }
             catch (err) {
                 const e = err;
+                if (execKey !== undefined && !opts.signal?.aborted && !provedUnsent(err)) {
+                    throw new HeldDrop(execKey, transportDetail(e));
+                }
                 if (keyedCreate && idempotencyKey && await this.againKeyed(attempt, opts.signal))
                     continue;
                 const cutOff = e?.cause?.code === "UND_ERR_HEADERS_TIMEOUT" || e?.cause?.code === "UND_ERR_BODY_TIMEOUT";
                 const timedOut = e?.name === "TimeoutError" || e?.name === "AbortError" || cutOff;
-                if (!timedOut && replayable && attempt < this.maxRetries && !opts.signal?.aborted) {
+                if (!timedOut && replayable && attempt < maxRetries && !opts.signal?.aborted) {
                     this.stats.retries++;
                     if (opts.method === "DELETE")
                         resentDelete = true;
@@ -1075,9 +1259,8 @@ export class PromigenceClient {
                     await sleep(backoff * (0.5 + Math.random() * 0.5));
                     continue;
                 }
-                if (opts.signal?.aborted) {
-                    throw new TimeoutError(`request ${opts.method} ${opts.path} was aborted by the caller's signal`, "Nothing to fix: the caller aborted it.");
-                }
+                if (opts.signal?.aborted)
+                    throw abortedByCaller(opts.method, opts.path);
                 if (e?.name === "TimeoutError" || e?.name === "AbortError") {
                     throw new TimeoutError(`request ${opts.method} ${opts.path} timed out after ${timeoutMs} ms`, "Retry. To allow longer: promigence --request-timeout 120s <command>, " +
                         "PROMIGENCE_REQUEST_TIMEOUT=120s, or requestTimeoutMs in the SDK. A --wait is bounded " +
@@ -1094,6 +1277,11 @@ export class PromigenceClient {
             const gateway = (res.status === 502 || res.status === 504) && replayable;
             let retryable = (res.status === 429 || res.status === 503 || gateway) && (retryAfterMs ?? 0) <= MAX_RETRY_SLEEP_MS;
             let body;
+            if (execKey !== undefined && (res.status === 502 || res.status === 504)) {
+                body = await res.text().catch(() => "");
+                if (errorCodeOf(body) === undefined)
+                    throw new HeldDrop(execKey, `HTTP ${res.status}`);
+            }
             if (retryable && res.status === 503 && !replayable && retryAfterMs === undefined) {
                 body = await res.text().catch(() => "");
                 retryable = REFUSED_503_CODES.has(errorCodeOf(body) ?? "");
@@ -1102,7 +1290,7 @@ export class PromigenceClient {
                 body = await res.text().catch(() => "");
                 retryable = errorCodeOf(body) === undefined;
             }
-            if (retryable && attempt < this.maxRetries) {
+            if (retryable && attempt < maxRetries) {
                 const backoff = Math.min(MAX_RETRY_SLEEP_MS, this.retryBaseMs * 2 ** attempt);
                 const jitter = Math.random();
                 const waitMs = retryAfterMs !== undefined
@@ -1135,8 +1323,10 @@ export class PromigenceClient {
                         text = "";
                     else {
                         const e = err;
+                        if (execKey !== undefined && !opts.signal?.aborted)
+                            throw new HeldDrop(execKey, transportDetail(e));
                         const timedOut = e?.name === "TimeoutError" || e?.name === "AbortError";
-                        if (!timedOut && replayable && attempt < this.maxRetries && !opts.signal?.aborted) {
+                        if (!timedOut && replayable && attempt < maxRetries && !opts.signal?.aborted) {
                             this.stats.retries++;
                             if (opts.method === "DELETE")
                                 resentDelete = true;
@@ -1513,6 +1703,7 @@ export class PromigenceClient {
                     method: "PATCH", path: `/orgs/${orgSeg(org)}/projects/${encodeURIComponent(project)}`, body: projectUpdateBody(update),
                 }),
             },
+            update: async (org, update) => this.request({ method: "PATCH", path: `/orgs/${orgSeg(org)}`, body: orgUpdateBody(update) }),
         };
     }
     group_credits() {
@@ -1592,6 +1783,8 @@ export class PromigenceClient {
                     body["network"] = opts.network;
                 if (opts.optimizeFor !== undefined)
                     body["optimize_for"] = opts.optimizeFor;
+                if (opts.capture !== undefined)
+                    body["capture"] = opts.capture;
                 if (opts.secrets?.length)
                     body["secrets"] = opts.secrets;
                 if (opts.labels && Object.keys(opts.labels).length)
@@ -1825,7 +2018,9 @@ export class PromigenceClient {
                     req.idempotencyKey = opts.idempotencyKey;
                 if (opts.signal)
                     req.signal = opts.signal;
-                return this.request(req);
+                if (opts.terminal)
+                    return this.request(req);
+                return this.heldExec(sandboxId, req);
             },
             execStream: async (sandboxId, opts, on = {}) => {
                 if (!opts.argv?.length && !opts.sh) {
@@ -1854,7 +2049,9 @@ export class PromigenceClient {
                 body["timeout_ms"] = commandTimeoutMs(opts.timeoutMs) ?? 15 * 60_000;
                 body["max_output_bytes"] = opts.maxOutputBytes ?? MAX_OUTPUT_BYTES_DEFAULT;
                 const url = `${this.apiUrl}/sandboxes/${encodeURIComponent(sandboxId)}/execs`;
-                const idle = new AbortController();
+                const keyed = opts.terminal !== true;
+                let key;
+                let idle = new AbortController();
                 let silent = false;
                 let reader;
                 let timer;
@@ -1866,23 +2063,33 @@ export class PromigenceClient {
                         reader?.cancel().catch(() => { });
                     }, STREAM_IDLE_MS);
                 };
+                const fresh = () => { idle = new AbortController(); silent = false; reader = undefined; };
+                const abortedBefore = opts.signal?.aborted === true;
+                const abandon = async (err) => {
+                    if (key !== undefined && !abortedBefore)
+                        await this.cancelHeldExec(sandboxId, key);
+                    throw err;
+                };
                 let stream;
                 const bearer = `Bearer ${this.requireKey()}`;
                 const dispatcher = this.guardDispatcher();
+                const headers = () => ({
+                    accept: "text/event-stream",
+                    "content-type": "application/json",
+                    "user-agent": this.userAgent,
+                    authorization: bearer,
+                    ...(this.caller ? { [CALLER_HEADER]: this.caller } : {}),
+                });
                 for (let attempt = 0; stream === undefined; attempt++) {
                     let r;
                     this.stats.requests++;
+                    if (keyed)
+                        key = newExecKey();
                     arm();
                     try {
                         r = await this.fetchImpl(url, {
                             method: "POST",
-                            headers: {
-                                accept: "text/event-stream",
-                                "content-type": "application/json",
-                                "user-agent": this.userAgent,
-                                authorization: bearer,
-                                ...(this.caller ? { [CALLER_HEADER]: this.caller } : {}),
-                            },
+                            headers: { ...headers(), ...(key === undefined ? {} : { [EXEC_KEY_HEADER]: key }) },
                             body: JSON.stringify(body),
                             signal: opts.signal ? AbortSignal.any([opts.signal, idle.signal]) : idle.signal,
                             ...(dispatcher === undefined ? {} : { dispatcher }),
@@ -1892,7 +2099,10 @@ export class PromigenceClient {
                         clearTimeout(timer);
                         const e = err;
                         const why = silent ? `no answer in ${STREAM_IDLE_MS / 1000} s` : transportDetail(e);
-                        throw new UnreachableError(`cannot reach ${url}: ${why}`);
+                        const failed = new UnreachableError(`cannot reach ${url}: ${why}`);
+                        if (opts.signal?.aborted)
+                            return abandon(failed);
+                        throw failed;
                     }
                     if (r.ok && r.body) {
                         stream = r.body;
@@ -1923,87 +2133,166 @@ export class PromigenceClient {
                 const errOut = [];
                 let exit;
                 let failure;
-                const dec = new TextDecoder();
-                let buf = "";
+                let holdS;
+                const received = { stdout: 0, stderr: 0 };
                 const lost = (how) => new UnreachableError(`the exec stream for sandbox ${sandboxId} ${how}`, "The connection dropped before the command finished; its result is unknown. " +
                     `Read it by id (\`promigence exec output <exec_id> --sandbox ${hintArg(sandboxId)}\`) or re-run.`);
-                reader = stream.getReader();
-                try {
-                    for (;;) {
+                const consume = async (from) => {
+                    const dec = new TextDecoder();
+                    let buf = "";
+                    const rd = from.getReader();
+                    reader = rd;
+                    try {
+                        for (;;) {
+                            arm();
+                            let chunk;
+                            try {
+                                chunk = await rd.read();
+                            }
+                            catch (err) {
+                                if (opts.signal?.aborted)
+                                    return abandon(err);
+                                if (holdS !== undefined && (exit || failure))
+                                    return { done: true };
+                                if (silent)
+                                    return { done: false, why: `was silent for ${STREAM_IDLE_MS / 1000} s`, restart: false };
+                                const e = err;
+                                return { done: false, why: `was cut off (${e?.cause?.code ?? e?.message ?? "connection lost"})`, restart: false };
+                            }
+                            if (chunk.done)
+                                break;
+                            const value = chunk.value;
+                            buf += dec.decode(value, { stream: true });
+                            let sep;
+                            while ((sep = buf.indexOf("\n\n")) >= 0) {
+                                const block = buf.slice(0, sep);
+                                buf = buf.slice(sep + 2);
+                                let event = "";
+                                let data = "";
+                                for (const line of block.split("\n")) {
+                                    if (line.startsWith("event:"))
+                                        event = line.slice(6).trim();
+                                    else if (line.startsWith("data:"))
+                                        data = line.slice(5).replace(/^ /, "");
+                                }
+                                if (!event)
+                                    continue;
+                                if (event === "heartbeat") {
+                                    on.heartbeat?.();
+                                    continue;
+                                }
+                                if (event === "reconnect") {
+                                    rd.cancel().catch(() => { });
+                                    return { done: false, why: "was ended by a service restart", restart: true };
+                                }
+                                let parsed = {};
+                                try {
+                                    parsed = JSON.parse(data);
+                                }
+                                catch { }
+                                if (event === "held") {
+                                    const h = parsed["hold_s"];
+                                    holdS = typeof h === "number" && Number.isFinite(h) && h >= 0 ? h : EXEC_HOLD_DEFAULT_S;
+                                }
+                                else if (event === "stdout" || event === "stderr") {
+                                    const bytes = b64ToBytes(String(parsed["data"] ?? ""));
+                                    if (event === "stdout") {
+                                        received.stdout += bytes.length;
+                                        out.push(bytes);
+                                        on.stdout?.(bytes);
+                                    }
+                                    else {
+                                        received.stderr += bytes.length;
+                                        errOut.push(bytes);
+                                        on.stderr?.(bytes);
+                                    }
+                                }
+                                else if (event === "exit") {
+                                    exit = { ...(exit ?? {}), ...parsed };
+                                }
+                                else if (event === "error") {
+                                    failure = parsed;
+                                }
+                            }
+                        }
+                    }
+                    finally {
+                        clearTimeout(timer);
+                    }
+                    if (exit || failure)
+                        return { done: true };
+                    if (silent)
+                        return { done: false, why: `was silent for ${STREAM_IDLE_MS / 1000} s`, restart: false };
+                    return { done: false, why: "ended without an exit event", restart: false };
+                };
+                const reattach = async () => {
+                    const droppedAt = Date.now();
+                    const hold = holdS ?? EXEC_HOLD_DEFAULT_S;
+                    const budgetMs = hold * 1000 + REATTACH_SLACK_MS;
+                    for (let attempt = 0;; attempt++) {
+                        if (opts.signal?.aborted)
+                            return abandon(opts.signal.reason);
+                        fresh();
+                        this.stats.requests++;
+                        this.stats.retries++;
                         arm();
-                        let chunk;
+                        let retryAfterMs;
+                        let r;
                         try {
-                            chunk = await reader.read();
+                            r = await this.fetchImpl(`${url}/attach`, {
+                                method: "POST",
+                                headers: headers(),
+                                body: JSON.stringify({ key, stdout_offset: received.stdout, stderr_offset: received.stderr }),
+                                signal: opts.signal ? AbortSignal.any([opts.signal, idle.signal]) : idle.signal,
+                                ...(dispatcher === undefined ? {} : { dispatcher }),
+                            });
                         }
                         catch (err) {
+                            clearTimeout(timer);
                             if (opts.signal?.aborted)
-                                throw err;
-                            if (silent)
-                                throw lost(`was silent for ${STREAM_IDLE_MS / 1000} s`);
-                            const e = err;
-                            throw lost(`was cut off (${e?.cause?.code ?? e?.message ?? "connection lost"})`);
+                                return abandon(err);
                         }
-                        if (chunk.done)
-                            break;
-                        const value = chunk.value;
-                        buf += dec.decode(value, { stream: true });
-                        let sep;
-                        while ((sep = buf.indexOf("\n\n")) >= 0) {
-                            const block = buf.slice(0, sep);
-                            buf = buf.slice(sep + 2);
-                            let event = "";
-                            let data = "";
-                            for (const line of block.split("\n")) {
-                                if (line.startsWith("event:"))
-                                    event = line.slice(6).trim();
-                                else if (line.startsWith("data:"))
-                                    data = line.slice(5).replace(/^ /, "");
-                            }
-                            if (!event)
-                                continue;
-                            if (event === "heartbeat") {
-                                on.heartbeat?.();
-                                continue;
-                            }
-                            if (event === "reconnect") {
-                                reader.cancel().catch(() => { });
-                                throw new ServiceRestartError(`a service restart stopped the command on sandbox ${sandboxId} while its output was streaming`, "Not billed: the time the command ran is credited back. The sandbox kept running; the command may have done part of its work before it stopped: run it again if that is safe.");
-                            }
-                            let parsed = {};
+                        if (r !== undefined) {
+                            if (r.ok && r.body)
+                                return r.body;
+                            clearTimeout(timer);
+                            const text = await r.text().catch(() => "");
+                            let payload = {};
                             try {
-                                parsed = JSON.parse(data);
+                                payload = text ? JSON.parse(text) : {};
                             }
                             catch { }
-                            if (event === "stdout" || event === "stderr") {
-                                const bytes = b64ToBytes(String(parsed["data"] ?? ""));
-                                if (event === "stdout") {
-                                    out.push(bytes);
-                                    on.stdout?.(bytes);
-                                }
-                                else {
-                                    errOut.push(bytes);
-                                    on.stderr?.(bytes);
-                                }
-                            }
-                            else if (event === "exit") {
-                                exit = { ...(exit ?? {}), ...parsed };
+                            retryAfterMs = parseRetryAfter(r.headers.get("retry-after"));
+                            if (r.status === 404) {
+                                throw new ExecLostError(`the connection to the service dropped while the command on sandbox ${sandboxId} was running, and it could not be picked up again: it is no longer kept (it ended, or this service cannot keep it)`, { sandboxId }, "gone");
                             }
-                            else if (event === "error") {
-                                failure = parsed;
+                            if (!REATTACH_AGAIN.has(r.status)) {
+                                throw errorFor(payload.error?.code ?? codeForStatus(r.status), payload.error?.message ?? `HTTP ${r.status} from POST /sandboxes/${sandboxId}/execs/attach`, payload.error?.fix ?? "See the API response for details.", { status: r.status, ...bodyExtra(payload), ...(retryAfterMs === undefined ? {} : { retryAfterMs }) });
                             }
                         }
+                        const left = budgetMs - (Date.now() - droppedAt);
+                        if (left <= 0) {
+                            throw new ExecLostError(`the connection to the service dropped while the command on sandbox ${sandboxId} was running, and the service could not be reached again in time: the command is stopped ${hold} s after the drop`, { sandboxId }, "unreachable");
+                        }
+                        await sleepUnless(Math.min(left, reattachWait(attempt, retryAfterMs)), opts.signal);
                     }
-                }
-                finally {
-                    clearTimeout(timer);
+                };
+                for (let ended = await consume(stream); !ended.done; ended = await consume(stream)) {
+                    if (holdS === undefined || key === undefined) {
+                        if (ended.restart) {
+                            throw new ServiceRestartError(`a service restart stopped the command on sandbox ${sandboxId} while its output was streaming`, "Not billed: the time the command ran is credited back. The sandbox kept running; the command may have done part of its work before it stopped: run it again if that is safe.");
+                        }
+                        throw lost(ended.why);
+                    }
+                    stream = await reattach();
                 }
                 if (failure) {
                     throw errorFor(failure.code ?? "internal", failure.message ?? "the exec stream failed", failure.fix ?? "Re-run; if it repeats, include the exec_id.", { status: 200, ...bodyExtra({ error: failure }) });
                 }
-                if (!exit && silent)
-                    throw lost(`was silent for ${STREAM_IDLE_MS / 1000} s`);
-                if (!exit) {
+                if (!exit)
                     throw lost("ended without an exit event");
+                if (stoppedUnattended(exit)) {
+                    throw new ExecLostError(`the command on sandbox ${sandboxId} was stopped: its connection to the service dropped and no client came back for it ${holdS === undefined ? "in time" : `within ${holdS} s`}`, { sandboxId }, "not_reattached");
                 }
                 const rawCode = exit["exit_code"];
                 const signal = exit["signal"];
@@ -2124,12 +2413,14 @@ export class PromigenceClient {
                 }),
             },
             ports: {
-                expose: (sandboxId, port) => {
+                expose: (sandboxId, port, opts = {}) => {
                     if (!Number.isInteger(port) || port < 1 || port > 65535) {
                         throw new InvalidArgumentError(`port must be an integer 1..65535, got ${port}`, "Pass the port your process listens on inside the sandbox, e.g. 3000.");
                     }
+                    checkPreviewLength(opts.length);
                     return withPreviewsFix(this.request({
-                        method: "POST", path: `/sandboxes/${encodeURIComponent(sandboxId)}/ports`, body: { port },
+                        method: "POST", path: `/sandboxes/${encodeURIComponent(sandboxId)}/ports`,
+                        body: { port, ...(opts.length === undefined ? {} : { length: opts.length }) },
                     }), sandboxId, port);
                 },
                 list: async (sandboxId) => {
@@ -2755,5 +3046,6 @@ export function toCommandResult(r) {
         stderr: r.stderr ?? "",
         ...(r.truncated === undefined ? {} : { truncated: r.truncated }),
         ...(r.exec_id === undefined ? {} : { execId: r.exec_id }),
+        timedOut: answerTimedOut(r),
     };
 }