@blaxel/core 0.2.87-preview.174 → 0.2.87

package/dist/esm-browser/common/h2fetch.js

@@ -3,3 +3,4 @@ export function createH2Fetch(session) { return (input) => globalThis.fetch(inpu
 export function createPoolBackedH2Fetch(pool, domain) { return (input) => globalThis.fetch(input); }
 export function h2RequestDirect(session, url, init) { return globalThis.fetch(url, init); }
 export function h2RequestDirectFromPool(pool, domain, url, init) { return globalThis.fetch(url, init); }
+export async function withUploadSlot(domain, fn) { return fn(); }

package/dist/esm-browser/common/settings.js

@@ -22,8 +22,8 @@ function missingCredentialsMessage() {
     return "No Blaxel credentials found. Set the BL_API_KEY and BL_WORKSPACE environment variables, or run `bl login`.";
 }
 // Build info - these placeholders are replaced at build time by build:replace-imports
-const BUILD_VERSION = "0.2.87-preview.174";
-const BUILD_COMMIT = "d820c0d5ab85c3e0603e16cd29198e14055869ec";
+const BUILD_VERSION = "0.2.87";
+const BUILD_COMMIT = "31085636cd9c2557198ee7399e7794e945c21687";
 const BUILD_SENTRY_DSN = "https://fd5e60e1c9820e1eef5ccebb84a07127@o4508714045276160.ingest.us.sentry.io/4510465864564736";
 const BLAXEL_API_VERSION = "2026-04-16";
 // Cache for config.yaml tracking value
@@ -257,6 +257,19 @@ class Settings {
         }
         return 0;
     }
+    get maxConcurrentUploadH2Requests() {
+        if (typeof this.config.maxConcurrentUploadH2Requests === "number") {
+            return this.config.maxConcurrentUploadH2Requests;
+        }
+        const value = env.BL_MAX_UPLOAD_H2_INFLIGHT;
+        if (value) {
+            const parsed = parseInt(value, 10);
+            if (!Number.isNaN(parsed)) {
+                return parsed;
+            }
+        }
+        return 2;
+    }
     get fsPartRetries() {
         if (typeof this.config.fsPartRetries === "number") {
             return this.config.fsPartRetries;
@@ -268,7 +281,20 @@ class Settings {
                 return parsed;
             }
         }
-        return 0;
+        return 3;
+    }
+    get sandboxReadRetries() {
+        if (typeof this.config.sandboxReadRetries === "number") {
+            return this.config.sandboxReadRetries;
+        }
+        const value = env.BL_SANDBOX_READ_RETRIES;
+        if (value) {
+            const parsed = parseInt(value, 10);
+            if (!Number.isNaN(parsed)) {
+                return parsed;
+            }
+        }
+        return 5;
     }
     /**
      * Fail fast with a clear, actionable error when credentials are missing or

package/dist/esm-browser/common/transient-retry.js

@@ -0,0 +1,141 @@
+import { settings } from "./settings.js";
+// Markers that, when present anywhere in the error chain, are unambiguous
+// signals of a transient HTTP/2 stream reset or connection drop. These are
+// protocol/transport level codes, not application payloads, so substring
+// matching them does not over-match a server-sent error body. Each entry is
+// matched case-sensitively against the error message and its cause.
+//
+// Deliberately excluded: bare "INTERNAL_ERROR" and "fetch failed". Both are
+// too generic on their own (an application 500 body or any failed fetch would
+// match), so we only treat them as transient when paired with a transport
+// error code on the cause (see isTransientResetError).
+const TRANSIENT_RESET_MARKERS = [
+    "ENHANCE_YOUR_CALM", // H2 flow-control backpressure reset
+    "NGHTTP2_INTERNAL_ERROR", // H2 internal stream error (qualified form)
+    "ERR_HTTP2", // node http2 error code family
+    "GOAWAY", // peer is draining the connection
+    "HTTP/2 session closed before response", // thrown by our own h2 transport
+    "HTTP/2 session sent GOAWAY before response",
+];
+// Node-level error codes (from `error.code` / `error.cause.code`) that mean
+// the connection itself dropped mid-flight and the request never completed.
+// These are safe to retry for an idempotent request.
+const TRANSIENT_ERROR_CODES = new Set([
+    "ECONNRESET",
+    "ECONNREFUSED",
+    "ETIMEDOUT",
+    "EPIPE",
+    "ERR_HTTP2_STREAM_ERROR",
+    "ERR_HTTP2_GOAWAY_SESSION",
+    "ERR_HTTP2_SESSION_ERROR",
+]);
+function collectErrorText(error) {
+    const messages = [];
+    const codes = [];
+    // Walk the error -> cause chain (bounded) so a transport error wrapped by a
+    // higher-level "fetch failed" is still classified correctly.
+    let current = error;
+    for (let depth = 0; depth < 5 && current && typeof current === "object"; depth++) {
+        const node = current;
+        if (typeof node.message === "string")
+            messages.push(node.message);
+        if (typeof node.code === "string")
+            codes.push(node.code);
+        current = node.cause;
+    }
+    return { messages, codes };
+}
+// True when the error chain carries a real HTTP response (a status came back
+// from the server). Such an error is an APPLICATION response — never a transport
+// reset — even if the server's error body text happens to contain a reset marker
+// like "GOAWAY" or "ERR_HTTP2". Guarding on this stops a marker-bearing 4xx/5xx
+// body from being misread as transient and retried (the over-match Codex flagged
+// for the now default-on idempotent-read retry).
+function hasHttpResponseStatus(error) {
+    let current = error;
+    for (let depth = 0; depth < 5 && current && typeof current === "object"; depth++) {
+        const node = current;
+        if (typeof node.status === "number")
+            return true;
+        if (node.response && typeof node.response === "object" &&
+            typeof node.response.status === "number") {
+            return true;
+        }
+        current = node.cause;
+    }
+    return false;
+}
+/**
+ * True only for transport-level resets/drops that are safe to retry on an
+ * IDEMPOTENT request (transient HTTP/2 stream reset, GOAWAY, or a dropped
+ * connection). Application errors (4xx/5xx — even when their body text contains
+ * a marker word) and a bare "fetch failed" with no transport code are
+ * deliberately NOT transient, so auto-retry never masks a real server error or
+ * duplicates a non-idempotent call.
+ *
+ * This is the single classifier shared by the upload-part retry (ENG-2680) and
+ * the idempotent sandbox-op retry (read/list/etc.), so both judge "transient"
+ * identically.
+ */
+export function isTransientResetError(error) {
+    if (!error || typeof error !== "object") {
+        return false;
+    }
+    // An error that carries an HTTP response is an application error, not a
+    // transport reset — never retry it on the strength of a marker in its body.
+    if (hasHttpResponseStatus(error)) {
+        return false;
+    }
+    const { messages, codes } = collectErrorText(error);
+    // 1. An explicit transient transport error code anywhere in the chain.
+    if (codes.some((code) => TRANSIENT_ERROR_CODES.has(code))) {
+        return true;
+    }
+    // 2. An unambiguous protocol-level reset marker in any message.
+    if (messages.some((text) => TRANSIENT_RESET_MARKERS.some((marker) => text.includes(marker)))) {
+        return true;
+    }
+    return false;
+}
+const DEFAULT_BASE_DELAY_MS = 200;
+const DEFAULT_MAX_DELAY_MS = 2000;
+// Exponential backoff with full-jitter on top of one base delay, capped so a
+// single wait never blocks unreasonably long. Exponential (rather than linear)
+// gives a later attempt room to span a multi-second sandbox cold-start/standby
+// wake, which is the window a first-call reset falls into, while early attempts
+// stay fast for the common quick-reset case.
+function backoffDelayMs(attempt, baseDelayMs, maxDelayMs) {
+    const exponential = baseDelayMs * Math.pow(2, attempt - 1);
+    const capped = Math.min(exponential, maxDelayMs);
+    const jitter = Math.floor(Math.random() * baseDelayMs);
+    return capped + jitter;
+}
+/**
+ * Run `fn`, retrying only on transient transport resets (see
+ * isTransientResetError) with exponential backoff. Caller owns idempotency:
+ * ONLY wrap idempotent operations (GET-shaped reads/lists, or an idempotent
+ * PUT of the same bytes) — never a non-idempotent POST such as process.exec,
+ * which would duplicate the side effect (ENG-2340).
+ *
+ * Defaults to `settings.sandboxReadRetries` (the higher idempotent-read budget,
+ * sized for a multi-second standby wake). The upload path passes
+ * `{ retries: settings.fsPartRetries }` to keep its own (lower) budget.
+ */
+export async function retryOnTransientReset(fn, options = {}) {
+    const retries = options.retries ?? settings.sandboxReadRetries;
+    const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
+    const maxDelayMs = options.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
+    let attempt = 0;
+    for (;;) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            attempt++;
+            if (retries <= 0 || attempt > retries || !isTransientResetError(error)) {
+                throw error;
+            }
+            await new Promise((resolve) => setTimeout(resolve, backoffDelayMs(attempt, baseDelayMs, maxDelayMs)));
+        }
+    }
+}

package/dist/esm-browser/sandbox/drive/drive.js

@@ -1,4 +1,5 @@
 import { SandboxAction } from "../action.js";
+import { retryOnTransientReset } from "../../common/transient-retry.js";
 import { postDrivesMount, deleteDrivesMountByMountPath, getDrivesMount, } from "../client/index.js";
 export class SandboxDrive extends SandboxAction {
     constructor(sandbox) {
@@ -33,11 +34,15 @@ export class SandboxDrive extends SandboxAction {
      * List all mounted drives in the sandbox
      */
     async list() {
-        const { response, data, error } = await getDrivesMount(this.withClient({
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        const result = data;
-        return result.mounts ?? [];
+        // Idempotent GET: self-heal a transient connection reset after sandbox resume.
+        // mount/unmount stay un-retried (non-idempotent POST/DELETE).
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getDrivesMount(this.withClient({
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            const result = data;
+            return result.mounts ?? [];
+        });
     }
 }

package/dist/esm-browser/sandbox/filesystem/filesystem.js

@@ -1,98 +1,22 @@
 import { fs } from "../../common/node.js";
 import { settings } from "../../common/settings.js";
+import { withUploadSlot } from "../../common/h2fetch.js";
+import { isTransientResetError, retryOnTransientReset } from "../../common/transient-retry.js";
 import { SandboxAction } from "../action.js";
 import { deleteFilesystemByPath, deleteFilesystemMultipartByUploadIdAbort, getFilesystemByPath, getFilesystemContentSearchByPath, getFilesystemFindByPath, getFilesystemSearchByPath, getWatchFilesystemByPath, postFilesystemMultipartByUploadIdComplete, postFilesystemMultipartInitiateByPath, putFilesystemByPath, putFilesystemMultipartByUploadIdPart } from "../client/index.js";
 // Multipart upload constants
 const MULTIPART_THRESHOLD = 5 * 1024 * 1024; // 5MB
 const CHUNK_SIZE = 5 * 1024 * 1024; // 5MB per part
 const MAX_PARALLEL_UPLOADS = 3; // Number of parallel part uploads
-// Base backoff between part-upload retries, in milliseconds. Grows linearly
-// per attempt and is jittered to avoid synchronized retries (thundering herd)
-// when several parallel parts fail against the same edge at the same time.
-const RETRY_BASE_DELAY_MS = 200;
-// Markers that, when present anywhere in the error chain, are unambiguous
-// signals of a transient HTTP/2 stream reset or connection drop. These are
-// protocol/transport level codes, not application payloads, so substring
-// matching them does not over-match a server-sent error body. Each entry is
-// matched case-sensitively against the error message and its cause.
-//
-// Deliberately excluded: bare "INTERNAL_ERROR" and "fetch failed". Both are
-// too generic on their own (an application 500 body or any failed fetch would
-// match), so we only treat them as transient when paired with a transport
-// error code on the cause (see isTransientUploadError).
-const TRANSIENT_RESET_MARKERS = [
-    "ENHANCE_YOUR_CALM", // H2 flow-control backpressure reset
-    "NGHTTP2_INTERNAL_ERROR", // H2 internal stream error (qualified form)
-    "ERR_HTTP2", // node http2 error code family
-    "GOAWAY", // peer is draining the connection
-    "HTTP/2 session closed before response", // thrown by our own h2 transport
-    "HTTP/2 session sent GOAWAY before response",
-];
-// Node-level error codes (from `error.code` / `error.cause.code`) that mean
-// the connection itself dropped mid-flight and the request never completed.
-// These are safe to retry for an idempotent part upload.
-const TRANSIENT_ERROR_CODES = new Set([
-    "ECONNRESET",
-    "ECONNREFUSED",
-    "ETIMEDOUT",
-    "EPIPE",
-    "ERR_HTTP2_STREAM_ERROR",
-    "ERR_HTTP2_GOAWAY_SESSION",
-    "ERR_HTTP2_SESSION_ERROR",
-]);
-function collectErrorText(error) {
-    const messages = [];
-    const codes = [];
-    // Walk the error -> cause chain (bounded) so a transport error wrapped by a
-    // higher-level "fetch failed" is still classified correctly.
-    let current = error;
-    for (let depth = 0; depth < 5 && current && typeof current === "object"; depth++) {
-        const node = current;
-        if (typeof node.message === "string")
-            messages.push(node.message);
-        if (typeof node.code === "string")
-            codes.push(node.code);
-        current = node.cause;
-    }
-    return { messages, codes };
-}
-function isTransientUploadError(error) {
-    if (!error || typeof error !== "object") {
-        return false;
-    }
-    const { messages, codes } = collectErrorText(error);
-    // 1. An explicit transient transport error code anywhere in the chain.
-    if (codes.some((code) => TRANSIENT_ERROR_CODES.has(code))) {
-        return true;
-    }
-    // 2. An unambiguous protocol-level reset marker in any message.
-    if (messages.some((text) => TRANSIENT_RESET_MARKERS.some((marker) => text.includes(marker)))) {
-        return true;
-    }
-    return false;
-}
-function nextRetryDelayMs(attempt) {
-    // Linear backoff (200ms, 400ms, ...) plus up to one extra base delay of
-    // random jitter so concurrent part retries do not all fire on the same tick.
-    const base = RETRY_BASE_DELAY_MS * attempt;
-    const jitter = Math.floor(Math.random() * RETRY_BASE_DELAY_MS);
-    return base + jitter;
-}
-async function retryOnTransient(fn) {
-    const retries = settings.fsPartRetries; // 0 = off (current default behavior)
-    let attempt = 0;
-    for (;;) {
-        try {
-            return await fn();
-        }
-        catch (error) {
-            attempt++;
-            if (retries <= 0 || attempt > retries || !isTransientUploadError(error)) {
-                throw error;
-            }
-            await new Promise((resolve) => setTimeout(resolve, nextRetryDelayMs(attempt)));
-        }
-    }
+// The transient-reset classifier and retry loop live in
+// common/transient-retry.ts, shared with the idempotent sandbox-op retry
+// (read/list/etc.) so every path judges "transient" identically. These aliases
+// preserve the import surface the ENG-2680 fault tests already depend on.
+export const isTransientUploadError = isTransientResetError;
+export function retryOnTransient(fn) {
+    // Upload path keeps its own (lower) fsPartRetries budget; the shared helper
+    // otherwise defaults to the higher idempotent-read budget (sandboxReadRetries).
+    return retryOnTransientReset(fn, { retries: settings.fsPartRetries });
 }
 export class SandboxFileSystem extends SandboxAction {
     process;
@@ -174,28 +98,44 @@ export class SandboxFileSystem extends SandboxAction {
         if (fileBlob.size > MULTIPART_THRESHOLD) {
             return await this.uploadWithMultipart(path, fileBlob, "0644");
         }
-        // Use regular upload for small files
-        const formData = new FormData();
-        formData.append("file", fileBlob, "test-binary.bin");
-        formData.append("permissions", "0644");
-        formData.append("path", path);
-        // Build URL
+        // Use regular upload for small files. Run the PUT under the same upload
+        // reliability wrapper as multipart parts: bound concurrency (ENG-2680) and
+        // retry transient connection resets (ECONNRESET/GOAWAY/ENHANCE_YOUR_CALM). A
+        // PUT of the same bytes to the same path is idempotent, so retry is safe.
+        // The FormData is rebuilt per attempt so a retried request has a fresh body.
         let url = `${this.url}/filesystem/${path}`;
         if (this.forcedUrl) {
             url = `${this.forcedUrl.toString()}/filesystem/${path}`;
         }
-        const response = await this.h2Fetch(url, {
-            method: 'PUT',
-            headers: {
-                ...settings.headers,
-            },
-            body: formData,
-        });
-        if (!response.ok) {
-            const errorText = await response.text();
-            throw new Error(`Failed to write binary: ${response.status} ${errorText}`);
-        }
-        return await response.json();
+        const h2Domain = this.sandbox?.h2Domain;
+        const putOnce = async () => {
+            const formData = new FormData();
+            formData.append("file", fileBlob, "test-binary.bin");
+            formData.append("permissions", "0644");
+            formData.append("path", path);
+            // A forceUrl (session-token) sandbox carries its own headers and must not
+            // require global credentials; settings.headers now throws without them
+            // (ENG-2698). Mirror streamLogs/execWithStreaming, which already pick
+            // sandbox.headers for forceUrl sessions.
+            const headers = this.sandbox.forceUrl ? this.sandbox.headers : settings.headers;
+            const response = await this.h2Fetch(url, {
+                method: 'PUT',
+                headers: {
+                    ...headers,
+                },
+                body: formData,
+            });
+            if (!response.ok) {
+                const errorText = await response.text();
+                throw new Error(`Failed to write binary: ${response.status} ${errorText}`);
+            }
+            return await response.json();
+        };
+        // Acquire the upload slot per-attempt INSIDE the retry so the per-domain cap
+        // bounds in-flight streams, not retry sequences: the slot is released during
+        // backoff, so a failing PUT never pins a slot while it sleeps (Mendral review).
+        const putWithSlot = h2Domain ? () => withUploadSlot(h2Domain, putOnce) : putOnce;
+        return retryOnTransient(putWithSlot);
     }
     async writeTree(files, destinationPath = null) {
         const options = {
@@ -220,30 +160,37 @@ export class SandboxFileSystem extends SandboxAction {
     }
     async read(path) {
         path = this.formatPath(path);
-        const { response, data, error } = await getFilesystemByPath(this.withClient({
-            path: { path },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        if (data && 'content' in data) {
-            return data.content;
-        }
-        throw new Error("Unsupported file type");
+        // Idempotent GET: self-heal a transient connection reset (e.g. first call
+        // after sandbox resume). Non-transient errors (404, etc.) are not retried.
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getFilesystemByPath(this.withClient({
+                path: { path },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            if (data && 'content' in data) {
+                return data.content;
+            }
+            throw new Error("Unsupported file type");
+        });
     }
     async readBinary(path) {
         path = this.formatPath(path);
-        const { response, data, error } = await getFilesystemByPath(this.withClient({
-            path: { path },
-            baseUrl: this.url,
-            headers: {
-                'Accept': 'application/octet-stream',
-            },
-        }));
-        this.handleResponseError(response, data, error);
-        if (typeof data === 'string') {
-            return new Blob([data]);
-        }
-        return data;
+        // Idempotent GET: self-heal a transient connection reset.
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getFilesystemByPath(this.withClient({
+                path: { path },
+                baseUrl: this.url,
+                headers: {
+                    'Accept': 'application/octet-stream',
+                },
+            }));
+            this.handleResponseError(response, data, error);
+            if (typeof data === 'string') {
+                return new Blob([data]);
+            }
+            return data;
+        });
     }
     async download(src, destinationPath, { mode = 0o644 } = {}) {
         if (!fs) {
@@ -266,15 +213,19 @@ export class SandboxFileSystem extends SandboxAction {
     }
     async ls(path) {
         path = this.formatPath(path);
-        const { response, data, error } = await getFilesystemByPath(this.withClient({
-            path: { path },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        if (!data || !('files' in data || 'subdirectories' in data)) {
-            throw new Error(JSON.stringify({ error: "Directory not found" }));
-        }
-        return data;
+        // Idempotent GET: self-heal a transient connection reset (the file-tree
+        // refresh that customers hit as intermittent 500s after sandbox resume).
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getFilesystemByPath(this.withClient({
+                path: { path },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            if (!data || !('files' in data || 'subdirectories' in data)) {
+                throw new Error(JSON.stringify({ error: "Directory not found" }));
+            }
+            return data;
+        });
     }
     async search(query, path = "/", options) {
         const formattedPath = this.formatPath(path);
@@ -291,13 +242,15 @@ export class SandboxFileSystem extends SandboxAction {
         if (options?.excludeHidden !== undefined) {
             queryParams.excludeHidden = options.excludeHidden;
         }
-        const result = await getFilesystemSearchByPath(this.withClient({
-            path: { path: formattedPath },
-            query: queryParams,
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(result.response, result.data, result.error);
-        return result.data;
+        return retryOnTransientReset(async () => {
+            const result = await getFilesystemSearchByPath(this.withClient({
+                path: { path: formattedPath },
+                query: queryParams,
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(result.response, result.data, result.error);
+            return result.data;
+        });
     }
     async find(path, options) {
         const formattedPath = this.formatPath(path);
@@ -317,13 +270,15 @@ export class SandboxFileSystem extends SandboxAction {
         if (options?.excludeHidden !== undefined) {
             queryParams.excludeHidden = options.excludeHidden;
         }
-        const result = await getFilesystemFindByPath(this.withClient({
-            path: { path: formattedPath },
-            query: queryParams,
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(result.response, result.data, result.error);
-        return result.data;
+        return retryOnTransientReset(async () => {
+            const result = await getFilesystemFindByPath(this.withClient({
+                path: { path: formattedPath },
+                query: queryParams,
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(result.response, result.data, result.error);
+            return result.data;
+        });
     }
     async grep(query, path = "/", options) {
         const formattedPath = this.formatPath(path);
@@ -345,13 +300,15 @@ export class SandboxFileSystem extends SandboxAction {
         if (options?.excludeDirs && options.excludeDirs.length > 0) {
             queryParams.excludeDirs = options.excludeDirs.join(',');
         }
-        const result = await getFilesystemContentSearchByPath(this.withClient({
-            path: { path: formattedPath },
-            query: queryParams,
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(result.response, result.data, result.error);
-        return result.data;
+        return retryOnTransientReset(async () => {
+            const result = await getFilesystemContentSearchByPath(this.withClient({
+                path: { path: formattedPath },
+                query: queryParams,
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(result.response, result.data, result.error);
+            return result.data;
+        });
     }
     async cp(source, destination, { maxWait = 180000 } = {}) {
         let process = await this.process.exec({
@@ -503,6 +460,12 @@ export class SandboxFileSystem extends SandboxAction {
             const size = blob.size;
             const numParts = Math.ceil(size / CHUNK_SIZE);
             const parts = [];
+            // Bound concurrent upload-part streams on the shared H2 connection so many
+            // parts (within and across files) cannot burst past the server's
+            // rapid-reset limit (ENG-2680). Scoped to uploads; default cap is 2. With
+            // no h2Domain the parts go over globalThis.fetch on separate connections,
+            // so the shared-connection cap does not apply.
+            const h2Domain = this.sandbox?.h2Domain;
             // Upload parts in batches for parallel processing
             for (let i = 0; i < numParts; i += MAX_PARALLEL_UPLOADS) {
                 const batch = [];
@@ -511,7 +474,11 @@ export class SandboxFileSystem extends SandboxAction {
                     const start = (partNumber - 1) * CHUNK_SIZE;
                     const end = Math.min(start + CHUNK_SIZE, size);
                     const chunk = blob.slice(start, end);
-                    batch.push(retryOnTransient(() => this.uploadPart(uploadId, partNumber, chunk)));
+                    // Slot acquired per-attempt inside the retry (see writeBinary): bounds
+                    // concurrent part streams, not retry sequences; freed during backoff.
+                    const doPart = () => this.uploadPart(uploadId, partNumber, chunk);
+                    const partWithSlot = h2Domain ? () => withUploadSlot(h2Domain, doPart) : doPart;
+                    batch.push(retryOnTransient(partWithSlot));
                 }
                 // Wait for batch to complete
                 const batchResults = await Promise.all(batch);

package/dist/esm-browser/sandbox/process/process.js

@@ -1,5 +1,6 @@
 import { settings } from "../../common/settings.js";
 import { SandboxAction } from "../action.js";
+import { retryOnTransientReset } from "../../common/transient-retry.js";
 import { deleteProcessByIdentifier, deleteProcessByIdentifierKill, getProcess, getProcessByIdentifier, getProcessByIdentifierLogs, postProcess } from "../client/index.js";
 export class SandboxProcess extends SandboxAction {
     constructor(sandbox) {
@@ -299,19 +300,27 @@ export class SandboxProcess extends SandboxAction {
         return data;
     }
     async get(identifier) {
-        const { response, data, error } = await getProcessByIdentifier(this.withClient({
-            path: { identifier },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        return data;
+        // Idempotent GET: self-heal a transient connection reset (also makes the
+        // wait() poll loop resilient). exec() stays un-retried — it is a
+        // non-idempotent POST and retrying it duplicates the process (ENG-2340).
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getProcessByIdentifier(this.withClient({
+                path: { identifier },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            return data;
+        });
     }
     async list() {
-        const { response, data, error } = await getProcess(this.withClient({
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        return data;
+        // Idempotent GET: self-heal a transient connection reset.
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getProcess(this.withClient({
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            return data;
+        });
     }
     async stop(identifier) {
         const { response, data, error } = await deleteProcessByIdentifier(this.withClient({
@@ -330,11 +339,15 @@ export class SandboxProcess extends SandboxAction {
         return data;
     }
     async logs(identifier, type = "all") {
-        const { response, data, error } = await getProcessByIdentifierLogs(this.withClient({
-            path: { identifier },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
+        // Idempotent GET: self-heal a transient connection reset.
+        const data = await retryOnTransientReset(async () => {
+            const { response, data, error } = await getProcessByIdentifierLogs(this.withClient({
+                path: { identifier },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            return data;
+        });
         if (type === "all") {
             return data?.logs || "";
         }