@blaxel/core 0.2.87-preview.174 → 0.2.87-preview.175

package/dist/esm/common/h2fetch.js

@@ -26,30 +26,31 @@ const sessionsWithListenerBudget = new WeakSet();
 // freed early (the queue could then admit one extra stream); acceptable as a
 // pure backstop, since the slot is otherwise tied to the true stream terminal.
 const H2_SLOT_TIMEOUT_MS = 1_800_000; // 30 minutes
+// Global per-domain gate (the opt-in maxConcurrentH2Requests cap).
 const h2GatesByDomain = new Map();
-function getH2Gate(domain) {
-    let gate = h2GatesByDomain.get(domain);
+// Upload-scoped per-domain gate (the default-on multipart upload cap, ENG-2680).
+// Kept separate from the global gate so it only ever throttles upload parts.
+const h2UploadGatesByDomain = new Map();
+function getGate(gates, domain) {
+    let gate = gates.get(domain);
     if (!gate) {
         gate = { active: 0, queue: [] };
-        h2GatesByDomain.set(domain, gate);
+        gates.set(domain, gate);
     }
     return gate;
 }
 /**
- * Acquire an OPEN-STREAM slot for `domain`. Resolves with a release function
- * that is idempotent and FIFO-fair: releasing wakes the longest-waiting queued
- * caller for the same domain. The slot is held for the lifetime of an OPEN H2
- * stream — the send path releases it on the stream terminal — so the gate
- * bounds true concurrent streams on the one shared session, not merely requests
- * awaiting headers (ENG-2678). A backstop timer releases the slot if a request
- * neither opens a stream nor ever settles, preventing per-domain starvation;
- * it never aborts the underlying request (see `H2_SLOT_TIMEOUT_MS`).
+ * Core per-domain async semaphore, shared by the global and upload gates.
+ * Resolves with a release function that is idempotent and FIFO-fair: releasing
+ * wakes the longest-waiting queued caller for the same domain. When `max` is
+ * `0`/unset the gate is a no-op (unlimited). A backstop timer releases the slot
+ * if a holder never releases, preventing per-domain starvation; it never aborts
+ * the underlying request (see `H2_SLOT_TIMEOUT_MS`).
  */
-async function acquireH2Slot(domain) {
-    const max = settings.maxConcurrentH2Requests; // 0/undefined = unlimited
+async function acquireGateSlot(gates, domain, max) {
     if (!max || max <= 0)
         return () => { };
-    const gate = getH2Gate(domain);
+    const gate = getGate(gates, domain);
     while (gate.active >= max) {
         await new Promise((resolve) => gate.queue.push(resolve));
     }
@@ -69,9 +70,9 @@ async function acquireH2Slot(domain) {
             next();
         }
         else if (gate.active === 0 && gate.queue.length === 0) {
-            // No active requests and nothing waiting: drop the empty gate so the
-            // Map does not grow unbounded across many short-lived domains.
-            h2GatesByDomain.delete(domain);
+            // No active holders and nothing waiting: drop the empty gate so the Map
+            // does not grow unbounded across many short-lived domains.
+            gates.delete(domain);
         }
     };
     timer.handle = setTimeout(release, H2_SLOT_TIMEOUT_MS);
@@ -80,6 +81,41 @@ async function acquireH2Slot(domain) {
     timer.handle.unref?.();
     return release;
 }
+/**
+ * Acquire the global OPEN-STREAM slot for `domain` (the opt-in
+ * `maxConcurrentH2Requests` cap; `0`/unset = unlimited, the default). Held for
+ * the lifetime of an OPEN H2 stream — the send path releases it on the stream
+ * terminal — so the gate bounds true concurrent streams on the one shared
+ * session, not merely requests awaiting headers (ENG-2678).
+ */
+function acquireH2Slot(domain) {
+    return acquireGateSlot(h2GatesByDomain, domain, settings.maxConcurrentH2Requests);
+}
+/**
+ * Acquire an upload-scoped slot for `domain`, bounding concurrent multipart
+ * upload-part requests on the shared H2 connection (ENG-2680). Separate from the
+ * global gate so it only throttles uploads. Defaults to 2 — the measured value
+ * that stops concurrent large uploads tripping ENHANCE_YOUR_CALM — making it the
+ * one reliability mitigation on by default, scoped to the upload path.
+ */
+function acquireUploadSlot(domain) {
+    return acquireGateSlot(h2UploadGatesByDomain, domain, settings.maxConcurrentUploadH2Requests);
+}
+/**
+ * Run `fn` while holding an upload slot for `domain`, releasing it when `fn`
+ * settles (the part PUT completes or fails). Bounds concurrent in-flight upload
+ * parts per domain across all files sharing the connection. Used by the
+ * multipart upload path; a no-op wrapper when the upload cap is unset.
+ */
+export async function withUploadSlot(domain, fn) {
+    const release = await acquireUploadSlot(domain);
+    try {
+        return await fn();
+    }
+    finally {
+        release();
+    }
+}
 /**
  * Creates a fetch()-compatible function that sends requests over an existing
  * HTTP/2 session. Falls back to globalThis.fetch() only when the session is

package/dist/esm/common/settings.js

@@ -22,8 +22,8 @@ function missingCredentialsMessage() {
     return "No Blaxel credentials found. Set the BL_API_KEY and BL_WORKSPACE environment variables, or run `bl login`.";
 }
 // Build info - these placeholders are replaced at build time by build:replace-imports
-const BUILD_VERSION = "0.2.87-preview.174";
-const BUILD_COMMIT = "d820c0d5ab85c3e0603e16cd29198e14055869ec";
+const BUILD_VERSION = "0.2.87-preview.175";
+const BUILD_COMMIT = "31085636cd9c2557198ee7399e7794e945c21687";
 const BUILD_SENTRY_DSN = "https://fd5e60e1c9820e1eef5ccebb84a07127@o4508714045276160.ingest.us.sentry.io/4510465864564736";
 const BLAXEL_API_VERSION = "2026-04-16";
 // Cache for config.yaml tracking value
@@ -257,6 +257,19 @@ class Settings {
         }
         return 0;
     }
+    get maxConcurrentUploadH2Requests() {
+        if (typeof this.config.maxConcurrentUploadH2Requests === "number") {
+            return this.config.maxConcurrentUploadH2Requests;
+        }
+        const value = env.BL_MAX_UPLOAD_H2_INFLIGHT;
+        if (value) {
+            const parsed = parseInt(value, 10);
+            if (!Number.isNaN(parsed)) {
+                return parsed;
+            }
+        }
+        return 2;
+    }
     get fsPartRetries() {
         if (typeof this.config.fsPartRetries === "number") {
             return this.config.fsPartRetries;
@@ -268,7 +281,20 @@ class Settings {
                 return parsed;
             }
         }
-        return 0;
+        return 3;
+    }
+    get sandboxReadRetries() {
+        if (typeof this.config.sandboxReadRetries === "number") {
+            return this.config.sandboxReadRetries;
+        }
+        const value = env.BL_SANDBOX_READ_RETRIES;
+        if (value) {
+            const parsed = parseInt(value, 10);
+            if (!Number.isNaN(parsed)) {
+                return parsed;
+            }
+        }
+        return 5;
     }
     /**
      * Fail fast with a clear, actionable error when credentials are missing or

package/dist/esm/common/transient-retry.js

@@ -0,0 +1,141 @@
+import { settings } from "./settings.js";
+// Markers that, when present anywhere in the error chain, are unambiguous
+// signals of a transient HTTP/2 stream reset or connection drop. These are
+// protocol/transport level codes, not application payloads, so substring
+// matching them does not over-match a server-sent error body. Each entry is
+// matched case-sensitively against the error message and its cause.
+//
+// Deliberately excluded: bare "INTERNAL_ERROR" and "fetch failed". Both are
+// too generic on their own (an application 500 body or any failed fetch would
+// match), so we only treat them as transient when paired with a transport
+// error code on the cause (see isTransientResetError).
+const TRANSIENT_RESET_MARKERS = [
+    "ENHANCE_YOUR_CALM", // H2 flow-control backpressure reset
+    "NGHTTP2_INTERNAL_ERROR", // H2 internal stream error (qualified form)
+    "ERR_HTTP2", // node http2 error code family
+    "GOAWAY", // peer is draining the connection
+    "HTTP/2 session closed before response", // thrown by our own h2 transport
+    "HTTP/2 session sent GOAWAY before response",
+];
+// Node-level error codes (from `error.code` / `error.cause.code`) that mean
+// the connection itself dropped mid-flight and the request never completed.
+// These are safe to retry for an idempotent request.
+const TRANSIENT_ERROR_CODES = new Set([
+    "ECONNRESET",
+    "ECONNREFUSED",
+    "ETIMEDOUT",
+    "EPIPE",
+    "ERR_HTTP2_STREAM_ERROR",
+    "ERR_HTTP2_GOAWAY_SESSION",
+    "ERR_HTTP2_SESSION_ERROR",
+]);
+function collectErrorText(error) {
+    const messages = [];
+    const codes = [];
+    // Walk the error -> cause chain (bounded) so a transport error wrapped by a
+    // higher-level "fetch failed" is still classified correctly.
+    let current = error;
+    for (let depth = 0; depth < 5 && current && typeof current === "object"; depth++) {
+        const node = current;
+        if (typeof node.message === "string")
+            messages.push(node.message);
+        if (typeof node.code === "string")
+            codes.push(node.code);
+        current = node.cause;
+    }
+    return { messages, codes };
+}
+// True when the error chain carries a real HTTP response (a status came back
+// from the server). Such an error is an APPLICATION response — never a transport
+// reset — even if the server's error body text happens to contain a reset marker
+// like "GOAWAY" or "ERR_HTTP2". Guarding on this stops a marker-bearing 4xx/5xx
+// body from being misread as transient and retried (the over-match Codex flagged
+// for the now default-on idempotent-read retry).
+function hasHttpResponseStatus(error) {
+    let current = error;
+    for (let depth = 0; depth < 5 && current && typeof current === "object"; depth++) {
+        const node = current;
+        if (typeof node.status === "number")
+            return true;
+        if (node.response && typeof node.response === "object" &&
+            typeof node.response.status === "number") {
+            return true;
+        }
+        current = node.cause;
+    }
+    return false;
+}
+/**
+ * True only for transport-level resets/drops that are safe to retry on an
+ * IDEMPOTENT request (transient HTTP/2 stream reset, GOAWAY, or a dropped
+ * connection). Application errors (4xx/5xx — even when their body text contains
+ * a marker word) and a bare "fetch failed" with no transport code are
+ * deliberately NOT transient, so auto-retry never masks a real server error or
+ * duplicates a non-idempotent call.
+ *
+ * This is the single classifier shared by the upload-part retry (ENG-2680) and
+ * the idempotent sandbox-op retry (read/list/etc.), so both judge "transient"
+ * identically.
+ */
+export function isTransientResetError(error) {
+    if (!error || typeof error !== "object") {
+        return false;
+    }
+    // An error that carries an HTTP response is an application error, not a
+    // transport reset — never retry it on the strength of a marker in its body.
+    if (hasHttpResponseStatus(error)) {
+        return false;
+    }
+    const { messages, codes } = collectErrorText(error);
+    // 1. An explicit transient transport error code anywhere in the chain.
+    if (codes.some((code) => TRANSIENT_ERROR_CODES.has(code))) {
+        return true;
+    }
+    // 2. An unambiguous protocol-level reset marker in any message.
+    if (messages.some((text) => TRANSIENT_RESET_MARKERS.some((marker) => text.includes(marker)))) {
+        return true;
+    }
+    return false;
+}
+const DEFAULT_BASE_DELAY_MS = 200;
+const DEFAULT_MAX_DELAY_MS = 2000;
+// Exponential backoff with full-jitter on top of one base delay, capped so a
+// single wait never blocks unreasonably long. Exponential (rather than linear)
+// gives a later attempt room to span a multi-second sandbox cold-start/standby
+// wake, which is the window a first-call reset falls into, while early attempts
+// stay fast for the common quick-reset case.
+function backoffDelayMs(attempt, baseDelayMs, maxDelayMs) {
+    const exponential = baseDelayMs * Math.pow(2, attempt - 1);
+    const capped = Math.min(exponential, maxDelayMs);
+    const jitter = Math.floor(Math.random() * baseDelayMs);
+    return capped + jitter;
+}
+/**
+ * Run `fn`, retrying only on transient transport resets (see
+ * isTransientResetError) with exponential backoff. Caller owns idempotency:
+ * ONLY wrap idempotent operations (GET-shaped reads/lists, or an idempotent
+ * PUT of the same bytes) — never a non-idempotent POST such as process.exec,
+ * which would duplicate the side effect (ENG-2340).
+ *
+ * Defaults to `settings.sandboxReadRetries` (the higher idempotent-read budget,
+ * sized for a multi-second standby wake). The upload path passes
+ * `{ retries: settings.fsPartRetries }` to keep its own (lower) budget.
+ */
+export async function retryOnTransientReset(fn, options = {}) {
+    const retries = options.retries ?? settings.sandboxReadRetries;
+    const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
+    const maxDelayMs = options.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
+    let attempt = 0;
+    for (;;) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            attempt++;
+            if (retries <= 0 || attempt > retries || !isTransientResetError(error)) {
+                throw error;
+            }
+            await new Promise((resolve) => setTimeout(resolve, backoffDelayMs(attempt, baseDelayMs, maxDelayMs)));
+        }
+    }
+}

package/dist/esm/sandbox/drive/drive.js

@@ -1,4 +1,5 @@
 import { SandboxAction } from "../action.js";
+import { retryOnTransientReset } from "../../common/transient-retry.js";
 import { postDrivesMount, deleteDrivesMountByMountPath, getDrivesMount, } from "../client/index.js";
 export class SandboxDrive extends SandboxAction {
     constructor(sandbox) {
@@ -33,11 +34,15 @@ export class SandboxDrive extends SandboxAction {
      * List all mounted drives in the sandbox
      */
     async list() {
-        const { response, data, error } = await getDrivesMount(this.withClient({
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        const result = data;
-        return result.mounts ?? [];
+        // Idempotent GET: self-heal a transient connection reset after sandbox resume.
+        // mount/unmount stay un-retried (non-idempotent POST/DELETE).
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getDrivesMount(this.withClient({
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            const result = data;
+            return result.mounts ?? [];
+        });
     }
 }