@blaxel/core 0.2.85-preview.159 → 0.2.86-preview.161

package/dist/esm/common/h2fetch.js

@@ -1,6 +1,74 @@
+import { settings } from "./settings.js";
 import { refH2SessionForActiveRequest } from "./h2ref.js";
 const MIN_H2_SESSION_MAX_LISTENERS = 64;
 const sessionsWithListenerBudget = new WeakSet();
+/**
+ * Per-domain async semaphore that bounds the number of in-flight HTTP/2
+ * requests against a single edge domain (one H2 session). The cap is keyed
+ * by domain so throttling one sandbox's uploads never blocks requests to an
+ * unrelated sandbox served by a different edge.
+ *
+ * When `settings.maxConcurrentH2Requests` is `0`/unset the gate is a no-op
+ * (current default behavior: unlimited concurrency).
+ */
+// Safety timeout for a held slot. A request that never resolves (no response,
+// no error, no abort) would otherwise hold its slot forever and starve every
+// queued request for the same domain. After this window we release the slot
+// and let the underlying request continue or reject on its own; we never
+// cancel the in-flight request here, we only stop it from blocking the queue.
+// Kept deliberately conservative (well above any realistic part-upload time)
+// so it does not interfere with legitimately slow but healthy requests.
+const H2_SLOT_TIMEOUT_MS = 120_000;
+const h2GatesByDomain = new Map();
+function getH2Gate(domain) {
+    let gate = h2GatesByDomain.get(domain);
+    if (!gate) {
+        gate = { active: 0, queue: [] };
+        h2GatesByDomain.set(domain, gate);
+    }
+    return gate;
+}
+/**
+ * Acquire an in-flight slot for `domain`. Resolves with a release function
+ * that is idempotent and FIFO-fair: releasing wakes the longest-waiting
+ * queued caller for the same domain. A safety timer also releases the slot
+ * if the caller never does, preventing per-domain starvation.
+ */
+async function acquireH2Slot(domain) {
+    const max = settings.maxConcurrentH2Requests; // 0/undefined = unlimited
+    if (!max || max <= 0)
+        return () => { };
+    const gate = getH2Gate(domain);
+    while (gate.active >= max) {
+        await new Promise((resolve) => gate.queue.push(resolve));
+    }
+    gate.active++;
+    let released = false;
+    // Holder so `release` can clear the safety timer that is created after it.
+    const timer = {};
+    const release = () => {
+        if (released)
+            return;
+        released = true;
+        if (timer.handle !== undefined)
+            clearTimeout(timer.handle);
+        gate.active--;
+        const next = gate.queue.shift();
+        if (next) {
+            next();
+        }
+        else if (gate.active === 0 && gate.queue.length === 0) {
+            // No active requests and nothing waiting: drop the empty gate so the
+            // Map does not grow unbounded across many short-lived domains.
+            h2GatesByDomain.delete(domain);
+        }
+    };
+    timer.handle = setTimeout(release, H2_SLOT_TIMEOUT_MS);
+    // unref() so a pending safety timer never keeps the process alive. Guarded
+    // because not every runtime's timer handle exposes unref().
+    timer.handle.unref?.();
+    return release;
+}
 /**
  * Creates a fetch()-compatible function that sends requests over an existing
  * HTTP/2 session. Falls back to globalThis.fetch() only when the session is
@@ -26,11 +94,40 @@ export function createH2Fetch(session) {
  */
 export function createPoolBackedH2Fetch(pool, domain) {
     return async (input) => {
+        const rel = await acquireH2Slot(domain);
+        try {
+            const session = await pool.get(domain);
+            if (session) {
+                let h2RequestCreated = false;
+                try {
+                    return await _h2Request(session, input, {
+                        onH2RequestCreated: () => {
+                            h2RequestCreated = true;
+                        },
+                    });
+                }
+                catch (err) {
+                    if (h2RequestCreated) {
+                        pool.evictSession(domain, session);
+                    }
+                    throw err;
+                }
+            }
+            return await globalThis.fetch(input);
+        }
+        finally {
+            rel();
+        }
+    };
+}
+export async function h2RequestDirectFromPool(pool, domain, url, init) {
+    const rel = await acquireH2Slot(domain);
+    try {
         const session = await pool.get(domain);
         if (session) {
             let h2RequestCreated = false;
             try {
-                return await _h2Request(session, input, {
+                return await h2RequestDirectInternal(session, url, init, {
                     onH2RequestCreated: () => {
                         h2RequestCreated = true;
                     },
@@ -43,28 +140,11 @@ export function createPoolBackedH2Fetch(pool, domain) {
                 throw err;
             }
         }
-        return globalThis.fetch(input);
-    };
-}
-export async function h2RequestDirectFromPool(pool, domain, url, init) {
-    const session = await pool.get(domain);
-    if (session) {
-        let h2RequestCreated = false;
-        try {
-            return await h2RequestDirectInternal(session, url, init, {
-                onH2RequestCreated: () => {
-                    h2RequestCreated = true;
-                },
-            });
-        }
-        catch (err) {
-            if (h2RequestCreated) {
-                pool.evictSession(domain, session);
-            }
-            throw err;
-        }
+        return await globalThis.fetch(url, init);
+    }
+    finally {
+        rel();
     }
-    return globalThis.fetch(url, init);
 }
 /**
  * Low-level H2 request that takes raw URL + init, skipping Request construction.

package/dist/esm/common/settings.js

@@ -5,8 +5,8 @@ import { authentication } from "../authentication/index.js";
 import { env } from "../common/env.js";
 import { fs, os, path } from "../common/node.js";
 // Build info - these placeholders are replaced at build time by build:replace-imports
-const BUILD_VERSION = "0.2.85-preview.159";
-const BUILD_COMMIT = "a9c088b49239125f1b129fbc95d1887556112a33";
+const BUILD_VERSION = "0.2.86-preview.161";
+const BUILD_COMMIT = "a5bdac720290868983bb67563ff7068c1cf8c67f";
 const BUILD_SENTRY_DSN = "https://fd5e60e1c9820e1eef5ccebb84a07127@o4508714045276160.ingest.us.sentry.io/4510465864564736";
 const BLAXEL_API_VERSION = "2026-04-16";
 // Cache for config.yaml tracking value
@@ -226,6 +226,32 @@ class Settings {
         }
         return isDenoRuntime();
     }
+    get maxConcurrentH2Requests() {
+        if (typeof this.config.maxConcurrentH2Requests === "number") {
+            return this.config.maxConcurrentH2Requests;
+        }
+        const value = env.BL_MAX_H2_INFLIGHT;
+        if (value) {
+            const parsed = parseInt(value, 10);
+            if (!Number.isNaN(parsed)) {
+                return parsed;
+            }
+        }
+        return 0;
+    }
+    get fsPartRetries() {
+        if (typeof this.config.fsPartRetries === "number") {
+            return this.config.fsPartRetries;
+        }
+        const value = env.BL_FS_PART_RETRIES;
+        if (value) {
+            const parsed = parseInt(value, 10);
+            if (!Number.isNaN(parsed)) {
+                return parsed;
+            }
+        }
+        return 0;
+    }
     async authenticate() {
         await this.credentials.authenticate();
     }

package/dist/esm/sandbox/filesystem/filesystem.js

@@ -6,6 +6,94 @@ import { deleteFilesystemByPath, deleteFilesystemMultipartByUploadIdAbort, getFi
 const MULTIPART_THRESHOLD = 5 * 1024 * 1024; // 5MB
 const CHUNK_SIZE = 5 * 1024 * 1024; // 5MB per part
 const MAX_PARALLEL_UPLOADS = 3; // Number of parallel part uploads
+// Base backoff between part-upload retries, in milliseconds. Grows linearly
+// per attempt and is jittered to avoid synchronized retries (thundering herd)
+// when several parallel parts fail against the same edge at the same time.
+const RETRY_BASE_DELAY_MS = 200;
+// Markers that, when present anywhere in the error chain, are unambiguous
+// signals of a transient HTTP/2 stream reset or connection drop. These are
+// protocol/transport level codes, not application payloads, so substring
+// matching them does not over-match a server-sent error body. Each entry is
+// matched case-sensitively against the error message and its cause.
+//
+// Deliberately excluded: bare "INTERNAL_ERROR" and "fetch failed". Both are
+// too generic on their own (an application 500 body or any failed fetch would
+// match), so we only treat them as transient when paired with a transport
+// error code on the cause (see isTransientUploadError).
+const TRANSIENT_RESET_MARKERS = [
+    "ENHANCE_YOUR_CALM", // H2 flow-control backpressure reset
+    "NGHTTP2_INTERNAL_ERROR", // H2 internal stream error (qualified form)
+    "ERR_HTTP2", // node http2 error code family
+    "GOAWAY", // peer is draining the connection
+    "HTTP/2 session closed before response", // thrown by our own h2 transport
+    "HTTP/2 session sent GOAWAY before response",
+];
+// Node-level error codes (from `error.code` / `error.cause.code`) that mean
+// the connection itself dropped mid-flight and the request never completed.
+// These are safe to retry for an idempotent part upload.
+const TRANSIENT_ERROR_CODES = new Set([
+    "ECONNRESET",
+    "ECONNREFUSED",
+    "ETIMEDOUT",
+    "EPIPE",
+    "ERR_HTTP2_STREAM_ERROR",
+    "ERR_HTTP2_GOAWAY_SESSION",
+    "ERR_HTTP2_SESSION_ERROR",
+]);
+function collectErrorText(error) {
+    const messages = [];
+    const codes = [];
+    // Walk the error -> cause chain (bounded) so a transport error wrapped by a
+    // higher-level "fetch failed" is still classified correctly.
+    let current = error;
+    for (let depth = 0; depth < 5 && current && typeof current === "object"; depth++) {
+        const node = current;
+        if (typeof node.message === "string")
+            messages.push(node.message);
+        if (typeof node.code === "string")
+            codes.push(node.code);
+        current = node.cause;
+    }
+    return { messages, codes };
+}
+function isTransientUploadError(error) {
+    if (!error || typeof error !== "object") {
+        return false;
+    }
+    const { messages, codes } = collectErrorText(error);
+    // 1. An explicit transient transport error code anywhere in the chain.
+    if (codes.some((code) => TRANSIENT_ERROR_CODES.has(code))) {
+        return true;
+    }
+    // 2. An unambiguous protocol-level reset marker in any message.
+    if (messages.some((text) => TRANSIENT_RESET_MARKERS.some((marker) => text.includes(marker)))) {
+        return true;
+    }
+    return false;
+}
+function nextRetryDelayMs(attempt) {
+    // Linear backoff (200ms, 400ms, ...) plus up to one extra base delay of
+    // random jitter so concurrent part retries do not all fire on the same tick.
+    const base = RETRY_BASE_DELAY_MS * attempt;
+    const jitter = Math.floor(Math.random() * RETRY_BASE_DELAY_MS);
+    return base + jitter;
+}
+async function retryOnTransient(fn) {
+    const retries = settings.fsPartRetries; // 0 = off (current default behavior)
+    let attempt = 0;
+    for (;;) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            attempt++;
+            if (retries <= 0 || attempt > retries || !isTransientUploadError(error)) {
+                throw error;
+            }
+            await new Promise((resolve) => setTimeout(resolve, nextRetryDelayMs(attempt)));
+        }
+    }
+}
 export class SandboxFileSystem extends SandboxAction {
     process;
     constructor(sandbox, process) {
@@ -423,7 +511,7 @@ export class SandboxFileSystem extends SandboxAction {
                     const start = (partNumber - 1) * CHUNK_SIZE;
                     const end = Math.min(start + CHUNK_SIZE, size);
                     const chunk = blob.slice(start, end);
-                    batch.push(this.uploadPart(uploadId, partNumber, chunk));
+                    batch.push(retryOnTransient(() => this.uploadPart(uploadId, partNumber, chunk)));
                 }
                 // Wait for batch to complete
                 const batchResults = await Promise.all(batch);