@blaxel/core 0.2.87-preview.171 → 0.2.87-preview.175

package/dist/esm/sandbox/filesystem/filesystem.js

@@ -1,98 +1,22 @@
 import { fs } from "../../common/node.js";
 import { settings } from "../../common/settings.js";
+import { withUploadSlot } from "../../common/h2fetch.js";
+import { isTransientResetError, retryOnTransientReset } from "../../common/transient-retry.js";
 import { SandboxAction } from "../action.js";
 import { deleteFilesystemByPath, deleteFilesystemMultipartByUploadIdAbort, getFilesystemByPath, getFilesystemContentSearchByPath, getFilesystemFindByPath, getFilesystemSearchByPath, getWatchFilesystemByPath, postFilesystemMultipartByUploadIdComplete, postFilesystemMultipartInitiateByPath, putFilesystemByPath, putFilesystemMultipartByUploadIdPart } from "../client/index.js";
 // Multipart upload constants
 const MULTIPART_THRESHOLD = 5 * 1024 * 1024; // 5MB
 const CHUNK_SIZE = 5 * 1024 * 1024; // 5MB per part
 const MAX_PARALLEL_UPLOADS = 3; // Number of parallel part uploads
-// Base backoff between part-upload retries, in milliseconds. Grows linearly
-// per attempt and is jittered to avoid synchronized retries (thundering herd)
-// when several parallel parts fail against the same edge at the same time.
-const RETRY_BASE_DELAY_MS = 200;
-// Markers that, when present anywhere in the error chain, are unambiguous
-// signals of a transient HTTP/2 stream reset or connection drop. These are
-// protocol/transport level codes, not application payloads, so substring
-// matching them does not over-match a server-sent error body. Each entry is
-// matched case-sensitively against the error message and its cause.
-//
-// Deliberately excluded: bare "INTERNAL_ERROR" and "fetch failed". Both are
-// too generic on their own (an application 500 body or any failed fetch would
-// match), so we only treat them as transient when paired with a transport
-// error code on the cause (see isTransientUploadError).
-const TRANSIENT_RESET_MARKERS = [
-    "ENHANCE_YOUR_CALM", // H2 flow-control backpressure reset
-    "NGHTTP2_INTERNAL_ERROR", // H2 internal stream error (qualified form)
-    "ERR_HTTP2", // node http2 error code family
-    "GOAWAY", // peer is draining the connection
-    "HTTP/2 session closed before response", // thrown by our own h2 transport
-    "HTTP/2 session sent GOAWAY before response",
-];
-// Node-level error codes (from `error.code` / `error.cause.code`) that mean
-// the connection itself dropped mid-flight and the request never completed.
-// These are safe to retry for an idempotent part upload.
-const TRANSIENT_ERROR_CODES = new Set([
-    "ECONNRESET",
-    "ECONNREFUSED",
-    "ETIMEDOUT",
-    "EPIPE",
-    "ERR_HTTP2_STREAM_ERROR",
-    "ERR_HTTP2_GOAWAY_SESSION",
-    "ERR_HTTP2_SESSION_ERROR",
-]);
-function collectErrorText(error) {
-    const messages = [];
-    const codes = [];
-    // Walk the error -> cause chain (bounded) so a transport error wrapped by a
-    // higher-level "fetch failed" is still classified correctly.
-    let current = error;
-    for (let depth = 0; depth < 5 && current && typeof current === "object"; depth++) {
-        const node = current;
-        if (typeof node.message === "string")
-            messages.push(node.message);
-        if (typeof node.code === "string")
-            codes.push(node.code);
-        current = node.cause;
-    }
-    return { messages, codes };
-}
-function isTransientUploadError(error) {
-    if (!error || typeof error !== "object") {
-        return false;
-    }
-    const { messages, codes } = collectErrorText(error);
-    // 1. An explicit transient transport error code anywhere in the chain.
-    if (codes.some((code) => TRANSIENT_ERROR_CODES.has(code))) {
-        return true;
-    }
-    // 2. An unambiguous protocol-level reset marker in any message.
-    if (messages.some((text) => TRANSIENT_RESET_MARKERS.some((marker) => text.includes(marker)))) {
-        return true;
-    }
-    return false;
-}
-function nextRetryDelayMs(attempt) {
-    // Linear backoff (200ms, 400ms, ...) plus up to one extra base delay of
-    // random jitter so concurrent part retries do not all fire on the same tick.
-    const base = RETRY_BASE_DELAY_MS * attempt;
-    const jitter = Math.floor(Math.random() * RETRY_BASE_DELAY_MS);
-    return base + jitter;
-}
-async function retryOnTransient(fn) {
-    const retries = settings.fsPartRetries; // 0 = off (current default behavior)
-    let attempt = 0;
-    for (;;) {
-        try {
-            return await fn();
-        }
-        catch (error) {
-            attempt++;
-            if (retries <= 0 || attempt > retries || !isTransientUploadError(error)) {
-                throw error;
-            }
-            await new Promise((resolve) => setTimeout(resolve, nextRetryDelayMs(attempt)));
-        }
-    }
+// The transient-reset classifier and retry loop live in
+// common/transient-retry.ts, shared with the idempotent sandbox-op retry
+// (read/list/etc.) so every path judges "transient" identically. These aliases
+// preserve the import surface the ENG-2680 fault tests already depend on.
+export const isTransientUploadError = isTransientResetError;
+export function retryOnTransient(fn) {
+    // Upload path keeps its own (lower) fsPartRetries budget; the shared helper
+    // otherwise defaults to the higher idempotent-read budget (sandboxReadRetries).
+    return retryOnTransientReset(fn, { retries: settings.fsPartRetries });
 }
 export class SandboxFileSystem extends SandboxAction {
     process;
@@ -174,28 +98,44 @@ export class SandboxFileSystem extends SandboxAction {
         if (fileBlob.size > MULTIPART_THRESHOLD) {
             return await this.uploadWithMultipart(path, fileBlob, "0644");
         }
-        // Use regular upload for small files
-        const formData = new FormData();
-        formData.append("file", fileBlob, "test-binary.bin");
-        formData.append("permissions", "0644");
-        formData.append("path", path);
-        // Build URL
+        // Use regular upload for small files. Run the PUT under the same upload
+        // reliability wrapper as multipart parts: bound concurrency (ENG-2680) and
+        // retry transient connection resets (ECONNRESET/GOAWAY/ENHANCE_YOUR_CALM). A
+        // PUT of the same bytes to the same path is idempotent, so retry is safe.
+        // The FormData is rebuilt per attempt so a retried request has a fresh body.
         let url = `${this.url}/filesystem/${path}`;
         if (this.forcedUrl) {
             url = `${this.forcedUrl.toString()}/filesystem/${path}`;
         }
-        const response = await this.h2Fetch(url, {
-            method: 'PUT',
-            headers: {
-                ...settings.headers,
-            },
-            body: formData,
-        });
-        if (!response.ok) {
-            const errorText = await response.text();
-            throw new Error(`Failed to write binary: ${response.status} ${errorText}`);
-        }
-        return await response.json();
+        const h2Domain = this.sandbox?.h2Domain;
+        const putOnce = async () => {
+            const formData = new FormData();
+            formData.append("file", fileBlob, "test-binary.bin");
+            formData.append("permissions", "0644");
+            formData.append("path", path);
+            // A forceUrl (session-token) sandbox carries its own headers and must not
+            // require global credentials; settings.headers now throws without them
+            // (ENG-2698). Mirror streamLogs/execWithStreaming, which already pick
+            // sandbox.headers for forceUrl sessions.
+            const headers = this.sandbox.forceUrl ? this.sandbox.headers : settings.headers;
+            const response = await this.h2Fetch(url, {
+                method: 'PUT',
+                headers: {
+                    ...headers,
+                },
+                body: formData,
+            });
+            if (!response.ok) {
+                const errorText = await response.text();
+                throw new Error(`Failed to write binary: ${response.status} ${errorText}`);
+            }
+            return await response.json();
+        };
+        // Acquire the upload slot per-attempt INSIDE the retry so the per-domain cap
+        // bounds in-flight streams, not retry sequences: the slot is released during
+        // backoff, so a failing PUT never pins a slot while it sleeps (Mendral review).
+        const putWithSlot = h2Domain ? () => withUploadSlot(h2Domain, putOnce) : putOnce;
+        return retryOnTransient(putWithSlot);
     }
     async writeTree(files, destinationPath = null) {
         const options = {
@@ -220,30 +160,37 @@ export class SandboxFileSystem extends SandboxAction {
     }
     async read(path) {
         path = this.formatPath(path);
-        const { response, data, error } = await getFilesystemByPath(this.withClient({
-            path: { path },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        if (data && 'content' in data) {
-            return data.content;
-        }
-        throw new Error("Unsupported file type");
+        // Idempotent GET: self-heal a transient connection reset (e.g. first call
+        // after sandbox resume). Non-transient errors (404, etc.) are not retried.
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getFilesystemByPath(this.withClient({
+                path: { path },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            if (data && 'content' in data) {
+                return data.content;
+            }
+            throw new Error("Unsupported file type");
+        });
     }
     async readBinary(path) {
         path = this.formatPath(path);
-        const { response, data, error } = await getFilesystemByPath(this.withClient({
-            path: { path },
-            baseUrl: this.url,
-            headers: {
-                'Accept': 'application/octet-stream',
-            },
-        }));
-        this.handleResponseError(response, data, error);
-        if (typeof data === 'string') {
-            return new Blob([data]);
-        }
-        return data;
+        // Idempotent GET: self-heal a transient connection reset.
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getFilesystemByPath(this.withClient({
+                path: { path },
+                baseUrl: this.url,
+                headers: {
+                    'Accept': 'application/octet-stream',
+                },
+            }));
+            this.handleResponseError(response, data, error);
+            if (typeof data === 'string') {
+                return new Blob([data]);
+            }
+            return data;
+        });
     }
     async download(src, destinationPath, { mode = 0o644 } = {}) {
         if (!fs) {
@@ -266,15 +213,19 @@ export class SandboxFileSystem extends SandboxAction {
     }
     async ls(path) {
         path = this.formatPath(path);
-        const { response, data, error } = await getFilesystemByPath(this.withClient({
-            path: { path },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        if (!data || !('files' in data || 'subdirectories' in data)) {
-            throw new Error(JSON.stringify({ error: "Directory not found" }));
-        }
-        return data;
+        // Idempotent GET: self-heal a transient connection reset (the file-tree
+        // refresh that customers hit as intermittent 500s after sandbox resume).
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getFilesystemByPath(this.withClient({
+                path: { path },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            if (!data || !('files' in data || 'subdirectories' in data)) {
+                throw new Error(JSON.stringify({ error: "Directory not found" }));
+            }
+            return data;
+        });
     }
     async search(query, path = "/", options) {
         const formattedPath = this.formatPath(path);
@@ -291,13 +242,15 @@ export class SandboxFileSystem extends SandboxAction {
         if (options?.excludeHidden !== undefined) {
             queryParams.excludeHidden = options.excludeHidden;
         }
-        const result = await getFilesystemSearchByPath(this.withClient({
-            path: { path: formattedPath },
-            query: queryParams,
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(result.response, result.data, result.error);
-        return result.data;
+        return retryOnTransientReset(async () => {
+            const result = await getFilesystemSearchByPath(this.withClient({
+                path: { path: formattedPath },
+                query: queryParams,
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(result.response, result.data, result.error);
+            return result.data;
+        });
     }
     async find(path, options) {
         const formattedPath = this.formatPath(path);
@@ -317,13 +270,15 @@ export class SandboxFileSystem extends SandboxAction {
         if (options?.excludeHidden !== undefined) {
             queryParams.excludeHidden = options.excludeHidden;
         }
-        const result = await getFilesystemFindByPath(this.withClient({
-            path: { path: formattedPath },
-            query: queryParams,
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(result.response, result.data, result.error);
-        return result.data;
+        return retryOnTransientReset(async () => {
+            const result = await getFilesystemFindByPath(this.withClient({
+                path: { path: formattedPath },
+                query: queryParams,
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(result.response, result.data, result.error);
+            return result.data;
+        });
     }
     async grep(query, path = "/", options) {
         const formattedPath = this.formatPath(path);
@@ -345,13 +300,15 @@ export class SandboxFileSystem extends SandboxAction {
         if (options?.excludeDirs && options.excludeDirs.length > 0) {
             queryParams.excludeDirs = options.excludeDirs.join(',');
         }
-        const result = await getFilesystemContentSearchByPath(this.withClient({
-            path: { path: formattedPath },
-            query: queryParams,
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(result.response, result.data, result.error);
-        return result.data;
+        return retryOnTransientReset(async () => {
+            const result = await getFilesystemContentSearchByPath(this.withClient({
+                path: { path: formattedPath },
+                query: queryParams,
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(result.response, result.data, result.error);
+            return result.data;
+        });
     }
     async cp(source, destination, { maxWait = 180000 } = {}) {
         let process = await this.process.exec({
@@ -503,6 +460,12 @@ export class SandboxFileSystem extends SandboxAction {
             const size = blob.size;
             const numParts = Math.ceil(size / CHUNK_SIZE);
             const parts = [];
+            // Bound concurrent upload-part streams on the shared H2 connection so many
+            // parts (within and across files) cannot burst past the server's
+            // rapid-reset limit (ENG-2680). Scoped to uploads; default cap is 2. With
+            // no h2Domain the parts go over globalThis.fetch on separate connections,
+            // so the shared-connection cap does not apply.
+            const h2Domain = this.sandbox?.h2Domain;
             // Upload parts in batches for parallel processing
             for (let i = 0; i < numParts; i += MAX_PARALLEL_UPLOADS) {
                 const batch = [];
@@ -511,7 +474,11 @@ export class SandboxFileSystem extends SandboxAction {
                     const start = (partNumber - 1) * CHUNK_SIZE;
                     const end = Math.min(start + CHUNK_SIZE, size);
                     const chunk = blob.slice(start, end);
-                    batch.push(retryOnTransient(() => this.uploadPart(uploadId, partNumber, chunk)));
+                    // Slot acquired per-attempt inside the retry (see writeBinary): bounds
+                    // concurrent part streams, not retry sequences; freed during backoff.
+                    const doPart = () => this.uploadPart(uploadId, partNumber, chunk);
+                    const partWithSlot = h2Domain ? () => withUploadSlot(h2Domain, doPart) : doPart;
+                    batch.push(retryOnTransient(partWithSlot));
                 }
                 // Wait for batch to complete
                 const batchResults = await Promise.all(batch);

package/dist/esm/sandbox/process/process.js

@@ -1,5 +1,6 @@
 import { settings } from "../../common/settings.js";
 import { SandboxAction } from "../action.js";
+import { retryOnTransientReset } from "../../common/transient-retry.js";
 import { deleteProcessByIdentifier, deleteProcessByIdentifierKill, getProcess, getProcessByIdentifier, getProcessByIdentifierLogs, postProcess } from "../client/index.js";
 export class SandboxProcess extends SandboxAction {
     constructor(sandbox) {
@@ -299,19 +300,27 @@ export class SandboxProcess extends SandboxAction {
         return data;
     }
     async get(identifier) {
-        const { response, data, error } = await getProcessByIdentifier(this.withClient({
-            path: { identifier },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        return data;
+        // Idempotent GET: self-heal a transient connection reset (also makes the
+        // wait() poll loop resilient). exec() stays un-retried — it is a
+        // non-idempotent POST and retrying it duplicates the process (ENG-2340).
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getProcessByIdentifier(this.withClient({
+                path: { identifier },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            return data;
+        });
     }
     async list() {
-        const { response, data, error } = await getProcess(this.withClient({
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        return data;
+        // Idempotent GET: self-heal a transient connection reset.
+        return retryOnTransientReset(async () => {
+            const { response, data, error } = await getProcess(this.withClient({
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            return data;
+        });
     }
     async stop(identifier) {
         const { response, data, error } = await deleteProcessByIdentifier(this.withClient({
@@ -330,11 +339,15 @@ export class SandboxProcess extends SandboxAction {
         return data;
     }
     async logs(identifier, type = "all") {
-        const { response, data, error } = await getProcessByIdentifierLogs(this.withClient({
-            path: { identifier },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
+        // Idempotent GET: self-heal a transient connection reset.
+        const data = await retryOnTransientReset(async () => {
+            const { response, data, error } = await getProcessByIdentifierLogs(this.withClient({
+                path: { identifier },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            return data;
+        });
         if (type === "all") {
             return data?.logs || "";
         }