@blaxel/core 0.2.87-preview.171 → 0.2.87-preview.175

package/dist/cjs/sandbox/filesystem/filesystem.js

@@ -1,101 +1,26 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SandboxFileSystem = void 0;
+exports.SandboxFileSystem = exports.isTransientUploadError = void 0;
+exports.retryOnTransient = retryOnTransient;
 const node_js_1 = require("../../common/node.js");
 const settings_js_1 = require("../../common/settings.js");
+const h2fetch_js_1 = require("../../common/h2fetch.js");
+const transient_retry_js_1 = require("../../common/transient-retry.js");
 const action_js_1 = require("../action.js");
 const index_js_1 = require("../client/index.js");
 // Multipart upload constants
 const MULTIPART_THRESHOLD = 5 * 1024 * 1024; // 5MB
 const CHUNK_SIZE = 5 * 1024 * 1024; // 5MB per part
 const MAX_PARALLEL_UPLOADS = 3; // Number of parallel part uploads
-// Base backoff between part-upload retries, in milliseconds. Grows linearly
-// per attempt and is jittered to avoid synchronized retries (thundering herd)
-// when several parallel parts fail against the same edge at the same time.
-const RETRY_BASE_DELAY_MS = 200;
-// Markers that, when present anywhere in the error chain, are unambiguous
-// signals of a transient HTTP/2 stream reset or connection drop. These are
-// protocol/transport level codes, not application payloads, so substring
-// matching them does not over-match a server-sent error body. Each entry is
-// matched case-sensitively against the error message and its cause.
-//
-// Deliberately excluded: bare "INTERNAL_ERROR" and "fetch failed". Both are
-// too generic on their own (an application 500 body or any failed fetch would
-// match), so we only treat them as transient when paired with a transport
-// error code on the cause (see isTransientUploadError).
-const TRANSIENT_RESET_MARKERS = [
-    "ENHANCE_YOUR_CALM", // H2 flow-control backpressure reset
-    "NGHTTP2_INTERNAL_ERROR", // H2 internal stream error (qualified form)
-    "ERR_HTTP2", // node http2 error code family
-    "GOAWAY", // peer is draining the connection
-    "HTTP/2 session closed before response", // thrown by our own h2 transport
-    "HTTP/2 session sent GOAWAY before response",
-];
-// Node-level error codes (from `error.code` / `error.cause.code`) that mean
-// the connection itself dropped mid-flight and the request never completed.
-// These are safe to retry for an idempotent part upload.
-const TRANSIENT_ERROR_CODES = new Set([
-    "ECONNRESET",
-    "ECONNREFUSED",
-    "ETIMEDOUT",
-    "EPIPE",
-    "ERR_HTTP2_STREAM_ERROR",
-    "ERR_HTTP2_GOAWAY_SESSION",
-    "ERR_HTTP2_SESSION_ERROR",
-]);
-function collectErrorText(error) {
-    const messages = [];
-    const codes = [];
-    // Walk the error -> cause chain (bounded) so a transport error wrapped by a
-    // higher-level "fetch failed" is still classified correctly.
-    let current = error;
-    for (let depth = 0; depth < 5 && current && typeof current === "object"; depth++) {
-        const node = current;
-        if (typeof node.message === "string")
-            messages.push(node.message);
-        if (typeof node.code === "string")
-            codes.push(node.code);
-        current = node.cause;
-    }
-    return { messages, codes };
-}
-function isTransientUploadError(error) {
-    if (!error || typeof error !== "object") {
-        return false;
-    }
-    const { messages, codes } = collectErrorText(error);
-    // 1. An explicit transient transport error code anywhere in the chain.
-    if (codes.some((code) => TRANSIENT_ERROR_CODES.has(code))) {
-        return true;
-    }
-    // 2. An unambiguous protocol-level reset marker in any message.
-    if (messages.some((text) => TRANSIENT_RESET_MARKERS.some((marker) => text.includes(marker)))) {
-        return true;
-    }
-    return false;
-}
-function nextRetryDelayMs(attempt) {
-    // Linear backoff (200ms, 400ms, ...) plus up to one extra base delay of
-    // random jitter so concurrent part retries do not all fire on the same tick.
-    const base = RETRY_BASE_DELAY_MS * attempt;
-    const jitter = Math.floor(Math.random() * RETRY_BASE_DELAY_MS);
-    return base + jitter;
-}
-async function retryOnTransient(fn) {
-    const retries = settings_js_1.settings.fsPartRetries; // 0 = off (current default behavior)
-    let attempt = 0;
-    for (;;) {
-        try {
-            return await fn();
-        }
-        catch (error) {
-            attempt++;
-            if (retries <= 0 || attempt > retries || !isTransientUploadError(error)) {
-                throw error;
-            }
-            await new Promise((resolve) => setTimeout(resolve, nextRetryDelayMs(attempt)));
-        }
-    }
+// The transient-reset classifier and retry loop live in
+// common/transient-retry.ts, shared with the idempotent sandbox-op retry
+// (read/list/etc.) so every path judges "transient" identically. These aliases
+// preserve the import surface the ENG-2680 fault tests already depend on.
+exports.isTransientUploadError = transient_retry_js_1.isTransientResetError;
+function retryOnTransient(fn) {
+    // Upload path keeps its own (lower) fsPartRetries budget; the shared helper
+    // otherwise defaults to the higher idempotent-read budget (sandboxReadRetries).
+    return (0, transient_retry_js_1.retryOnTransientReset)(fn, { retries: settings_js_1.settings.fsPartRetries });
 }
 class SandboxFileSystem extends action_js_1.SandboxAction {
     process;
@@ -177,28 +102,44 @@ class SandboxFileSystem extends action_js_1.SandboxAction {
         if (fileBlob.size > MULTIPART_THRESHOLD) {
             return await this.uploadWithMultipart(path, fileBlob, "0644");
         }
-        // Use regular upload for small files
-        const formData = new FormData();
-        formData.append("file", fileBlob, "test-binary.bin");
-        formData.append("permissions", "0644");
-        formData.append("path", path);
-        // Build URL
+        // Use regular upload for small files. Run the PUT under the same upload
+        // reliability wrapper as multipart parts: bound concurrency (ENG-2680) and
+        // retry transient connection resets (ECONNRESET/GOAWAY/ENHANCE_YOUR_CALM). A
+        // PUT of the same bytes to the same path is idempotent, so retry is safe.
+        // The FormData is rebuilt per attempt so a retried request has a fresh body.
         let url = `${this.url}/filesystem/${path}`;
         if (this.forcedUrl) {
             url = `${this.forcedUrl.toString()}/filesystem/${path}`;
         }
-        const response = await this.h2Fetch(url, {
-            method: 'PUT',
-            headers: {
-                ...settings_js_1.settings.headers,
-            },
-            body: formData,
-        });
-        if (!response.ok) {
-            const errorText = await response.text();
-            throw new Error(`Failed to write binary: ${response.status} ${errorText}`);
-        }
-        return await response.json();
+        const h2Domain = this.sandbox?.h2Domain;
+        const putOnce = async () => {
+            const formData = new FormData();
+            formData.append("file", fileBlob, "test-binary.bin");
+            formData.append("permissions", "0644");
+            formData.append("path", path);
+            // A forceUrl (session-token) sandbox carries its own headers and must not
+            // require global credentials; settings.headers now throws without them
+            // (ENG-2698). Mirror streamLogs/execWithStreaming, which already pick
+            // sandbox.headers for forceUrl sessions.
+            const headers = this.sandbox.forceUrl ? this.sandbox.headers : settings_js_1.settings.headers;
+            const response = await this.h2Fetch(url, {
+                method: 'PUT',
+                headers: {
+                    ...headers,
+                },
+                body: formData,
+            });
+            if (!response.ok) {
+                const errorText = await response.text();
+                throw new Error(`Failed to write binary: ${response.status} ${errorText}`);
+            }
+            return await response.json();
+        };
+        // Acquire the upload slot per-attempt INSIDE the retry so the per-domain cap
+        // bounds in-flight streams, not retry sequences: the slot is released during
+        // backoff, so a failing PUT never pins a slot while it sleeps (Mendral review).
+        const putWithSlot = h2Domain ? () => (0, h2fetch_js_1.withUploadSlot)(h2Domain, putOnce) : putOnce;
+        return retryOnTransient(putWithSlot);
     }
     async writeTree(files, destinationPath = null) {
         const options = {
@@ -223,30 +164,37 @@ class SandboxFileSystem extends action_js_1.SandboxAction {
     }
     async read(path) {
         path = this.formatPath(path);
-        const { response, data, error } = await (0, index_js_1.getFilesystemByPath)(this.withClient({
-            path: { path },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        if (data && 'content' in data) {
-            return data.content;
-        }
-        throw new Error("Unsupported file type");
+        // Idempotent GET: self-heal a transient connection reset (e.g. first call
+        // after sandbox resume). Non-transient errors (404, etc.) are not retried.
+        return (0, transient_retry_js_1.retryOnTransientReset)(async () => {
+            const { response, data, error } = await (0, index_js_1.getFilesystemByPath)(this.withClient({
+                path: { path },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            if (data && 'content' in data) {
+                return data.content;
+            }
+            throw new Error("Unsupported file type");
+        });
     }
     async readBinary(path) {
         path = this.formatPath(path);
-        const { response, data, error } = await (0, index_js_1.getFilesystemByPath)(this.withClient({
-            path: { path },
-            baseUrl: this.url,
-            headers: {
-                'Accept': 'application/octet-stream',
-            },
-        }));
-        this.handleResponseError(response, data, error);
-        if (typeof data === 'string') {
-            return new Blob([data]);
-        }
-        return data;
+        // Idempotent GET: self-heal a transient connection reset.
+        return (0, transient_retry_js_1.retryOnTransientReset)(async () => {
+            const { response, data, error } = await (0, index_js_1.getFilesystemByPath)(this.withClient({
+                path: { path },
+                baseUrl: this.url,
+                headers: {
+                    'Accept': 'application/octet-stream',
+                },
+            }));
+            this.handleResponseError(response, data, error);
+            if (typeof data === 'string') {
+                return new Blob([data]);
+            }
+            return data;
+        });
     }
     async download(src, destinationPath, { mode = 0o644 } = {}) {
         if (!node_js_1.fs) {
@@ -269,15 +217,19 @@ class SandboxFileSystem extends action_js_1.SandboxAction {
     }
     async ls(path) {
         path = this.formatPath(path);
-        const { response, data, error } = await (0, index_js_1.getFilesystemByPath)(this.withClient({
-            path: { path },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        if (!data || !('files' in data || 'subdirectories' in data)) {
-            throw new Error(JSON.stringify({ error: "Directory not found" }));
-        }
-        return data;
+        // Idempotent GET: self-heal a transient connection reset (the file-tree
+        // refresh that customers hit as intermittent 500s after sandbox resume).
+        return (0, transient_retry_js_1.retryOnTransientReset)(async () => {
+            const { response, data, error } = await (0, index_js_1.getFilesystemByPath)(this.withClient({
+                path: { path },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            if (!data || !('files' in data || 'subdirectories' in data)) {
+                throw new Error(JSON.stringify({ error: "Directory not found" }));
+            }
+            return data;
+        });
     }
     async search(query, path = "/", options) {
         const formattedPath = this.formatPath(path);
@@ -294,13 +246,15 @@ class SandboxFileSystem extends action_js_1.SandboxAction {
         if (options?.excludeHidden !== undefined) {
             queryParams.excludeHidden = options.excludeHidden;
         }
-        const result = await (0, index_js_1.getFilesystemSearchByPath)(this.withClient({
-            path: { path: formattedPath },
-            query: queryParams,
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(result.response, result.data, result.error);
-        return result.data;
+        return (0, transient_retry_js_1.retryOnTransientReset)(async () => {
+            const result = await (0, index_js_1.getFilesystemSearchByPath)(this.withClient({
+                path: { path: formattedPath },
+                query: queryParams,
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(result.response, result.data, result.error);
+            return result.data;
+        });
     }
     async find(path, options) {
         const formattedPath = this.formatPath(path);
@@ -320,13 +274,15 @@ class SandboxFileSystem extends action_js_1.SandboxAction {
         if (options?.excludeHidden !== undefined) {
             queryParams.excludeHidden = options.excludeHidden;
         }
-        const result = await (0, index_js_1.getFilesystemFindByPath)(this.withClient({
-            path: { path: formattedPath },
-            query: queryParams,
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(result.response, result.data, result.error);
-        return result.data;
+        return (0, transient_retry_js_1.retryOnTransientReset)(async () => {
+            const result = await (0, index_js_1.getFilesystemFindByPath)(this.withClient({
+                path: { path: formattedPath },
+                query: queryParams,
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(result.response, result.data, result.error);
+            return result.data;
+        });
     }
     async grep(query, path = "/", options) {
         const formattedPath = this.formatPath(path);
@@ -348,13 +304,15 @@ class SandboxFileSystem extends action_js_1.SandboxAction {
         if (options?.excludeDirs && options.excludeDirs.length > 0) {
             queryParams.excludeDirs = options.excludeDirs.join(',');
         }
-        const result = await (0, index_js_1.getFilesystemContentSearchByPath)(this.withClient({
-            path: { path: formattedPath },
-            query: queryParams,
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(result.response, result.data, result.error);
-        return result.data;
+        return (0, transient_retry_js_1.retryOnTransientReset)(async () => {
+            const result = await (0, index_js_1.getFilesystemContentSearchByPath)(this.withClient({
+                path: { path: formattedPath },
+                query: queryParams,
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(result.response, result.data, result.error);
+            return result.data;
+        });
     }
     async cp(source, destination, { maxWait = 180000 } = {}) {
         let process = await this.process.exec({
@@ -506,6 +464,12 @@ class SandboxFileSystem extends action_js_1.SandboxAction {
             const size = blob.size;
             const numParts = Math.ceil(size / CHUNK_SIZE);
             const parts = [];
+            // Bound concurrent upload-part streams on the shared H2 connection so many
+            // parts (within and across files) cannot burst past the server's
+            // rapid-reset limit (ENG-2680). Scoped to uploads; default cap is 2. With
+            // no h2Domain the parts go over globalThis.fetch on separate connections,
+            // so the shared-connection cap does not apply.
+            const h2Domain = this.sandbox?.h2Domain;
             // Upload parts in batches for parallel processing
             for (let i = 0; i < numParts; i += MAX_PARALLEL_UPLOADS) {
                 const batch = [];
@@ -514,7 +478,11 @@ class SandboxFileSystem extends action_js_1.SandboxAction {
                     const start = (partNumber - 1) * CHUNK_SIZE;
                     const end = Math.min(start + CHUNK_SIZE, size);
                     const chunk = blob.slice(start, end);
-                    batch.push(retryOnTransient(() => this.uploadPart(uploadId, partNumber, chunk)));
+                    // Slot acquired per-attempt inside the retry (see writeBinary): bounds
+                    // concurrent part streams, not retry sequences; freed during backoff.
+                    const doPart = () => this.uploadPart(uploadId, partNumber, chunk);
+                    const partWithSlot = h2Domain ? () => (0, h2fetch_js_1.withUploadSlot)(h2Domain, doPart) : doPart;
+                    batch.push(retryOnTransient(partWithSlot));
                 }
                 // Wait for batch to complete
                 const batchResults = await Promise.all(batch);

package/dist/cjs/sandbox/process/process.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.SandboxProcess = void 0;
 const settings_js_1 = require("../../common/settings.js");
 const action_js_1 = require("../action.js");
+const transient_retry_js_1 = require("../../common/transient-retry.js");
 const index_js_1 = require("../client/index.js");
 class SandboxProcess extends action_js_1.SandboxAction {
     constructor(sandbox) {
@@ -302,19 +303,27 @@ class SandboxProcess extends action_js_1.SandboxAction {
         return data;
     }
     async get(identifier) {
-        const { response, data, error } = await (0, index_js_1.getProcessByIdentifier)(this.withClient({
-            path: { identifier },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        return data;
+        // Idempotent GET: self-heal a transient connection reset (also makes the
+        // wait() poll loop resilient). exec() stays un-retried — it is a
+        // non-idempotent POST and retrying it duplicates the process (ENG-2340).
+        return (0, transient_retry_js_1.retryOnTransientReset)(async () => {
+            const { response, data, error } = await (0, index_js_1.getProcessByIdentifier)(this.withClient({
+                path: { identifier },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            return data;
+        });
     }
     async list() {
-        const { response, data, error } = await (0, index_js_1.getProcess)(this.withClient({
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
-        return data;
+        // Idempotent GET: self-heal a transient connection reset.
+        return (0, transient_retry_js_1.retryOnTransientReset)(async () => {
+            const { response, data, error } = await (0, index_js_1.getProcess)(this.withClient({
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            return data;
+        });
     }
     async stop(identifier) {
         const { response, data, error } = await (0, index_js_1.deleteProcessByIdentifier)(this.withClient({
@@ -333,11 +342,15 @@ class SandboxProcess extends action_js_1.SandboxAction {
         return data;
     }
     async logs(identifier, type = "all") {
-        const { response, data, error } = await (0, index_js_1.getProcessByIdentifierLogs)(this.withClient({
-            path: { identifier },
-            baseUrl: this.url,
-        }));
-        this.handleResponseError(response, data, error);
+        // Idempotent GET: self-heal a transient connection reset.
+        const data = await (0, transient_retry_js_1.retryOnTransientReset)(async () => {
+            const { response, data, error } = await (0, index_js_1.getProcessByIdentifierLogs)(this.withClient({
+                path: { identifier },
+                baseUrl: this.url,
+            }));
+            this.handleResponseError(response, data, error);
+            return data;
+        });
         if (type === "all") {
             return data?.logs || "";
         }

package/dist/cjs/types/common/h2fetch.d.ts

@@ -1,5 +1,12 @@
 import http2 from "http2";
 import type { H2Pool } from "./h2pool.js";
+/**
+ * Run `fn` while holding an upload slot for `domain`, releasing it when `fn`
+ * settles (the part PUT completes or fails). Bounds concurrent in-flight upload
+ * parts per domain across all files sharing the connection. Used by the
+ * multipart upload path; a no-op wrapper when the upload cap is unset.
+ */
+export declare function withUploadSlot<T>(domain: string, fn: () => Promise<T>): Promise<T>;
 /**
  * Creates a fetch()-compatible function that sends requests over an existing
  * HTTP/2 session. Falls back to globalThis.fetch() only when the session is

package/dist/cjs/types/common/settings.d.ts

@@ -18,10 +18,28 @@ export type Config = {
      */
     maxConcurrentH2Requests?: number;
     /**
-     * Number of retry attempts for transient resets on multipart part uploads.
-     * `0` or `undefined` disables retry (current behavior).
+     * Maximum number of concurrent in-flight multipart upload-part requests per
+     * edge domain on the shared H2 connection. Defaults to 2 (the measured value
+     * that stops concurrent large uploads tripping ENHANCE_YOUR_CALM). Scoped to
+     * the upload-part path only; non-upload traffic is unaffected. `0` disables it.
+     */
+    maxConcurrentUploadH2Requests?: number;
+    /**
+     * Retry attempts for transient connection resets (ECONNRESET, GOAWAY,
+     * ENHANCE_YOUR_CALM, etc.) on file uploads. Covers both small single-PUT
+     * uploads and multipart parts; both are idempotent writes and safe to retry.
+     * Defaults to 3. Set `0` to disable.
      */
     fsPartRetries?: number;
+    /**
+     * Retry attempts for transient connection resets on IDEMPOTENT sandbox reads
+     * (fs.read/readBinary/ls/search/find/grep, drives.list, process.get/list/logs).
+     * Higher than the upload default so a later attempt can span a multi-second
+     * sandbox cold-start/standby wake (the window a first-call read reset falls in).
+     * Defaults to 5. Set `0` to disable. Never applied to non-idempotent ops
+     * (process.exec, drives.mount, etc.).
+     */
+    sandboxReadRetries?: number;
     /**
      * Client credentials for OAuth2 client_credentials flow.
      *
@@ -62,7 +80,9 @@ declare class Settings {
     get region(): string | undefined;
     get disableH2(): boolean;
     get maxConcurrentH2Requests(): number;
+    get maxConcurrentUploadH2Requests(): number;
     get fsPartRetries(): number;
+    get sandboxReadRetries(): number;
     /**
      * Fail fast with a clear, actionable error when credentials are missing or
      * incomplete, instead of sending empty workspace/authorization headers and

package/dist/cjs/types/common/transient-retry.d.ts

@@ -0,0 +1,30 @@
+/**
+ * True only for transport-level resets/drops that are safe to retry on an
+ * IDEMPOTENT request (transient HTTP/2 stream reset, GOAWAY, or a dropped
+ * connection). Application errors (4xx/5xx — even when their body text contains
+ * a marker word) and a bare "fetch failed" with no transport code are
+ * deliberately NOT transient, so auto-retry never masks a real server error or
+ * duplicates a non-idempotent call.
+ *
+ * This is the single classifier shared by the upload-part retry (ENG-2680) and
+ * the idempotent sandbox-op retry (read/list/etc.), so both judge "transient"
+ * identically.
+ */
+export declare function isTransientResetError(error: unknown): boolean;
+export type RetryOptions = {
+    retries?: number;
+    baseDelayMs?: number;
+    maxDelayMs?: number;
+};
+/**
+ * Run `fn`, retrying only on transient transport resets (see
+ * isTransientResetError) with exponential backoff. Caller owns idempotency:
+ * ONLY wrap idempotent operations (GET-shaped reads/lists, or an idempotent
+ * PUT of the same bytes) — never a non-idempotent POST such as process.exec,
+ * which would duplicate the side effect (ENG-2340).
+ *
+ * Defaults to `settings.sandboxReadRetries` (the higher idempotent-read budget,
+ * sized for a multi-second standby wake). The upload path passes
+ * `{ retries: settings.fsPartRetries }` to keep its own (lower) budget.
+ */
+export declare function retryOnTransientReset<T>(fn: () => Promise<T>, options?: RetryOptions): Promise<T>;

package/dist/cjs/types/sandbox/filesystem/filesystem.d.ts

@@ -1,8 +1,11 @@
 import { Sandbox } from "../../client/types.gen.js";
+import { isTransientResetError } from "../../common/transient-retry.js";
 import { SandboxAction } from "../action.js";
 import { ContentSearchResponse, Directory, FindResponse, FuzzySearchResponse, SuccessResponse } from "../client/index.js";
 import { SandboxProcess } from "../process/index.js";
 import { CopyResponse, FilesystemFindOptions, FilesystemGrepOptions, FilesystemSearchOptions, SandboxFilesystemFile, WatchEvent } from "./types.js";
+export declare const isTransientUploadError: typeof isTransientResetError;
+export declare function retryOnTransient<T>(fn: () => Promise<T>): Promise<T>;
 export declare class SandboxFileSystem extends SandboxAction {
     private process;
     constructor(sandbox: Sandbox, process: SandboxProcess);