@shardflux/sdk 0.12.0 → 0.13.1

package/dist/generated/cell-api.d.ts

@@ -80,6 +80,45 @@ export interface paths {
          *     `503 service_unavailable` (`no_execution_host`: no host with
          *     protocol feature `file_first` has room; retry), secret resolution
          *     errors as for processful starts.
+         *
+         *     Burst execution (contracts §33, not public yet): `burst: always` runs
+         *     the command on a larger, short-lived burst VM on the workspace's
+         *     host over a copy of the workspace disk and applies its file changes
+         *     back when it exits. The session is read, streamed and attached like
+         *     any other; it carries `burst` (BurstSummary) and cannot be signalled
+         *     or canceled (`409 conflict`, `details.reason burst_not_supported`).
+         *     Refusals: `422 validation_failed` with `details.reason`
+         *     `burst_mode_not_supported` (`auto`), `burst_not_supported` (stdin),
+         *     `burst_size_exceeds_plan` (`burst_vcpus`/`burst_memory_mib` above the
+         *     plan's per-workspace ceilings; `details.field`, `details.limit`);
+         *     `409 burst_unavailable` with `details.reason` `not_available` (no
+         *     entitlement `policy.burst_exec`, or the host lacks protocol feature
+         *     `burst_exec`), `layout_unsupported`, `shared_volumes`,
+         *     `host_capacity`, `fence_not_drained`, `workspace_fenced`,
+         *     `apply_pending`, `park_failed`, `workspace_resumed`, `interrupted`;
+         *     `409 burst_apply_failed` (`details.reason` `disk_full` or
+         *     `apply_failed`, `details.applied_entries`, `details.pending_entries`;
+         *     the workspace refuses writes until a retry with the same
+         *     `session_id` completes the apply). A refusal the host gives within
+         *     the first seconds is the start's answer; a later one ends the
+         *     session (`failed_to_start` for `burst_unavailable`, `lost` for
+         *     `burst_apply_failed`), is recorded in `burst.error`, and is the
+         *     output stream's final `error` event. A retry with the same
+         *     `session_id` replays the recorded outcome (the host journals the
+         *     burst by the exec's operation id) or resumes an unfinished apply.
+         *
+         *     Start and follow in one request: a processful start sent with
+         *     `Accept: application/x-ndjson` (or `text/event-stream`) answers 200
+         *     with the session's output stream, as `GET .../output?follow=true`
+         *     from offset 0 (output events, `exit`, a final `error`), and
+         *     `X-Exec-Session-Id` names the session. The stream slot is taken
+         *     before the command starts (`503 service_unavailable` when the
+         *     gateway has no stream slot: nothing ran). Headers are sent as soon
+         *     as the command started, before any output. A `failed_to_start`
+         *     session, a refusal and a file-first execution keep their JSON
+         *     answers. A replayed `session_id` streams the existing session and
+         *     never runs it again; after a dropped stream, reconnect with
+         *     `GET .../output` from the byte offsets already read.
          */
         post: operations["execStart"];
         delete?: never;
@@ -152,7 +191,7 @@ export interface paths {
          *     Client messages are JSON `ExecClientMessage`. Close codes: 1000 after
          *     the `exit` event, 1001 on gateway shutdown (reconnect with offsets),
          *     4401 unauthenticated, 4403 forbidden, 4409 stale epoch / workspace
-         *     busy / workspace not running, 4410 workspace gone, 4429 rate limited,
+         *     busy / workspace not running, 4410 workspace gone, 4429 rate limited / quota exceeded,
          *     4500/4503/4504 internal/unavailable/timeout (4000 + HTTP status; see
          *     "WebSocket close codes" above; the close reason is compact JSON with
          *     the error `code`). A refused upgrade from an allowed Origin is
@@ -225,7 +264,20 @@ export interface paths {
         };
         get?: never;
         put?: never;
-        /** SIGTERM the process group, SIGKILL after grace */
+        /**
+         * Stop the command and every process it started (SIGTERM, SIGKILL after grace)
+         * @description Sends SIGTERM to every process the command started, background and
+         *     setsid processes included, and SIGKILL after `grace_ms` (0 to 60000;
+         *     0 or omitted is 5000). Answers when they have all exited: at once when
+         *     SIGTERM ends them, otherwise after the grace and the SIGKILL. A lost
+         *     command is stopped the same way; a command that already exited keeps
+         *     its result and what it left running is stopped. `503
+         *     dependency_unavailable` (retryable) while a process cannot exit yet.
+         *     Template versions before python-node-browser 13, ubuntu-24.04 4,
+         *     ubuntu-22.04 2 and debian-12 2 signal the command's process group and
+         *     return an ended session as it is. A `grace_ms` outside the range is `422 validation_failed`
+         *     (`details.field` `grace_ms`, `details.minimum`, `details.maximum`).
+         */
         post: operations["execCancel"];
         delete?: never;
         options?: never;
@@ -293,7 +345,11 @@ export interface paths {
         get: operations["ptyGet"];
         put?: never;
         post?: never;
-        /** Close (SIGHUP, then SIGKILL after grace) */
+        /**
+         * Close (SIGHUP, then SIGKILL after grace)
+         * @description Sends SIGHUP to the session's process group and SIGKILL 2 s later
+         *     if it is still running; answers once it has ended.
+         */
         delete: operations["ptyClose"];
         options?: never;
         head?: never;
@@ -744,8 +800,9 @@ export interface paths {
          * @description `idle_since` = max(`last_tool_at`, `last_attach_at`, `keepalive_until`,
          *     `running_since`, the work horizon of running exec commands) (null when
          *     none is known; in the future while a keepalive or a running command is
-         *     active). A running exec command counts as work until min(its start +
-         *     `IDLE_COMMAND_MAX_SECONDS`, its start + `timeout_ms`), and its end is
+         *     active). A running exec command counts as work until it ends or its
+         *     timeout passes: its start + `timeout_ms` when it has one, else its
+         *     start + `IDLE_COMMAND_MAX_SECONDS` (1 hour by default). Its end is
          *     activity (contracts §20.6): an end seen without a client call is
          *     recorded in `last_attach_at`, which is work but not a tool call.
          *     `suspend_at` = `idle_since` + `idle_minutes` when
@@ -823,7 +880,7 @@ export type webhooks = Record<string, never>;
 export interface components {
     schemas: {
         /** @enum {string} */
-        ErrorCode: "bad_request" | "unauthenticated" | "forbidden" | "not_found" | "conflict" | "stale_epoch" | "workspace_not_running" | "validation_failed" | "payload_too_large" | "range_not_satisfiable" | "rate_limited" | "capacity_pending" | "dependency_unavailable" | "timeout" | "internal_error" | "service_unavailable" | "workspace_busy";
+        ErrorCode: "bad_request" | "unauthenticated" | "forbidden" | "not_found" | "conflict" | "stale_epoch" | "workspace_not_running" | "validation_failed" | "payload_too_large" | "range_not_satisfiable" | "rate_limited" | "capacity_pending" | "dependency_unavailable" | "timeout" | "internal_error" | "service_unavailable" | "workspace_busy" | "burst_unavailable" | "burst_apply_failed" | "quota_exceeded";
         ErrorBody: {
             error: {
                 code: components["schemas"]["ErrorCode"];
@@ -872,7 +929,10 @@ export interface components {
             stdin?: string;
             /** Format: int64 */
             timeout_ms?: number;
-            /** Format: int64 */
+            /**
+             * Format: int64
+             * @description Time between the SIGTERM and the SIGKILL when timeout_ms ends the command; 0 or omitted is 5000.
+             */
             kill_grace_ms?: number;
             secret_refs?: components["schemas"]["SecretRefs"];
             /** @description File-first workspaces (required there, refused otherwise) — the execution's idempotency key. */
@@ -883,6 +943,16 @@ export interface components {
              * @default 1048576
              */
             output_limit_bytes?: number;
+            /**
+             * @description Burst execution (contracts §33): `always` runs this run-to-completion command on a larger, short-lived burst VM and applies its file changes back. Needs entitlement `policy.burst_exec` and a layered processful workspace; not with stdin. `auto` is reserved (422 validation_failed, details.reason burst_mode_not_supported).
+             * @default never
+             * @enum {string}
+             */
+            burst?: "never" | "always";
+            /** @description The burst VM's vCPUs (burst always only; default the host's, 8), at most the plan's workspace CPU ceiling. */
+            burst_vcpus?: number;
+            /** @description The burst VM's memory in MiB (burst always only; default the host's, 8192), at most the plan's workspace memory ceiling. */
+            burst_memory_mib?: number;
         };
         /** @description Names of customer secrets (contracts §17) to inject as environment variables NAME=value of this process only. The cell resolves them at session start through the API with the caller's tool token (permission-checked and audited by the API); values are never logged, persisted or returned by the cell. Any name the caller may not use (unknown, not permitted for this workspace/project/tool) refuses the whole start with 403 forbidden (details.reason secret_not_available, details.names); nothing is started. A name that is also a key of `env` is refused (422 validation_failed). Requires a workspace tool token (browser stream tickets cannot resolve secrets: 403, details.reason secret_refs_require_tool_token). Resolution failures: 401 (token revoked/expired), 409 stale_epoch, 429 rate_limited, 503 dependency_unavailable. A retried start with the same session_id re-resolves the names; if a value changed since the first start the host refuses the different request (409 conflict). */
         SecretRefs: string[];
@@ -980,6 +1050,88 @@ export interface components {
             ended_at?: string | null;
             /** @description Why the session ended without an exit code. For failed_to_start the command never ran and this names the cause, e.g. `working directory "/home/user/app" is not a directory` or `executable "foo" not found in PATH`. */
             error?: string;
+            memory_grow?: components["schemas"]["MemoryGrow"];
+            burst?: components["schemas"]["BurstSummary"];
+        };
+        /** @description Burst execution (contracts §33.5): present on every session started with `burst: always`. The size, method and counts are set once the burst ended (the host's result); while it runs only host, applied and the requested size are known. */
+        BurstSummary: {
+            /**
+             * @description Where the burst VM ran; `local` = the workspace's host (`remote` is reserved).
+             * @enum {string}
+             */
+            host: "local" | "remote";
+            /** @description The burst VM's vCPUs (the requested count while it runs, absent for the host default). */
+            vcpus?: number;
+            /** @description The burst VM's memory in MiB (the requested size while it runs, absent for the host default). */
+            memory_mib?: number;
+            /**
+             * @description How the burst VM got the workspace copy.
+             * @enum {string}
+             */
+            method?: "layer" | "clone";
+            /** @description The command's file changes were applied to the workspace. */
+            applied: boolean;
+            /** Format: int64 */
+            written_files?: number;
+            /** Format: int64 */
+            written_dirs?: number;
+            /** Format: int64 */
+            removed?: number;
+            /** Format: int64 */
+            written_bytes?: number;
+            /** @description Processes the command left behind in the burst VM (killed; they do not survive a burst). */
+            leftover_killed?: number;
+            /**
+             * Format: int64
+             * @description Wall time of the burst minus the command's own run time.
+             */
+            overhead_ms?: number;
+            /** @description The burst VM's own paths whose changes are never carried back (the host's denylist). */
+            excluded_paths?: string[];
+            /** @description The host's phase timings (ms) and counters. */
+            timings?: {
+                [key: string]: number;
+            };
+            /** @description The recorded outcome of an earlier attempt with the same session_id. */
+            replayed?: boolean;
+            /** @description Set when the burst failed after the start answered: burst_unavailable (the workspace is unchanged) or burst_apply_failed (details.applied_entries, details.pending_entries). */
+            error?: components["schemas"]["BurstError"];
+        };
+        BurstError: {
+            code: components["schemas"]["ErrorCode"];
+            message: string;
+            retryable: boolean;
+            details?: {
+                [key: string]: unknown;
+            };
+        };
+        /** @description Elastic memory (contracts §32.3, §32.6): the host grew the workspace's memory before this exec started (the exec waited for it). Present only on the session returned by the start that grew the VM; absent when no grow ran (fixed workspaces, or already at the exec size). */
+        MemoryGrow: {
+            /**
+             * @description delivered: the whole grow; partial: less than wanted; missed: nothing could be plugged; failed: the plug did not converge or errored.
+             * @enum {string}
+             */
+            outcome: "delivered" | "partial" | "missed" | "failed";
+            /**
+             * Format: int64
+             * @description Guest memory before the grow (MiB).
+             */
+            from_mib: number;
+            /**
+             * Format: int64
+             * @description Guest memory the grow aimed for (MiB).
+             */
+            want_mib: number;
+            /**
+             * Format: int64
+             * @description Guest memory after the grow (MiB).
+             */
+            got_mib: number;
+            /**
+             * Format: int64
+             * @description Trigger to plugged and guest MemTotal settled (ms, rounded).
+             */
+            deliver_ms: number;
         };
         OutputEvent: {
             /** @enum {string} */
@@ -999,7 +1151,10 @@ export interface components {
             /** @enum {string} */
             type: "signal" | "cancel";
             signal?: components["schemas"]["SignalValue"];
-            /** Format: int64 */
+            /**
+             * Format: int64
+             * @description For cancel, as CancelRequest.grace_ms; out of range is an `error` event (validation_failed).
+             */
             grace_ms?: number;
         };
         /** @description Signal number (1-64) or name (e.g. "SIGTERM", "TERM"). */
@@ -1026,7 +1181,10 @@ export interface components {
             only_leader?: boolean;
         };
         CancelRequest: {
-            /** Format: int64 */
+            /**
+             * Format: int64
+             * @description Time between the SIGTERM and the SIGKILL; 0 or omitted is 5000.
+             */
             grace_ms?: number;
         };
         PtyOpenRequest: {
@@ -1545,12 +1703,16 @@ export interface operations {
             };
         };
         responses: {
-            /** @description The session already existed; nothing was run again (processful), or the recorded result of an existing execution (file-first). */
+            /** @description An existing session/result, or followed output when a processful start accepts NDJSON/SSE. A failed-to-start session stays JSON. */
             200: {
                 headers: {
+                    /** @description The streamed session id; replay/reconnect use the same session and byte offsets. */
+                    "X-Exec-Session-Id"?: string;
                     [name: string]: unknown;
                 };
                 content: {
+                    "application/x-ndjson": string;
+                    "text/event-stream": string;
                     "application/json": components["schemas"]["ExecSession"] | components["schemas"]["ExecutionResult"];
                 };
             };

package/dist/http.d.ts

@@ -1,5 +1,5 @@
 import type { RetryRecord } from './progress.js';
-export declare const SDK_VERSION = "0.12.0";
+export declare const SDK_VERSION = "0.13.1";
 export interface RequestOptions {
     query?: Record<string, string | number | boolean | undefined | null>;
     json?: unknown;
@@ -10,9 +10,19 @@ export interface RequestOptions {
     idempotencyKey?: string;
     /** The request has no effect a retry could duplicate (a read-only POST such as files/search): retried like a GET. */
     idempotent?: boolean;
+    /**
+     * false: the cell gateway's 429 `quota_exceeded` surfaces at once instead of being retried (the wake hint, which
+     * never waits). Default true.
+     */
+    quotaRetry?: boolean;
     signal?: AbortSignal;
     /** Override the client's default request timeout (ms); 0 disables it (streams). */
     timeoutMs?: number;
+    /**
+     * Bounds each attempt until the response headers arrive (ms), leaving the body unbounded: a stream that answers a
+     * start (exec.run's combined start and output). Usually with `timeoutMs: 0`.
+     */
+    headersTimeoutMs?: number;
     /** Called before each retry of a transient failure (lifecycle timing records these). */
     onRetry?: (retry: Omit<RetryRecord, 'atMs'>) => void;
 }
@@ -28,13 +38,45 @@ export interface HttpOptions {
     onSuccess?: (() => void) | undefined;
 }
 export declare const defaultSleep: (ms: number) => Promise<void>;
+/**
+ * How long an idle pooled connection stays reusable (ms): 5 minutes, instead of undici's 4 s default, so the request
+ * after an agent's usual pause between tool calls (15 s to a few minutes) skips a new TCP and TLS handshake. Far below
+ * the 3600 s idle timeout of the load balancer in front of the API and the cell endpoints, so the server side never
+ * closes a connection for idleness while the client still treats it as reusable; below the AWS NAT gateway's 350 s idle
+ * timeout; and the same limit Chrome keeps for used idle sockets. Also caps a server's `Keep-Alive: timeout` hint.
+ */
+export declare const KEEP_ALIVE_TIMEOUT_MS = 300000;
+/**
+ * TCP keepalive probes start after a pooled socket has been idle this long (ms), so NATs and firewalls with shorter
+ * idle timeouts than the pool's (Azure SNAT: 4 minutes) keep the connection instead of silently dropping it. Equal to
+ * undici's own default, set explicitly so it is visible and tested.
+ */
+export declare const TCP_KEEPALIVE_INITIAL_DELAY_MS = 60000;
+/** The private pool's undici Agent options (exported for tests). */
+export declare const POOL_OPTIONS: {
+    readonly allowH2: false;
+    readonly pipelining: 1;
+    readonly keepAliveTimeout: 300000;
+    readonly keepAliveMaxTimeout: 300000;
+    readonly connect: {
+        readonly keepAlive: true;
+        readonly keepAliveInitialDelay: 60000;
+    };
+};
 /**
  * Reuses TLS connections. Node 26's bundled undici 8.9 can stall reused connections: use pinned undici with a
- * private HTTP/1.1-only dispatcher. Other runtimes retain native fetch. SHARDFLUX_HTTP_KEEPALIVE=0 forces close
- * for diagnosis; =1 retains the historical opt-in to native pooling. An explicitly supplied fetch is untouched.
+ * private HTTP/1.1-only dispatcher whose idle connections stay reusable for KEEP_ALIVE_TIMEOUT_MS. Other runtimes
+ * retain native fetch. SHARDFLUX_HTTP_KEEPALIVE=0 forces close for diagnosis; =1 retains the historical opt-in to
+ * native pooling. An explicitly supplied fetch is untouched. Idle pooled sockets do not keep the process alive.
  */
 export declare function defaultFetch(env?: Record<string, string | undefined> | undefined, undiciVersion?: string | undefined): typeof fetch;
 export declare function buildUrl(base: string, path: string, query?: RequestOptions['query']): string;
+/**
+ * True when fetch failed because its connection closed or reset before a response arrived: what a request sees when
+ * it went out on a pooled connection the other side closed while it sat idle. Connect failures (ECONNREFUSED,
+ * ENOTFOUND, UND_ERR_CONNECT_TIMEOUT), timeouts and aborts are not this.
+ */
+export declare function isStaleConnectionError(err: unknown): boolean;
 /** `X-Tree-Revision` (file-first workspaces): a non-negative integer, else null. */
 export declare function treeRevisionOf(headers: Headers): number | null;
 /** Turns a non-2xx response into a ShardfluxApiError (or a protocol error for undocumented bodies). */

package/dist/http.js

@@ -4,18 +4,44 @@
  * retries only where a retry cannot duplicate an effect (safe methods,
  * requests carrying an Idempotency-Key, and read-only POSTs marked
  * `idempotent`). A retryable 429/502/503/504 (e.g. 503 `host_capacity`, 503
- * `wake_failed`) waits `Retry-After` (at most 5 s) before the retry.
+ * `wake_failed`) waits `Retry-After` (at most 5 s) before the retry. Such a request whose connection closed before
+ * any response (a pooled connection gone stale while idle) is sent again at once, once, outside `maxRetries`. The cell
+ * gateway's 429 `quota_exceeded` (every working slot of the plan is taken) is retried for any request (0.14.0+): it is
+ * refused at admission, so nothing ran.
  */
-import { ShardfluxApiError, ShardfluxProtocolError, apiError, isErrorBody } from "./errors.js";
+import { ShardfluxApiError, ShardfluxProtocolError, apiError, isErrorBody, isWorkingQuotaRefusal } from "./errors.js";
 import { describeFailure } from "./progress.js";
-export const SDK_VERSION = '0.12.0';
+export const SDK_VERSION = '0.13.1';
 export const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/**
+ * How long an idle pooled connection stays reusable (ms): 5 minutes, instead of undici's 4 s default, so the request
+ * after an agent's usual pause between tool calls (15 s to a few minutes) skips a new TCP and TLS handshake. Far below
+ * the 3600 s idle timeout of the load balancer in front of the API and the cell endpoints, so the server side never
+ * closes a connection for idleness while the client still treats it as reusable; below the AWS NAT gateway's 350 s idle
+ * timeout; and the same limit Chrome keeps for used idle sockets. Also caps a server's `Keep-Alive: timeout` hint.
+ */
+export const KEEP_ALIVE_TIMEOUT_MS = 300_000;
+/**
+ * TCP keepalive probes start after a pooled socket has been idle this long (ms), so NATs and firewalls with shorter
+ * idle timeouts than the pool's (Azure SNAT: 4 minutes) keep the connection instead of silently dropping it. Equal to
+ * undici's own default, set explicitly so it is visible and tested.
+ */
+export const TCP_KEEPALIVE_INITIAL_DELAY_MS = 60_000;
+/** The private pool's undici Agent options (exported for tests). */
+export const POOL_OPTIONS = {
+    allowH2: false,
+    pipelining: 1,
+    keepAliveTimeout: KEEP_ALIVE_TIMEOUT_MS,
+    keepAliveMaxTimeout: KEEP_ALIVE_TIMEOUT_MS,
+    connect: { keepAlive: true, keepAliveInitialDelay: TCP_KEEPALIVE_INITIAL_DELAY_MS },
+};
 /** One private HTTP/1.1 pool on Node 26+, shared by SDK clients. No global dispatcher changes. */
 let nodeFetch;
 /**
  * Reuses TLS connections. Node 26's bundled undici 8.9 can stall reused connections: use pinned undici with a
- * private HTTP/1.1-only dispatcher. Other runtimes retain native fetch. SHARDFLUX_HTTP_KEEPALIVE=0 forces close
- * for diagnosis; =1 retains the historical opt-in to native pooling. An explicitly supplied fetch is untouched.
+ * private HTTP/1.1-only dispatcher whose idle connections stay reusable for KEEP_ALIVE_TIMEOUT_MS. Other runtimes
+ * retain native fetch. SHARDFLUX_HTTP_KEEPALIVE=0 forces close for diagnosis; =1 retains the historical opt-in to
+ * native pooling. An explicitly supplied fetch is untouched. Idle pooled sockets do not keep the process alive.
  */
 export function defaultFetch(env = globalThis.process?.env, undiciVersion = globalThis.process?.versions?.undici) {
     // Why a fresh connection on Node 26 (its bundled HTTP client, measured): docs/progress/startup-latency.md.
@@ -30,7 +56,7 @@ export function defaultFetch(env = globalThis.process?.env, undiciVersion = glob
         return base;
     return async (input, init) => {
         nodeFetch ??= import('undici').then(({ Agent, fetch: pooledFetch }) => {
-            const dispatcher = new Agent({ allowH2: false, pipelining: 1 });
+            const dispatcher = new Agent({ ...POOL_OPTIONS, connect: { ...POOL_OPTIONS.connect } });
             return async (input, init) => {
                 const response = await pooledFetch(input, { ...init, dispatcher });
                 // Web-standard runtime shape; undici and DOM iterator declarations differ.
@@ -49,6 +75,25 @@ export function buildUrl(base, path, query) {
     }
     return u.toString();
 }
+/** Codes (on fetch's `cause` chain) of a connection that closed or reset before the response arrived. */
+const STALE_CONNECTION_CODES = new Set(['UND_ERR_SOCKET', 'ECONNRESET', 'EPIPE', 'UND_ERR_CLOSED']);
+/**
+ * True when fetch failed because its connection closed or reset before a response arrived: what a request sees when
+ * it went out on a pooled connection the other side closed while it sat idle. Connect failures (ECONNREFUSED,
+ * ENOTFOUND, UND_ERR_CONNECT_TIMEOUT), timeouts and aborts are not this.
+ */
+export function isStaleConnectionError(err) {
+    if (!(err instanceof TypeError))
+        return false;
+    const seen = new Set();
+    for (let c = err.cause; typeof c === 'object' && c !== null && !seen.has(c); c = c.cause) {
+        seen.add(c);
+        const code = c.code;
+        if (typeof code === 'string' && STALE_CONNECTION_CODES.has(code))
+            return true;
+    }
+    return false;
+}
 function retryAfterSeconds(res) {
     const v = res.headers.get('retry-after');
     if (v === null)
@@ -89,6 +134,9 @@ export class HttpClient {
         const safe = method === 'GET' || method === 'HEAD';
         const retriable = safe || init.idempotencyKey !== undefined || init.idempotent === true;
         const sleep = this.opts.sleep ?? defaultSleep;
+        // Retries counted against maxRetries (they also set the backoff); `attempt` counts every send.
+        let counted = 0;
+        let staleReplayed = false;
         for (let attempt = 0;; attempt += 1) {
             const headers = {
                 accept: init.accept ?? 'application/json',
@@ -108,7 +156,13 @@ export class HttpClient {
             }
             const timeoutMs = init.timeoutMs ?? this.opts.timeoutMs;
             const timeout = timeoutMs > 0 ? AbortSignal.timeout(timeoutMs) : undefined;
-            const signal = timeout && init.signal ? AbortSignal.any([timeout, init.signal]) : (timeout ?? init.signal);
+            const headersMs = init.headersTimeoutMs ?? 0;
+            const headersAbort = headersMs > 0 ? new AbortController() : undefined;
+            const headersTimer = headersAbort
+                ? setTimeout(() => headersAbort.abort(new DOMException('The response headers timed out.', 'TimeoutError')), headersMs)
+                : undefined;
+            const signals = [timeout, init.signal, headersAbort?.signal].filter((s) => s !== undefined);
+            const signal = signals.length > 1 ? AbortSignal.any(signals) : signals[0];
             let res;
             try {
                 res = await this.opts.fetch(url, { method, headers, ...(body === undefined ? {} : { body: body }), ...(signal ? { signal } : {}) });
@@ -116,13 +170,27 @@ export class HttpClient {
             catch (err) {
                 if (init.signal?.aborted)
                     throw err;
-                if (!retriable || attempt >= this.opts.maxRetries)
+                if (!retriable)
                     throw err;
-                const delayMs = Math.min(2_000, 200 * 2 ** attempt);
-                init.onRetry?.({ request: `${method} ${path}`, attempt: attempt + 1, cause: describeFailure(err, timeoutMs), delayMs });
+                if (!staleReplayed && !signal?.aborted && isStaleConnectionError(err)) {
+                    // The connection closed before any response, typically a pooled one the server or a NAT closed while idle:
+                    // send again at once on a fresh connection, once per request and outside maxRetries (as Go's net/http
+                    // replays a request that failed on a reused connection when it is idempotent or carries an Idempotency-Key).
+                    staleReplayed = true;
+                    init.onRetry?.({ request: `${method} ${path}`, attempt: attempt + 1, cause: describeFailure(err), delayMs: 0 });
+                    continue;
+                }
+                if (counted >= this.opts.maxRetries)
+                    throw err;
+                const delayMs = Math.min(2_000, 200 * 2 ** counted);
+                counted += 1;
+                init.onRetry?.({ request: `${method} ${path}`, attempt: attempt + 1, cause: describeFailure(err, headersAbort?.signal.aborted ? headersMs : timeoutMs), delayMs });
                 await sleep(delayMs);
                 continue;
             }
+            finally {
+                clearTimeout(headersTimer);
+            }
             if (res.ok) {
                 if (this.opts.onSuccess) {
                     try {
@@ -137,10 +205,15 @@ export class HttpClient {
             const error = await errorFrom(res, this.opts.source);
             const retryableStatus = res.status === 429 || res.status === 502 || res.status === 503 || res.status === 504;
             const retryableError = error instanceof ShardfluxApiError ? error.retryable && retryableStatus : retryableStatus;
-            if (!retriable || !retryableError || attempt >= this.opts.maxRetries)
+            // The working-at-once quota (cell 429 quota_exceeded): the gateway refuses the call at admission, before the
+            // workspace is touched, so nothing ran and even a non-idempotent request (exec start, stdin, PTY create,
+            // keepalive) is safe to send again. Same Retry-After wait and retry budget as every other retry.
+            const admissionRefusal = init.quotaRetry !== false && isWorkingQuotaRefusal(error);
+            if ((!retriable && !admissionRefusal) || !retryableError || counted >= this.opts.maxRetries)
                 throw error;
             const hinted = error instanceof ShardfluxApiError ? error.retryAfterSeconds : undefined;
-            const delayMs = Math.min(5_000, hinted !== undefined ? hinted * 1000 : 200 * 2 ** attempt);
+            const delayMs = Math.min(5_000, hinted !== undefined ? hinted * 1000 : 200 * 2 ** counted);
+            counted += 1;
             init.onRetry?.({ request: `${method} ${path}`, attempt: attempt + 1, cause: describeFailure(error), delayMs });
             await sleep(delayMs);
         }

package/dist/index.d.ts

@@ -15,10 +15,10 @@
 export type { components, operations, paths } from './generated/app-api.js';
 export type { components as CellComponents, paths as CellPaths } from './generated/cell-api.js';
 export { BillingApi, Shardflux, WorkspacesApi, fetchBillingCatalog, pickByKey } from './client.js';
-export type { AgentSession, BillingCatalog, BillingSubscription, Caps, CheckoutSession, DiskLayout, Entitlements, FindByKeyOptions, ForkTarget, Invoice, InvoicePage, LifetimeFilter, ListParams, Me, OpenParams, OpenResponse, Operation, Page, PortalSession, PurposeFilter, ResetWorkspaceBody, ResumeAnswer, ResumeRequestOptions, ResumeResponse, ShardfluxOptions, SuspendRequest, SuspendWhenIdleOptions, SuspendWhenIdleResponse, SuspendWhenIdleResult, UpdatePolicy, WaitOptions, WorkspaceInputs, WorkspaceLifetime, WorkspaceOrigin, WorkspacePurpose, WorkspaceView, } from './client.js';
-export type { FinishedOperation, LifecycleOptions, ResumeOptions, SuspendOptions, WaitedLifecycleOptions, WaitedResumeOptions, WaitedSuspendOptions } from './lifecycle.js';
-export { durabilityOf, formatTiming, isDurable, lostSuspendOf } from './progress.js';
-export type { Durability, LostSuspend, LifecycleAction, LifecyclePhase, LifecycleTiming, ProgressEvent, ProgressListener, RetryRecord, ServerTiming, TimingOutcome, TimingPhase, } from './progress.js';
+export type { AgentSession, AllocationMode, BillingCatalog, BillingSubscription, Caps, CheckoutSession, DiskLayout, Entitlements, FindByKeyOptions, ForkTarget, Invoice, InvoicePage, LifetimeFilter, ListParams, Me, OpenParams, OpenResponse, Operation, Page, PortalSession, PurposeFilter, ResetWorkspaceBody, ResumeAnswer, ResumeRequestOptions, ResumeResponse, ShardfluxOptions, SuspendRequest, SuspendWhenIdleOptions, SuspendWhenIdleResponse, SuspendWhenIdleResult, WaitOptions, WorkspaceInputs, WorkspaceLifetime, WorkspaceMemory, WorkspaceOrigin, WorkspacePurpose, WorkspaceView, } from './client.js';
+export type { FinishedOperation, ForkOptions, LifecycleOptions, ResumeOptions, SuspendOptions, WaitedForkOptions, WaitedLifecycleOptions, WaitedResumeOptions, WaitedSuspendOptions } from './lifecycle.js';
+export { durabilityOf, formatTiming, hostLostOf, isDurable, lostSuspendOf } from './progress.js';
+export type { ColdBootReason, Durability, HostLost, LostSuspend, LifecycleAction, LifecyclePhase, LifecycleTiming, ProgressEvent, ProgressListener, RetryRecord, ServerTiming, TimingOutcome, TimingPhase, } from './progress.js';
 export { FEEDBACK_CATEGORIES, FEEDBACK_MESSAGE_MAX_LENGTH } from './feedback.js';
 export type { AccountFeedbackParams, FeedbackCategory, FeedbackContext, FeedbackReceipt, SendFeedbackParams } from './feedback.js';
 export { UsageApi } from './usage.js';
@@ -28,7 +28,7 @@ export { TemplateFileError, packDirectory, parseTemplateText, readTemplateFile }
 export type { PackedFile, YamlParser } from './template-file.js';
 export { tarEnd, tarHeader, tarPadding } from './tar.js';
 export type { TarEntry, TarEntryType } from './tar.js';
-export type { BuildFromFileEvent, BuildFromFileOptions, BuildFromFileResult, BuildFromRecipeOptions, BuilderAvailability, CreateDraftBody, CreateDraftParams, CreateTemplateBuildParams, CreateTestInstanceBody, CreateVersionTestInstanceBody, CreateVersionTestInstanceParams, DraftOpened, LocalUpload, DraftState, OpenTestInstanceParams, OrgTemplateStorage, PublishDraftBody, PublishDraftParams, PutUploadOptions, SaveAsTemplateBody, SaveAsTemplateParams, SaveAsTemplateResponse, TemplateBuild, TemplateBuildLogUrl, TemplateBuildRecipeV2, TemplateBuildRegistrationState, TemplateBuildState, TemplateCategory, TemplateDefaults, TemplateDefaultsInput, TemplateDetail, TemplateDiffChange, TemplateDiffEntry, TemplateDiffPage, TemplateDiffParams, TemplateDraft, TemplateDraftSummary, TemplateEgressDefault, TemplateFileEntry, TemplateFilePage, TemplateFilesParams, TemplateFilesSummary, TemplateInput, TemplateLanguage, TemplateLanguages, TemplateOwner, TemplatePackage, TemplatePackageEcosystem, TemplatePackagePage, TemplateRecipe, TemplateRecipeV2, TemplateRecipeV2File, TemplateService, TemplateSettings, TemplateSettingsInput, TemplateSource, TemplateStartCommand, TemplateStorage, TemplateStorageWarning, TemplateSummary, TemplateUpload, TemplateUploadRequest, TemplateUploadResponse, TemplateUploadResult, TemplateVersion, TemplateVersionRecipe, TemplateVersionState, UploadData, WaitForBuildOptions, WorkspaceStartup, } from './templates.js';
+export type { BuildFromFileEvent, BuildFromFileOptions, BuildFromFileResult, BuildFromRecipeOptions, BuilderAvailability, CreateDraftBody, CreateDraftParams, CreateTemplateBuildParams, CreateTestInstanceBody, CreateVersionTestInstanceBody, CreateVersionTestInstanceParams, DraftOpened, LocalUpload, DraftState, OpenTestInstanceParams, OrgTemplateStorage, PublishDraftBody, PublishDraftParams, PutUploadOptions, SaveAsTemplateBody, SaveAsTemplateParams, SaveAsTemplateResponse, TemplateBuild, TemplateBuildLogUrl, TemplateBuildRecipeV2, TemplateBuildRegistrationState, TemplateBuildState, TemplateCategory, TemplateDefaults, TemplateDefaultsInput, TemplateDetail, TemplateDiffChange, TemplateDiffEntry, TemplateDiffPage, TemplateDiffParams, TemplateDraft, TemplateDraftSummary, TemplateEgressDefault, TemplateFileEntry, TemplateFilePage, TemplateFilesParams, TemplateFilesSummary, TemplateInput, TemplateLanguage, TemplateLanguages, TemplateOwner, TemplatePackage, TemplatePackageEcosystem, TemplatePackagePage, TemplateRecipe, TemplateRecipeV2, TemplateRecipeV2File, TemplateService, TemplateSettings, TemplateSettingsInput, TemplateSource, TemplateStartCommand, TemplateStorage, TemplateStorageWarning, TemplateSummary, TemplateUpload, TemplateUploadRequest, TemplateUploadResponse, TemplateUploadResult, TemplateVersion, TemplateVersionImmutable, TemplateVersionRecipe, TemplateVersionState, UploadData, WaitForBuildOptions, WorkspaceStartup, } from './templates.js';
 export { SecretsApi, WorkspaceSecrets } from './secrets.js';
 export type { BoundSecretStatus, CreateOrganizationSecretParams, CreateSecretParams, Secret, SecretAccessEvent, SecretPermissions, SecretScope, SecretVersion, UpdateSecretParams, WorkspaceSecretBindings, } from './secrets.js';
 export { EgressPolicyApi } from './egress.js';
@@ -42,7 +42,7 @@ export type { HintOptions, HintResult, WakeOptions } from './workspace.js';
 export { CellClient, DEFAULT_TRANSITION_TIMEOUT_MS, cellPath, ndjson } from './cell.js';
 export { EXECUTION_ID, ExecutionResult, newExecutionId } from './executions.js';
 export type { ExecutionChange, ExecutionError, ExecutionGetOptions, ExecutionResultBody, ExecutionRunOptions, ExecutionState } from './executions.js';
-export type { TreeRevisionOptions, BrowserContent, BrowserContentRequest, BrowserScreenshotRequest, CellClientOptions, ExecSession, ExecStartRequest, FileEdit, FileInfo, FileList, FilePatchEdit, FilePatchParams, FilePatchRequest, FilePatchResult, FileReadResult, FileRevision, FileSearchMatch, FileSearchOptions, FileSearchRequest, FileSearchResponse, FileSearchResult, FileWriteResult, GitResult, GitStatus, OutputEvent, ProcessList, PtyOpenRequest, PtySession, Residency, RunOptions, RunResult, ServedFrom, Signal, WakeHintResult, WorkspaceChange, WorkspaceChangeKind, WorkspaceChangesPage, WorkspaceChangesParams, WorkspaceChangesSummary, } from './cell.js';
+export type { TreeRevisionOptions, BrowserContent, BrowserContentRequest, BrowserScreenshotRequest, CellClientOptions, ExecSession, ExecStartRequest, MemoryGrow, BurstSummary, BurstError, FileEdit, FileInfo, FileList, FilePatchEdit, FilePatchParams, FilePatchRequest, FilePatchResult, FileReadResult, FileRevision, FileSearchMatch, FileSearchOptions, FileSearchRequest, FileSearchResponse, FileSearchResult, FileWriteResult, GitResult, GitStatus, OutputEvent, ProcessList, PtyOpenRequest, PtySession, Residency, RunOptions, RunResult, ServedFrom, Signal, WakeHintResult, WorkspaceChange, WorkspaceChangeKind, WorkspaceChangesPage, WorkspaceChangesParams, WorkspaceChangesSummary, } from './cell.js';
 export { ToolTokenManager } from './tokens.js';
 export type { ToolName, ToolToken, ToolTokenOptions } from './tokens.js';
 export { ToolArgumentError, executeToolCall, toAnthropicTools, toOpenAITools, validateArgs, workspaceTools } from './tools.js';

package/dist/index.js

@@ -1,5 +1,5 @@
 export { BillingApi, Shardflux, WorkspacesApi, fetchBillingCatalog, pickByKey } from "./client.js";
-export { durabilityOf, formatTiming, isDurable, lostSuspendOf } from "./progress.js";
+export { durabilityOf, formatTiming, hostLostOf, isDurable, lostSuspendOf } from "./progress.js";
 export { FEEDBACK_CATEGORIES, FEEDBACK_MESSAGE_MAX_LENGTH } from "./feedback.js";
 export { UsageApi } from "./usage.js";
 export { TemplateBuildTimeoutError, TemplateBuildsApi, TemplateDraftApi, TemplatePackagesApi, TemplateUploadError, TemplateUploadsApi, TemplateVersionTestInstancesApi, TemplateVersionsApi, TemplatesApi, buildSettled, saveAsTemplateBody, } from "./templates.js";

package/dist/lifecycle.d.ts

@@ -61,6 +61,11 @@ export interface ResumeOptions extends LifecycleOptions {
 export type WaitedResumeOptions = ResumeOptions & {
     wait: true | WaitOptions;
 };
+/** A waited fork (0.13.0+) returns the target view and token together when the API supports it. */
+export type ForkOptions = ResumeOptions;
+export type WaitedForkOptions = ForkOptions & {
+    wait: true | WaitOptions;
+};
 /** Internal: the trace a wait continues instead of starting its own (open(), wake() and waited lifecycle calls). */
 export declare const TRACE: unique symbol;
 /** Internal: work a workspace handle does after the operation finished, inside the same trace (refreshing its view). */
@@ -88,6 +93,7 @@ export declare function waitOptionsOf(opts: LifecycleOptions): WaitOptions | nul
  * Starts a lifecycle operation with `start` and, when `opts.wait` asks for it, waits for it to finish. The trace covers
  * the request, every observed state, and `AFTER_WAIT`; it ends with `done` either way.
  */
-export declare function runLifecycle(ctx: ClientContext, kind: LifecycleAction, workspaceId: string, start: (init: Pick<RequestOptions, 'onRetry'>) => Promise<Operation>, opts: InternalLifecycleOptions, capture?: {
+export declare function runLifecycle(ctx: ClientContext, kind: LifecycleAction, workspaceId: string, start: (init: Pick<RequestOptions, 'onRetry'>, trace: Trace) => Promise<Operation>, opts: InternalLifecycleOptions, capture?: {
     settle?: boolean;
+    requestReason?: string | null;
 }): Promise<Operation>;

package/dist/lifecycle.js

@@ -28,8 +28,8 @@ export async function runLifecycle(ctx, kind, workspaceId, start, opts, capture
         const pending = capture.settle ? ctx.captures.settle(workspaceId) : undefined;
         if (pending)
             await trace.span('capture_flush', () => pending);
-        trace.phase('request');
-        const operation = await start({ onRetry: trace.onRetry });
+        trace.phase('request', capture.requestReason ?? null);
+        const operation = await start({ onRetry: trace.onRetry }, trace);
         trace.observe(operation);
         if (!waitOpts)
             return operation;