@shardflux/sdk 0.12.0 → 0.13.1

package/dist/cell.js

@@ -1,7 +1,10 @@
-import { ExecStartError, NotSupportedForModeError, ShardfluxApiError, ShardfluxProtocolError } from "./errors.js";
+import { ExecStartError, NotSupportedForModeError, ShardfluxApiError, ShardfluxProtocolError, apiError, isErrorBody, isWorkingQuotaRefusal } from "./errors.js";
 import { EXECUTION_ID, ExecutionResult, newExecutionId } from "./executions.js";
 import { HttpClient, defaultSleep, randomId, treeRevisionOf } from "./http.js";
 import { describeFailure, emitTo } from "./progress.js";
+/** Error codes of a burst's own failure: the burst's terminal outcome, never a dropped stream. */
+const BURST_FAILURES = new Set(['burst_unavailable', 'burst_apply_failed']);
+const TERMINAL = Symbol('shardflux.terminal');
 /** Fills a path template that must exist in cell-api.yaml. */
 export function cellPath(template, params) {
     return template.replace(/\{([a-z_]+)\}/g, (_, name) => {
@@ -17,8 +20,20 @@ export const DEFAULT_TRANSITION_TIMEOUT_MS = 120_000;
  * capture writes recorded before them (read-your-writes). Workspace.cell() sets it; the capture's own client does not.
  */
 export const CAPTURE_BARRIER = Symbol('shardflux.captureBarrier');
-/** Wakes per call at most: a workspace that keeps being suspended again surfaces the refusal. */
+/**
+ * Wakes per call at most: a workspace that keeps being suspended again, or keeps refusing the call while the API reports
+ * it running, surfaces the refusal.
+ */
 const MAX_WAKES = 3;
+/** The pause before retrying a call refused again while the API reports the workspace running (from the second time). */
+const RUNNING_AGAIN_PAUSE_MS = 500;
+/** exec.cancel grace: the workspace waits 5 s without one and takes at most 60 s. */
+const DEFAULT_CANCEL_GRACE_MS = 5_000;
+const MAX_CANCEL_GRACE_MS = 60_000;
+/** The API error of a burst's recorded failure (`burst.error` of its session). Internal (the agent tools use it too). */
+export function burstFailure(e) {
+    return new ShardfluxApiError(409, { error: { code: e.code, message: e.message, request_id: '', retryable: e.retryable, ...(e.details ? { details: e.details } : {}) } }, 'cell');
+}
 const b64 = (bytes) => Buffer.from(typeof bytes === 'string' ? Buffer.from(bytes, 'utf8') : bytes).toString('base64');
 const unb64 = (s) => (s ? new Uint8Array(Buffer.from(s, 'base64')) : new Uint8Array());
 /** Parses an NDJSON byte stream into objects (tolerates CRLF and a final unterminated line). */
@@ -53,6 +68,30 @@ export async function* ndjson(body) {
         reader.releaseLock();
     }
 }
+/**
+ * A refused attach the gateway had to accept (cell-api.yaml "WebSocket close codes": an allowed Origin cannot read the
+ * HTTP answer to a failed upgrade): its only text frame is exactly the ErrorBody, then a close with 4000 + the HTTP
+ * status (4410 for not_found). A 4429 without a body (its close reason is `{"code","retryable",...}`) is the same
+ * refusal in short. Undefined for any other close.
+ */
+function attachRefusal(code, reason, body) {
+    const status = code === 4410 ? 404 : code !== undefined && code >= 4000 && code < 5000 ? code - 4000 : undefined;
+    if (body) {
+        const hinted = body.error.details?.retry_after_seconds;
+        return apiError(status ?? 502, body, 'cell', typeof hinted === 'number' && hinted >= 0 ? hinted : undefined);
+    }
+    if (code !== 4429)
+        return undefined;
+    let short = {};
+    try {
+        short = JSON.parse(reason ?? '');
+    }
+    catch {
+        // Not the compact JSON: the close code alone says it.
+    }
+    const errCode = typeof short.code === 'string' ? short.code : 'rate_limited';
+    return apiError(429, { error: { code: errCode, message: `The attach was refused (${errCode}); retry shortly.`, request_id: typeof short.request_id === 'string' ? short.request_id : '', retryable: short.retryable !== false } }, 'cell');
+}
 class ByteSink {
     #max;
     #chunks = [];
@@ -98,6 +137,10 @@ const executionRetryDelayMs = (failures, retryAfterSeconds) => retryAfterSeconds
 const attemptTimedOut = (err) => err instanceof Error && err.name === 'TimeoutError';
 /** A transport failure or a retryable 429/5xx: the execution request may be sent again with the same id. */
 function transientExecutionFailure(err) {
+    // The working-at-once refusal was already retried by the HTTP layer within the client's maxRetries: a second loop
+    // here would multiply that budget, so it surfaces.
+    if (isWorkingQuotaRefusal(err))
+        return null;
     if (err instanceof ShardfluxApiError) {
         const status = err.status === 429 || err.status === 502 || err.status === 503 || err.status === 504;
         return status && err.retryable ? { retryAfterSeconds: err.retryAfterSeconds } : null;
@@ -203,8 +246,10 @@ export class CellClient {
      * One authorized request. Refreshes the token once on stale_epoch / 401. Lifecycle transitions,
      * bounded in total by `transitionTimeoutMs`: a call refused with `workspace_busy` is retried after `Retry-After`; one
      * refused with `workspace_not_running` (or whose token cannot be minted because the workspace is not running) wakes
-     * the workspace through `wake`, given the time left, and is retried with a fresh token, at most 3 wakes per call.
-     * Refused calls were never executed, so retrying is safe. When the budget is spent the refusal surfaces.
+     * the workspace through `wake`, given the time left, and is retried with a fresh token, at most 3 wakes per call. A
+     * wake that finds the workspace already running (`false`) is retried the same way: the transition that refused the
+     * call ended meanwhile (0.13.1+). Refused calls were never executed, so retrying is safe. When the budget is spent
+     * the refusal surfaces.
      */
     async request(method, path, init = {}) {
         const closer = this.#closer.signal;
@@ -227,6 +272,7 @@ export class CellClient {
         const retry = (cause, delayMs, attempt) => emit?.({ type: 'retry', retry: { atMs: Math.round((performance.now() - t0) * 10) / 10, request: `${method} ${path}`, attempt, cause, delayMs } });
         let refreshed = false;
         let wakes = 0;
+        let runningAgain = 0;
         let attempts = 0;
         for (;;) {
             try {
@@ -260,8 +306,17 @@ export class CellClient {
                 if (notRunning && wakeAllowed && this.#wake && wakes < MAX_WAKES && left > 0) {
                     wakes += 1;
                     const before = this.tokens.current;
-                    if ((await this.#wake(left, signal)) === false)
-                        throw err; // running per the API: nothing to wait for
+                    if ((await this.#wake(left, signal)) === false) {
+                        // Running per the API: the refusal was answered from a transition that ended before the wake looked (a
+                        // resume or a move committed in between: the token request was refused while the workspace was resuming,
+                        // and the wake found it running). The refused call never ran, so it is retried with a current token. A
+                        // refusal that comes back while the API keeps reporting the workspace running is retried after a pause,
+                        // within the same wakes and time budget, then surfaces.
+                        const pause = runningAgain++ === 0 ? 0 : Math.max(0, Math.min(deadline - Date.now(), RUNNING_AGAIN_PAUSE_MS));
+                        retry(`${err.code === 'conflict' ? 'conflict workspace_not_running' : err.code} (the workspace runs; new token)`, pause, (attempts += 1));
+                        if (pause > 0)
+                            await this.#opts.sleep(pause);
+                    }
                     // A held resume handed this client a token of the woken workspace: use it. Otherwise the
                     // old token is of the previous epoch: fetch a new one.
                     if (this.tokens.current === before)
@@ -349,11 +404,18 @@ export class CellClient {
                 return Promise.reject(refusal);
             return this.#json('POST', this.#p('/v1/workspaces/{workspace_id}/exec/{session_id}/signal', { session_id: sessionId }), { json: { signal, only_leader: onlyLeader } });
         },
+        /**
+         * SIGTERM to the process group, SIGKILL after `graceMs` (0 to 60000; 0 or omitted is 5000). Resolves once the
+         * command has ended; the request waits for the grace.
+         */
         cancel: (sessionId, graceMs) => {
             const refusal = this.#noSessions('exec.cancel');
             if (refusal)
                 return Promise.reject(refusal);
-            return this.#json('POST', this.#p('/v1/workspaces/{workspace_id}/exec/{session_id}/cancel', { session_id: sessionId }), { json: graceMs === undefined ? {} : { grace_ms: graceMs } });
+            return this.#json('POST', this.#p('/v1/workspaces/{workspace_id}/exec/{session_id}/cancel', { session_id: sessionId }), {
+                json: graceMs === undefined ? {} : { grace_ms: graceMs },
+                timeoutMs: Math.max(this.#opts.timeoutMs, (graceMs || DEFAULT_CANCEL_GRACE_MS) + 15_000),
+            });
         },
         /**
          * Starts (or re-attaches to) a session and collects its output until it exits, reconnecting
@@ -382,21 +444,53 @@ export class CellClient {
                 req.kill_grace_ms = opts.killGraceMs;
             if (opts.secretRefs !== undefined)
                 req.secret_refs = opts.secretRefs;
-            const started = await this.exec.start(req, opts.signal);
-            if (started.state === 'failed_to_start')
-                throw new ExecStartError(started);
+            if (opts.burst !== undefined)
+                req.burst = opts.burst;
+            if (opts.burstVcpus !== undefined)
+                req.burst_vcpus = opts.burstVcpus;
+            if (opts.burstMemoryMib !== undefined)
+                req.burst_memory_mib = opts.burstMemoryMib;
+            const burst = opts.burst === 'always';
+            // One request starts the command and follows its output (0.13.0+). Each attempt's response headers are bounded
+            // like the old start request; the output stream then runs as long as the command. `initialAbort` ends that
+            // stream once it is drained.
+            const initialAbort = new AbortController();
+            const signal = opts.signal ? AbortSignal.any([opts.signal, initialAbort.signal]) : initialAbort.signal;
+            const res = await this.request('POST', this.#p('/v1/workspaces/{workspace_id}/exec'), {
+                json: req, accept: 'application/x-ndjson', timeoutMs: 0, headersTimeoutMs: this.#opts.timeoutMs, signal,
+            });
+            const streamed = res.headers.get('content-type')?.includes('application/x-ndjson') === true;
+            let started;
+            if (!streamed) {
+                // A cell without the combined start, and a start that failed, answer with the session JSON: start, then output.
+                const text = await res.text();
+                try {
+                    started = JSON.parse(text);
+                }
+                catch {
+                    throw new ShardfluxProtocolError('POST exec: response is not JSON', res.status, 'cell');
+                }
+                if (started.state === 'failed_to_start')
+                    throw started.burst?.error ? burstFailure(started.burst.error) : new ExecStartError(started);
+            }
             try {
-                return await this.#collect(sessionId, opts);
+                const r = await this.#collect(sessionId, opts, streamed ? { response: res, abort: initialAbort } : undefined);
+                // The start's session carries the grow (a combined stream carries it on its exit event); later reads may not.
+                const grow = started?.memory_grow ?? r.session.memory_grow;
+                return grow ? { ...r, memoryGrow: grow } : r;
             }
             catch (e) {
                 // Aborting the caller must stop the command too, not only our HTTP calls (MCP cancellation, timeouts):
-                // best-effort cancel (SIGTERM, SIGKILL after the grace), bounded so the abort stays prompt.
-                if (opts.signal?.aborted && opts.cancelOnAbort !== false) {
+                // best-effort cancel (SIGTERM, SIGKILL after the grace), bounded so the abort stays prompt. A burst cannot be
+                // canceled (it runs to its end or its timeout).
+                if (opts.signal?.aborted && opts.cancelOnAbort !== false && !burst) {
                     let timer;
                     const bound = new Promise((resolve) => {
                         timer = setTimeout(resolve, 5_000);
                     });
-                    await Promise.race([this.exec.cancel(sessionId, opts.killGraceMs).then(() => undefined, () => undefined), bound]);
+                    // killGraceMs goes up to 600000; a cancel takes at most 60000.
+                    const grace = opts.killGraceMs === undefined ? undefined : Math.min(opts.killGraceMs, MAX_CANCEL_GRACE_MS);
+                    await Promise.race([this.exec.cancel(sessionId, grace).then(() => undefined, () => undefined), bound]);
                     clearTimeout(timer);
                 }
                 throw e;
@@ -404,7 +498,7 @@ export class CellClient {
         },
     };
     /** exec.run's output loop: offsets, reconnects, exit. */
-    async #collect(sessionId, opts) {
+    async #collect(sessionId, opts, initial) {
         let session;
         const max = opts.maxOutputBytes ?? 1_048_576;
         const out = new ByteSink(max);
@@ -415,11 +509,15 @@ export class CellClient {
         const maxReconnects = opts.maxReconnects ?? 10;
         for (;;) {
             let exited;
-            const streamAbort = new AbortController();
+            const first = initial;
+            initial = undefined; // A dropped combined response reconnects only by GET, never re-starting the command.
+            const streamAbort = first?.abort ?? new AbortController();
             const signal = opts.signal ? AbortSignal.any([opts.signal, streamAbort.signal]) : streamAbort.signal;
             let drainTimer;
             try {
-                const events = await this.exec.output(sessionId, { stdoutOffset: so, stderrOffset: se, follow: true, signal });
+                if (first && !first.response.body)
+                    throw new ShardfluxProtocolError('exec output: empty body', first.response.status, 'cell');
+                const events = first ? ndjson(first.response.body) : await this.exec.output(sessionId, { stdoutOffset: so, stderrOffset: se, follow: true, signal });
                 for await (const ev of events) {
                     if (exited)
                         continue;
@@ -447,12 +545,15 @@ export class CellClient {
                         drainTimer = setTimeout(() => streamAbort.abort(), 250);
                     }
                     else if (ev.type === 'error' && ev.error) {
+                        // A burst's failure is its terminal outcome, not a dropped stream: never reconnect.
+                        if (BURST_FAILURES.has(ev.error.error.code))
+                            throw Object.assign(new ShardfluxApiError(409, ev.error, 'cell'), { [TERMINAL]: true });
                         throw new ShardfluxApiError(502, ev.error, 'cell');
                     }
                 }
             }
             catch (e) {
-                if (opts.signal?.aborted || this.closed)
+                if (opts.signal?.aborted || this.closed || (e instanceof ShardfluxApiError && TERMINAL in e))
                     throw e;
                 const transient = !(e instanceof ShardfluxApiError) || e.retryable;
                 if (!exited && (!transient || reconnects >= maxReconnects))
@@ -478,6 +579,9 @@ export class CellClient {
                 break;
             }
         }
+        // A burst that failed after its start answered (the session ended with burst.error).
+        if (session.burst?.error)
+            throw burstFailure(session.burst.error);
         // A start answered while the session was still starting can end without starting the command.
         if (session.state === 'failed_to_start')
             throw new ExecStartError(session);
@@ -494,6 +598,8 @@ export class CellClient {
             truncated: out.total > out.kept || err.total > err.kept,
             session,
             reconnects,
+            memoryGrow: null,
+            burst: session.burst ?? null,
         };
     }
     // ---- executions (file-first workspaces) -----------------------------
@@ -683,6 +789,7 @@ export class CellClient {
             let next = opts.offset ?? 0;
             let session = null;
             let exited = false;
+            let refusal;
             await new Promise((resolve, reject) => {
                 const onClose = () => finish(this.#closer.signal.reason instanceof Error ? this.#closer.signal.reason : new Error('closed'));
                 const ws = new WS(url, { headers: { authorization: `Bearer ${token}` } });
@@ -699,8 +806,10 @@ export class CellClient {
                     catch {
                         // already closed
                     }
-                    if (err)
-                        reject(err);
+                    // A refusal body whose close did not arrive within the read: the body alone says it.
+                    const failure = err ?? (refusal ? attachRefusal(undefined, undefined, refusal) : undefined);
+                    if (failure)
+                        reject(failure);
                     else
                         resolve();
                 };
@@ -714,6 +823,12 @@ export class CellClient {
                     if (typeof e.data !== 'string')
                         return;
                     const m = JSON.parse(e.data);
+                    // A refused attach (e.g. 4429 quota_exceeded: every working slot is taken): the bare ErrorBody, then the close.
+                    if (!('type' in m)) {
+                        if (isErrorBody(m))
+                            refusal = m;
+                        return;
+                    }
                     if (m.type === 'output' && m.data) {
                         const bytes = unb64(m.data);
                         sink.push(bytes);
@@ -730,7 +845,7 @@ export class CellClient {
                     }
                 });
                 ws.addEventListener('error', () => finish(new ShardfluxProtocolError('pty attach WebSocket failed', 0, 'cell')));
-                ws.addEventListener('close', () => finish());
+                ws.addEventListener('close', (e) => finish(attachRefusal(e.code, e.reason, refusal)));
                 if (this.closed)
                     onClose();
                 else
@@ -894,7 +1009,8 @@ export class CellClient {
         // Nothing of a file-first workspace sleeps: the cell would answer `resident`.
         if (this.mode === 'file_first')
             return Promise.resolve({ residency: 'resident' });
-        return this.#json('POST', this.#p('/v1/workspaces/{workspace_id}/wake-hint'), { wake: false, busy: false, timeoutMs: 10_000, ...(signal ? { signal } : {}) });
+        // A hint never waits: a 429 quota_exceeded (every working slot is taken) surfaces at once, like its other refusals.
+        return this.#json('POST', this.#p('/v1/workspaces/{workspace_id}/wake-hint'), { wake: false, busy: false, quotaRetry: false, timeoutMs: 10_000, ...(signal ? { signal } : {}) });
     }
     /** Read idle signals without recording activity or waking the workspace. */
     idle(signal) {

package/dist/client.d.ts

@@ -22,7 +22,7 @@ import type { SaveAsTemplateParams, SaveAsTemplateResponse } from './templates.j
 import { UsageApi } from './usage.js';
 import { VolumesApi } from './volumes.js';
 import type { VersionCheckOption } from './version-check.js';
-import type { FinishedOperation, InternalLifecycleOptions, LifecycleOptions, ResumeOptions, SuspendOptions, WaitedLifecycleOptions, WaitedResumeOptions, WaitedSuspendOptions } from './lifecycle.js';
+import type { FinishedOperation, ForkOptions, InternalLifecycleOptions, LifecycleOptions, ResumeOptions, SuspendOptions, WaitedForkOptions, WaitedLifecycleOptions, WaitedResumeOptions, WaitedSuspendOptions } from './lifecycle.js';
 import type { ProgressListener } from './progress.js';
 import { CaptureRegistry } from './capture.js';
 import type { FeedbackReceipt, SendFeedbackParams } from './feedback.js';
@@ -34,8 +34,6 @@ export type WorkspaceLifetime = components['schemas']['WorkspaceLifetime'];
 export type DiskLayout = components['schemas']['DiskLayout'];
 /** standard, template_draft (a template's dev-mode draft) or template_test (a test instance of a draft state). */
 export type WorkspacePurpose = components['schemas']['WorkspacePurpose'];
-/** Reserved (T2): always `pinned` in T1. */
-export type UpdatePolicy = components['schemas']['UpdatePolicy'];
 /** Where the workspace's disk came from: null, a fork, or a draft state (test instances). */
 export type WorkspaceOrigin = components['schemas']['WorkspaceOrigin'];
 export type ResetWorkspaceBody = components['schemas']['ResetWorkspaceBody'];
@@ -143,10 +141,35 @@ export interface ShardfluxOptions {
      */
     versionCheck?: VersionCheckOption;
 }
+/**
+ * How a workspace holds its memory (0.13.0): `fixed` boots `memory_mib` and holds it; `elastic` makes
+ * `memory_mib` a promise: the VM holds `memory_mib_held` while idle and the host grows it when a command needs it.
+ */
+export type AllocationMode = 'fixed' | 'elastic';
+/**
+ * Memory of a workspace (0.13.0): `promised_mib` (what it may grow to), `held_mib` (the idle floor;
+ * for a fixed workspace the same as the promise) and `plugged_mib` (plugged above the held floor on the live
+ * allocation, refreshed about every 60 s; null without one). `allocation_mode` is the running VM's, else the next start's.
+ */
+export type WorkspaceMemory = WorkspaceView['memory'];
 export interface Caps {
     cpu_millis?: number;
+    /** Memory in MiB; for an elastic workspace the promise (what it may grow to). */
     memory_mib?: number;
     disk_gib?: number;
+    /**
+     * `fixed` (default) or `elastic` (0.13.0). Given caps replace the stored ones: caps without it make
+     * the workspace fixed again; omitted caps keep the stored layout. Elastic needs the organization's entitlement, else
+     * ShardfluxApiError 422 `validation_failed` reason `allocation_mode_not_available` (nothing is created or changed);
+     * a file-first workspace gets `not_supported_for_mode`. A promise that does not exceed the held floor by at least
+     * 512 MiB is fixed at the promise (the returned `caps.allocation_mode` says which). Takes effect at the next VM start.
+     */
+    allocation_mode?: AllocationMode;
+    /**
+     * Elastic only (0.13.0): the memory held while idle, in MiB (at least 512, default 1024 server side, at most
+     * `memory_mib`). Without elastic: 422 reason `requires_elastic`; above `memory_mib`: 422 `exceeds_memory_mib`.
+     */
+    memory_mib_held?: number;
 }
 export interface ForkTarget {
     key: string;
@@ -420,13 +443,14 @@ export declare class WorkspacesApi {
     /**
      * Forks into a new key. `lifetime` is the fork's own (default persistent): forking a session is how it is kept.
      * Resolves when the fork is requested (the copy's handle is returned at once); with `wait`, once the copy exists,
-     * with its handle refreshed.
+     * with its handle ready. From 0.13.0, a supporting API returns the target view and final-epoch token together.
+     * `agentLabel`/`tools` choose that token; an older API ignoring Prefer falls back to polling and refreshing.
      */
-    fork(workspaceId: string, target: ForkTarget, opts: WaitedLifecycleOptions): Promise<{
+    fork(workspaceId: string, target: ForkTarget, opts: WaitedForkOptions): Promise<{
         operation: FinishedOperation;
         workspace: Workspace;
     }>;
-    fork(workspaceId: string, target: ForkTarget, opts?: LifecycleOptions): Promise<{
+    fork(workspaceId: string, target: ForkTarget, opts?: ForkOptions): Promise<{
         operation: Operation;
         workspace: Workspace;
     }>;

package/dist/client.js

@@ -543,17 +543,45 @@ export class WorkspacesApi {
     }
     async fork(workspaceId, target, opts = {}) {
         let copy;
-        const operation = await runLifecycle(this.#ctx(), 'fork', workspaceId, async (init) => {
-            const out = await this.#lifecycle('POST', `/v1/workspaces/${encodeURIComponent(workspaceId)}/fork`, target, opts.idempotencyKey, init);
-            copy = this.#wrap(out.workspace);
+        let heldReady = false;
+        const requestedWait = waitOptionsOf(opts);
+        const wait = requestedWait ? { ...requestedWait } : null;
+        const timeoutMs = wait?.timeoutMs ?? 300_000;
+        const started = Date.now();
+        const operation = await runLifecycle(this.#ctx(), 'fork', workspaceId, async (init, trace) => {
+            const waitS = wait && wait.serverWait !== false ? Math.min(SERVER_WAIT_MAX_S, Math.floor(timeoutMs / 1000)) : 0;
+            const body = { ...target };
+            if (waitS >= 1 && opts.agentLabel !== undefined)
+                body.agent_label = opts.agentLabel;
+            if (waitS >= 1 && opts.tools !== undefined)
+                body.tools = [...opts.tools];
+            const res = await this.#http.jsonWithHeaders('POST', `/v1/workspaces/${encodeURIComponent(workspaceId)}/fork`, {
+                ...init, json: body, idempotencyKey: opts.idempotencyKey ?? randomId('op-'),
+                ...(wait?.signal ? { signal: wait.signal } : {}),
+                ...(waitS >= 1 ? { headers: { prefer: `wait=${waitS}` }, timeoutMs: Math.max(this.#http.opts.timeoutMs, waitS * 1000 + 10_000) } : {}),
+            }, this.#auth);
+            const out = res.body;
+            if (!out?.workspace || !out.operation)
+                throw new ShardfluxProtocolError('fork: response has no workspace or operation', res.status, 'api');
+            heldReady = waitS >= 1 && res.status === 200 && /\bwait\s*=/i.test(res.headers.get('preference-applied') ?? '');
+            if (heldReady && (out.operation.state !== 'succeeded' || out.workspace.observed_state !== 'running')) {
+                throw new ShardfluxProtocolError('fork: held response is not a succeeded running target', res.status, 'api');
+            }
+            const token = heldReady ? out.tool_token ?? null : null;
+            this.#noteToken(token);
+            copy = this.#wrap(out.workspace, { agentLabel: opts.agentLabel, tools: opts.tools, token, trace });
+            if (wait && waitS >= 1)
+                wait.timeoutMs = Math.max(1, timeoutMs - (Date.now() - started));
             return out.operation;
         }, {
             ...opts,
+            ...(wait ? { wait } : {}),
             [AFTER_WAIT]: async (trace, op) => {
-                await trace.span('view', () => copy.refresh());
+                if (!heldReady)
+                    await trace.span('view', () => copy.refresh());
                 await opts[AFTER_WAIT]?.(trace, op);
             },
-        }, { settle: true });
+        }, { settle: true, requestReason: wait && wait.serverWait !== false && timeoutMs >= 1000 ? 'held' : null });
         return { operation, workspace: copy };
     }
     /** The workspace's text inputs `{NAME: value}` (0.7.0). Secret inputs are bound secrets, never listed. */

package/dist/errors.d.ts

@@ -11,7 +11,7 @@ export type ErrorCode = AppErrorCode | CellErrorCode;
  * added: 409 conflict legacy_disk_layout, not_session, session_lifetime, lifetime_mismatch,
  * not_resettable, template_not_layered, draft_exists, draft_stale, build_in_progress, file_list_unavailable,
  * file_list_indexing (retryable), guest_feature_unavailable; 422 validation_failed confirm_destructive_required,
- * reserved_key_prefix, invalid_defaults, update_policy_not_available, invalid_path, too_many_acknowledged_findings;
+ * reserved_key_prefix, invalid_defaults, invalid_path, too_many_acknowledged_findings;
  * 403 forbidden template_dev_mode_role; 404 not_found draft_not_found, version_not_found, path_not_found.
  * The template editor (0.7.0) added: 422 validation_failed invalid_recipe, base_not_layered,
  * language_unavailable, language_conflict, invalid_package, too_many_files, platform_owned_path, upload_required,
@@ -51,8 +51,38 @@ export type ErrorCode = AppErrorCode | CellErrorCode;
  * overage_unavailable, spend_cap_required, spend_cap_below_minimum (details.min_minor), spend_cap_above_plan_price
  * (details.max_minor) and spend_cap_below_charges (details.charges_minor); with `ifMatch`, 409 conflict
  * version_mismatch (details.current_version), as egress puts with `ifMatch` answer too.
+ * Elastic memory (0.13.0): 422 validation_failed allocation_mode_not_available (details.field
+ * `caps.allocation_mode`; the organization does not have elastic memory, nothing is created or changed),
+ * not_supported_for_mode (elastic with file_first), requires_elastic and exceeds_memory_mib (details.field
+ * `caps.memory_mib_held`).
+ * Burst execution (0.13.0): the codes `burst_unavailable` (409; details.reason
+ * not_available, layout_unsupported, shared_volumes, host_capacity, fence_not_drained, workspace_fenced, apply_pending,
+ * park_failed, workspace_resumed, interrupted; details.replayed on a journaled failure answered again) and
+ * `burst_apply_failed` (409; details.reason disk_full, apply_failed, reverted, revert_failed with
+ * details.applied_entries and details.pending_entries: retry with the same session id to finish an apply_failed one);
+ * 422 validation_failed burst_mode_not_supported (`auto`), burst_not_supported (with stdin; also 409 conflict for
+ * signal/cancel of a burst session) and burst_size_exceeds_plan (details.field, details.limit); a burst session nobody
+ * followed any more ends with burst.error service_unavailable burst_lost.
+ * Immutable paths (0.13.0): 422 validation_failed immutable_path_removed (a recipe drops a path of the template's open
+ * version: details.removed, details.open_version; a version keeps every immutable path), immutable_paths_unsupported_base
+ * (the base's guest agent lacks the feature, or a Dockerfile build of a template with immutable paths: details.base
+ * {name, version}, details.required_feature, details.paths) and invalid_path at details.field `recipe.immutable[<i>]`;
+ * 409 conflict read_only_path (a files write under an immutable path, from the cell gateway). The reason
+ * `update_policy_not_available` is gone with the update policy. A build that fails on them carries `failure.code`
+ * immutable_path_missing (details.path) or immutable_image_too_large.
+ * Host loss (0.13.1; the machine a workspace ran on failed, and the workspace restores itself on its next use): the
+ * operation error `workspace_storage_unavailable` carries details.reason host_lost (and details.workspace_state
+ * `suspended`) when neither the workspace's disk nor a checkpoint could be restored (not retryable). A fork or snapshot
+ * of such a workspace before its resume fails with the operation error `resume_required` (details.reason host_lost;
+ * not retryable): resume the workspace first.
+ * Workspaces working at once (0.14.0): the cell gateway answers a tool call that finds every working slot of the plan
+ * taken with 429 `quota_exceeded` (retryable, Retry-After; details.limit `concurrent_workspaces`, limit_value, current,
+ * retry_after_seconds); the call did not run and the SDK retries it like any transient refusal. The API's 403
+ * `quota_exceeded` (not retryable) names details.limit `concurrent_workspaces` on open, resume and fork, and
+ * `retained_state` (details.limit_value and details.current in GiB) when opening a new key or forking with the plan's
+ * Retained state used up. `details.limit` is not a reason: these are not in this union.
  */
-export type KnownErrorReason = 'invalid_recipe' | 'base_not_layered' | 'language_unavailable' | 'language_conflict' | 'invalid_package' | 'too_many_files' | 'platform_owned_path' | 'upload_required' | 'upload_missing' | 'upload_digest_mismatch' | 'upload_too_large' | 'extra_hosts_without_auto' | 'invalid_settings' | 'services_unsupported' | 'input_required' | 'input_unknown' | 'input_invalid' | 'egress_widening' | 'reserved_session_id' | 'env_collision' | 'reserved_template_slug' | 'package_index_unavailable' | 'package_not_found' | 'startup_failed' | 'service_not_ready' | 'secrets_unavailable' | 'workspace_not_running' | 'operation_in_progress' | 'workspace_deleted' | 'secret_not_available' | 'legacy_disk_layout' | 'not_session' | 'session_lifetime' | 'lifetime_mismatch' | 'not_resettable' | 'template_not_layered' | 'draft_exists' | 'draft_stale' | 'build_in_progress' | 'file_list_unavailable' | 'file_list_indexing' | 'guest_feature_unavailable' | 'confirm_destructive_required' | 'reserved_key_prefix' | 'invalid_defaults' | 'update_policy_not_available' | 'invalid_path' | 'too_many_acknowledged_findings' | 'template_dev_mode_role' | 'draft_not_found' | 'version_not_found' | 'path_not_found' | 'revision_mismatch' | 'edit_not_found' | 'edit_ambiguous' | 'edit_not_text' | 'patch_invalid' | 'host_capacity' | 'wake_failed' | 'workspace_fenced' | 'offline_unavailable' | 'offline_budget' | 'offline_changed' | 'host_feature_unavailable' | 'not_supported_for_mode' | 'mode_mismatch' | 'mode_not_available' | 'layout_unsupported' | 'tree_revision_mismatch' | 'outside_tree_root' | 'execution_in_progress' | 'execution_id_reused' | 'operation_id_reused' | 'no_execution_host' | 'lease_expired' | 'host_unreachable' | 'host_restarted' | 'tree_moved' | 'blob_missing' | 'blob_corrupt' | 'exec_failed_to_start' | 'invalid_cwd' | 'allowance_used' | 'overage_paused' | 'spend_cap_reached' | 'overage_unavailable' | 'spend_cap_required' | 'spend_cap_below_minimum' | 'spend_cap_above_plan_price' | 'spend_cap_below_charges' | 'version_mismatch';
+export type KnownErrorReason = 'invalid_recipe' | 'base_not_layered' | 'language_unavailable' | 'language_conflict' | 'invalid_package' | 'too_many_files' | 'platform_owned_path' | 'upload_required' | 'upload_missing' | 'upload_digest_mismatch' | 'upload_too_large' | 'extra_hosts_without_auto' | 'invalid_settings' | 'services_unsupported' | 'input_required' | 'input_unknown' | 'input_invalid' | 'egress_widening' | 'reserved_session_id' | 'env_collision' | 'reserved_template_slug' | 'package_index_unavailable' | 'package_not_found' | 'startup_failed' | 'service_not_ready' | 'secrets_unavailable' | 'workspace_not_running' | 'operation_in_progress' | 'workspace_deleted' | 'secret_not_available' | 'legacy_disk_layout' | 'not_session' | 'session_lifetime' | 'lifetime_mismatch' | 'not_resettable' | 'template_not_layered' | 'draft_exists' | 'draft_stale' | 'build_in_progress' | 'file_list_unavailable' | 'file_list_indexing' | 'guest_feature_unavailable' | 'confirm_destructive_required' | 'reserved_key_prefix' | 'invalid_defaults' | 'invalid_path' | 'too_many_acknowledged_findings' | 'template_dev_mode_role' | 'draft_not_found' | 'version_not_found' | 'path_not_found' | 'revision_mismatch' | 'edit_not_found' | 'edit_ambiguous' | 'edit_not_text' | 'patch_invalid' | 'host_capacity' | 'wake_failed' | 'workspace_fenced' | 'offline_unavailable' | 'offline_budget' | 'offline_changed' | 'host_feature_unavailable' | 'not_supported_for_mode' | 'mode_mismatch' | 'mode_not_available' | 'layout_unsupported' | 'tree_revision_mismatch' | 'outside_tree_root' | 'execution_in_progress' | 'execution_id_reused' | 'operation_id_reused' | 'no_execution_host' | 'lease_expired' | 'host_unreachable' | 'host_restarted' | 'tree_moved' | 'blob_missing' | 'blob_corrupt' | 'exec_failed_to_start' | 'invalid_cwd' | 'allowance_used' | 'overage_paused' | 'spend_cap_reached' | 'overage_unavailable' | 'spend_cap_required' | 'spend_cap_below_minimum' | 'spend_cap_above_plan_price' | 'spend_cap_below_charges' | 'version_mismatch' | 'allocation_mode_not_available' | 'requires_elastic' | 'exceeds_memory_mib' | 'burst_mode_not_supported' | 'burst_not_supported' | 'burst_size_exceeds_plan' | 'not_available' | 'shared_volumes' | 'fence_not_drained' | 'apply_pending' | 'park_failed' | 'workspace_resumed' | 'interrupted' | 'burst_lost' | 'disk_full' | 'apply_failed' | 'reverted' | 'revert_failed' | 'immutable_path_removed' | 'immutable_paths_unsupported_base' | 'read_only_path' | 'host_lost';
 /** A known reason, or any other string the server sends (reasons are open-ended). */
 export type ErrorReason = KnownErrorReason | (string & {});
 export interface ErrorBodyLike {
@@ -87,6 +117,13 @@ export declare class ShardfluxApiError extends Error {
     readonly treeRevision: number | undefined;
     constructor(status: number, body: ErrorBodyLike, source: 'api' | 'cell', retryAfterSeconds?: number, treeRevision?: number);
 }
+/**
+ * The cell gateway's working-at-once refusal (0.14.0+): 429 `quota_exceeded`, retryable, with Retry-After. The gateway
+ * refuses the call at admission, before the workspace is touched, so nothing ran and any request (exec start, stdin,
+ * PTY create, keepalive included) may be sent again. The API's 403 `quota_exceeded` is a different refusal (not
+ * retryable) and never matches.
+ */
+export declare function isWorkingQuotaRefusal(err: unknown): err is ShardfluxApiError;
 /** processful (the default: one VM keeps processes, memory and files) or file_first. */
 export type WorkspaceMode = AppComponents['schemas']['WorkspaceMode'];
 /**
@@ -178,6 +215,7 @@ export declare class OperationTimeoutError extends Error {
  * The operation reached `failed` or `canceled`. A suspend-when-idle that found the workspace active (a tool call after
  * the request, an attached stream) is `canceled` with `errorCode` `workspace_active` (`workspaceActive` true, 0.12.0+):
  * nothing changed and the workspace keeps running. `waitUntilReady()` and `wake()` treat it as running, not as a failure.
+ * `resume_required` (0.13.1+): a fork or snapshot of a workspace whose machine failed; resume it, then call again.
  */
 export declare class OperationFailedError extends Error {
     readonly operation: Operation;

package/dist/errors.js

@@ -40,6 +40,15 @@ export class ShardfluxApiError extends Error {
         this.reason = typeof reason === 'string' ? reason : undefined;
     }
 }
+/**
+ * The cell gateway's working-at-once refusal (0.14.0+): 429 `quota_exceeded`, retryable, with Retry-After. The gateway
+ * refuses the call at admission, before the workspace is touched, so nothing ran and any request (exec start, stdin,
+ * PTY create, keepalive included) may be sent again. The API's 403 `quota_exceeded` is a different refusal (not
+ * retryable) and never matches.
+ */
+export function isWorkingQuotaRefusal(err) {
+    return err instanceof ShardfluxApiError && err.source === 'cell' && err.status === 429 && err.code === 'quota_exceeded' && err.retryable;
+}
 /**
  * The call does not exist for the workspace's mode: 409 `conflict` with details.reason
  * `not_supported_for_mode`, `details.mode` (the workspace's mode) and `details.operation`. File-first workspaces have no
@@ -187,6 +196,7 @@ export class OperationTimeoutError extends Error {
  * The operation reached `failed` or `canceled`. A suspend-when-idle that found the workspace active (a tool call after
  * the request, an attached stream) is `canceled` with `errorCode` `workspace_active` (`workspaceActive` true, 0.12.0+):
  * nothing changed and the workspace keeps running. `waitUntilReady()` and `wake()` treat it as running, not as a failure.
+ * `resume_required` (0.13.1+): a fork or snapshot of a workspace whose machine failed; resume it, then call again.
  */
 export class OperationFailedError extends Error {
     operation;