@shardflux/cli 0.6.2 → 0.8.0

package/dist/commands.js

@@ -8,15 +8,15 @@
 import { randomUUID } from 'node:crypto';
 import { access, readFile, writeFile } from 'node:fs/promises';
 import { basename, dirname, resolve as resolvePath } from 'node:path';
-import { EXECUTION_ID, FEEDBACK_CATEGORIES, FEEDBACK_MESSAGE_MAX_LENGTH, NotSupportedForModeError, SDK_VERSION, ShardfluxApiError, newExecutionId } from '@shardflux/sdk';
+import { EXECUTION_ID, ExecStartError, FEEDBACK_CATEGORIES, FEEDBACK_MESSAGE_MAX_LENGTH, NotSupportedForModeError, SDK_VERSION, ShardfluxApiError, ShardfluxProtocolError, newExecutionId } from '@shardflux/sdk';
 import { parse as parseYaml } from 'yaml';
 import { CLI_VERSION, SECRET_NAME, UUID, parseDuration, parseEnvPairs, parsePositiveInt } from "./args.js";
-import { ExecutionFailedError, ExecutionPendingError, InterruptedError, NotFoundError, UsageError } from "./errors.js";
+import { ExecSessionRunningError, ExecutionFailedError, ExecutionPendingError, InterruptedError, NotFoundError, UsageError } from "./errors.js";
 import { updateCheckDisabled } from "./update-check.js";
 import { initTemplateYaml, recipeYaml } from "./template-yaml.js";
 import { BuildTimer, buildTimingJson, buildTimingText, progressLine } from "./timing.js";
 import { timingJson } from "./timing.js";
-import { bindingsText, buildTable, buildText, changesTable, diffSummaryText, diffTable, draftText, draftStateTable, executionJson, executionSummary, fileInfoText, fileTable, languageTable, json, kv, modeText, operationDetail, pairs, operationTable, packageTable, searchText, secretAccessTable, secretDetail, secretTable, secretVersionTable, sessionTable, suspendRequestText, suspendWhenIdleText, templateDetailText, templateEntryText, templateFileTable, templateTable, usageEstimateText, usageSeriesText, usageText, workspaceDetail, workspaceTable, } from "./format.js";
+import { bindingsText, buildTable, buildText, changesTable, diffSummaryText, diffTable, draftText, draftStateTable, executionJson, executionSummary, fileInfoText, fileTable, languageTable, json, kv, memoryGrowText, burstText, modeText, operationDetail, pairs, operationTable, packageTable, searchText, secretAccessTable, secretDetail, secretTable, secretVersionTable, sessionTable, durableText, suspendRequestText, suspendWhenIdleText, templateDetailText, templateEntryText, templateFileTable, templateTable, usageEstimateText, usageSeriesText, usageText, workspaceDetail, workspaceTable, } from "./format.js";
 const DEFAULT_WAIT_MS = 300_000;
 const MAX_JSON_OUTPUT = 16 * 1024 * 1024;
 const MAX_STDIN = 1024 * 1024;
@@ -66,6 +66,47 @@ function caps(values) {
         c.disk_gib = parsePositiveInt(disk, 'disk-gib', 1_000_000);
     return Object.keys(c).length ? c : undefined;
 }
+const ALLOCATION_MODES = ['fixed', 'elastic'];
+/**
+ * The caps of `workspaces open` and `workspaces fork` (0.8.0): the resource caps plus the memory allocation. Given caps
+ * replace the stored ones, so they are sent only when a flag was given.
+ */
+function workspaceCaps(values) {
+    const c = caps(values) ?? {};
+    const mode = oneOf(values, 'allocation-mode', ALLOCATION_MODES);
+    const held = str(values['memory-held']);
+    if (held !== undefined && mode !== 'elastic')
+        throw new UsageError('--memory-held needs --allocation-mode elastic (a fixed workspace holds all of --memory-mib)');
+    if (mode !== undefined)
+        c.allocation_mode = mode;
+    if (held !== undefined)
+        c.memory_mib_held = parsePositiveInt(held, 'memory-held', 16_777_216);
+    return Object.keys(c).length ? c : undefined;
+}
+/**
+ * `--burst [--burst-vcpus N] [--burst-memory MIB]` of `workspaces exec` (0.8.0): the exec.run options,
+ * or undefined without --burst.
+ */
+function burstOptions(values, withStdin) {
+    const vcpus = str(values['burst-vcpus']);
+    const mem = str(values['burst-memory']);
+    if (values.burst !== true) {
+        if (vcpus !== undefined || mem !== undefined)
+            throw new UsageError('--burst-vcpus and --burst-memory need --burst');
+        return undefined;
+    }
+    if (withStdin)
+        throw new UsageError('--burst runs the command to completion without stdin; drop --stdin');
+    const o = { burst: 'always' };
+    if (vcpus !== undefined)
+        o.burstVcpus = parsePositiveInt(vcpus, 'burst-vcpus', 32);
+    if (mem !== undefined) {
+        o.burstMemoryMib = parsePositiveInt(mem, 'burst-memory', 65_536);
+        if (o.burstMemoryMib < 512)
+            throw new UsageError('--burst-memory must be between 512 and 65536');
+    }
+    return o;
+}
 /** `--when-idle <duration>`: whole seconds from 0 (as soon as it is idle; 0.6.1) to 3600 (the API's after_seconds bounds). */
 export function whenIdleSeconds(text) {
     if (/^0+(?:\.0+)?(?:ms|s|m|h)?$/.test(text.trim()))
@@ -414,6 +455,75 @@ export function execExitCode(r) {
         return 128 + r.termSignal;
     return 1;
 }
+/** The longest cancel grace the workspace applies (it waits 5 s without one). */
+const MAX_CANCEL_GRACE_MS = 60_000;
+/** Error codes of a burst's own failure (an output stream's final error event; the session carries it too). */
+const BURST_FAILURES = new Set(['burst_unavailable', 'burst_apply_failed']);
+/**
+ * `shard ws exec-read` (0.8.0): an exec session's output from offset 0, and the session. Without `follow`, the output
+ * that exists now; with it, until the session ends, resuming from the byte offsets read after a dropped stream.
+ * `maxBytes` of each stream are kept for the result (the rest is counted); `onOutput` gets every new chunk.
+ */
+async function readSession(cell, sessionId, opts) {
+    let session = await cell.exec.get(sessionId);
+    const kept = { stdout: [], stderr: [] };
+    const size = { stdout: 0, stderr: 0 };
+    let total = { stdout: 0, stderr: 0 };
+    let reconnects = 0;
+    const live = (s) => s.state === 'starting' || s.state === 'running';
+    for (;;) {
+        let ended;
+        try {
+            const events = await cell.exec.output(sessionId, { stdoutOffset: total.stdout, stderrOffset: total.stderr, follow: opts.follow && live(session), signal: opts.signal });
+            for await (const ev of events) {
+                if (ev.type === 'output' && ev.data !== undefined) {
+                    const stream = ev.stream === 'stderr' ? 'stderr' : 'stdout';
+                    const bytes = Buffer.from(ev.data, 'base64');
+                    const start = ev.offset ?? total[stream];
+                    // A reconnect may resend bytes already printed.
+                    const fresh = start < total[stream] ? bytes.subarray(Math.min(total[stream] - start, bytes.length)) : bytes;
+                    if (fresh.length === 0)
+                        continue;
+                    total = { ...total, [stream]: Math.max(total[stream], start + bytes.length) };
+                    const room = opts.maxBytes - size[stream];
+                    if (room > 0) {
+                        kept[stream].push(fresh.subarray(0, room));
+                        size[stream] += Math.min(room, fresh.length);
+                    }
+                    opts.onOutput?.(stream, fresh);
+                }
+                else if (ev.type === 'exit') {
+                    ended = ev.session ?? (await cell.exec.get(sessionId));
+                    break;
+                }
+                else if (ev.type === 'error' && ev.error && !BURST_FAILURES.has(ev.error.error.code)) {
+                    throw new ShardfluxApiError(502, ev.error, 'cell');
+                }
+            }
+        }
+        catch (err) {
+            if (opts.signal.aborted)
+                throw err;
+            const transient = !(err instanceof ShardfluxApiError) || err.retryable;
+            if (!opts.follow || !transient || reconnects >= 10)
+                throw err;
+        }
+        if (ended)
+            session = ended;
+        else if (opts.follow) {
+            // The stream ended without the exit (a dropped connection, a restarted gateway): continue from the offsets.
+            reconnects += 1;
+            if (reconnects > 10)
+                throw new ShardfluxProtocolError(`exec ${sessionId}: output stream kept dropping`, 0, 'cell');
+            await new Promise((r) => setTimeout(r, Math.min(2_000, 100 * 2 ** reconnects)).unref());
+            session = await cell.exec.get(sessionId);
+            if (live(session) || total.stdout < session.stdout_size || total.stderr < session.stderr_size)
+                continue;
+        }
+        const text = (b) => new TextDecoder().decode(Buffer.concat(b));
+        return { session, stdout: text(kept.stdout), stderr: text(kept.stderr), truncated: total.stdout > size.stdout || total.stderr > size.stderr, reconnects };
+    }
+}
 async function bestEffortCancel(cell, sessionId) {
     await Promise.race([cell.exec.cancel(sessionId).catch(() => undefined), new Promise((r) => setTimeout(r, 3_000).unref())]);
 }
@@ -496,7 +606,7 @@ function reportExecution(ctx, ref, r, opts) {
 }
 function lifecycleText(verb, w, op) {
     const finished = op.state === 'succeeded' || op.state === 'failed' || op.state === 'canceled';
-    return `${verb} ${w.key} (${w.id}): operation ${op.id} ${op.kind} ${op.state}${op.state_reason ? ` (${op.state_reason})` : ''}\n${finished ? '' : `wait with: shard operations wait ${op.id}\n`}`;
+    return `${verb} ${w.key !== undefined ? `${w.key} (${w.id})` : w.id}: operation ${op.id} ${op.kind} ${op.state}${op.state_reason ? ` (${op.state_reason})` : ''}\n${finished ? durableText(op) : `wait with: shard operations wait ${op.id}\n`}`;
 }
 /** The project-key commands; handlers.ts adds the account plane's. */
 export const KEY_HANDLERS = {
@@ -565,11 +675,119 @@ export const KEY_HANDLERS = {
         out(ctx, usageText(summary), summary);
         return 0;
     },
+    async 'workspaces labels'(ctx, values, [ref]) {
+        const ws = await resolveWorkspace(ctx, ref);
+        const labels = labelMap(str(values.set));
+        if (labels !== undefined)
+            await ws.setLabels(labels);
+        out(ctx, JSON.stringify(ws.labels, null, 2) + '\n', ws.labels);
+        return 0;
+    },
+    async 'workspaces idle'(ctx, values, [ref]) {
+        const ws = await resolveWorkspace(ctx, ref);
+        const policy = str(values.policy);
+        const keepalive = str(values.keepalive);
+        if (policy !== undefined && keepalive !== undefined)
+            throw new UsageError('--policy and --keepalive cannot be combined');
+        let result;
+        if (policy !== undefined)
+            result = (await ws.setIdlePolicy(policy === 'default' ? null : policy)).data;
+        else if (keepalive !== undefined)
+            result = await ws.keepalive(parsePositiveInt(keepalive, 'keepalive', 86400));
+        else
+            result = await ws.idle(ctx.signal);
+        out(ctx, JSON.stringify(result, null, 2) + '\n', result);
+        return 0;
+    },
+    async 'workspaces exec-start'(ctx, values, [ref, ...argv]) {
+        const timeout = str(values.timeout);
+        const timeoutMs = timeout === undefined ? undefined : parseDuration(timeout, 'timeout');
+        const ws = await resolveWorkspace(ctx, ref);
+        const session = await cellOf(ctx, ws).exec.start({ argv, stdin_open: values['stdin-open'] === true,
+            ...(str(values['session-id']) ? { session_id: str(values['session-id']) } : {}), ...(str(values.cwd) ? { cwd: str(values.cwd) } : {}),
+            ...(timeoutMs !== undefined ? { timeout_ms: timeoutMs } : {}) }, ctx.signal);
+        out(ctx, JSON.stringify(session, null, 2) + '\n', session);
+        // 0.8.0: what to do next (stderr, so stdout stays the session's JSON).
+        if (!ctx.json && session.state !== 'failed_to_start')
+            ctx.io.stderr.write(`read it with: shard ws exec-read ${ref} ${session.session_id} --follow; stop it with: shard ws exec-cancel ${ref} ${session.session_id}\n`);
+        return 0;
+    },
+    async 'workspaces exec-read'(ctx, values, [ref, sessionId]) {
+        const ws = await resolveWorkspace(ctx, ref);
+        const cell = cellOf(ctx, ws);
+        const follow = values.follow === true;
+        let r;
+        try {
+            r = await readSession(cell, sessionId, {
+                follow,
+                signal: ctx.signal,
+                maxBytes: ctx.json ? MAX_JSON_OUTPUT : 0,
+                ...(ctx.json ? {} : { onOutput: (stream, chunk) => void (stream === 'stderr' ? ctx.io.stderr : ctx.io.stdout).write(chunk) }),
+            });
+        }
+        catch (err) {
+            if (ctx.signal.aborted && follow) {
+                throw new InterruptedError(`Stopped following exec session ${sessionId}; the command keeps running.`, undefined, { hint: `stop it with: shard ws exec-cancel ${ref} ${sessionId}` });
+            }
+            throw err;
+        }
+        const s = r.session;
+        // Nothing ran (a cwd that is not a directory, a program not on PATH): exit 6 with the workspace's reason, as ws exec.
+        if (s.burst?.error)
+            throw new ShardfluxApiError(409, { error: { code: s.burst.error.code, message: s.burst.error.message, request_id: '', retryable: s.burst.error.retryable, ...(s.burst.error.details ? { details: s.burst.error.details } : {}) } }, 'cell');
+        if (s.state === 'failed_to_start')
+            throw new ExecStartError(s);
+        if (ctx.json) {
+            ctx.io.stdout.write(json(withTiming(ctx, {
+                session_id: s.session_id,
+                state: s.state,
+                exit_code: s.exit_code ?? null,
+                term_signal: s.term_signal ?? null,
+                timed_out: s.timed_out ?? false,
+                canceled: s.canceled ?? false,
+                stdout: r.stdout,
+                stderr: r.stderr,
+                truncated: r.truncated,
+                reconnects: r.reconnects,
+                ...(s.error ? { error: s.error } : {}),
+                ...(s.burst ? { burst: s.burst } : {}),
+            })));
+        }
+        if (s.state === 'starting' || s.state === 'running') {
+            throw new ExecSessionRunningError(`Exec session ${s.session_id} is still ${s.state}.`, s, `follow it to its end with: shard ws exec-read ${ref} ${s.session_id} --follow`);
+        }
+        const result = { exitCode: s.exit_code ?? null, termSignal: s.term_signal ?? null, timedOut: s.timed_out ?? false };
+        if (!ctx.json && result.exitCode === null && !result.timedOut && !(result.termSignal !== null && result.termSignal > 0)) {
+            ctx.io.stderr.write(`exec session ${s.session_id} ended ${s.state} without an exit code${s.error ? `: ${s.error}` : ''}\n`);
+        }
+        return execExitCode(result);
+    },
+    async 'workspaces exec-cancel'(ctx, values, [ref, sessionId]) {
+        const grace = str(values.grace);
+        const graceMs = grace === undefined ? undefined : parseDuration(grace, 'grace');
+        if (graceMs !== undefined && graceMs > MAX_CANCEL_GRACE_MS)
+            throw new UsageError(`--grace must be at most 60s (got "${grace}")`);
+        const ws = await resolveWorkspace(ctx, ref);
+        const s = await cellOf(ctx, ws).exec.cancel(sessionId, graceMs);
+        const ended = s.state !== 'starting' && s.state !== 'running';
+        const status = !ended ? `${s.state} (cancel sent)` : s.canceled ? `canceled${s.term_signal ? ` (signal ${s.term_signal})` : ''}` : `already ${s.state}${typeof s.exit_code === 'number' ? ` (exit code ${s.exit_code})` : ''}`;
+        out(ctx, `exec session ${s.session_id}: ${status}\n`, s);
+        return 0;
+    },
+    async 'workspaces exec-input'(ctx, values, [ref, sessionId, data]) {
+        const offset = Number(str(values.offset));
+        if (str(values.offset) === undefined || !Number.isSafeInteger(offset) || offset < 0)
+            throw new UsageError('--offset must be a non-negative integer');
+        const ws = await resolveWorkspace(ctx, ref);
+        const ack = await cellOf(ctx, ws).exec.input(sessionId, data ?? '', { offset, close: values.close === true, signal: ctx.signal });
+        out(ctx, JSON.stringify(ack) + '\n', ack);
+        return 0;
+    },
     async 'workspaces open'(ctx, values, [key]) {
         const template = str(values.template);
         if (!template)
             throw new UsageError('--template <slug> is required for "shard workspaces open"');
-        const c = caps(values);
+        const c = workspaceCaps(values);
         const bind = secretNames(strs(values.secret), '--secret');
         const lifetime = oneOf(values, 'lifetime', LIFETIMES);
         const mode = modeFlag(values);
@@ -583,6 +801,8 @@ export const KEY_HANDLERS = {
         const ws = await waited(ctx, 'the open continues server side (open the key again to keep waiting).', () => cloud.workspaces.open({
             key: key,
             template,
+            ...(str(values.labels) !== undefined ? { labels: labelMap(str(values.labels)) } : {}),
+            ...(str(values['idle-policy']) ? { idlePolicy: str(values['idle-policy']) } : {}),
             ...(c ? { caps: c } : {}),
             ...(bind.length ? { secrets: bind } : {}),
             ...(inputs ? { inputs } : {}),
@@ -609,6 +829,7 @@ export const KEY_HANDLERS = {
         const params = {
             ...(state ? { state: state } : {}),
             ...(prefix ? { keyPrefix: prefix } : {}),
+            ...(str(values.labels) !== undefined ? { labels: labelMap(str(values.labels)) } : {}),
             ...(lifetime ? { lifetime } : {}),
             ...(purpose ? { purpose } : {}),
             ...(values['include-deleted'] === true ? { includeDeleted: true } : {}),
@@ -656,11 +877,14 @@ export const KEY_HANDLERS = {
         const limitText = str(values['output-limit']);
         const outputLimit = limitText === undefined ? undefined : parsePositiveInt(limitText, 'output-limit', MAX_OUTPUT_LIMIT);
         const stdinSource = str(values.stdin);
+        const burst = burstOptions(values, stdinSource !== undefined);
         const stdin = stdinSource === undefined ? undefined : await readInput(ctx, stdinSource, MAX_STDIN, '--stdin');
         const ws = await resolveWorkspace(ctx, ref);
         const cell = cellOf(ctx, ws);
         const cwd = str(values.cwd);
         if (ws.mode === 'file_first') {
+            if (burst)
+                throw new UsageError(`--burst applies to processful workspaces only; ${ws.key} is ${modeText(ws.mode)}`);
             // An execution: answered when the command ended. The id is ours, so Ctrl-C can name it.
             const id = executionId ?? newExecutionId();
             let r;
@@ -696,6 +920,7 @@ export const KEY_HANDLERS = {
                 ...(Object.keys(env).length ? { env } : {}),
                 ...(timeoutMs !== undefined ? { timeoutMs } : {}),
                 ...(stdin !== undefined ? { stdin } : {}),
+                ...(burst ?? {}),
                 maxOutputBytes: ctx.json ? MAX_JSON_OUTPUT : 0,
                 ...(ctx.json ? {} : { onOutput: (stream, chunk) => void (stream === 'stderr' ? ctx.io.stderr : ctx.io.stdout).write(chunk) }),
                 signal: ctx.signal,
@@ -703,9 +928,14 @@ export const KEY_HANDLERS = {
         }
         catch (err) {
             if (ctx.signal.aborted) {
+                if (burst)
+                    throw new InterruptedError(`Interrupted; burst exec session ${sessionId} keeps running server side (a burst cannot be canceled).`);
                 await bestEffortCancel(cell, sessionId);
                 throw new InterruptedError(`Interrupted; sent cancel to exec session ${sessionId}.`);
             }
+            // A burst's apply that did not complete is finished by a retry of the same session (SDK exec.run / the API).
+            if (burst && err instanceof ShardfluxApiError && err.code === 'burst_apply_failed')
+                ctx.io.stderr.write(`burst exec session ${sessionId}\n`);
             throw err;
         }
         const code = execExitCode(r);
@@ -714,7 +944,14 @@ export const KEY_HANDLERS = {
         const unexplained = r.exitCode === null && !r.timedOut && !(r.termSignal !== null && r.termSignal > 0);
         if (values.verbose === true && !ctx.json) {
             const status = r.timedOut ? 'timed out' : r.canceled ? 'canceled' : r.termSignal ? `killed by signal ${r.termSignal}` : unexplained ? r.session.state : `exit code ${r.exitCode ?? '-'}`;
-            ctx.io.stderr.write(kv([['session', r.sessionId], ['status', status], ['output', `${r.stdoutBytes} bytes stdout, ${r.stderrBytes} bytes stderr`], ['reconnects', String(r.reconnects)]]));
+            ctx.io.stderr.write(kv([
+                ['session', r.sessionId],
+                ['status', status],
+                ['output', `${r.stdoutBytes} bytes stdout, ${r.stderrBytes} bytes stderr`],
+                ['reconnects', String(r.reconnects)],
+                ...(r.memoryGrow ? [['memory grow', memoryGrowText(r.memoryGrow)]] : []),
+                ...(r.burst ? [['burst', burstText(r.burst)]] : []),
+            ]));
         }
         if (ctx.json) {
             ctx.io.stdout.write(json(withTiming(ctx, {
@@ -729,6 +966,8 @@ export const KEY_HANDLERS = {
                 truncated: r.truncated,
                 reconnects: r.reconnects,
                 ...(r.session.error ? { error: r.session.error } : {}),
+                ...(r.memoryGrow ? { memory_grow: r.memoryGrow } : {}),
+                ...(r.burst ? { burst: r.burst } : {}),
             })));
         }
         else if (unexplained) {
@@ -854,8 +1093,26 @@ export const KEY_HANDLERS = {
     async 'workspaces suspend'(ctx, values, [ref]) {
         const whenIdle = str(values['when-idle']);
         const cancel = values['cancel-when-idle'] === true;
-        if (whenIdle === undefined && !cancel)
-            return lifecycle(ctx, values, ref, 'Suspending', (ws, o) => ctx.cloud().workspaces.suspend(ws.id, o), ['suspend', 'Suspend']);
+        const durable = values.durable === true;
+        if (whenIdle === undefined && !cancel) {
+            const suspend = (id, o) => ctx.cloud().workspaces.suspend(id, durable ? { ...o, durable: true, wait: o.wait ?? { timeoutMs: waitTimeout(values), signal: ctx.signal } } : o);
+            if (UUID.test(ref)) {
+                // An id needs no resolve request: the suspend goes straight to it (the API refuses a file-first workspace
+                // itself). Only a UUID that is no workspace id here (404, nothing started) is looked up as a key.
+                try {
+                    const op = await waited(ctx, 'the operation continues server side.', () => suspend(ref, lifecycleOptions(ctx, values, waitTimeout(values))));
+                    out(ctx, lifecycleText('Suspending', { id: ref }, op), op);
+                    return 0;
+                }
+                catch (err) {
+                    if (!(err instanceof ShardfluxApiError && err.status === 404 && err.operationId === undefined))
+                        throw err;
+                }
+            }
+            return lifecycle(ctx, values, ref, 'Suspending', (ws, o) => suspend(ws.id, o), ['suspend', 'Suspend']);
+        }
+        if (durable)
+            throw new UsageError(`${whenIdle !== undefined ? '--when-idle' : '--cancel-when-idle'} cannot be combined with --durable (nothing is suspended now)`);
         if (whenIdle !== undefined && cancel)
             throw new UsageError('--when-idle and --cancel-when-idle cannot be combined');
         const flag = cancel ? '--cancel-when-idle' : '--when-idle';
@@ -1376,7 +1633,7 @@ export const KEY_HANDLERS = {
         return 0;
     },
     async 'workspaces fork'(ctx, values, [ref, newKey]) {
-        const c = caps(values);
+        const c = workspaceCaps(values);
         const timeoutMs = waitTimeout(values);
         const source = await resolveWorkspace(ctx, ref);
         requireProcessful(source, 'fork', 'A fork');
@@ -1526,7 +1783,10 @@ export const KEY_HANDLERS = {
     async 'operations wait'(ctx, values, [id]) {
         if (!UUID.test(id))
             throw new UsageError(`<operation-id> must be a UUID (got "${id}")`);
-        const op = await waitOperation(ctx, id, waitTimeout(values));
+        const timeoutMs = waitTimeout(values);
+        const op = values.durable === true
+            ? await waited(ctx, `operation ${id} continues server side.`, () => ctx.cloud().workspaces.waitForDurable(id, { timeoutMs, signal: ctx.signal }))
+            : await waitOperation(ctx, id, timeoutMs);
         out(ctx, operationDetail(op), op);
         return 0;
     },
@@ -1681,3 +1941,17 @@ async function lifecycle(ctx, values, ref, verb, start, processfulOnly) {
     out(ctx, lifecycleText(verb, ws, op), op);
     return 0;
 }
+function labelMap(value) {
+    if (value === undefined)
+        return undefined;
+    let parsed;
+    try {
+        parsed = JSON.parse(value);
+    }
+    catch {
+        throw new UsageError('labels must be a JSON object of strings');
+    }
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed) || Object.values(parsed).some((v) => typeof v !== 'string'))
+        throw new UsageError('labels must be a JSON object of strings');
+    return parsed;
+}

package/dist/errors.d.ts

@@ -1,4 +1,4 @@
-import type { CheckoutStatus, ExecutionResult } from '@shardflux/sdk';
+import type { CheckoutStatus, ExecSession, ExecutionResult } from '@shardflux/sdk';
 export declare const EXIT: {
     readonly ok: 0;
     /** Any other failure: API refusal (conflict, quota, ...), network, protocol. */
@@ -76,6 +76,15 @@ export declare class ExecutionPendingError extends Error {
     readonly hint: string;
     constructor(message: string, result: ExecutionResult, hint: string);
 }
+/**
+ * `shard ws exec-read` without --follow (0.8.0): the background command still runs (exit 5, as a file-first execution
+ * that still runs); `hint` names the command that follows it to its end.
+ */
+export declare class ExecSessionRunningError extends HintedError {
+    name: string;
+    readonly session: ExecSession;
+    constructor(message: string, session: ExecSession, hint: string);
+}
 export interface ErrorInfo {
     code: string;
     message: string;
@@ -84,7 +93,7 @@ export interface ErrorInfo {
     retryable?: boolean;
     operation_id?: string;
     details?: Record<string, unknown>;
-    source?: 'api' | 'cell';
+    source?: 'api' | 'cell' | 'unknown';
     last_state?: string;
     operation?: unknown;
     /** A file-first execution the error is about (failed, lost, still running, or left running by Ctrl-C). */

package/dist/errors.js

@@ -3,7 +3,7 @@
  * stderr with the API's error code, message and request id; the process exit
  * code tells scripts what kind of failure it was (README "Exit codes").
  */
-import { CheckoutTimeoutError, ExecStartError, OperationFailedError, OperationTimeoutError, ShardfluxApiError, ShardfluxProtocolError, TemplateBuildTimeoutError, TemplateFileError, TemplateUploadError } from '@shardflux/sdk';
+import { CheckoutTimeoutError, DurabilityLostError, ExecStartError, OperationFailedError, OperationTimeoutError, ShardfluxApiError, ShardfluxProtocolError, TemplateBuildTimeoutError, TemplateFileError, TemplateUploadError } from '@shardflux/sdk';
 export const EXIT = {
     ok: 0,
     /** Any other failure: API refusal (conflict, quota, ...), network, protocol. */
@@ -97,6 +97,18 @@ export class ExecutionPendingError extends Error {
         this.hint = hint;
     }
 }
+/**
+ * `shard ws exec-read` without --follow (0.8.0): the background command still runs (exit 5, as a file-first execution
+ * that still runs); `hint` names the command that follows it to its end.
+ */
+export class ExecSessionRunningError extends HintedError {
+    name = 'ExecSessionRunningError';
+    session;
+    constructor(message, session, hint) {
+        super(message, hint);
+        this.session = session;
+    }
+}
 const AUTH_CODES = new Set(['unauthenticated', 'invalid_credentials', 'token_invalid', 'forbidden', 'mfa_required', 'email_unverified', 'step_up_required', 'csrf_failed']);
 const KEY_PATTERN = /sfk_[A-Za-z0-9]{8,}_[A-Za-z0-9]+/g;
 /** User session tokens (`sfu_` + 43 base64url characters), also cut short. */
@@ -137,15 +149,22 @@ export function describeError(err) {
             source: err.source,
         };
     }
-    if (err instanceof OperationTimeoutError)
-        return { code: 'timeout', message: err.message, operation_id: err.operationId, last_state: err.lastState };
+    if (err instanceof OperationTimeoutError) {
+        return { code: 'timeout', message: err.message, operation_id: err.operationId, last_state: err.lastState, ...(err.durable ? { details: { durable: false, state: 'pending' } } : {}) };
+    }
+    // --durable (0.7.0+): the suspend succeeded; its durable copy could not be made (see the lifecycle reference).
+    if (err instanceof DurabilityLostError) {
+        return { code: 'durability_lost', message: err.message, operation_id: err.operationId, retryable: false, details: { state: 'lost', ...(err.durability.reason ? { reason: err.durability.reason } : {}), checkpoint_id: err.durability.checkpointId }, operation: err.operation };
+    }
     if (err instanceof OperationFailedError) {
         // retryable: the operation error's own flag (true for capacity_unavailable: nothing was started, retry later).
         // details.reason: the operation error's reason (e.g. the scheduler's no_ready_hosts), printed after the message.
         const opDetails = err.operation.error?.details;
         const reason = typeof opDetails === 'object' && opDetails !== null ? opDetails.reason : undefined;
         const details = { ...(err.errorCode ? { error_code: err.errorCode } : {}), ...(typeof reason === 'string' ? { reason } : {}) };
-        return { code: 'operation_failed', message: err.message, operation_id: err.operationId, retryable: err.retryable, ...(Object.keys(details).length ? { details } : {}), operation: err.operation };
+        // A suspend-when-idle canceled workspace_active (0.7.0+): the workspace was in use and keeps running.
+        const message = err.workspaceActive ? `${err.message}: the workspace was in use, so the suspend-when-idle was canceled; it keeps running` : err.message;
+        return { code: 'operation_failed', message, operation_id: err.operationId, retryable: err.retryable, ...(Object.keys(details).length ? { details } : {}), operation: err.operation };
     }
     // File-first executions (0.5.0): failed or lost like a failed operation (details.reason says why;
     // retryable: a new execution id may succeed), still running like a wait that gave up.
@@ -156,6 +175,8 @@ export function describeError(err) {
     }
     if (err instanceof ExecutionPendingError)
         return { code: 'timeout', message: err.message, execution_id: err.result.executionId, last_state: err.result.state };
+    if (err instanceof ExecSessionRunningError)
+        return { code: 'timeout', message: err.message, last_state: err.session.state, details: { session_id: err.session.session_id } };
     if (err instanceof UsageError)
         return { code: 'usage', message: err.message };
     if (err instanceof CheckoutTimeoutError) {
@@ -180,7 +201,7 @@ export function describeError(err) {
         return { code: 'interrupted', message: err.message, ...(err.operationId ? { operation_id: err.operationId } : {}), ...(err.executionId ? { execution_id: err.executionId } : {}) };
     }
     if (err instanceof ShardfluxProtocolError)
-        return { code: 'protocol_error', message: err.message, status: err.status };
+        return { code: 'protocol_error', message: err.message, status: err.status, source: err.source };
     if (isNamed(err, 'TimeoutError'))
         return { code: 'timeout', message: 'The request timed out.' };
     if (isNamed(err, 'AbortError'))
@@ -205,9 +226,9 @@ export function exitCodeFor(err, interrupted = false) {
         return EXIT.notFound;
     if (err instanceof OperationTimeoutError || err instanceof CheckoutTimeoutError || isNamed(err, 'TimeoutError'))
         return EXIT.timeout;
-    if (err instanceof OperationFailedError || err instanceof ExecutionFailedError || err instanceof CheckoutEndedError)
+    if (err instanceof OperationFailedError || err instanceof DurabilityLostError || err instanceof ExecutionFailedError || err instanceof CheckoutEndedError)
         return EXIT.operationFailed;
-    if (err instanceof ExecutionPendingError)
+    if (err instanceof ExecutionPendingError || err instanceof ExecSessionRunningError)
         return EXIT.timeout;
     // A command that could not start exits 6 in both modes, as a failed execution does (0.5.1).
     if (err instanceof ExecStartError)

package/dist/format.d.ts

@@ -1,10 +1,7 @@
-/**
- * Human-readable output (aligned tables, key/value blocks) and the error line.
- * `--json` output is the API's own objects (see README "JSON output"), so
- * these functions only shape what people read.
- */
-import type { AgentSession, ApiKey, AuditEvent, AuthSessionInfo, BillingCatalog, BillingInvoicePage, BillingSubscription, DataExport, DraftState, ExecutionChange, ExecutionResult, FileInfo, EgressPolicyVersion, Invitation, Member, Organization, Project, ProjectEgressPolicy, Volume, VolumeAttachment, WorkspaceEgressPolicy, FileList, FileSearchResponse, Operation, Secret, SecretAccessEvent, SecretVersion, SpendPolicy, SuspendRequest, TemplateBuild, TemplateDetail, TemplateDiffEntry, TemplateDraft, TemplateFileEntry, TemplateLanguages, TemplatePackagePage, TemplateSettings, TemplateSummary, WorkspaceChange, WorkspaceSecretBindings, WorkspaceView } from '@shardflux/sdk';
+import type { AgentSession, ApiKey, AuditEvent, AuthSessionInfo, BillingCatalog, BillingInvoicePage, BillingSubscription, DataExport, DraftState, ExecutionChange, ExecutionResult, FileInfo, EgressPolicyVersion, Invitation, Member, Organization, Project, ProjectEgressPolicy, Volume, VolumeAttachment, WorkspaceEgressPolicy, FileList, FileSearchResponse, Operation, Secret, SecretAccessEvent, SecretVersion, SpendPolicy, SuspendRequest, TemplateBuild, TemplateDetail, TemplateDiffEntry, TemplateDraft, TemplateFileEntry, TemplateLanguages, TemplatePackagePage, TemplateSettings, TemplateSummary, TemplateVersion, WorkspaceChange, WorkspaceSecretBindings, WorkspaceMemory, WorkspaceView, MemoryGrow, BurstSummary } from '@shardflux/sdk';
 import type { ErrorInfo } from './errors.js';
+/** A size for people: `512 B`, `10.0 KiB`, `128.0 MiB`, `1.50 GiB`. */
+export declare function byteSize(n: number): string;
 export declare function json(value: unknown): string;
 export declare function table(headers: readonly string[], rows: readonly (readonly string[])[]): string;
 export declare function kv(pairs: readonly (readonly [string, string])[]): string;
@@ -38,9 +35,20 @@ export declare function suspendWhenIdleText(w: {
     suspend_request: SuspendRequest | null;
     operation: Operation | null;
 }): string;
+/** `fixed 4096 MiB`, or `elastic: promised 8192 MiB, held 1024 MiB, plugged +2048 MiB` (plugged: above the held floor now). */
+export declare function memoryText(m: WorkspaceMemory): string;
+/** The grow an exec's start waited for: `delivered 1024 -> 4096 MiB in 175 ms`, `partial 1024 -> 3072 of 4096 MiB in 175 ms`. */
+/** The burst line of `ws exec -v` (0.8.0). */
+export declare function burstText(b: BurstSummary): string;
+export declare function memoryGrowText(g: MemoryGrow): string;
 export declare function workspaceDetail(w: WorkspaceView): string;
 export declare const OPERATION_HEADERS: readonly ["ID", "KIND", "STATE", "REASON", "CREATED", "COMPLETED"];
 export declare function operationTable(ops: readonly Operation[]): string;
+/**
+ * A finished suspend or fork (0.7.0+): whether its copy is in durable storage, when the result says (`durable`,
+ * `durability`). Empty when the result has no durability block (stored durably before the operation completed).
+ */
+export declare function durableText(op: Operation): string;
 export declare function operationDetail(op: Operation): string;
 export declare function sessionTable(sessions: readonly AgentSession[]): string;
 /** `files stat`: one path's metadata; the revision only when requested and returned. */
@@ -90,6 +98,8 @@ export declare function languageTable(l: TemplateLanguages): string;
 export declare function packageTable(page: TemplatePackagePage): string;
 /** A template: identity, what open resolves to, its versions, and the open version's settings. */
 export declare function templateDetailText(t: TemplateDetail): string;
+/** A version's immutable paths and the size of their image: `/opt/acme, /srv/tools (128.0 MiB)`, or `-`. */
+export declare function immutablePathsText(i: TemplateVersion['immutable'] | undefined): string;
 /** A version's settings, one line per part. */
 export declare function settingsText(s: TemplateSettings): string;
 /** How many changed paths the human output lists before "... N more" (--json lists every one the result has). */