@wpelevator/tug 0.12.0 → 0.13.0

package/README.md

@@ -6,9 +6,11 @@ A VPS is a durable computer with enough resources to host multiple sites or a me
 
 Tug runs a small control plane and an agent, which can share the same VPS with your applications. It builds application images on the control server or your machine and delivers them to your nodes, so you do not need to run or configure an image registry for those builds. Published images are still pulled from their upstream registries.
 
+Existing service and resource targets can be chosen directly in an interactive terminal: try `tug logs`, `tug deploy cancel` or `tug restore run`. Use `--select` for backup and move-hold subsets; see [interactive CLI selection](docs/cli.md#interactive-selection) for all supported commands and scripting behavior.
+
 ## What you can do with it
 
-- **Deploy a Compose file to your servers.** `tug deploy` builds or pulls your images, starts them on the right nodes, waits for health checks and tells you why if something fails. `tug rollback` returns to the previous version, and `x-tug: { rollback: auto }` does it for you.
+- **Deploy a Compose file to your servers.** `tug deploy` builds or pulls your images, starts them on the right nodes, waits for health checks and tells you why if something fails. It reports deployment stages and elapsed time while work is pending. `tug rollback` returns to the previous version, and `x-tug: { rollback: auto }` does it for you.
 - **Choose where things run.** Pin a database and its volume to one node, spread web containers over several, and let Tug's managed Caddy ingress serve them over HTTPS with certificates it obtains for you.
 - **Update without downtime.** Opt-in rolling rollouts start the new version, wait until it is healthy and only then switch traffic.
 - **Keep secrets out of your repository.** `tug secret set DB_PASSWORD` stores them encrypted and sends each one only to the nodes whose services use it.
@@ -34,6 +36,8 @@ The control plane and agent share the VPS with your application. Once your servi
 
 In a terminal Tug asks before it changes anything: whether to install Docker, which private address the node should use and whether it should handle ingress. It shows the plan and waits for you to confirm. `tug deploy` asks for any Compose variables, such as `${DB_PASSWORD}`, that no secret or `.env` provides. In CI or any other script nothing is ever asked, so pass the choices as flags (`--address`, `--ingress`, `--install-docker`, `-y`).
 
+Long-running deployments, rollbacks, setup, image transfers, backups, restores and agent command waits report their current stage and elapsed time to stderr every ten seconds. Progress also works in CI and keeps JSON results separate. See [CLI progress reporting](docs/cli.md).
+
 ## Your Compose file, plus a little `x-tug`
 
 This example builds on the [quickstart](docs/getting-started.md). It pins the app, database and database volume to `app1`. Placement can be omitted with one node, but explicit placement keeps the database on its chosen server when you add more nodes. Enable ingress with `tug node ingress app1`, point the hostname's DNS at the VPS, and allow public ports 80 and 443. Save this as `compose.yaml` and replace the hostname with yours:
@@ -94,7 +98,7 @@ tug logs wordpress --follow
 
 The named volume holds data on `app1`; it is not copied to another node automatically. Back it up and restore or move its data deliberately if you replace the server.
 
-See the [examples directory](examples/) for stack-specific applications. Everything `x-tug` can express is in [Compose extensions](docs/compose.md).
+See the [examples directory](examples/) for stack-specific applications and [local stack benchmarks](docs/testing.md#example-stack-benchmarks) for measured behavior of those examples. Everything `x-tug` can express is in [Compose extensions](docs/compose.md).
 
 ## Everyday commands
 
@@ -199,4 +203,4 @@ npm run lint
 npm run test:integration
 ```
 
-See [Development](docs/development.md) for details.
+See [Development](docs/development.md) for details. Database code uses direct SQLite queries; [SQLite storage](docs/architecture.md#sqlite-storage) describes query boundaries and synchronous transactions.

package/dist/backup-restore.d.ts

@@ -69,6 +69,6 @@ export declare function createRestoreService(deps: RestoreDeps): {
         status: number;
     };
     holds: () => Record<string, string[]>;
-    list: (project: string, limit?: number) => RestoreRunRow[];
+    list: (project: string, limit?: number, interruptedOnly?: boolean) => RestoreRunRow[];
 };
 export type RestoreService = ReturnType<typeof createRestoreService>;

package/dist/backup-restore.js

@@ -374,7 +374,9 @@ export function createRestoreService(deps) {
             (result[row.project_id] ??= []).push(row.service);
         return result;
     }
-    function list(project, limit = 20) {
+    function list(project, limit = 20, interruptedOnly = false) {
+        if (interruptedOnly)
+            return db.all("SELECT * FROM restore_runs WHERE project_id=? AND state='interrupted' ORDER BY started_at DESC,id", project);
         return db.all('SELECT * FROM restore_runs WHERE project_id=? ORDER BY started_at DESC LIMIT ?', project, limit);
     }
     return { plan, start, advance, releaseInterrupted, resume, holds, list };

package/dist/backup-scheduler.js

@@ -4,6 +4,7 @@ import { projectRepository } from "./backup-destination.js";
 import { volumeName } from "./container-spec.js";
 import { id } from "./util.js";
 import { isUniqueViolation } from "./db-driver.js";
+import { backupDestinationQueries } from "./db-backup-destination.js";
 /** Pinned by tag; override on control with `TUG_RESTIC_IMAGE` (a digest reference is recommended for production). */
 export const DEFAULT_RESTIC_IMAGE = 'restic/restic:0.17.3';
 const MINUTE = 60_000;
@@ -63,7 +64,8 @@ export function createBackupService(deps) {
     const started = clock().getTime();
     const stamp = () => clock().toISOString();
     const heartbeats = new Map();
-    const destination = () => db.get("SELECT * FROM backup_destination WHERE id='default'");
+    const destinationQueries = backupDestinationQueries(db);
+    const destination = () => destinationQueries.find();
     const nodeId = (name) => db.get('SELECT id FROM nodes WHERE name=?', name)?.id;
     const nodeName = (nodeIdValue) => nodeIdValue
         ? (db.get('SELECT name FROM nodes WHERE id=?', nodeIdValue)?.name ?? null)

package/dist/cli-build.d.ts

@@ -14,4 +14,4 @@ export declare function createSourceArchive(contextDir: string, archive: string,
 /** Build from the same filtered, validated source snapshot used for remote builds. */
 export declare function createLocalBuildSnapshot(contextDir: string, directory: string, configFiles: string[]): Promise<string>;
 /** Build only application build services; callers own temporary files and upload transport. */
-export declare function prepareLocalImages(compose: ComposeFile, contextDir: string, directory: string, dependencies?: LocalBuildDependencies): Promise<LocalImageUpload[]>;
+export declare function prepareLocalImages(compose: ComposeFile, contextDir: string, directory: string, dependencies?: LocalBuildDependencies, onProgress?: (message: string) => void): Promise<LocalImageUpload[]>;

package/dist/cli-build.js

@@ -143,7 +143,7 @@ export async function prepareLocalImages(compose, contextDir, directory, depende
     remove: async (imageRef) => {
         await run('docker', ['image', 'rm', imageRef]);
     },
-}) {
+}, onProgress) {
     const images = [];
     const nonce = dependencies.nonce();
     for (const [service, spec] of Object.entries(compose.services)) {
@@ -152,16 +152,19 @@ export async function prepareLocalImages(compose, contextDir, directory, depende
         const field = `image-${images.length}`;
         const imageRef = `tug-build/${nonce}/${field}:local`;
         try {
+            onProgress?.(`Building image for ${service} locally`);
             await dependencies.build({
                 contextDir,
                 service,
                 buildSpec: spec.build,
                 imageRef,
             });
+            onProgress?.(`Exporting image for ${service}`);
             const archive = await dependencies.save(imageRef, path.join(directory, `${field}.tar.gz`));
             images.push({ service, imageRef, ...archive, field });
         }
         finally {
+            onProgress?.(`Cleaning up temporary image for ${service}`);
             try {
                 await dependencies.remove(imageRef);
             }

package/dist/cli-process.d.ts

@@ -0,0 +1,13 @@
+/** Keep the event loop free for progress timers during SSH, Docker and file transfers. */
+export declare function runProcess(cmd: string, args: string[], { capture, input }?: {
+    capture?: boolean;
+    input?: string;
+}): Promise<{
+    status: number | null;
+    stdout: string;
+    stderr: string;
+}>;
+export declare function runLocal(cmd: string, args: string[]): Promise<void>;
+export declare function runCapture(cmd: string, args: string[], opts?: {
+    input?: string;
+}): Promise<string>;

package/dist/cli-process.js

@@ -0,0 +1,46 @@
+import { spawn } from 'node:child_process';
+import { RUN_OUTPUT_LIMIT } from "./build.js";
+/** Keep the event loop free for progress timers during SSH, Docker and file transfers. */
+export function runProcess(cmd, args, { capture = false, input } = {}) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(cmd, args, {
+            stdio: [
+                input === undefined ? 'inherit' : 'pipe',
+                capture ? 'pipe' : 'inherit',
+                capture ? 'pipe' : 'inherit',
+            ],
+        });
+        let stdout = '';
+        let stderr = '';
+        child.stdout?.on('data', (chunk) => {
+            stdout = (stdout + String(chunk)).slice(-RUN_OUTPUT_LIMIT);
+        });
+        child.stderr?.on('data', (chunk) => {
+            stderr = (stderr + String(chunk)).slice(-RUN_OUTPUT_LIMIT);
+        });
+        child.on('error', reject);
+        child.stdin?.on('error', (error) => {
+            if (error.code !== 'EPIPE')
+                reject(error);
+        });
+        if (input !== undefined)
+            child.stdin?.end(input);
+        child.on('close', (status, signal) => {
+            if (signal)
+                reject(new Error(`${cmd} terminated by ${signal}`));
+            else
+                resolve({ status, stdout, stderr });
+        });
+    });
+}
+export async function runLocal(cmd, args) {
+    const result = await runProcess(cmd, args);
+    if (result.status !== 0)
+        throw new Error(`${cmd} failed (exit ${result.status})`);
+}
+export async function runCapture(cmd, args, opts = {}) {
+    const result = await runProcess(cmd, args, { ...opts, capture: true });
+    if (result.status !== 0)
+        throw new Error(`${cmd} failed (exit ${result.status})\n${result.stderr}`);
+    return result.stdout;
+}

package/dist/cli-progress.d.ts

@@ -0,0 +1,11 @@
+/** Line-based stderr progress works in terminals and CI without changing stdout results. */
+export declare function createProgress({ write, now, intervalMs, }?: {
+    write?: ((line: string) => void) | undefined;
+    now?: (() => number) | undefined;
+    intervalMs?: number | undefined;
+}): {
+    stage(next: string): void;
+    stop(): void;
+};
+/** Own the timer for one operation, including failures and early returns. */
+export declare function withProgress<T>(label: string, operation: (progress: ReturnType<typeof createProgress>) => Promise<T>): Promise<T>;

package/dist/cli-progress.js

@@ -0,0 +1,37 @@
+/** Line-based stderr progress works in terminals and CI without changing stdout results. */
+export function createProgress({ write = (line) => {
+    process.stderr.write(`${line}\n`);
+}, now = Date.now, intervalMs = 10_000, } = {}) {
+    let label = '';
+    let started = now();
+    let timer;
+    const report = () => write(`[${Math.floor((now() - started) / 1000)}s] ${label}`);
+    return {
+        stage(next) {
+            if (next === label)
+                return;
+            label = next;
+            started = now();
+            report();
+            if (!timer) {
+                timer = setInterval(report, intervalMs);
+                timer.unref();
+            }
+        },
+        stop() {
+            clearInterval(timer);
+            timer = undefined;
+        },
+    };
+}
+/** Own the timer for one operation, including failures and early returns. */
+export async function withProgress(label, operation) {
+    const progress = createProgress();
+    try {
+        progress.stage(label);
+        return await operation(progress);
+    }
+    finally {
+        progress.stop();
+    }
+}

package/dist/cli-prompt.js

@@ -10,24 +10,25 @@ export const ask = async (question, hidden = false) => {
     const output = new Writable({
         write(chunk, _encoding, callback) {
             if (!muted.on)
-                process.stdout.write(chunk);
+                process.stderr.write(chunk);
             callback();
         },
     });
     const rl = readline.createInterface({ input: process.stdin, output, terminal: true });
     try {
         return await new Promise((resolve, reject) => {
-            rl.once('close', () => reject(new Error('input closed before an answer was given')));
+            const onClose = () => reject(new Error('input closed before an answer was given'));
+            rl.once('close', onClose);
             rl.question(question, (answer) => {
+                rl.off('close', onClose);
                 if (hidden)
-                    process.stdout.write('\n');
+                    process.stderr.write('\n');
                 resolve(answer);
             });
             muted.on = hidden;
         });
     }
     finally {
-        rl.removeAllListeners('close');
         rl.close();
     }
 };

package/dist/cli-selection.d.ts

@@ -0,0 +1,13 @@
+import type { Prompter } from './cli-prompt.ts';
+export interface Choice {
+    value: string;
+    label: string;
+}
+export interface SelectionIO {
+    interactive: boolean;
+    prompt: Prompter;
+    write: (message: string) => void;
+}
+/** Resolve lazily so explicit arguments never need a discovery request. */
+export declare function selectOne(value: string | undefined, question: string, load: () => Promise<Choice[]>, io?: SelectionIO): Promise<string>;
+export declare function selectMany(question: string, load: () => Promise<Choice[]>, io?: SelectionIO): Promise<string[]>;

package/dist/cli-selection.js

@@ -0,0 +1,39 @@
+import { ask, canPrompt } from "./cli-prompt.js";
+const terminal = () => ({
+    interactive: canPrompt(),
+    prompt: ask,
+    write: console.error,
+});
+/** Resolve lazily so explicit arguments never need a discovery request. */
+export async function selectOne(value, question, load, io = terminal()) {
+    if (value)
+        return value;
+    const selected = await selectChoices(question, load, false, io);
+    return selected[0];
+}
+export async function selectMany(question, load, io = terminal()) {
+    return selectChoices(question, load, true, io);
+}
+async function selectChoices(question, load, multiple, io) {
+    if (!io.interactive)
+        throw new Error(`${question}: pass an explicit target; selection requires a terminal outside CI`);
+    const options = await load();
+    if (!options.length)
+        throw new Error(`${question}: no eligible options`);
+    io.write(question);
+    options.forEach((option, index) => io.write(`  ${index + 1}) ${option.label}`));
+    for (;;) {
+        const answer = (await io.prompt(multiple
+            ? 'Choose numbers separated by commas (or q to cancel): '
+            : 'Choose a number (or q to cancel): ')).trim();
+        if (/^(q|quit|cancel)$/i.test(answer))
+            throw new Error('cancelled; nothing was changed');
+        const parts = answer.split(',').map((part) => part.trim());
+        if ((!multiple && parts.length !== 1) || parts.some((part) => !/^[1-9]\d*$/.test(part)))
+            continue;
+        const indexes = parts.map((part) => Number(part) - 1);
+        if (indexes.some((index) => index >= options.length))
+            continue;
+        return [...new Set(indexes)].map((index) => options[index].value);
+    }
+}

package/dist/commands/backup.js

@@ -1,3 +1,5 @@
+import { selectBackups } from "./selection.js";
+import { withProgress } from "../cli-progress.js";
 import { backupCheckProblems } from "../backup-check.js";
 import { collect, loadCfg, print, readStdin, request } from "./context.js";
 const formatBytes = (bytes) => bytes === null
@@ -13,42 +15,57 @@ export function registerBackup(program) {
     backup
         .command('run [targets...]')
         .description('back up the named services or volumes now, or every backup target of the project; waits and exits nonzero if any backup fails')
+        .option('--select', 'choose one or more backup targets instead of backing up all')
         .option('--no-wait', 'return once the backups are queued')
         .option('--timeout <seconds>', 'give up waiting after this many seconds', '3600')
         .action(async (targets, o) => {
-        const project = (await loadCfg()).project || 'site';
-        const queued = await request('POST', '/api/v1/backups/run', {
-            project,
-            targets: targets.length ? targets : undefined,
-        });
-        for (const message of queued.errors)
-            console.error(message);
-        for (const run of queued.runs)
-            console.log(`queued ${run.target} as ${run.runId}`);
-        let failed = queued.errors.length > 0;
-        if (o.wait) {
-            const deadline = Date.now() + Number(o.timeout) * 1000;
-            const pending = new Set(queued.runs.map((run) => run.runId));
-            while (pending.size && Date.now() < deadline) {
-                await new Promise((resolve) => setTimeout(resolve, 2000));
-                const listing = await request('GET', `/api/v1/backups?project=${encodeURIComponent(project)}&limit=200`);
-                for (const run of listing.runs)
-                    if (pending.has(run.id) && BACKUP_TERMINAL.includes(run.state)) {
-                        pending.delete(run.id);
-                        const ok = run.state === 'succeeded';
-                        failed ||= !ok;
-                        console.log(ok
-                            ? `${run.target}: snapshot ${run.snapshot_id?.slice(0, 8)}, ${formatBytes(run.added_bytes)} added`
-                            : `${run.target}: ${run.state}${run.error ? ` (${run.error})` : ''}`);
+        if (o.select && targets.length)
+            throw new Error('--select cannot be combined with explicit targets');
+        if (o.select)
+            targets = await selectBackups();
+        return withProgress('Queueing backups', async (progress) => {
+            const project = (await loadCfg()).project || 'site';
+            const queued = await request('POST', '/api/v1/backups/run', {
+                project,
+                targets: targets.length ? targets : undefined,
+            });
+            for (const message of queued.errors)
+                console.error(message);
+            for (const run of queued.runs)
+                console.log(`queued ${run.target} as ${run.runId}`);
+            let failed = queued.errors.length > 0;
+            if (o.wait) {
+                const deadline = Date.now() + Number(o.timeout) * 1000;
+                const pending = new Set(queued.runs.map((run) => run.runId));
+                if (pending.size)
+                    progress.stage(`Waiting for backups: ${queued.runs.map((run) => run.target).join(', ')}`);
+                const states = new Map();
+                while (pending.size && Date.now() < deadline) {
+                    await new Promise((resolve) => setTimeout(resolve, 2000));
+                    const listing = await request('GET', `/api/v1/backups?project=${encodeURIComponent(project)}&limit=200`);
+                    for (const run of listing.runs) {
+                        if (pending.has(run.id))
+                            states.set(run.id, `${run.target}: ${run.state}`);
+                        if (pending.has(run.id) && BACKUP_TERMINAL.includes(run.state)) {
+                            pending.delete(run.id);
+                            const ok = run.state === 'succeeded';
+                            failed ||= !ok;
+                            console.log(ok
+                                ? `${run.target}: snapshot ${run.snapshot_id?.slice(0, 8)}, ${formatBytes(run.added_bytes)} added`
+                                : `${run.target}: ${run.state}${run.error ? ` (${run.error})` : ''}`);
+                        }
                     }
+                    if (pending.size)
+                        progress.stage(`Backups: ${[...pending].map((id) => states.get(id) ?? id).join(', ')}`);
+                }
+                if (pending.size) {
+                    console.error(`timed out waiting for ${pending.size} backup(s); they keep running on the node`);
+                    failed = true;
+                }
             }
-            if (pending.size) {
-                console.error(`timed out waiting for ${pending.size} backup(s); they keep running on the node`);
-                failed = true;
-            }
-        }
-        if (failed)
-            process.exitCode = 1;
+            if (failed)
+                process.exitCode = 1;
+        });
     });
     backup
         .command('list')

package/dist/commands/context.d.ts

@@ -1,4 +1,4 @@
-import type { SpawnSyncOptions } from 'node:child_process';
+export { runCapture, runLocal } from '../cli-process.ts';
 import type { ResolvedCliConfig } from '../cli-config.ts';
 import type { LiveInput, LiveRequest } from '../cli-live.ts';
 import type { DeploymentState } from '../protocol.ts';
@@ -50,9 +50,6 @@ export declare function request<T = any>(method: string, route: string, body?: u
 /** The project a project-scoped command works on: `--project`, then the resolved config, then `site`. */
 export declare function projectOf(o: CommonOpts): Promise<string>;
 export declare const print: (value: unknown) => void;
-export declare function runLocal(cmd: string, args: string[], opts?: SpawnSyncOptions): void;
-export declare function runCapture(cmd: string, args: string[], opts?: SpawnSyncOptions): string;
-export declare function waitForDeployment(deploymentId: string, timeoutSeconds?: number): Promise<DeploymentRecord>;
+export declare function waitForDeployment(deploymentId: string, timeoutSeconds?: number, onProgress?: (deployment: DeploymentRecord) => void): Promise<DeploymentRecord>;
 export declare function waitForCommand(commandId: string, timeoutSeconds?: number): Promise<CommandRecord>;
 export declare function runLive(operation: LiveRequest['operation'], input: Omit<LiveInput, 'project'>): Promise<void>;
-export {};

package/dist/commands/context.js

@@ -1,10 +1,14 @@
-import { spawnSync } from 'node:child_process';
+import { canPrompt } from "../cli-prompt.js";
+import { selectOne } from "../cli-selection.js";
+import { withProgress } from "../cli-progress.js";
+export { runCapture, runLocal } from "../cli-process.js";
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { createTransport } from "../agent-credential.js";
 import { createCliLiveTransport } from "../cli-live-transport.js";
 import { parseLiveRequest, runCliLive } from "../cli-live.js";
 import { TERMINAL_DEPLOYMENT_STATES } from "../protocol.js";
+import { describeFetchError, errMessage } from "../util.js";
 // Templates live next to the commands' parent: src/templates in dev (`node src/cli.ts`), dist/templates in the
 // built image (`node dist/cli.js`; tools/scripts/copy-templates.js puts them there).
 export const templatesDir = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'templates');
@@ -53,11 +57,17 @@ export async function request(method, route, body, multipart = false) {
     if (!multipart)
         headers['content-type'] = 'application/json';
     try {
-        const res = await transport.http(`${transport.base}${route}`, {
-            method,
-            headers,
-            body: multipart ? body : body ? JSON.stringify(body) : undefined,
-        });
+        let res;
+        try {
+            res = await transport.http(`${transport.base}${route}`, {
+                method,
+                headers,
+                body: multipart ? body : body ? JSON.stringify(body) : undefined,
+            });
+        }
+        catch (error) {
+            throw new Error(`${method} ${route}: could not reach control at ${transport.base}: ${describeFetchError(error)}`, { cause: error });
+        }
         if (!res.ok)
             throw new Error(`${method} ${route}: ${res.status} ${await res.text()}`);
         return (await res.json());
@@ -71,21 +81,17 @@ export async function projectOf(o) {
     return o.project || (await loadCfg()).project || 'site';
 }
 export const print = (value) => console.log(JSON.stringify(value, null, 2));
-export function runLocal(cmd, args, opts = {}) {
-    const result = spawnSync(cmd, args, { stdio: 'inherit', ...opts });
-    if (result.status !== 0)
-        throw new Error(`${cmd} failed${result.error ? `: ${result.error.message}` : ` (exit ${result.status ?? result.signal})`}`);
-}
-export function runCapture(cmd, args, opts = {}) {
-    const result = spawnSync(cmd, args, { encoding: 'utf8', ...opts });
-    if (result.status !== 0)
-        throw new Error(`${cmd} ${args.join(' ')} failed\n${result.stderr || ''}`);
-    return String(result.stdout);
-}
-export async function waitForDeployment(deploymentId, timeoutSeconds = 300) {
+export async function waitForDeployment(deploymentId, timeoutSeconds = 300, onProgress) {
     const deadline = Date.now() + timeoutSeconds * 1000;
     while (Date.now() < deadline) {
-        const deployment = await request('GET', `/api/v1/deployments/${encodeURIComponent(deploymentId)}`);
+        let deployment;
+        try {
+            deployment = await request('GET', `/api/v1/deployments/${encodeURIComponent(deploymentId)}`);
+        }
+        catch (error) {
+            throw new Error(`lost contact with control while waiting for deployment ${deploymentId} (${errMessage(error)}); it may still be rolling out: check tug status, or stop it with: tug deploy cancel ${deploymentId}`, { cause: error });
+        }
+        onProgress?.(deployment);
         if (TERMINAL_DEPLOYMENT_STATES.includes(deployment.state)) {
             if (deployment.state !== 'succeeded')
                 throw new Error(`deployment ${deploymentId} ${deployment.state}${deployment.error_message ? `: ${deployment.error_message}` : ''}`);
@@ -93,19 +99,35 @@ export async function waitForDeployment(deploymentId, timeoutSeconds = 300) {
         }
         await new Promise((r) => setTimeout(r, 2000));
     }
-    throw new Error(`timed out waiting for deployment ${deploymentId}`);
+    throw new Error(`timed out waiting for deployment ${deploymentId}; it is still rolling out: check tug status, or stop it with: tug deploy cancel ${deploymentId}`);
 }
 export async function waitForCommand(commandId, timeoutSeconds = 60) {
-    const deadline = Date.now() + timeoutSeconds * 1000;
-    while (Date.now() < deadline) {
-        const cmd = await request('GET', `/api/v1/commands/${commandId}`);
-        if (cmd?.state === 'succeeded' || cmd?.state === 'failed')
-            return cmd;
-        await new Promise((r) => setTimeout(r, 1000));
-    }
-    throw new Error(`timed out waiting for command ${commandId}`);
+    return withProgress(`Waiting for command ${commandId}`, async (progress) => {
+        const deadline = Date.now() + timeoutSeconds * 1000;
+        while (Date.now() < deadline) {
+            const cmd = await request('GET', `/api/v1/commands/${commandId}`);
+            if (cmd)
+                progress.stage(`Command ${commandId}: ${cmd.state}`);
+            if (cmd?.state === 'succeeded' || cmd?.state === 'failed')
+                return cmd;
+            await new Promise((r) => setTimeout(r, 1000));
+        }
+        throw new Error(`timed out waiting for command ${commandId}`);
+    });
 }
 export async function runLive(operation, input) {
     const cfg = await loadCfg();
+    if (!input.node && canPrompt()) {
+        const { services: listing } = await request('GET', `/api/v1/services?project=${encodeURIComponent(cfg.project || 'site')}`);
+        const candidates = new Map(listing
+            .filter((service) => operation === 'ps' || service.name === input.service)
+            .flatMap((service) => service.nodes)
+            .map((node) => [node.name, node]));
+        if (candidates.size > 1)
+            input.node = await selectOne(undefined, 'Select a node for this operation', async () => [...candidates.values()].map((node) => ({
+                value: node.name,
+                label: `${node.name} (${node.status})`,
+            })));
+    }
     process.exitCode = await runCliLive(parseLiveRequest(operation, { ...input, project: cfg.project || 'site' }), createCliLiveTransport(cfg.credential, cfg.controlUrl));
 }