@wpelevator/tug 0.10.0 → 0.11.0

package/dist/commands/host.js

@@ -0,0 +1,99 @@
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { saveImageArchive } from "../artifacts.js";
+import { controlPortCheck, localImageBuildArgs, shellQuote as sh } from "../cli-install.js";
+import { LIST_ADDRESSES_COMMAND, parseAddressCandidates, sortCandidates, } from "../cli-onboarding.js";
+import { runCapture, runLocal, templatesDir } from "./context.js";
+export function preflightRemote(target, { controlUrl, ports = [], minDiskMb = 1024, compose = false, } = {}) {
+    const checks = [
+        [
+            'docker.version',
+            'docker version >/dev/null',
+            'Install Docker Engine and ensure the SSH user can access it.',
+        ],
+        ...(compose
+            ? [
+                [
+                    'docker.compose.version',
+                    'docker compose version >/dev/null',
+                    'Install Docker Compose v2.',
+                ],
+            ]
+            : []),
+        [
+            'paths',
+            'mkdir -p /opt/tug-preflight /var/lib/tug-preflight /etc/tug && test -w /opt/tug-preflight && test -w /var/lib/tug-preflight && test -w /etc/tug',
+            'Ensure /opt, /var/lib, and /etc/tug are writable by the installer.',
+        ],
+        [
+            'disk.space',
+            `[ "$(df -Pm /var/lib | tail -1 | awk '{print $4}')" -ge ${Number(minDiskMb)} ]`,
+            `Free at least ${minDiskMb}MB on the filesystem backing /var/lib.`,
+        ],
+    ];
+    for (const port of ports) {
+        checks.push([
+            `port.${port}.free`,
+            controlPortCheck(port),
+            `Port ${port} is already in use on the target host by something other than the Tug control container; stop the conflicting service or choose another port.`,
+        ]);
+    }
+    if (controlUrl)
+        checks.push([
+            'control.reachable',
+            // TCP reachability only: no credential is copied to the host, and real TLS authorization happens when the agent connects.
+            `timeout 5 bash -c ${sh(`exec 3<>/dev/tcp/${new URL(controlUrl).hostname.replace(/^\[|\]$/g, '')}/${new URL(controlUrl).port || 443}`)}`,
+            'Verify DNS and the firewall from the node to the control port (control speaks mTLS only, so only TCP reachability is checked here).',
+        ]);
+    const script = `set -e
+${checks.map(([name, cmd, fix]) => `echo "preflight: ${name}"; (${cmd}) || { echo "FAILED ${name}: ${fix}" >&2; exit 20; }`).join('\n')}
+`;
+    runLocal('ssh', ['--', target, script]);
+}
+export async function transferTugImage(target, imageTag) {
+    const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'tug-runtime-'));
+    const archive = path.join(directory, 'image.tar.gz');
+    const remote = `/tmp/${path.basename(directory)}.tar.gz`;
+    try {
+        const uname = runCapture('ssh', ['--', target, 'uname -m']);
+        runLocal('docker', localImageBuildArgs(imageTag, uname));
+        await saveImageArchive(imageTag, archive);
+        runLocal('scp', ['--', archive, `${target}:${remote}`]);
+        runLocal('ssh', [
+            '--',
+            target,
+            `set -eu\ntrap ${sh(`rm -f ${sh(remote)}`)} EXIT\ndocker load --input ${sh(remote)}`,
+        ]);
+    }
+    finally {
+        await fs.rm(directory, { recursive: true, force: true });
+    }
+}
+export async function dockerBootstrap() {
+    return fs.readFile(path.join(templatesDir, 'install-docker.sh'), 'utf8');
+}
+/** Installs Docker over SSH; `compose` also installs the Compose plugin, which only the control host needs. */
+export async function bootstrapDocker(target, compose = false) {
+    runLocal('ssh', [
+        '--',
+        target,
+        `${compose ? 'TUG_INSTALL_COMPOSE=1 ' : ''}sh -eu -c ${sh(await dockerBootstrap())}`,
+    ]);
+}
+/** Whether the host already has the Docker Engine (and the Compose plugin when `compose`). */
+export function hostHasDocker(target, compose = false) {
+    const check = `docker version >/dev/null 2>&1${compose ? ' && docker compose version >/dev/null 2>&1' : ''}`;
+    return spawnSync('ssh', ['--', target, check], { stdio: 'ignore' }).status === 0;
+}
+/** The host's short hostname, or undefined when it cannot be read. */
+export function remoteHostname(target) {
+    const result = spawnSync('ssh', ['--', target, 'hostname -s'], { encoding: 'utf8' });
+    return result.status === 0 ? String(result.stdout).trim() || undefined : undefined;
+}
+/** The host's global IPv4 addresses, private ones first; empty when the host cannot say. */
+export function discoverAddresses(target) {
+    const result = spawnSync('ssh', ['--', target, LIST_ADDRESSES_COMMAND], { encoding: 'utf8' });
+    return result.status === 0 ? sortCandidates(parseAddressCandidates(String(result.stdout))) : [];
+}

package/dist/commands/init.d.ts

@@ -0,0 +1,11 @@
+import type { Command } from 'commander';
+import type { ResolvedCliConfig } from '../cli-config.ts';
+/**
+ * Store a credential document in a 0600 file under the user's config directory and point the git-ignored overlay
+ * next to the tracked config at it, creating the tracked config when it is missing. Prints what it wrote.
+ */
+export declare function importCredential(cfg: ResolvedCliConfig, text: string, connection: {
+    controlUrl?: string;
+    project?: string;
+}): Promise<void>;
+export declare function registerInit(program: Command): void;

package/dist/commands/init.js

@@ -0,0 +1,40 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { writeInitConfig } from "../cli-config.js";
+import { decodeCredential, writeFileDurable } from "../pki.js";
+import { loadCfg, readStdin } from "./context.js";
+/**
+ * Store a credential document in a 0600 file under the user's config directory and point the git-ignored overlay
+ * next to the tracked config at it, creating the tracked config when it is missing. Prints what it wrote.
+ */
+export async function importCredential(cfg, text, connection) {
+    const credential = decodeCredential(text);
+    // TUG_CREDENTIALS_DIR keeps scripts and tests out of the real home directory.
+    const dir = process.env.TUG_CREDENTIALS_DIR ||
+        path.join(process.env.XDG_CONFIG_HOME || path.join(os.homedir(), '.config'), 'tug', 'credentials');
+    const source = path.join(dir, `${new URL(credential.url).host.replace(/[^A-Za-z0-9.-]/g, '_')}.json`);
+    await writeFileDurable(source, `${JSON.stringify(credential, null, 2)}\n`, 0o600);
+    for (const file of await writeInitConfig(cfg, source, connection))
+        console.log(`wrote ${file}`);
+    console.log(`imported credential to ${source}`);
+}
+export function registerInit(program) {
+    program
+        .command('init [credentialFile]')
+        .description('create the config file if missing and import a credential (a file, or - for stdin) into a 0600 user-config file that the git-ignored .local overlay points at')
+        .action(async (credentialFile, _o, command) => {
+        const cfg = await loadCfg();
+        const { controlUrl, project } = command.optsWithGlobals();
+        let text;
+        if (credentialFile === '-')
+            text = await readStdin();
+        else if (credentialFile)
+            text = await fs.readFile(credentialFile, 'utf8');
+        else if (cfg.credential)
+            text = JSON.stringify(cfg.credential);
+        if (!text)
+            throw new Error('tug init needs a credential file (or - for stdin)');
+        await importCredential(cfg, text, { controlUrl, project });
+    });
+}

package/dist/commands/node.d.ts

@@ -0,0 +1,28 @@
+import type { Command } from 'commander';
+import type { AddressCandidate } from '../cli-onboarding.ts';
+import type { Prompter } from '../cli-prompt.ts';
+import type { InstallOpts } from './context.ts';
+/** Everything the SSH-assisted node commands share. */
+export interface NodeIdentityOpts extends InstallOpts {
+    name?: string;
+    address?: string;
+    label: string[];
+    agentControlUrl?: string;
+    all?: boolean;
+    wait: boolean;
+    ingress?: boolean;
+    yes?: boolean;
+}
+/** The questions and probes `planNodeAdd` makes, injectable so the prompt flow is testable without a terminal or SSH. */
+export interface PlanDeps {
+    interactive: boolean;
+    hasDocker: (target: string) => boolean;
+    discover: (target: string) => AddressCandidate[];
+    prompt: Prompter;
+}
+/**
+ * What `node add` is about to do, with the choices a terminal can still make: Docker, the inter-node address and the
+ * ingress role are asked about, never assumed. Without a terminal nothing is probed or asked and only the plan is printed.
+ */
+export declare function planNodeAdd(sshTarget: string, name: string, o: NodeIdentityOpts, deps?: PlanDeps): Promise<void>;
+export declare function registerNode(program: Command): void;

package/dist/commands/node.js

@@ -0,0 +1,320 @@
+import { applyConfigDefaults, normalizeControlUrl, nodeSettings } from "../cli-config.js";
+import { formatNodes } from "../cli-status.js";
+import { dockerRunSnippet, enrollmentCloudInit, shellQuote as sh, HOST_NETWORK_FLAG, } from "../cli-install.js";
+import { formatNodePlan, inferNodeName, isValidNodeName } from "../cli-onboarding.js";
+import { ask, canPrompt, choose, confirm } from "../cli-prompt.js";
+import { isNodeHost } from "../node-address.js";
+import { resolveImage } from "../image.js";
+import { errMessage, loadTemplate } from "../util.js";
+import { collect, loadCfg, print, readStdin, request, runCapture, runLocal, templatesDir, transportFor, } from "./context.js";
+import { bootstrapDocker, discoverAddresses, remoteHostname, dockerBootstrap, hostHasDocker, preflightRemote, transferTugImage, } from "./host.js";
+import fs from 'node:fs/promises';
+const AGENT_STATE = '/var/lib/tug';
+function addNodeIdentityOptions(command) {
+    return command
+        .option('--address <addr>', 'private inter-node address for this node')
+        .option('--label <label>', 'node label (repeatable)', collect, [])
+        .option('--all', 'walk every node with an ssh target in the config file, continuing after failures')
+        .option('--agent-control-url <url>', 'connect-only URL the agent dials (for example a private address); the certificate is still verified against the credential URL')
+        .option('--image <tag>', "Tug image to run (default: ghcr.io/wpelevator/tug:<this CLI's version>)")
+        .option('--install-docker', 'install missing Docker Engine on supported Debian, Ubuntu, Fedora and RHEL-compatible hosts (root required)')
+        .option('--build-local-image', 'build the image from the current checkout and copy it to the host over SSH (tags it tug:local unless --image is set); for development')
+        .option('--no-wait', 'do not wait for the node to report after the install');
+}
+/**
+ * The SSH-assisted flow: the node makes its own key and CSR, an admin credential authorizes the identity at this moment and
+ * the certificate is installed back over SSH, so no bootstrap secret travels and the private key never leaves the node.
+ */
+async function installNodeIdentity(sshTarget, name, mode, o, repair = false, 
+/** False in batch mode, where only the final per-node results are printed. */
+printResult = true) {
+    const cfg = await loadCfg();
+    const connectUrl = o.agentControlUrl ? normalizeControlUrl(o.agentControlUrl) : undefined;
+    await transportFor(cfg).then((t) => t.close()); // fail before touching the host when there is no credential
+    if (o.installDocker)
+        await bootstrapDocker(sshTarget);
+    preflightRemote(sshTarget, {
+        controlUrl: connectUrl ?? cfg.credential?.url,
+    });
+    const image = resolveImage(o.image, o.buildLocalImage);
+    if (o.buildLocalImage)
+        await transferTugImage(sshTarget, image);
+    const started = new Date().toISOString();
+    if (repair)
+        runLocal('ssh', [
+            '--',
+            sshTarget,
+            `mkdir -p ${AGENT_STATE} && cd ${AGENT_STATE} && for f in agent.json agent-pending.json; do [ -e "$f" ] && mv "$f" "$f.corrupt-$(date +%s)"; done; true`,
+        ]);
+    const agent = (command, interactive = false) => `mkdir -p ${AGENT_STATE} && docker run --rm ${interactive ? '-i ' : ''}-v ${AGENT_STATE}:${AGENT_STATE} ${sh(image)} agent ${command}`;
+    const csr = runCapture('ssh', ['--', sshTarget, agent('identity')]);
+    const response = await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/identity`, { csr, mode, address: o.address, labels: o.label });
+    runCapture('ssh', ['--', sshTarget, agent('install-cert', true)], {
+        input: JSON.stringify(response),
+    });
+    const env = await loadTemplate(templatesDir, 'agent.env', {
+        CONTROL_URL_LINE: connectUrl ? `TUG_CONTROL_URL=${connectUrl}\n` : '',
+        NODE_NAME: name,
+    });
+    const remote = await loadTemplate(templatesDir, 'node-install.sh', {
+        ENV: env,
+        IMAGE_SH: sh(image),
+        RUN_FLAGS: `${HOST_NETWORK_FLAG} `,
+    });
+    runLocal('ssh', ['--', sshTarget, remote]);
+    if (o.ingress)
+        await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/ingress`, { enabled: true });
+    const result = { ok: true, node: response.node, ...(o.ingress ? { ingress: true } : {}) };
+    let reporting = !o.wait;
+    if (o.wait) {
+        // A report from the new identity: wait for a report after the install began, not a replaced node's stale status.
+        for (let i = 0; i < 30; i++) {
+            const { nodes } = await request('GET', '/api/v1/nodes');
+            const seen = nodes.find((n) => n.name === name)?.last_seen_at;
+            if (seen && seen > started) {
+                reporting = true;
+                break;
+            }
+            await new Promise((r) => setTimeout(r, 2000));
+        }
+        if (!reporting)
+            console.error(`node ${name} has not reported yet; check \`docker logs tug-agent\` on the host`);
+    }
+    if (printResult)
+        print(o.wait ? { ...result, reporting } : result);
+    return reporting;
+}
+const realPlanDeps = () => ({
+    interactive: canPrompt(),
+    hasDocker: (target) => hostHasDocker(target),
+    discover: discoverAddresses,
+    prompt: ask,
+});
+/**
+ * What `node add` is about to do, with the choices a terminal can still make: Docker, the inter-node address and the
+ * ingress role are asked about, never assumed. Without a terminal nothing is probed or asked and only the plan is printed.
+ */
+export async function planNodeAdd(sshTarget, name, o, deps = realPlanDeps()) {
+    const { interactive, prompt } = deps;
+    if (interactive && !o.installDocker && !deps.hasDocker(sshTarget))
+        o.installDocker = await confirm(`Docker is not installed on ${sshTarget}. Install it now (needs root)?`, true, prompt);
+    if (interactive && !o.address) {
+        const candidates = deps.discover(sshTarget);
+        if (candidates.length) {
+            const none = 'none: leave the address unset';
+            const options = [
+                ...candidates.map((c) => `${c.address} (${c.interface}, ${c.private ? 'private' : 'public'})`),
+                none,
+            ];
+            // A public address would carry cross-node traffic in the clear, so it is never the default.
+            const index = await choose(`Which address should other nodes use to reach ${name}?`, options, candidates[0].private ? 0 : options.length - 1, prompt);
+            if (index < candidates.length)
+                o.address = candidates[index].address;
+        }
+    }
+    if (interactive && o.ingress === undefined)
+        o.ingress = await confirm(`Should ${name} handle ingress (managed Caddy on ports 80 and 443)?`, false, prompt);
+    console.error(formatNodePlan({
+        name,
+        ssh: sshTarget,
+        installDocker: o.installDocker,
+        probed: interactive,
+        address: o.address,
+        ingress: o.ingress,
+        image: resolveImage(o.image, o.buildLocalImage),
+    }));
+    if (interactive && !o.yes && !(await confirm(`Add ${name}?`, true, prompt)))
+        throw new Error('cancelled; nothing was changed');
+}
+/**
+ * A name for a node when the SSH host does not suggest one (an IP address): in a terminal ask for it, offering the
+ * host's own hostname as the default. Without a terminal there is nobody to ask.
+ */
+async function promptNodeName(sshTarget) {
+    if (!canPrompt())
+        return undefined;
+    const suggestion = sshTarget ? remoteHostname(sshTarget) : undefined;
+    const fallback = suggestion && isValidNodeName(suggestion) ? suggestion : undefined;
+    for (;;) {
+        const answer = (await ask(`Node name${fallback ? ` (${fallback})` : ''}: `)).trim() || fallback;
+        if (!answer)
+            continue;
+        if (isValidNodeName(answer))
+            return answer;
+        console.error('Use letters, numbers, dots, underscores or hyphens, starting with a letter or number.');
+    }
+}
+async function runNodeIdentity(positional, o, mode, command, repair = false) {
+    const cfg = await loadCfg();
+    const verb = repair ? 'node repair' : mode === 'create' ? 'node add' : 'node replace';
+    if (o.all) {
+        if (positional.name || positional.ssh)
+            throw new Error(`${verb} --all walks every configured node; do not name one`);
+        const results = [];
+        for (const [name, settings] of Object.entries(cfg.nodes)) {
+            const target = settings.ssh;
+            if (!target)
+                continue;
+            const merged = { ...o, name };
+            applyConfigDefaults(command, merged, nodeSettings(cfg, name));
+            try {
+                const reporting = await installNodeIdentity(target, name, mode, merged, repair, false);
+                results.push({ node: name, ok: true, ...(merged.wait ? { reporting } : {}) });
+            }
+            catch (e) {
+                results.push({ node: name, ok: false, error: errMessage(e) });
+            }
+        }
+        print({ results });
+        if (results.some((r) => !r.ok || r.reporting === false))
+            process.exitCode = 1;
+        return;
+    }
+    let name = positional.name ?? (positional.ssh ? inferNodeName(positional.ssh) : undefined);
+    if (!name && mode === 'create')
+        name = await promptNodeName(positional.ssh);
+    if (!name)
+        throw new Error(mode === 'create'
+            ? `${verb} needs --name${positional.ssh ? ' (an IP address does not suggest one; in a terminal it asks)' : ''}`
+            : `${verb} needs the node name`);
+    if (mode === 'create') {
+        // The name may have come from the SSH host, after the config defaults were applied without one.
+        o.name = name;
+        applyConfigDefaults(command, o, nodeSettings(cfg, name));
+    }
+    const sshTarget = positional.ssh ?? nodeSettings(cfg, name).ssh;
+    if (!sshTarget)
+        throw new Error(`${verb} needs an SSH target: pass <ssh-target> or set nodes.${name}.ssh in ${cfg.file}`);
+    if (mode === 'create') {
+        if (o.agentControlUrl)
+            normalizeControlUrl(o.agentControlUrl);
+        await transportFor(cfg).then((t) => t.close()); // no questions about a host this CLI cannot enroll
+        await planNodeAdd(sshTarget, name, o);
+    }
+    if (!(await installNodeIdentity(sshTarget, name, mode, o, repair)))
+        process.exitCode = 1;
+}
+export function registerNode(program) {
+    const node = program.command('node').description('add, inspect, repair and remove nodes');
+    addNodeIdentityOptions(node
+        .command('add [sshTarget]')
+        .description('create a node and install its agent over SSH; the name defaults to the first label of the SSH host and the SSH target to nodes.<name>.ssh in the config file. In a terminal it offers to install Docker, choose the inter-node address and enable ingress, then shows a plan to confirm. Safe to rerun after a partial failure')
+        .option('--name <name>', 'node name (default: the first DNS label of the SSH host)')
+        .option('--ingress', 'also make the node an ingress node (managed Caddy on ports 80/443)')
+        .option('-y, --yes', 'do not ask for confirmation in a terminal')).action(async (ssh, o, command) => runNodeIdentity({ name: o.name, ssh }, o, 'create', command));
+    for (const [verb, description, repair] of [
+        [
+            'replace',
+            'install a new key on an existing node over SSH: for an expired, lost or suspected key, a disabled node, a CA loss or an older restore',
+            false,
+        ],
+        [
+            'repair',
+            'move aside corrupt agent credential documents on the node (never service volumes) and replace its identity',
+            true,
+        ],
+    ]) {
+        addNodeIdentityOptions(node
+            .command(`${verb} [name] [sshTarget]`)
+            .description(`${description} (the SSH target defaults to nodes.<name>.ssh in the config file; --all walks every configured node)`)).action(async (name, ssh, o, command) => runNodeIdentity({ name, ssh }, o, 'replace', command, repair));
+    }
+    node
+        .command('list')
+        .alias('ls')
+        .description('list the registered nodes with their state, address, ingress role and last contact')
+        .option('--json', 'print the raw API result as JSON')
+        .action(async (o) => {
+        const result = await request('GET', '/api/v1/nodes');
+        if (o.json)
+            print(result);
+        else
+            console.log(formatNodes(result.nodes));
+    });
+    node
+        .command('enroll')
+        .description('mint a single-use, name-bound join key so a host you cannot SSH to can self-join with one docker run; a saved cloud-init document contains a credential')
+        .option('--name <name>', 'the node name the key is bound to; mint one key per instance (asked for in a terminal when omitted)')
+        .option('--address <addr>', 'private inter-node address for this node')
+        .option('--agent-control-url <url>', 'connect-only URL the agent dials; the certificate is still verified against the credential URL')
+        .option('--ttl <seconds>', 'join key lifetime', '3600')
+        .option('--image <tag>', "agent image used in the printed docker run (default: ghcr.io/wpelevator/tug:<this CLI's version>)")
+        .option('--label <label>', 'node label (repeatable)', collect, [])
+        .option('--install-docker', 'print cloud-init that installs Docker and starts the agent')
+        .option('--json', 'print JSON including the command or cloud-init payload')
+        .action(async (o, cmd) => {
+        if (!o.name) {
+            o.name = await promptNodeName();
+            if (!o.name)
+                throw new Error('node enroll needs --name');
+            applyConfigDefaults(cmd, o, nodeSettings(await loadCfg(), o.name));
+        }
+        if (o.address !== undefined && !isNodeHost(o.address))
+            throw new Error('address must be an IP address or hostname without a port');
+        const connectUrl = o.agentControlUrl ? normalizeControlUrl(o.agentControlUrl) : undefined;
+        const minted = await request('POST', '/api/v1/nodes/enrollment-credential', {
+            name: o.name,
+            ttlSeconds: Number(o.ttl),
+            labels: o.label.length ? o.label : undefined,
+        });
+        const keyPayload = {
+            v: 2,
+            credential: minted.credential,
+            name: minted.name,
+            ...(connectUrl ? { connectUrl } : {}),
+        };
+        const joinKey = Buffer.from(JSON.stringify(keyPayload)).toString('base64url');
+        const command = dockerRunSnippet(resolveImage(o.image), joinKey, o.address);
+        const cloudInit = o.installDocker
+            ? enrollmentCloudInit(await dockerBootstrap(), command)
+            : undefined;
+        if (o.json) {
+            print({
+                name: minted.name,
+                expiresAt: minted.expiresAt,
+                joinKey,
+                command,
+                ...(cloudInit ? { cloudInit } : {}),
+            });
+            return;
+        }
+        if (cloudInit) {
+            process.stdout.write(cloudInit);
+            return;
+        }
+        console.log(`# Join key for ${minted.name} valid until ${minted.expiresAt}, single use. Run on the fresh host:\n`);
+        console.log(command);
+    });
+    node
+        .command('register <name>')
+        .description('sign a node-generated CSR and print the identity response (the primitive behind node add; the private key stays with whoever made the CSR)')
+        .requiredOption('--csr <file>', 'PEM certificate request generated on the node (- for stdin)')
+        .option('--mode <mode>', 'create a new node or replace an existing node key', 'create')
+        .option('--address <addr>')
+        .option('--label <label>', 'node label (repeatable)', collect, [])
+        .action(async (name, o) => {
+        if (o.mode !== 'create' && o.mode !== 'replace')
+            throw new Error("--mode must be 'create' or 'replace'");
+        const csr = o.csr === '-' ? await readStdin() : await fs.readFile(o.csr, 'utf8');
+        print(await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/identity`, { csr, mode: o.mode, address: o.address, labels: o.label }));
+    });
+    node
+        .command('address <name> <address>')
+        .description("update a node's inter-node address")
+        .action(async (name, address) => print(await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/address`, { address })));
+    node
+        .command('disable <name>')
+        .description("block a node's key immediately without deleting its metadata or placement (no SSH needed); bring it back with node replace")
+        .action(async (name) => print(await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/disable`, {})));
+    node
+        .command('remove <name>')
+        .description('remove a retired node from the control plane (its identity, statuses and queued commands); stop its agent and remove its containers on the host yourself')
+        .option('--force', 'remove even if the node is still reporting')
+        .action(async (name, o) => print(await request('DELETE', `/api/v1/nodes/${encodeURIComponent(name)}${o.force ? '?force=true' : ''}`)));
+    node
+        .command('ingress <name>')
+        .description('mark a node as an ingress node (managed Caddy on ports 80/443), or remove the role with --disable')
+        .option('--disable', 'stop acting as an ingress node')
+        .action(async (name, o) => print(await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/ingress`, {
+        enabled: !o.disable,
+    })));
+}

package/dist/commands/observe.d.ts

@@ -0,0 +1,2 @@
+import type { Command } from 'commander';
+export declare function registerObserve(program: Command): void;

package/dist/commands/observe.js

@@ -0,0 +1,71 @@
+import { errMessage } from "../util.js";
+import { formatStatus } from "../cli-status.js";
+import { loadCfg, print, request, runCapture, waitForCommand } from "./context.js";
+export function registerObserve(program) {
+    program
+        .command('status')
+        .description('summarize projects, deployments and nodes')
+        .option('--json', 'print the raw status as JSON for scripts')
+        .action(async (o) => {
+        const status = await request('GET', '/api/v1/status');
+        console.log(o.json ? JSON.stringify(status, null, 2) : formatStatus(status));
+    });
+    program
+        .command('events')
+        .description('list recent events with the credential that caused each control action; defaults to the resolved project')
+        .option('--limit <n>', 'number of events', '100')
+        .option('--all', 'list events across all projects (limited to your projects for scoped credentials)')
+        .action(async (o) => {
+        if (o.all && o.project)
+            throw new Error('--all and --project cannot be combined');
+        const cfg = await loadCfg();
+        const project = o.all ? undefined : o.project || cfg.project;
+        const query = `limit=${encodeURIComponent(o.limit)}${project ? `&project=${encodeURIComponent(project)}` : ''}`;
+        print(await request('GET', `/api/v1/events?${query}`));
+    });
+    program
+        .command('doctor')
+        .description('check this machine and the control plane: certificate expiry, node connectivity and clock skew')
+        .option('--nodes', 'also run host diagnostics on every node through its agent and wait')
+        .option('--node <name>', 'run host diagnostics on one node through its agent and wait')
+        .option('--timeout <seconds>', 'wait timeout for node diagnostics', '60')
+        .action(async (o) => {
+        const local = [];
+        for (const [name, cmd, args] of [
+            ['node', 'node', ['--version']],
+            ['docker', 'docker', ['version', '--format', '{{.Server.Version}}']],
+            ['docker_compose', 'docker', ['compose', 'version']],
+        ]) {
+            try {
+                local.push({ name, ok: true, output: runCapture(cmd, args).trim() });
+            }
+            catch (e) {
+                local.push({ name, ok: false, error: errMessage(e) });
+            }
+        }
+        const controlDiagnostics = await request('GET', '/api/v1/diagnostics');
+        const warnings = (controlDiagnostics.checks ?? []).flatMap((check) => (check.warnings ?? []).map((warning) => `${check.name}: ${warning}`));
+        const output = {
+            ok: local.every((c) => c.ok) && controlDiagnostics.ok,
+            ...(warnings.length && { warnings }),
+            local,
+            control: controlDiagnostics,
+        };
+        if (o.nodes || o.node) {
+            const queued = await request('POST', '/api/v1/diagnostics/nodes', { node: o.node || null });
+            const commands = [];
+            for (const command of queued.commands) {
+                const finished = await waitForCommand(command.commandId, Number(o.timeout));
+                commands.push({
+                    nodeId: command.nodeId,
+                    state: finished.state,
+                    result: JSON.parse(finished.result_json || '{}'),
+                });
+            }
+            output.nodeDiagnostics = { commands };
+            if (commands.some((command) => command.state !== 'succeeded'))
+                output.ok = false;
+        }
+        print(output);
+    });
+}

package/dist/commands/restore.d.ts

@@ -0,0 +1,2 @@
+import type { Command } from 'commander';
+export declare function registerRestore(program: Command): void;