@wpelevator/tug 0.6.0 → 0.8.0

package/dist/cli.js

@@ -2,7 +2,7 @@
 import { Command, Option } from 'commander';
 import { openAsBlob } from 'node:fs';
 import { applyConfigDefaults, integerOption, normalizeControlUrl, nodeSettings, resolveCliConfig, writeInitConfig, } from "./cli-config.js";
-import { dockerRunSnippet, enrollmentCloudInit, shellQuote as sh, HOST_NETWORK_FLAG, MESH_RUN_FLAGS, controlPortCheck, localImageBuildArgs, meshPortCheck, } from "./cli-install.js";
+import { dockerRunSnippet, enrollmentCloudInit, shellQuote as sh, HOST_NETWORK_FLAG, MESH_RUN_FLAGS, CONTROL_CONTAINER, controlPortCheck, localImageBuildArgs, meshPortCheck, } from "./cli-install.js";
 import { createSourceArchive, prepareLocalImages } from "./cli-build.js";
 import { saveImageArchive } from "./artifacts.js";
 import { spawnSync } from 'node:child_process';
@@ -12,8 +12,11 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { composeHash } from "./compose.js";
 import { normalizeCompose } from "./normalize.js";
-import { errMessage, token, loadTemplate } from "./util.js";
-import { PROJECT_NAME_PATTERN, ROLES, TOKEN_NAME_PATTERN } from "./tokens.js";
+import { errMessage, loadTemplate } from "./util.js";
+import { createTransport } from "./agent-credential.js";
+import { decodeCredential, writeFileDurable } from "./pki.js";
+import { backupCheckProblems } from "./backup-check.js";
+import { PROJECT_NAME_PATTERN, ROLES, CREDENTIAL_NAME_PATTERN } from "./credentials.js";
 import { packageVersion, resolveImage } from "./image.js";
 import { TERMINAL_DEPLOYMENT_STATES } from "./protocol.js";
 import { CliLiveError, parseLiveRequest, runCliLive } from "./cli-live.js";
@@ -26,11 +29,12 @@ const CONFIG_HELP = 'CLI config file, YAML or JSON (default: TUG_CONFIG, else tu
 const program = new Command();
 program
     .name('tug')
-    .description('Portable VPS deployments with Docker Compose')
+    .description('Deploy Docker Compose applications to your own servers')
     .version(packageVersion())
     .enablePositionalOptions()
     .option('--config <file>', CONFIG_HELP)
-    .option('--control-url <url>', 'control API URL (overrides TUG_CONTROL_URL and config)')
+    .option('--control-url <url>', 'connect target for the control API (the credential still verifies its own public URL)')
+    .option('--credential <file>', 'credential file (overrides TUG_CREDENTIAL, TUG_CREDENTIAL_FILE and config)')
     .option('--project <name>', 'project (overrides TUG_PROJECT and config; default: site)');
 let resolvedConfig;
 async function loadCfg() {
@@ -43,22 +47,32 @@ async function readStdin() {
         chunks.push(Buffer.from(chunk));
     return Buffer.concat(chunks).toString('utf8');
 }
+/** The pinned mTLS transport for the resolved credential; an address override only changes the connect target. */
+async function transportFor(c) {
+    if (!c.credential)
+        throw new Error('no credential: run `tug init <credential-file>`, set TUG_CREDENTIAL_FILE or pass --credential');
+    return createTransport(c.credential, c.controlUrl);
+}
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 async function request(method, route, body, multipart = false) {
     const c = await loadCfg();
-    const base = c.controlUrl;
-    const tok = c.token;
-    const headers = tok ? { Authorization: `Bearer ${tok}` } : {};
+    const transport = await transportFor(c);
+    const headers = {};
     if (!multipart)
         headers['content-type'] = 'application/json';
-    const res = await fetch(`${base}${route}`, {
-        method,
-        headers,
-        body: multipart ? body : body ? JSON.stringify(body) : undefined,
-    });
-    if (!res.ok)
-        throw new Error(`${method} ${route}: ${res.status} ${await res.text()}`);
-    return res.json();
+    try {
+        const res = await transport.http(`${transport.base}${route}`, {
+            method,
+            headers,
+            body: multipart ? body : body ? JSON.stringify(body) : undefined,
+        });
+        if (!res.ok)
+            throw new Error(`${method} ${route}: ${res.status} ${await res.text()}`);
+        return (await res.json());
+    }
+    finally {
+        void transport.close();
+    }
 }
 function runLocal(cmd, args, opts = {}) {
     const result = spawnSync(cmd, args, { stdio: 'inherit', ...opts });
@@ -71,13 +85,6 @@ function runCapture(cmd, args, opts = {}) {
         throw new Error(`${cmd} ${args.join(' ')} failed\n${result.stderr || ''}`);
     return String(result.stdout);
 }
-function remoteHost(sshTarget) {
-    return sshTarget.replace(/^[^@]+@/, '').split(':')[0];
-}
-function assertHttps(url, allowHttp) {
-    if (!allowHttp && !String(url).startsWith('https://'))
-        throw new Error(`${url} must use https:// outside local development (pass --http-only-dev to override)`);
-}
 function preflightRemote(target, { controlUrl, ports = [], udpPorts = [], minDiskMb = 1024, mesh = false, compose = false, } = {}) {
     const checks = [
         [
@@ -129,8 +136,9 @@ function preflightRemote(target, { controlUrl, ports = [], udpPorts = [], minDis
     if (controlUrl)
         checks.push([
             'control.reachable',
-            `curl -fsS ${sh(controlUrl)}/healthz >/dev/null`,
-            'Verify DNS/firewall/TLS for the control URL from the node.',
+            // TCP reachability only: no credential is copied to the host, and real TLS authorization happens when the agent connects.
+            `timeout 5 bash -c ${sh(`exec 3<>/dev/tcp/${new URL(controlUrl).hostname.replace(/^\[|\]$/g, '')}/${new URL(controlUrl).port || 443}`)}`,
+            'Verify DNS and the firewall from the node to the control port (control speaks mTLS only, so only TCP reachability is checked here).',
         ]);
     const script = `set -e
 ${checks.map(([name, cmd, fix]) => `echo "preflight: ${name}"; (${cmd}) || { echo "FAILED ${name}: ${fix}" >&2; exit 20; }`).join('\n')}
@@ -181,33 +189,49 @@ async function waitForDeployment(deploymentId, timeoutSeconds = 300) {
     throw new Error(`timed out waiting for deployment ${deploymentId}`);
 }
 program
-    .command('init')
-    .option('--token <token>', 'operator token (prefer TUG_TOKEN to avoid process-list exposure)')
-    .description('create the config file if missing and store the operator token in its git-ignored .local overlay')
-    .action(async (o, command) => {
+    .command('init [credentialFile]')
+    .description('create the config file if missing and import a credential (a file, or - for stdin) into a 0600 user-config file that the git-ignored .local overlay points at')
+    .action(async (credentialFile, _o, command) => {
     const cfg = await loadCfg();
     const { controlUrl, project } = command.optsWithGlobals();
-    for (const file of await writeInitConfig(cfg, o.token || cfg.token || token('tug_op'), {
-        controlUrl,
-        project,
-    }))
+    let text;
+    if (credentialFile === '-')
+        text = await readStdin();
+    else if (credentialFile)
+        text = await fs.readFile(credentialFile, 'utf8');
+    else if (cfg.credential)
+        text = JSON.stringify(cfg.credential);
+    if (!text)
+        throw new Error('tug init needs a credential file (or - for stdin)');
+    const credential = decodeCredential(text);
+    // TUG_CREDENTIALS_DIR keeps scripts and tests out of the real home directory.
+    const dir = process.env.TUG_CREDENTIALS_DIR ||
+        path.join(process.env.XDG_CONFIG_HOME || path.join(os.homedir(), '.config'), 'tug', 'credentials');
+    const source = path.join(dir, `${new URL(credential.url).host.replace(/[^A-Za-z0-9.-]/g, '_')}.json`);
+    await writeFileDurable(source, `${JSON.stringify(credential, null, 2)}\n`, 0o600);
+    for (const file of await writeInitConfig(cfg, source, { controlUrl, project }))
         console.log(`wrote ${file}`);
+    console.log(`imported credential to ${source}`);
 });
 program
     .command('node-register <name>')
-    .description('register a node with the control plane')
+    .description('sign a node-generated CSR and print the identity response (the primitive behind node-install; the private key stays with whoever made the CSR)')
+    .requiredOption('--csr <file>', 'PEM certificate request generated on the node (- for stdin)')
+    .option('--mode <mode>', 'create a new node or replace an existing node key', 'create')
     .option('--address <addr>')
     .option('--public-address <addr>')
     .option('--label <label>', 'node label (repeatable)', collect, [])
     .action(async (name, o) => {
-    const r = await request('POST', '/api/v1/nodes/register', {
-        name,
-        address: o.address,
-        publicAddress: o.publicAddress,
-        labels: o.label,
-    });
+    if (o.mode !== 'create' && o.mode !== 'replace')
+        throw new Error("--mode must be 'create' or 'replace'");
+    const csr = o.csr === '-' ? await readStdin() : await fs.readFile(o.csr, 'utf8');
+    const r = await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/identity`, { csr, mode: o.mode, address: o.address, publicAddress: o.publicAddress, labels: o.label });
     console.log(JSON.stringify(r, null, 2));
 });
+program
+    .command('node-disable <name>')
+    .description("block a node's key immediately without deleting its metadata or placement (no SSH needed); bring it back with node-replace-identity")
+    .action(async (name) => console.log(JSON.stringify(await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/disable`, {}), null, 2)));
 program
     .command('node-rotate-wg-key <name>')
     .description('generate a new WireGuard keypair on the node; peers pick up the new public key on their next poll')
@@ -221,8 +245,8 @@ program
 });
 program
     .command('node-remove <name>')
-    .description('remove a retired node from control (its agent token, WireGuard peer and queued commands)')
-    .option('--force', 'remove even if the agent is still reporting')
+    .description('remove a retired node from the control plane (its identity, WireGuard peer, statuses and queued commands); stop its agent and remove its containers on the host yourself')
+    .option('--force', 'remove even if the node is still reporting')
     .action(async (name, o) => {
     console.log(JSON.stringify(await request('DELETE', `/api/v1/nodes/${encodeURIComponent(name)}${o.force ? '?force=true' : ''}`), null, 2));
 });
@@ -245,21 +269,20 @@ program
         throw new Error('replicas must be a positive integer');
     console.log(JSON.stringify(await request('POST', '/api/v1/scale', { project, service, replicas: Number(replicas) }), null, 2));
 });
-program
-    .command('control-token')
-    .description('print local dev operator token if available')
-    .action(async () => console.log((await fs.readFile('./data/operator.token', 'utf8')).trim()));
 program
     .command('control-install <sshTarget>')
-    .description('install the control plane on a Docker-capable host over SSH; front it with your own reverse proxy/TLS if exposing it publicly')
+    .description('install the control plane on a Docker-capable host over SSH and issue the root credential into a local 0600 file; it listens for mTLS only, so front it with TCP passthrough rather than a terminating proxy')
+    .requiredOption('--public-url <url>', 'credential-free https origin clients use, including the published port')
+    .option('--extra-san <name>', 'additional DNS name or IP for the server certificate (repeatable)', collect, [])
     .option('--image <tag>', "Tug image to run (default: ghcr.io/wpelevator/tug:<this CLI's version>)")
     .option('--install-docker', 'install missing Docker Engine and Compose on supported Debian, Ubuntu, Fedora and RHEL-compatible hosts (root required)')
     .option('--build-local-image', 'build the image from the current checkout and copy it to the host over SSH (tags it tug:local unless --image is set); for development')
-    .option('--token <token>', 'operator token')
-    .option('--port <port>', 'published API port', '8080')
+    .option('--port <port>', 'published mTLS port', '8443')
+    .option('--credential-out <file>', 'where to write the root credential', 'tug-root.credential.json')
+    .option('--recover-root', "replace root's key from the control host without the API (lost credential); other admins are unaffected")
     .option('--mesh', 'enable the WireGuard mesh on control (TUG_MESH=true); install nodes with --mesh too')
     .action(async (sshTarget, o) => {
-    const operatorToken = o.token || process.env.TUG_TOKEN || token('tug_op');
+    const publicUrl = normalizeControlUrl(o.publicUrl);
     if (o.installDocker)
         await bootstrapDocker(sshTarget, true);
     preflightRemote(sshTarget, { ports: [Number(o.port)], minDiskMb: 2048, compose: true });
@@ -269,63 +292,79 @@ program
     const yamlValue = (value) => JSON.stringify(value);
     const compose = await loadTemplate(templatesDir, 'control-compose.yml', {
         IMAGE: yamlValue(image),
-        OPERATOR_TOKEN: yamlValue(operatorToken),
+        PUBLIC_URL: yamlValue(new URL(publicUrl).origin),
+        EXTRA_SANS: yamlValue(o.extraSan.join(',')),
         PORT: o.port,
         MESH: yamlValue(o.mesh ? 'true' : 'false'),
     });
     const remote = await loadTemplate(templatesDir, 'control-install.sh', {
         COMPOSE: compose,
-        OPERATOR_TOKEN_SH: sh(operatorToken),
+        CONTAINER_SH: sh(CONTROL_CONTAINER),
     });
     runLocal('ssh', ['--', sshTarget, remote]);
-    console.log(JSON.stringify({
-        ok: true,
-        sshTarget,
-        operatorToken,
-        url: `http://${remoteHost(sshTarget)}:${o.port}`,
-    }, null, 2));
+    // Upgrades never reissue root: fresh installs create it, `--recover-root` replaces its key. Output goes straight to a file.
+    const issue = `docker exec ${sh(CONTROL_CONTAINER)} node dist/main.js control issue-credential --name root --role admin${o.recoverRoot ? ' --replace' : ''}`;
+    const out = spawnSync('ssh', ['--', sshTarget, issue], { encoding: 'utf8' });
+    if (out.status !== 0) {
+        if (!o.recoverRoot && /already used/.test(String(out.stderr)))
+            console.error('control already has a root credential; upgrade complete (pass --recover-root to replace it)');
+        else
+            throw new Error(`issuing the root credential failed: ${String(out.stderr).trim()}`);
+    }
+    else {
+        decodeCredential(out.stdout);
+        await writeFileDurable(o.credentialOut, `${String(out.stdout).trim()}\n`, 0o600);
+        console.error(`root credential written to ${o.credentialOut}; import it with: tug init ${o.credentialOut}`);
+    }
+    console.log(JSON.stringify({ ok: true, sshTarget, url: publicUrl }, null, 2));
 });
-program
-    .command('node-install [sshTarget]')
-    .description('register and install an agent container on a Docker-capable host over SSH (the SSH target defaults to nodes.<name>.ssh in the config file)')
-    .requiredOption('--name <name>')
-    .option('--address <addr>')
-    .option('--public-address <addr>')
-    .option('--label <label>', 'node label (repeatable)', collect, [])
-    .option('--agent-control-url <url>', 'control URL reachable by the agent (default: resolved operator control URL)')
-    .option('--http-only-dev', 'allow plaintext HTTP control URL for local/private development')
-    .option('--mesh', 'join the WireGuard mesh (agent runs with host networking and NET_ADMIN; control needs TUG_MESH=true)')
-    .option('--wg-port <port>', 'WireGuard UDP port checked by preflight', '51820')
-    .option('--image <tag>', "Tug image to run (default: ghcr.io/wpelevator/tug:<this CLI's version>)")
-    .option('--install-docker', 'install missing Docker Engine and Compose on supported Debian, Ubuntu, Fedora and RHEL-compatible hosts (root required)')
-    .option('--build-local-image', 'build the image from the current checkout and copy it to the host over SSH (tags it tug:local unless --image is set); for development')
-    .action(async (sshArgument, o) => {
+const AGENT_STATE = '/var/lib/tug';
+function addNodeIdentityOptions(command) {
+    return command
+        .option('--name <name>', 'node name')
+        .option('--all', 'walk every node with an ssh target in the config file, continuing after failures')
+        .option('--agent-control-url <url>', 'connect-only URL the agent dials (for example a private address); the certificate is still verified against the credential URL')
+        .option('--mesh', 'join the WireGuard mesh (agent runs with host networking and NET_ADMIN; control needs TUG_MESH=true)')
+        .option('--wg-port <port>', 'WireGuard UDP port checked by preflight', '51820')
+        .option('--image <tag>', "Tug image to run (default: ghcr.io/wpelevator/tug:<this CLI's version>)")
+        .option('--install-docker', 'install missing Docker Engine and Compose on supported Debian, Ubuntu, Fedora and RHEL-compatible hosts (root required)')
+        .option('--build-local-image', 'build the image from the current checkout and copy it to the host over SSH (tags it tug:local unless --image is set); for development')
+        .option('--no-wait', 'do not wait for the node to report after the install');
+}
+/**
+ * The SSH-assisted flow: the node makes its own key and CSR, an admin credential authorizes the identity at this moment and
+ * the certificate is installed back over SSH, so no bootstrap secret travels and the private key never leaves the node.
+ */
+async function installNodeIdentity(sshTarget, name, mode, o, repair = false) {
     const cfg = await loadCfg();
-    const sshTarget = sshArgument ?? nodeSettings(cfg, o.name).ssh;
-    if (!sshTarget)
-        throw new Error(`node-install needs an SSH target: pass <ssh-target> or set nodes.${o.name}.ssh in ${cfg.file}`);
-    const controlUrl = normalizeControlUrl(o.agentControlUrl ?? cfg.controlUrl);
-    assertHttps(controlUrl, o.httpOnlyDev);
+    const connectUrl = o.agentControlUrl ? normalizeControlUrl(o.agentControlUrl) : undefined;
+    await transportFor(cfg).then((t) => t.close()); // fail before touching the host when there is no credential
     if (o.installDocker)
         await bootstrapDocker(sshTarget);
     preflightRemote(sshTarget, {
-        controlUrl,
+        controlUrl: connectUrl ?? cfg.credential?.url,
         mesh: o.mesh,
         udpPorts: o.mesh ? [Number(o.wgPort)] : [],
     });
-    const registered = await request('POST', '/api/v1/nodes/register', {
-        name: o.name,
-        address: o.address,
-        publicAddress: o.publicAddress,
-        labels: o.label,
-    });
     const image = resolveImage(o.image, o.buildLocalImage);
     if (o.buildLocalImage)
         await transferTugImage(sshTarget, image);
+    const started = new Date().toISOString();
+    if (repair)
+        runLocal('ssh', [
+            '--',
+            sshTarget,
+            `mkdir -p ${AGENT_STATE} && cd ${AGENT_STATE} && for f in agent.json agent-pending.json; do [ -e "$f" ] && mv "$f" "$f.corrupt-$(date +%s)"; done; true`,
+        ]);
+    const agent = (command, interactive = false) => `mkdir -p ${AGENT_STATE} && docker run --rm ${interactive ? '-i ' : ''}-v ${AGENT_STATE}:${AGENT_STATE} ${sh(image)} agent ${command}`;
+    const csr = runCapture('ssh', ['--', sshTarget, agent('identity')]);
+    const response = await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/identity`, { csr, mode, address: o.address, publicAddress: o.publicAddress, labels: o.label });
+    runCapture('ssh', ['--', sshTarget, agent('install-cert', true)], {
+        input: JSON.stringify(response),
+    });
     const env = (await loadTemplate(templatesDir, 'agent.env', {
-        CONTROL_URL: controlUrl,
-        NODE_NAME: o.name,
-        AGENT_TOKEN: registered.agentToken,
+        CONTROL_URL_LINE: connectUrl ? `TUG_CONTROL_URL=${connectUrl}\n` : '',
+        NODE_NAME: name,
     })) + (o.mesh ? 'TUG_MESH=true\n' : '');
     const remote = await loadTemplate(templatesDir, 'node-install.sh', {
         ENV: env,
@@ -333,54 +372,131 @@ program
         RUN_FLAGS: `${o.mesh ? MESH_RUN_FLAGS : HOST_NETWORK_FLAG} `,
     });
     runLocal('ssh', ['--', sshTarget, remote]);
-    console.log(JSON.stringify({ ok: true, node: registered.node }, null, 2));
+    if (o.wait) {
+        // A report from the new identity: wait for a report after the install began, not a replaced node's stale status.
+        for (let i = 0; i < 30; i++) {
+            const { nodes } = await request('GET', '/api/v1/nodes');
+            const seen = nodes.find((n) => n.name === name)?.last_seen_at;
+            if (seen && seen > started)
+                return void console.log(JSON.stringify({ ok: true, node: response.node }, null, 2));
+            await new Promise((r) => setTimeout(r, 2000));
+        }
+        console.error(`node ${name} has not reported yet; check \`docker logs tug-agent\` on the host`);
+    }
+    console.log(JSON.stringify({ ok: true, node: response.node }, null, 2));
+}
+async function runNodeIdentity(sshArgument, o, mode, repair = false) {
+    const cfg = await loadCfg();
+    const verb = repair
+        ? 'node-repair-identity'
+        : mode === 'create'
+            ? 'node-install'
+            : 'node-replace-identity';
+    if (o.all) {
+        const results = [];
+        for (const [name, settings] of Object.entries(cfg.nodes)) {
+            const target = settings.ssh;
+            if (!target)
+                continue;
+            const merged = { ...o, ...nodeOverrides(settings), name };
+            try {
+                await installNodeIdentity(target, name, mode, merged, repair);
+                results.push({ node: name, ok: true });
+            }
+            catch (e) {
+                results.push({ node: name, ok: false, error: errMessage(e) });
+            }
+        }
+        console.log(JSON.stringify({ results }, null, 2));
+        if (results.some((r) => !r.ok))
+            process.exitCode = 1;
+        return;
+    }
+    if (!o.name)
+        throw new Error(`${verb} needs --name`);
+    const sshTarget = sshArgument ?? nodeSettings(cfg, o.name).ssh;
+    if (!sshTarget)
+        throw new Error(`${verb} needs an SSH target: pass <ssh-target> or set nodes.${o.name}.ssh in ${cfg.file}`);
+    await installNodeIdentity(sshTarget, o.name, mode, o, repair);
+}
+const nodeOverrides = (settings) => ({
+    ...(settings.address ? { address: settings.address } : {}),
+    ...(settings.publicAddress ? { publicAddress: settings.publicAddress } : {}),
+    ...(Array.isArray(settings.label) ? { label: settings.label } : {}),
+    ...(settings.mesh ? { mesh: true } : {}),
 });
+for (const [verb, description, mode, repair] of [
+    [
+        'node-install',
+        'create a node and install its agent over SSH (the SSH target defaults to nodes.<name>.ssh in the config file); safe to rerun after a partial failure',
+        'create',
+        false,
+    ],
+    [
+        'node-replace-identity',
+        'install a new key on an existing node over SSH: for an expired, lost or suspected key, a disabled node, a CA loss or an older restore',
+        'replace',
+        false,
+    ],
+    [
+        'node-repair-identity',
+        'move aside corrupt agent credential documents on the node (never service volumes) and replace its identity',
+        'replace',
+        true,
+    ],
+]) {
+    const command = program
+        .command(`${verb} [sshTarget]`)
+        .description(description)
+        .option('--address <addr>')
+        .option('--public-address <addr>')
+        .option('--label <label>', 'node label (repeatable)', collect, []);
+    addNodeIdentityOptions(command).action(async (sshArgument, o) => runNodeIdentity(sshArgument, o, mode, repair));
+}
 program
-    .command('node-enroll')
-    .description('mint a single-use enrollment key so a fresh host can self-join with one docker run (no SSH)')
-    .option('--name <name>', 'pre-bind the node name (default: the agent uses its own hostname)')
-    .option('--agent-control-url <url>', 'control URL reachable by the agent (default: resolved operator control URL)')
-    .option('--ttl <seconds>', 'enrollment key lifetime', '3600')
-    .option('--uses <n>', 'how many hosts may enroll with this key', '1')
-    .option('--ca <file>', 'PEM CA cert to embed so agents trust a private/self-signed control')
-    .option('--http-only-dev', 'allow a plaintext HTTP control URL for local/private development')
+    .command('node-prepare-enrollment')
+    .description('mint a single-use, name-bound join key so a host you cannot SSH to can self-join with one docker run; a saved cloud-init document contains a credential')
+    .requiredOption('--name <name>', 'the node name the key is bound to; mint one key per instance')
+    .option('--agent-control-url <url>', 'connect-only URL the agent dials; the certificate is still verified against the credential URL')
+    .option('--ttl <seconds>', 'join key lifetime', '3600')
     .option('--image <tag>', "agent image used in the printed docker run (default: ghcr.io/wpelevator/tug:<this CLI's version>)")
     .option('--mesh', 'print a docker run that joins the WireGuard mesh (host networking, NET_ADMIN)')
     .option('--label <label>', 'node label (repeatable)', collect, [])
     .option('--install-docker', 'print cloud-init that installs Docker and starts the agent')
     .option('--json', 'print JSON including the command or cloud-init payload')
     .action(async (o) => {
-    const cfg = await loadCfg();
-    const controlUrl = normalizeControlUrl(o.agentControlUrl ?? cfg.controlUrl);
-    assertHttps(controlUrl, o.httpOnlyDev);
-    const ca = o.ca ? await fs.readFile(o.ca, 'utf8') : undefined;
-    const minted = await request('POST', '/api/v1/nodes/enroll-token', {
-        name: o.name || null,
+    const connectUrl = o.agentControlUrl ? normalizeControlUrl(o.agentControlUrl) : undefined;
+    const minted = await request('POST', '/api/v1/nodes/enrollment-credential', {
+        name: o.name,
         ttlSeconds: Number(o.ttl),
-        uses: Number(o.uses),
+        labels: o.label.length ? o.label : undefined,
     });
     const keyPayload = {
-        v: 1,
-        controlUrl,
-        enrollToken: minted.enrollToken,
-        ...(o.label.length ? { labels: o.label } : {}),
+        v: 2,
+        credential: minted.credential,
+        name: minted.name,
+        ...(connectUrl ? { connectUrl } : {}),
     };
-    if (ca)
-        keyPayload.ca = ca;
     const joinKey = Buffer.from(JSON.stringify(keyPayload)).toString('base64url');
     const command = dockerRunSnippet(resolveImage(o.image), joinKey, o.mesh);
     const cloudInit = o.installDocker
         ? enrollmentCloudInit(await dockerBootstrap(), command)
         : undefined;
     if (o.json) {
-        console.log(JSON.stringify({ ...minted, controlUrl, joinKey, command, ...(cloudInit ? { cloudInit } : {}) }, null, 2));
+        console.log(JSON.stringify({
+            name: minted.name,
+            expiresAt: minted.expiresAt,
+            joinKey,
+            command,
+            ...(cloudInit ? { cloudInit } : {}),
+        }, null, 2));
         return;
     }
     if (cloudInit) {
         process.stdout.write(cloudInit);
         return;
     }
-    console.log(`# Enrollment key valid until ${minted.expiresAt} (${minted.usesRemaining} use(s)). Run on each fresh host:\n`);
+    console.log(`# Join key for ${minted.name} valid until ${minted.expiresAt}, single use. Run on the fresh host:\n`);
     console.log(command);
 });
 program
@@ -451,6 +567,7 @@ program
     .command('backup-list')
     .description('show each backup target with its last success and next run, then recent runs')
     .option('--limit <n>', 'number of recent runs to show', '20')
+    .option('--check', 'exit nonzero when a target is stale, or its last run failed or was missed, or there is no destination; for cron and monitors')
     .action(async (o) => {
     const project = (await loadCfg()).project || 'site';
     const listing = await request('GET', `/api/v1/backups?project=${encodeURIComponent(project)}&limit=${encodeURIComponent(o.limit)}`);
@@ -462,6 +579,13 @@ program
         console.log(`${t.target}\t${t.node ?? '-'}\t${t.schedule}\tlast success ${t.lastSuccessAt ?? 'never'}\tnext ${t.enabled ? t.nextRunAt : 'disabled'}`);
     for (const r of listing.runs)
         console.log(`${r.finished_at ?? r.queued_at}\t${r.target}\t${r.trigger}\t${r.state}\t${r.snapshot_id?.slice(0, 8) ?? '-'}\t${formatBytes(r.size_bytes)}${r.error ? `\t${r.error}` : ''}`);
+    if (o.check) {
+        const problems = backupCheckProblems(listing);
+        for (const problem of problems)
+            console.error(`PROBLEM ${problem}`);
+        if (problems.length)
+            process.exitCode = 1;
+    }
 });
 program
     .command('restore <target>')
@@ -504,6 +628,8 @@ program
         const { restores } = await request('GET', `/api/v1/restores?project=${encodeURIComponent(project)}`);
         const restore = restores.find((item) => item.id === result.restoreId);
         if (restore?.state === 'succeeded') {
+            if (restore.error)
+                console.error(`warning: ${restore.error}`);
             console.log(`restore ${restore.id} completed${restore.scratch_volume ? `; files are in volume ${restore.scratch_volume}` : ''}`);
             return;
         }
@@ -563,41 +689,41 @@ program
     .description('remove the backup destination and its stored credentials; existing backups stay in the repository')
     .action(async () => console.log(JSON.stringify(await request('DELETE', '/api/v1/backup/destination'))));
 program
-    .command('backup-key')
+    .command('backup-repository-password')
     .description("print the project's repository and its password for offline escrow; anyone holding the password and destination credentials can read the project's backups")
     .action(async () => {
     const c = await loadCfg();
     const project = c.project || 'site';
-    console.log(JSON.stringify(await request('GET', `/api/v1/backup/key?project=${encodeURIComponent(project)}`), null, 2));
+    console.log(JSON.stringify(await request('GET', `/api/v1/backup/repository-password?project=${encodeURIComponent(project)}`), null, 2));
 });
 program
-    .command('token-create <name>')
-    .description('create a named API token for CI or a teammate; the secret is printed once and cannot be shown again')
+    .command('credential-create <name>')
+    .description('create a named operator credential for CI or a teammate; it is printed once, control keeps no copy of its private key, and it expires after one year at most')
     .addOption(new Option('--role <role>', 'read: status and listings; deploy: read plus deploy, rollback, cancel, restart, logs; admin: everything')
     .choices(ROLES)
     .makeOptionMandatory())
-    .addOption(new Option('--project <name>', 'limit a read or deploy token to this project; repeat for several')
+    .addOption(new Option('--project <name>', 'limit a read or deploy credential to this project; repeat for several')
     .default([], 'all projects')
     .argParser((value, previous) => [...previous, value]))
-    .option('--expires-in-days <n>', 'expire the token after this many days (default: never)')
+    .option('--expires-in-days <n>', 'expire the credential after this many days (default and maximum: 365)')
     .action(async (name, o) => {
-    const created = await request('POST', '/api/v1/tokens', {
+    const created = await request('POST', '/api/v1/credentials', {
         name,
         role: o.role,
         projects: o.project.length ? o.project : undefined,
         expiresInSeconds: o.expiresInDays ? Number(o.expiresInDays) * 86400 : undefined,
     });
-    console.error('Store this token now (for example as the TUG_TOKEN secret in CI); it is not shown again.');
+    console.error('Store this credential now (for example as the TUG_CREDENTIAL secret in CI); it is not shown again. Replace one by creating a new name, verifying it, then revoking the old name.');
     console.log(JSON.stringify(created, null, 2));
 });
 program
-    .command('token-list')
-    .description('list API tokens with role, projects, expiry and last use; secrets are never shown')
-    .action(async () => console.log(JSON.stringify(await request('GET', '/api/v1/tokens'), null, 2)));
+    .command('credential-list')
+    .description('list operator credentials with role, projects, expiry and last use; secrets are never shown')
+    .action(async () => console.log(JSON.stringify(await request('GET', '/api/v1/credentials'), null, 2)));
 program
-    .command('token-revoke <name>')
-    .description('revoke an API token immediately; its name stays reserved for the audit trail')
-    .action(async (name) => console.log(JSON.stringify(await request('DELETE', `/api/v1/tokens/${encodeURIComponent(name)}`), null, 2)));
+    .command('credential-revoke <name>')
+    .description('revoke an operator credential immediately; its name stays reserved for the audit trail')
+    .action(async (name) => console.log(JSON.stringify(await request('DELETE', `/api/v1/credentials/${encodeURIComponent(name)}`), null, 2)));
 async function makeContext() {
     const tmp = path.join(os.tmpdir(), `tug-${Date.now()}.tgz`);
     try {
@@ -668,9 +794,9 @@ program
     .action(async () => console.log(JSON.stringify(await request('GET', '/api/v1/status'), null, 2)));
 program
     .command('events')
-    .description('list recent events with the token that caused each control action; defaults to the resolved project')
+    .description('list recent events with the credential that caused each control action; defaults to the resolved project')
     .option('--limit <n>', 'number of events', '100')
-    .option('--all', 'list events across all projects (limited to your projects for scoped tokens)')
+    .option('--all', 'list events across all projects (limited to your projects for scoped credentials)')
     .action(async (o) => {
     if (o.all && o.project)
         throw new Error('--all and --project cannot be combined');
@@ -681,9 +807,9 @@ program
 });
 program
     .command('doctor')
-    .description('run local and control-plane diagnostics')
-    .option('--nodes', 'also run diagnostics on every registered agent and wait')
-    .option('--node <name>', 'run diagnostics on one agent and wait')
+    .description('check this machine and the control plane: certificate expiry, node connectivity and clock skew')
+    .option('--nodes', 'also run host diagnostics on every node through its agent and wait')
+    .option('--node <name>', 'run host diagnostics on one node through its agent and wait')
     .option('--timeout <seconds>', 'wait timeout for node diagnostics', '60')
     .action(async (o) => {
     const local = [];
@@ -700,8 +826,10 @@ program
         }
     }
     const controlDiagnostics = await request('GET', '/api/v1/diagnostics');
+    const warnings = (controlDiagnostics.checks ?? []).flatMap((check) => (check.warnings ?? []).map((warning) => `${check.name}: ${warning}`));
     const output = {
         ok: local.every((c) => c.ok) && controlDiagnostics.ok,
+        ...(warnings.length && { warnings }),
         local,
         control: controlDiagnostics,
     };
@@ -754,11 +882,11 @@ program
 });
 program
     .command('control-backup')
-    .description('create a control-plane SQLite/master-key backup')
+    .description('create a control-plane backup (state, PKI, master key and retained image archives; not workload volumes)')
     .action(async () => console.log(JSON.stringify(await request('POST', '/api/v1/control/backups/run', {}), null, 2)));
 program
     .command('control-restore <artifactRef>')
-    .description('restore a control-plane backup; requires control restart')
+    .description('restore a control-plane backup on the control-plane host; requires a control restart')
     .option('--confirm')
     .action(async (artifactRef, o) => console.log(JSON.stringify(await request('POST', '/api/v1/control/restore', {
     artifactRef,
@@ -788,7 +916,7 @@ program
 });
 async function runLive(operation, input) {
     const cfg = await loadCfg();
-    process.exitCode = await runCliLive(parseLiveRequest(operation, { ...input, project: cfg.project || 'site' }), createCliLiveTransport(cfg.controlUrl, cfg.token));
+    process.exitCode = await runCliLive(parseLiveRequest(operation, { ...input, project: cfg.project || 'site' }), createCliLiveTransport(cfg.credential, cfg.controlUrl));
 }
 program
     .command('exec <service> [command...]')
@@ -824,7 +952,7 @@ program
 });
 program
     .command('logs <service>')
-    .description('fetch logs from the assigned node agent')
+    .description('fetch service container logs from the node that runs the service')
     .option('--tail <n>', 'number of lines', '200')
     .option('-f, --follow', 'follow live logs')
     .option('--node <name>', 'select a node for followed logs')
@@ -877,15 +1005,15 @@ program.hook('preAction', async (_root, command) => {
     if (options.name && !/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(options.name))
         throw new Error('--name must contain only letters, numbers, dots, underscores or hyphens');
     const connection = command.optsWithGlobals();
-    if (command.name() === 'token-create') {
-        // --project is repeatable here and scopes the new token, so it must not also select the connection project.
-        if (!TOKEN_NAME_PATTERN.test(command.args[0]))
-            throw new Error('token name must be 1-64 letters, numbers, dots, underscores or hyphens, starting with a letter or number');
+    if (command.name() === 'credential-create') {
+        // --project is repeatable here and scopes the new credential, so it must not also select the connection project.
+        if (!CREDENTIAL_NAME_PATTERN.test(command.args[0]))
+            throw new Error('credential name must be 1-64 letters, numbers, dots, underscores or hyphens, starting with a letter or number');
         for (const project of options.project)
             if (!PROJECT_NAME_PATTERN.test(project))
                 throw new Error('--project must contain only letters, numbers, dots, underscores or hyphens');
         if (options.role === 'admin' && options.project.length)
-            throw new Error('admin tokens cannot be limited to projects; use --role read or --role deploy with --project');
+            throw new Error('admin credentials cannot be limited to projects; use --role read or --role deploy with --project');
         connection.project = undefined;
     }
     resolvedConfig = await resolveCliConfig(connection, process.env, process.cwd(), command.name() === 'init');
@@ -893,7 +1021,12 @@ program.hook('preAction', async (_root, command) => {
     // commands that create or register one, never by commands that talk to an existing node.
     const nodeName = command.name() === 'node-register'
         ? command.args[0]
-        : ['node-install', 'node-enroll'].includes(command.name())
+        : [
+            'node-install',
+            'node-replace-identity',
+            'node-repair-identity',
+            'node-prepare-enrollment',
+        ].includes(command.name())
             ? options.name
             : undefined;
     applyConfigDefaults(command, options, {
@@ -904,7 +1037,8 @@ program.hook('preAction', async (_root, command) => {
 for (const command of program.commands) {
     command
         .option('--config <file>', CONFIG_HELP)
-        .option('--control-url <url>', 'control API URL (overrides TUG_CONTROL_URL and config)');
+        .option('--control-url <url>', 'connect target for the control API (the credential still verifies its own public URL)')
+        .option('--credential <file>', 'credential file (overrides TUG_CREDENTIAL, TUG_CREDENTIAL_FILE and config)');
     if (!command.options.some((option) => option.long === '--project'))
         command.option('--project <name>', 'project (overrides TUG_PROJECT and config; default: site)');
 }