@wpelevator/tug 0.0.0 → 0.2.0

package/README.md

@@ -0,0 +1,63 @@
+# Tug
+
+The name: tugboats pull ships into place, like the pull-based agents here; `tug` is the CLI.
+
+Deploy Docker Compose projects to one or more ordinary Linux VPSes through a small self-hosted control plane and a pull-based agent on each node, without Kubernetes or a hosted PaaS. The original specification is in [plan.md](plan.md).
+
+## Features
+
+- Control API with bearer-token auth, scoped `read`/`deploy`/`admin` API tokens for CI and teammates, an audit trail naming who acted, and SQLite metadata.
+- Nodes join over SSH (`node-install`) or enroll themselves with a single `docker run` (`node-enroll`), with per-node agent and registry tokens. Agents only dial out, so nodes behind NAT work.
+- Agents pull desired state from control, apply a generated per-node Compose file, run TCP/HTTP/Docker health checks and report status.
+- `x-tug` placement of services and volumes on nodes, with immutable desired-state records per deployment.
+- Build application images on control from uploaded source, or locally with `deploy --build-local`. Agents download saved images directly from control by default; a bring-your-own registry is optional.
+- Secrets encrypted at rest, released only to the nodes whose services reference them, with targeted service recreation when only a secret changes.
+- Deployment convergence, per-project operation locks, cancellation, rollback, per-service rollout events and log fetching through the agents.
+- Opt-in zero-downtime rolling rollouts for ingress-fronted services (`x-tug.rollout`) and automatic rollback of failed deployments (`x-tug.rollback: auto`). See [Compose extensions](docs/compose.md#rolling-rollouts).
+- Per-node registry pull tokens enforced by an htpasswd-backed registry you provide, with rotation and revocation.
+- Optional WireGuard mesh between nodes, health-aware cross-node service names and a managed Caddy ingress with automatic HTTPS, each opt-in. See [Networking](docs/networking.md).
+- Control-plane backup and restore, and `tug doctor` diagnostics.
+
+## Quick start
+
+Install the CLI (Node >= 22.18), or run it without installing through `npx @wpelevator/tug`:
+
+```bash
+npm install -g @wpelevator/tug
+```
+
+The CLI installs the matching `ghcr.io/wpelevator/tug:<version>` image on your hosts.
+
+1. Install the control plane: `tug control-install root@control.example.com --install-docker`. The flag installs missing Docker Engine and Compose on supported Debian, Ubuntu, Fedora and RHEL-compatible hosts; omit it when Docker is ready. Put HTTPS in front of the control API as described in [Control plane](docs/control-plane.md).
+2. Point the CLI at it: `tug init --control-url https://control.example.com --token "$TUG_TOKEN"`.
+3. Set the secrets your services need: `tug secret-set DB_PASSWORD`.
+4. Add nodes with `tug node-install root@app1.example.com --name app1 --install-docker`, or `tug node-enroll --name app1` and run the printed Docker command. For a bare cloud host, `tug node-enroll --name app1 --install-docker` prints cloud-init user-data.
+5. Deploy: `tug deploy -f docker-compose.yml`. It waits until every node reports the new version healthy.
+6. Operate with `tug status`, `events`, `logs`, `restart`, `rollback` and `doctor`.
+
+## Documentation
+
+- [Architecture](docs/architecture.md): components, reconciliation, workflow diagrams, secrets and runtime paths.
+- [Compose extensions](docs/compose.md): `x-tug` placement, health checks, volumes and limitations.
+- [Control plane](docs/control-plane.md): install, TLS, registry credentials, backup and restore.
+- [Access control](docs/access-control.md): scoped API tokens, roles, audit trail and a GitHub Actions deploy example.
+- [Networking](docs/networking.md): WireGuard mesh, service DNS, ingress and TLS, firewall notes.
+- [Nodes](docs/nodes.md): SSH install, self-service enrollment, registration and status.
+- [CLI reference](docs/cli.md): every `tug` command and option.
+- [Configuration](docs/configuration.md): CLI, control and agent environment variables.
+- [Development](docs/development.md): local setup, CI, Compose stack, host mode and the integration test.
+- [Testing](docs/testing.md): test layers, the numbers rollouts are held to, and the backlog of mocked tests.
+- [Releases](docs/releases.md): changesets, npm and Docker image publishing.
+- [Standalone binary](docs/standalone-binary.md): proposed Bun-compiled CLI binaries and Homebrew distribution.
+- [Roadmap](docs/roadmap.md): planned milestones toward parity with and beyond uncloud.
+
+## Development
+
+```bash
+npm install
+npm test
+npm run lint
+npm run test:integration
+```
+
+See [Development](docs/development.md) for details.

package/dist/agent.d.ts

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+import { parseDuration } from './util.ts';
+import type { DockerRunner, HostRunner, HttpClient } from './runners.ts';
+import type { Command, CommandResult, DesiredState, HealthCheck, ProjectStatus, ImageArtifact, ServiceEndpoint } from './protocol.ts';
+export interface AgentContext {
+    docker: DockerRunner;
+    http: HttpClient;
+    nodeName: string;
+    stateDir: string;
+    apply: boolean;
+    controlUrl: string;
+    token: string | undefined;
+    /** Runs `wg` / `ip` for the mesh. */
+    host?: HostRunner;
+    /** Manage the WireGuard mesh (generate a key, report it, reconcile `wg0`). */
+    mesh?: boolean;
+    caddyImage?: string;
+    /** Host port range (`LOW-HIGH`) for rolling-rollout revisions; default 20000-20999. */
+    rolloutPorts?: string;
+}
+export declare function defaultContext(): AgentContext;
+export declare function containerIp(ctx: AgentContext, project: string, service: string, cwd: string, composeFile?: string): Promise<string>;
+export declare function waitForHealth(ctx: AgentContext, project: string, service: string, health: HealthCheck | null | undefined, cwd: string, composeFile?: string): Promise<void>;
+export { parseDuration };
+export declare function executeCommand(command: Command, ctx?: AgentContext): Promise<CommandResult>;
+export declare function ensureArtifact(ctx: AgentContext, artifact: ImageArtifact): Promise<void>;
+export declare function pruneManagedImages(ctx: AgentContext, retained: string[]): Promise<void>;
+export declare function reconcile(desiredState: DesiredState, ctx?: AgentContext, endpoints?: Record<string, ServiceEndpoint[]>): Promise<ProjectStatus>;
+/** Reports this node's WireGuard public key when control has not seen the current one yet. */
+export declare function reportWgKey(ctx: AgentContext): Promise<void>;
+export declare function pollOnce(ctx?: AgentContext): Promise<ProjectStatus[]>;
+interface Creds {
+    agentToken: string;
+    nodeName?: string;
+    controlUrl?: string;
+    registry?: string | null;
+    registryToken?: string | null;
+}
+export declare function loadOrEnroll(): Promise<Creds>;
+export declare function main(): Promise<void>;

package/dist/agent.js

@@ -0,0 +1,547 @@
+#!/usr/bin/env node
+import fs from 'node:fs/promises';
+import { createWriteStream } from 'node:fs';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { validateImageArchive, verifyDownload } from "./artifacts.js";
+import path from 'node:path';
+import { spawn, spawnSync } from 'node:child_process';
+import net from 'node:net';
+import { composeForNode, dumpYaml, isRolling, serviceSecretVersions } from "./compose.js";
+import { activeRollingStates, composeBase, hasLegacyContainers, reconcileRolling, retireAll } from "./rollout.js";
+import { ensureDir, errMessage, now, parseDuration } from "./util.js";
+import { openAgentDb } from "./db.js";
+import { createDockerRunner, createHostRunner } from "./runners.js";
+import { CADDY_CONTAINER, reconcileIngress, removeIngress } from "./ingress.js";
+import { WG_INTERFACE, ensureWgKeys, readReportedKey, reconcileMesh, rotateWgKeys, writeReportedKey } from "./mesh.js";
+const CONTROL_URL_FROM_ENV = Boolean(process.env.TUG_CONTROL_URL);
+let CONTROL_URL = process.env.TUG_CONTROL_URL || 'http://localhost:8080';
+let TOKEN = process.env.TUG_AGENT_TOKEN;
+let NODE_NAME = process.env.TUG_NODE_NAME || process.env.HOSTNAME || 'node1';
+const STATE_DIR = process.env.TUG_STATE_DIR || '/var/lib/tug';
+const APPLY = process.env.TUG_APPLY !== 'false';
+const INTERVAL = Number(process.env.TUG_POLL_INTERVAL_MS || 5000);
+// Opt-in WireGuard mesh: needs `wg`/`ip` in the agent image, host networking and NET_ADMIN.
+const MESH = process.env.TUG_MESH === 'true';
+export function defaultContext() {
+    return { docker: createDockerRunner(), http: fetch, nodeName: NODE_NAME, stateDir: STATE_DIR, apply: APPLY, controlUrl: CONTROL_URL, token: TOKEN, host: createHostRunner(), mesh: MESH, caddyImage: process.env.TUG_CADDY_IMAGE, rolloutPorts: process.env.TUG_ROLLOUT_PORTS };
+}
+const dbsByStateDir = new Map();
+async function agentDb(stateDir) {
+    let db = dbsByStateDir.get(stateDir);
+    if (!db) {
+        await ensureDir(stateDir);
+        db = await openAgentDb(path.join(stateDir, 'agent.db'));
+        dbsByStateDir.set(stateDir, db);
+    }
+    return db;
+}
+function authHeaders(ctx) { if (!ctx.token)
+    throw new Error('TUG_AGENT_TOKEN required'); return { Authorization: `Bearer ${ctx.token}` }; }
+async function api(ctx, route, opts = {}) {
+    const res = await ctx.http(`${ctx.controlUrl}${route}`, { ...opts, headers: { ...authHeaders(ctx), ...(opts.headers || {}) } });
+    if (!res.ok)
+        throw new Error(`${opts.method || 'GET'} ${route}: ${res.status} ${await res.text()}`);
+    return res.json();
+}
+async function writeEnv(file, secrets) {
+    const body = Object.entries(secrets).map(([k, v]) => `${k}=${String(v).replace(/\n/g, '\\n')}`).join('\n') + '\n';
+    await fs.writeFile(file, body, { mode: 0o600 });
+}
+/** `docker compose` prefix for a project; rolling revision projects pass their own Compose file. */
+const composeCmd = (project, cwd, composeFile) => composeFile ? composeBase(project, cwd, composeFile) : ['compose', '-p', project];
+export async function containerIp(ctx, project, service, cwd, composeFile) {
+    const ids = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out.trim().split(/\s+/).filter(Boolean);
+    if (!ids.length)
+        throw new Error(`${service}: no container found`);
+    const inspected = JSON.parse((await ctx.docker(['inspect', ids[0]], { cwd })).out)[0];
+    const networks = inspected?.NetworkSettings?.Networks || {};
+    return Object.values(networks)[0]?.IPAddress || '127.0.0.1';
+}
+function tcpCheck(host, port, timeoutMs = 2000) {
+    return new Promise((resolve, reject) => {
+        const s = net.createConnection({ host, port: Number(port), timeout: timeoutMs }, () => { s.destroy(); resolve(true); });
+        s.on('timeout', () => { s.destroy(); reject(new Error(`tcp timeout ${host}:${port}`)); });
+        s.on('error', reject);
+    });
+}
+export async function waitForHealth(ctx, project, service, health, cwd, composeFile) {
+    if (!ctx.apply || !health)
+        return;
+    const retries = Number(health.retries || 10);
+    const intervalMs = parseDuration(health.interval || '2s');
+    let last;
+    for (let i = 0; i < retries; i++) {
+        try {
+            if (health.type === 'docker') {
+                const id = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out.trim().split(/\s+/)[0];
+                const state = JSON.parse((await ctx.docker(['inspect', id], { cwd })).out)[0]?.State;
+                if (!state?.Running)
+                    throw new Error(`${service}: not running`);
+                if (state.Health && state.Health.Status !== 'healthy')
+                    throw new Error(`${service}: docker health ${state.Health.Status}`);
+            }
+            else if (health.type === 'tcp') {
+                await tcpCheck(await containerIp(ctx, project, service, cwd, composeFile), health.port, parseDuration(health.timeout || '2s'));
+            }
+            else if (health.type === 'http') {
+                const ip = await containerIp(ctx, project, service, cwd, composeFile);
+                const res = await ctx.http(`http://${ip}:${health.port || 80}${health.path || '/'}`, { signal: AbortSignal.timeout(parseDuration(health.timeout || '2s')) });
+                if (!res.ok)
+                    throw new Error(`${service}: http ${res.status}`);
+            }
+            return;
+        }
+        catch (e) {
+            last = e;
+            await new Promise(r => setTimeout(r, intervalMs));
+        }
+    }
+    throw new Error(`${service}: ${describeHealthCheck(health)} failed after ${retries} attempt${retries === 1 ? '' : 's'}: ${describeError(last)}`);
+}
+/** Node's fetch reports every network failure as "fetch failed"; the system error that explains it is on `cause`. */
+function describeError(error) {
+    const e = error;
+    const cause = e?.cause?.code || e?.cause?.message;
+    return cause ? `${e?.message ?? 'error'} (${cause})` : (e?.message ?? 'no response');
+}
+function describeHealthCheck(health) {
+    if (health.type === 'http')
+        return `http health check on port ${health.port || 80}${health.path || '/'}`;
+    if (health.type === 'tcp')
+        return `tcp health check on port ${health.port}`;
+    return 'docker health check';
+}
+export { parseDuration };
+export async function executeCommand(command, ctx = defaultContext()) {
+    const payload = typeof command.payload_json === 'string' ? JSON.parse(command.payload_json || '{}') : command.payload || {};
+    if (command.type === 'diagnostics.run') {
+        const checks = [];
+        for (const [name, args] of [
+            ['docker.info', ['info']],
+            ['docker.version', ['version']],
+            ['docker.compose.version', ['compose', 'version']]
+        ]) {
+            try {
+                const result = await ctx.docker([...args]);
+                checks.push({ name, ok: true, stdout: result.out.slice(-4000), stderr: result.err.slice(-4000) });
+            }
+            catch (e) {
+                checks.push({ name, ok: false, error: errMessage(e).slice(-4000) });
+            }
+        }
+        const extra = [];
+        if (ctx.mesh && ctx.host) {
+            const host = ctx.host;
+            extra.push(['wireguard.version', () => host('wg', ['--version'])], ['wireguard.interface', () => host('wg', ['show', WG_INTERFACE])]);
+        }
+        if (await fs.stat(path.join(ctx.stateDir, 'ingress')).then(() => true, () => false)) {
+            extra.push(['ingress.caddy', () => ctx.docker(['container', 'inspect', '-f', '{{.State.Status}}', CADDY_CONTAINER])]);
+        }
+        for (const [name, fn] of extra) {
+            try {
+                const result = await fn();
+                checks.push({ name, ok: true, stdout: result.out.slice(-4000), stderr: result.err.slice(-4000) });
+            }
+            catch (e) {
+                checks.push({ name, ok: false, error: errMessage(e).slice(-4000) });
+            }
+        }
+        return { commandId: command.id, state: checks.every(c => c.ok) ? 'succeeded' : 'failed', result: { checks, node: ctx.nodeName } };
+    }
+    if (command.type === 'service.restart') {
+        const project = payload.project;
+        const service = payload.service;
+        const dir = path.join(ctx.stateDir, 'projects', project);
+        await ctx.docker(['compose', '-p', project, 'restart', service], { cwd: dir });
+        return { commandId: command.id, state: 'succeeded', result: { project, service, restarted: true } };
+    }
+    if (command.type === 'logs.fetch') {
+        const project = payload.project;
+        const service = payload.service;
+        const tail = String(payload.tail || 200);
+        const dir = path.join(ctx.stateDir, 'projects', project);
+        const result = await ctx.docker(['compose', '-p', project, 'logs', '--no-color', '--tail', tail, service], { cwd: dir });
+        return { commandId: command.id, state: 'succeeded', result: { stdout: result.out, stderr: result.err } };
+    }
+    if (command.type === 'volume.delete') {
+        const project = payload.project;
+        const volume = payload.volume;
+        const composeVolume = `${project}_${volume}`;
+        try {
+            await ctx.docker(['volume', 'rm', composeVolume]);
+        }
+        catch (e) {
+            if (!errMessage(e).includes('No such volume'))
+                throw e;
+        }
+        return { commandId: command.id, state: 'succeeded', result: { project, volume, deleted: true } };
+    }
+    if (command.type === 'wg.rotate-key') {
+        if (!ctx.mesh)
+            throw new Error('the WireGuard mesh is not enabled on this agent (TUG_MESH=true)');
+        const { publicKey } = await rotateWgKeys(ctx.stateDir);
+        await api(ctx, '/agent/v1/wireguard/key', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ publicKey }) });
+        await writeReportedKey(ctx.stateDir, publicKey);
+        return { commandId: command.id, state: 'succeeded', result: { publicKey } };
+    }
+    return { commandId: command.id, state: 'failed', result: { error: `unsupported command ${command.type}` } };
+}
+async function managedImageFile(ctx) { await ensureDir(ctx.stateDir); return path.join(ctx.stateDir, 'managed-images.json'); }
+async function managedImages(ctx) {
+    try {
+        const refs = JSON.parse(await fs.readFile(await managedImageFile(ctx), 'utf8'));
+        return Array.isArray(refs) ? refs.filter((ref) => typeof ref === 'string' && ref.startsWith('tug-build/')) : [];
+    }
+    catch {
+        return [];
+    }
+}
+export async function ensureArtifact(ctx, artifact) {
+    const identityFile = path.join(ctx.stateDir, 'image-identities.json');
+    let identities = {};
+    try {
+        identities = JSON.parse(await fs.readFile(identityFile, 'utf8'));
+    }
+    catch { }
+    const remember = async (daemonId) => {
+        identities[artifact.imageRef] = { daemonId, artifactId: artifact.imageId, sha256: artifact.sha256 };
+        await fs.writeFile(identityFile, JSON.stringify(identities), { mode: 0o600 });
+        const refs = await managedImages(ctx);
+        if (!refs.includes(artifact.imageRef))
+            await fs.writeFile(await managedImageFile(ctx), JSON.stringify([...refs, artifact.imageRef]), { mode: 0o600 });
+    };
+    const inspect = async () => JSON.parse((await ctx.docker(['image', 'inspect', artifact.imageRef])).out)[0];
+    try {
+        const image = await inspect();
+        if ((image?.Id === artifact.imageId || (identities[artifact.imageRef]?.daemonId === image?.Id && identities[artifact.imageRef]?.artifactId === artifact.imageId && identities[artifact.imageRef]?.sha256 === artifact.sha256)) && `${image.Os}/${image.Architecture}` === artifact.platform) {
+            await remember(image.Id);
+            return;
+        }
+    }
+    catch { }
+    const host = JSON.parse((await ctx.docker(['info', '--format', '{{json .}}'])).out);
+    const arch = { x86_64: 'amd64', aarch64: 'arm64' }[host.Architecture] || host.Architecture;
+    if (host.OSType && host.Architecture && `${host.OSType}/${arch}` !== artifact.platform)
+        throw new Error(`image platform ${artifact.platform} does not match node ${host.OSType}/${arch}; build for the target platform`);
+    const downloadDir = await fs.mkdtemp(path.join(ctx.stateDir, 'image-download-'));
+    const file = path.join(downloadDir, 'image.tar.gz');
+    try {
+        const response = await ctx.http(`${ctx.controlUrl}/agent/v1/images/${encodeURIComponent(artifact.buildId)}`, { headers: authHeaders(ctx), redirect: 'error', signal: AbortSignal.timeout(10 * 60 * 1000) });
+        if (!response.ok || !response.body)
+            throw new Error(`image download failed: HTTP ${response.status}`);
+        await pipeline(Readable.fromWeb(response.body), verifyDownload(artifact), createWriteStream(file, { mode: 0o600 }));
+        const archive = await validateImageArchive(file, artifact.imageRef);
+        if (archive.imageId !== artifact.imageId || archive.platform !== artifact.platform)
+            throw new Error('downloaded archive image metadata differs from desired artifact');
+        await ctx.docker(['load', '-i', file]);
+        const image = await inspect();
+        if (!image?.Id || `${image.Os}/${image.Architecture}` !== artifact.platform)
+            throw new Error('loaded image identity or platform does not match desired artifact');
+        await remember(image.Id);
+    }
+    finally {
+        await fs.rm(downloadDir, { recursive: true, force: true });
+    }
+}
+export async function pruneManagedImages(ctx, retained) {
+    if (!ctx.apply)
+        return;
+    const keep = new Set(retained);
+    const remaining = [];
+    for (const ref of await managedImages(ctx)) {
+        if (keep.has(ref)) {
+            remaining.push(ref);
+            continue;
+        }
+        try {
+            const used = (await ctx.docker(['ps', '-aq', '--filter', `ancestor=${ref}`])).out.trim();
+            if (used) {
+                remaining.push(ref);
+                continue;
+            }
+            await ctx.docker(['image', 'rm', ref]);
+        }
+        catch {
+            remaining.push(ref);
+        }
+    }
+    await fs.writeFile(await managedImageFile(ctx), JSON.stringify(remaining), { mode: 0o600 });
+}
+export async function reconcile(desiredState, ctx = defaultContext(), endpoints = {}) {
+    const dir = path.join(ctx.stateDir, 'projects', desiredState.project);
+    await ensureDir(dir);
+    const plannedCompose = composeForNode(desiredState, ctx.nodeName, {});
+    const serviceNames = Object.keys(plannedCompose.services || {});
+    if (!serviceNames.length) {
+        if (ctx.apply)
+            await retireAll(ctx, dir, desiredState).catch(() => { });
+        return { project: desiredState.project, observedVersion: desiredState.version, ...(desiredState.revision ? { observedRevision: desiredState.revision } : {}), health: 'skipped', services: [], serviceStates: {} };
+    }
+    const serviceStates = Object.fromEntries(serviceNames.map(service => [service, { rolloutState: 'pending', image: desiredState.services[service]?.image }]));
+    const emit = (service, rolloutState, message = rolloutState) => api(ctx, '/agent/v1/events', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ projectId: desiredState.project, level: 'info', type: `service.${rolloutState}`, message, metadata: { service, version: desiredState.version, node: ctx.nodeName } }) }).catch(() => { });
+    const sec = await api(ctx, `/agent/v1/secrets?project=${encodeURIComponent(desiredState.project)}&services=${encodeURIComponent(serviceNames.join(','))}`);
+    const currentSecretVersions = serviceSecretVersions(desiredState.compose, serviceNames, sec.versions || {});
+    const db = await agentDb(ctx.stateDir);
+    const previousRow = await db.get('SELECT version, revision, secret_versions_json FROM applied_state WHERE project = ?', desiredState.project);
+    const previous = previousRow ? { version: previousRow.version, revision: previousRow.revision, secretVersions: JSON.parse(previousRow.secret_versions_json) } : {};
+    const rolling = serviceNames.filter(service => isRolling(desiredState.compose.services[service]));
+    const rollingSet = new Set(rolling);
+    const mainServices = serviceNames.filter(service => !rollingSet.has(service));
+    const secretChangedServices = mainServices.filter(service => JSON.stringify(previous.secretVersions?.[service] || {}) !== JSON.stringify(currentSecretVersions[service] || {}));
+    const sameDesiredVersion = desiredState.revision ? previous.revision === desiredState.revision : previous.version === desiredState.version;
+    // Rolling services run in their own revision projects; the main project holds only the recreated ones.
+    const nodeCompose = composeForNode(desiredState, ctx.nodeName, sec.secrets || {}, endpoints, rollingSet);
+    await fs.writeFile(path.join(dir, 'docker-compose.yml'), dumpYaml(nodeCompose));
+    await writeEnv(path.join(dir, '.env'), sec.secrets || {});
+    const failedStatus = (error) => ({ project: desiredState.project, observedVersion: desiredState.version, ...(desiredState.revision ? { observedRevision: desiredState.revision } : {}), health: 'failed', services: serviceNames, serviceStates, error });
+    if (ctx.apply) {
+        try {
+            for (const service of mainServices) {
+                serviceStates[service].rolloutState = 'pulling';
+                await emit(service, 'pulling');
+            }
+            for (const service of mainServices) {
+                const artifact = desiredState.services[service].artifact;
+                if (artifact)
+                    await ensureArtifact(ctx, artifact);
+            }
+            const pullServices = mainServices.filter(service => !desiredState.services[service].artifact);
+            if (pullServices.length)
+                await ctx.docker(['compose', '-p', desiredState.project, 'pull', ...(pullServices.length === mainServices.length ? [] : pullServices)], { cwd: dir });
+            for (const service of mainServices) {
+                serviceStates[service].rolloutState = 'starting';
+                await emit(service, 'starting');
+            }
+            if (mainServices.length) {
+                // Services that just became rolling still run here until their replacement takes over; keep them.
+                const keepLegacy = rolling.length > 0 && await hasLegacyContainers(ctx, desiredState.project, rolling);
+                await ctx.docker(['compose', '-p', desiredState.project, 'up', '-d', ...(keepLegacy ? [] : ['--remove-orphans'])], { cwd: dir });
+            }
+            if (sameDesiredVersion && secretChangedServices.length) {
+                for (const service of secretChangedServices)
+                    await emit(service, 'secret_recreate', 'secret version changed; recreating service');
+                await ctx.docker(['compose', '-p', desiredState.project, 'up', '-d', '--no-deps', '--force-recreate', ...secretChangedServices], { cwd: dir });
+            }
+            for (const service of mainServices) {
+                serviceStates[service].rolloutState = 'health_checking';
+                await emit(service, 'health_checking');
+                await waitForHealth(ctx, desiredState.project, service, desiredState.services[service]?.health, dir);
+                serviceStates[service].rolloutState = 'serving';
+                serviceStates[service].healthyAt = now();
+                await emit(service, 'serving');
+            }
+        }
+        catch (e) {
+            // Rolling services are untouched by a failure here, so keep reporting their current revision as serving.
+            if (!rolling.length)
+                throw e;
+            Object.assign(serviceStates, await activeRollingStates(dir, rolling));
+            return failedStatus(errMessage(e));
+        }
+        if (rolling.length) {
+            const rolled = await reconcileRolling(ctx, {
+                desired: desiredState, dir, services: rolling, secretVersions: currentSecretVersions,
+                full: composeForNode(desiredState, ctx.nodeName, sec.secrets || {}, endpoints),
+                deps: {
+                    waitForHealth: (project, service, composeFile) => waitForHealth(ctx, project, service, desiredState.services[service]?.health, dir, composeFile),
+                    ensureArtifact: async (service) => { const artifact = desiredState.services[service].artifact; if (artifact)
+                        await ensureArtifact(ctx, artifact); },
+                    emit
+                }
+            });
+            Object.assign(serviceStates, rolled.serviceStates);
+            const errors = Object.entries(rolled.errors);
+            if (errors.length)
+                return failedStatus(errors.map(([service, error]) => `${service}: ${error}`).join('; '));
+        }
+        else
+            await retireAll(ctx, dir, desiredState);
+    }
+    else {
+        for (const service of serviceNames)
+            serviceStates[service].rolloutState = 'serving';
+    }
+    await db.run(`INSERT INTO applied_state(project, version, revision, applied_at, service_states_json, secret_versions_json)
+     VALUES(?,?,?,?,?,?)
+     ON CONFLICT(project) DO UPDATE SET version=excluded.version, revision=excluded.revision, applied_at=excluded.applied_at,
+       service_states_json=excluded.service_states_json, secret_versions_json=excluded.secret_versions_json`, desiredState.project, desiredState.version, desiredState.revision || null, now(), JSON.stringify(serviceStates), JSON.stringify(currentSecretVersions));
+    return { project: desiredState.project, observedVersion: desiredState.version, ...(desiredState.revision ? { observedRevision: desiredState.revision } : {}), health: 'serving', services: serviceNames, serviceStates };
+}
+/** Reports this node's WireGuard public key when control has not seen the current one yet. */
+export async function reportWgKey(ctx) {
+    const { publicKey } = await ensureWgKeys(ctx.stateDir);
+    if (await readReportedKey(ctx.stateDir) === publicKey)
+        return;
+    await api(ctx, '/agent/v1/wireguard/key', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ publicKey }) });
+    await writeReportedKey(ctx.stateDir, publicKey);
+}
+let lastNetworkError = '';
+async function networkStep(label, fn) {
+    try {
+        await fn();
+        lastNetworkError = '';
+    }
+    catch (e) {
+        const message = `${label}: ${errMessage(e)}`;
+        if (message !== lastNetworkError)
+            console.error(message);
+        lastNetworkError = message;
+    }
+}
+export async function pollOnce(ctx = defaultContext()) {
+    if (ctx.mesh)
+        await networkStep('wireguard key report failed', () => reportWgKey(ctx));
+    const work = await api(ctx, '/agent/v1/work');
+    // The mesh comes up first so cross-node service endpoints are routable when containers start.
+    if (ctx.mesh && work.network) {
+        const network = work.network;
+        await networkStep('wireguard reconcile failed', () => reconcileMesh(ctx.host ?? createHostRunner(), ctx.stateDir, network));
+    }
+    const projects = [];
+    for (const desired of work.desiredStates || (work.desiredState ? [work.desiredState] : [])) {
+        try {
+            projects.push(await reconcile(desired, ctx, work.serviceEndpoints?.[desired.project]));
+        }
+        catch (e) {
+            projects.push({ project: desired.project, observedVersion: desired.version, ...(desired.revision ? { observedRevision: desired.revision } : {}), health: 'failed', error: errMessage(e) });
+        }
+    }
+    let ingress;
+    if (ctx.apply) {
+        const ingressCtx = { docker: ctx.docker, stateDir: ctx.stateDir, caddyImage: ctx.caddyImage };
+        if (work.ingress)
+            ingress = await reconcileIngress(ingressCtx, work.ingress);
+        else
+            await removeIngress(ingressCtx).catch(() => false);
+    }
+    const commandResults = [];
+    for (const command of work.commands || []) {
+        try {
+            commandResults.push(await executeCommand(command, ctx));
+        }
+        catch (e) {
+            commandResults.push({ commandId: command.id, state: 'failed', result: { error: errMessage(e) } });
+        }
+    }
+    const report = { node: ctx.nodeName, projects, commandResults };
+    if (ingress)
+        report.ingress = ingress;
+    if (work.retainedImages && projects.every(project => project.health !== 'failed'))
+        await pruneManagedImages(ctx, work.retainedImages);
+    await api(ctx, '/agent/v1/status', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(report) });
+    return projects;
+}
+const CREDS_FILE = () => path.join(STATE_DIR, 'agent.json');
+async function loadCreds() {
+    try {
+        return JSON.parse(await fs.readFile(CREDS_FILE(), 'utf8'));
+    }
+    catch {
+        return null;
+    }
+}
+async function saveCreds(creds) {
+    await ensureDir(STATE_DIR);
+    await fs.writeFile(CREDS_FILE(), JSON.stringify(creds), { mode: 0o600 });
+}
+async function readJoinKey() {
+    if (process.env.TUG_JOIN_KEY)
+        return process.env.TUG_JOIN_KEY.trim();
+    const file = process.env.TUG_JOIN_KEY_FILE;
+    if (file) {
+        try {
+            return (await fs.readFile(file, 'utf8')).trim();
+        }
+        catch (e) {
+            throw new Error(`cannot read TUG_JOIN_KEY_FILE ${file}: ${errMessage(e)}`, { cause: e });
+        }
+    }
+    return null;
+}
+function dockerLogin(registry, user, password) {
+    return new Promise((resolve, reject) => {
+        const p = spawn('docker', ['login', registry, '-u', user, '--password-stdin'], { stdio: ['pipe', 'ignore', 'pipe'] });
+        let err = '';
+        p.stderr.on('data', d => err += d);
+        p.on('error', reject);
+        p.on('close', code => code === 0 ? resolve() : reject(new Error(err.trim() || `exit ${code}`)));
+        p.stdin.write(password);
+        p.stdin.end();
+    });
+}
+async function enroll(joinKey) {
+    let decoded;
+    try {
+        decoded = JSON.parse(Buffer.from(joinKey, 'base64url').toString('utf8'));
+    }
+    catch (e) {
+        throw new Error('invalid TUG_JOIN_KEY (expected base64url-encoded JSON)', { cause: e });
+    }
+    if (!decoded.enrollToken)
+        throw new Error('join key is missing enrollToken');
+    const controlUrl = (!CONTROL_URL_FROM_ENV && decoded.controlUrl) ? decoded.controlUrl : CONTROL_URL;
+    // node's fetch reads NODE_EXTRA_CA_CERTS only at startup, so apply an embedded CA by
+    // persisting it and re-execing ourselves once with the env var set.
+    if (decoded.ca && !process.env.NODE_EXTRA_CA_CERTS) {
+        await ensureDir(STATE_DIR);
+        const caPath = path.join(STATE_DIR, 'control-ca.pem');
+        await fs.writeFile(caPath, decoded.ca);
+        const r = spawnSync(process.execPath, process.argv.slice(1), { stdio: 'inherit', env: { ...process.env, NODE_EXTRA_CA_CERTS: caPath } });
+        process.exit(r.status ?? 1);
+    }
+    console.log(`enrolling with ${controlUrl} as ${NODE_NAME}`);
+    const wgPublicKey = MESH ? (await ensureWgKeys(STATE_DIR)).publicKey : undefined;
+    const res = await fetch(`${controlUrl}/agent/v1/enroll`, {
+        method: 'POST', headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({ enrollToken: decoded.enrollToken, name: NODE_NAME, address: process.env.TUG_NODE_ADDRESS, publicAddress: process.env.TUG_NODE_PUBLIC_ADDRESS, wgPublicKey })
+    });
+    if (!res.ok)
+        throw new Error(`enroll failed: ${res.status} ${await res.text()}`);
+    const out = await res.json();
+    const creds = { agentToken: out.agentToken, nodeName: out.node?.name || NODE_NAME, controlUrl, registry: out.registry || null, registryToken: out.registryToken || null };
+    await saveCreds(creds);
+    if (wgPublicKey)
+        await writeReportedKey(STATE_DIR, wgPublicKey);
+    console.log(`enrolled as node ${creds.nodeName}`);
+    return creds;
+}
+// Establish durable credentials before polling: explicit env token (SSH/push install) wins,
+// then credentials persisted on the state volume, then a one-time enroll via the join key.
+export async function loadOrEnroll() {
+    let creds = TOKEN ? { agentToken: TOKEN, nodeName: NODE_NAME, controlUrl: CONTROL_URL } : await loadCreds();
+    if (!creds) {
+        const joinKey = await readJoinKey();
+        if (!joinKey)
+            throw new Error('no TUG_AGENT_TOKEN, no persisted credentials, and no TUG_JOIN_KEY to enroll with');
+        creds = await enroll(joinKey);
+    }
+    TOKEN = creds.agentToken;
+    NODE_NAME = creds.nodeName || NODE_NAME;
+    if (!CONTROL_URL_FROM_ENV && creds.controlUrl)
+        CONTROL_URL = creds.controlUrl;
+    if (APPLY && creds.registry && creds.registryToken) {
+        try {
+            await dockerLogin(creds.registry, NODE_NAME, creds.registryToken);
+        }
+        catch (e) {
+            console.error(`registry login failed (pulls from ${creds.registry} may fail): ${errMessage(e)}`);
+        }
+    }
+    return creds;
+}
+export async function main() {
+    await loadOrEnroll();
+    console.log(`tug-agent node=${NODE_NAME} control=${CONTROL_URL} apply=${APPLY}`);
+    for (;;) {
+        try {
+            await pollOnce();
+        }
+        catch (e) {
+            console.error(errMessage(e));
+        }
+        await new Promise(r => setTimeout(r, INTERVAL));
+    }
+}
+if (import.meta.url === `file://${process.argv[1]}`)
+    main();

package/dist/archive.d.ts

@@ -0,0 +1,2 @@
+import * as tar from 'tar';
+export declare function scanArchive(file: string, inspect: (entry: tar.ReadEntry) => void): Promise<void>;