@wpelevator/tug 0.1.0 → 0.2.1

package/README.md

@@ -6,13 +6,14 @@ Deploy Docker Compose projects to one or more ordinary Linux VPSes through a sma
 
 ## Features
 
-- Control API with bearer-token auth and SQLite metadata.
-- Nodes join over SSH (`node install`) or enroll themselves with a single `docker run` (`node enroll`), with per-node agent and registry tokens. Agents only dial out, so nodes behind NAT work.
+- Control API with bearer-token auth, scoped `read`/`deploy`/`admin` API tokens for CI and teammates, an audit trail naming who acted, and SQLite metadata.
+- Nodes join over SSH (`node-install`) or enroll themselves with a single `docker run` (`node-enroll`), with per-node agent and registry tokens. Agents only dial out, so nodes behind NAT work.
 - Agents pull desired state from control, apply a generated per-node Compose file, run TCP/HTTP/Docker health checks and report status.
 - `x-tug` placement of services and volumes on nodes, with immutable desired-state records per deployment.
-- Optional remote Docker builds on the control host from an uploaded source tarball.
+- Build application images on control from uploaded source, or locally with `deploy --build-local`. Agents download saved images directly from control by default; a bring-your-own registry is optional.
 - Secrets encrypted at rest, released only to the nodes whose services reference them, with targeted service recreation when only a secret changes.
 - Deployment convergence, per-project operation locks, cancellation, rollback, per-service rollout events and log fetching through the agents.
+- Opt-in zero-downtime rolling rollouts for ingress-fronted services (`x-tug.rollout`) and automatic rollback of failed deployments (`x-tug.rollback: auto`). See [Compose extensions](docs/compose.md#rolling-rollouts).
 - Per-node registry pull tokens enforced by an htpasswd-backed registry you provide, with rotation and revocation.
 - Optional WireGuard mesh between nodes, health-aware cross-node service names and a managed Caddy ingress with automatic HTTPS, each opt-in. See [Networking](docs/networking.md).
 - Control-plane backup and restore, and `tug doctor` diagnostics.
@@ -27,10 +28,10 @@ npm install -g @wpelevator/tug
 
 The CLI installs the matching `ghcr.io/wpelevator/tug:<version>` image on your hosts.
 
-1. Install the control plane on a Docker host, pointing it at a registry you already run: `tug control install root@control.example.com --registry registry.example.com`.
+1. Install the control plane: `tug control-install root@control.example.com --install-docker`. The flag installs missing Docker Engine and Compose on supported Debian, Ubuntu, Fedora and RHEL-compatible hosts; omit it when Docker is ready. Put HTTPS in front of the control API as described in [Control plane](docs/control-plane.md).
 2. Point the CLI at it: `tug init --control-url https://control.example.com --token "$TUG_TOKEN"`.
-3. Set the secrets your services need: `tug secret:set DB_PASSWORD`.
-4. Add nodes, either `tug node install root@app1.example.com --name app1 --control-url https://control.example.com --registry registry.example.com`, or `tug node enroll --name app1` and run the printed `docker run` on the host.
+3. Set the secrets your services need: `tug secret-set DB_PASSWORD`.
+4. Add nodes with `tug node-install root@app1.example.com --name app1 --install-docker`, or `tug node-enroll --name app1` and run the printed Docker command. For a bare cloud host, `tug node-enroll --name app1 --install-docker` prints cloud-init user-data.
 5. Deploy: `tug deploy -f docker-compose.yml`. It waits until every node reports the new version healthy.
 6. Operate with `tug status`, `events`, `logs`, `restart`, `rollback` and `doctor`.
 
@@ -39,12 +40,16 @@ The CLI installs the matching `ghcr.io/wpelevator/tug:<version>` image on your h
 - [Architecture](docs/architecture.md): components, reconciliation, workflow diagrams, secrets and runtime paths.
 - [Compose extensions](docs/compose.md): `x-tug` placement, health checks, volumes and limitations.
 - [Control plane](docs/control-plane.md): install, TLS, registry credentials, backup and restore.
+- [Access control](docs/access-control.md): scoped API tokens, roles, audit trail and a GitHub Actions deploy example.
 - [Networking](docs/networking.md): WireGuard mesh, service DNS, ingress and TLS, firewall notes.
 - [Nodes](docs/nodes.md): SSH install, self-service enrollment, registration and status.
 - [CLI reference](docs/cli.md): every `tug` command and option.
 - [Configuration](docs/configuration.md): CLI, control and agent environment variables.
 - [Development](docs/development.md): local setup, CI, Compose stack, host mode and the integration test.
+- [Testing](docs/testing.md): test layers, the numbers rollouts are held to, and the backlog of mocked tests.
+- [Observability](docs/observability.md): what can be observed today, and the proposed logs, metrics and alerting.
 - [Releases](docs/releases.md): changesets, npm and Docker image publishing.
+- [Standalone binary](docs/standalone-binary.md): proposed Bun-compiled CLI binaries and Homebrew distribution.
 - [Roadmap](docs/roadmap.md): planned milestones toward parity with and beyond uncloud.
 
 ## Development

package/dist/agent.d.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
+import { parseDuration } from './util.ts';
 import type { DockerRunner, HostRunner, HttpClient } from './runners.ts';
-import type { Command, CommandResult, DesiredState, HealthCheck, ProjectStatus, ServiceEndpoint } from './protocol.ts';
+import type { Command, CommandResult, DesiredState, HealthCheck, ProjectStatus, ImageArtifact, ServiceEndpoint } from './protocol.ts';
 export interface AgentContext {
     docker: DockerRunner;
     http: HttpClient;
@@ -14,12 +15,16 @@ export interface AgentContext {
     /** Manage the WireGuard mesh (generate a key, report it, reconcile `wg0`). */
     mesh?: boolean;
     caddyImage?: string;
+    /** Host port range (`LOW-HIGH`) for rolling-rollout revisions; default 20000-20999. */
+    rolloutPorts?: string;
 }
 export declare function defaultContext(): AgentContext;
-export declare function containerIp(ctx: AgentContext, project: string, service: string, cwd: string): Promise<string>;
-export declare function waitForHealth(ctx: AgentContext, project: string, service: string, health: HealthCheck | null | undefined, cwd: string): Promise<void>;
-export declare function parseDuration(value: string | number): number;
+export declare function containerIp(ctx: AgentContext, project: string, service: string, cwd: string, composeFile?: string): Promise<string>;
+export declare function waitForHealth(ctx: AgentContext, project: string, service: string, health: HealthCheck | null | undefined, cwd: string, composeFile?: string): Promise<void>;
+export { parseDuration };
 export declare function executeCommand(command: Command, ctx?: AgentContext): Promise<CommandResult>;
+export declare function ensureArtifact(ctx: AgentContext, artifact: ImageArtifact): Promise<void>;
+export declare function pruneManagedImages(ctx: AgentContext, retained: string[]): Promise<void>;
 export declare function reconcile(desiredState: DesiredState, ctx?: AgentContext, endpoints?: Record<string, ServiceEndpoint[]>): Promise<ProjectStatus>;
 /** Reports this node's WireGuard public key when control has not seen the current one yet. */
 export declare function reportWgKey(ctx: AgentContext): Promise<void>;
@@ -33,4 +38,3 @@ interface Creds {
 }
 export declare function loadOrEnroll(): Promise<Creds>;
 export declare function main(): Promise<void>;
-export {};

package/dist/agent.js

@@ -1,10 +1,16 @@
 #!/usr/bin/env node
 import fs from 'node:fs/promises';
+import { createWriteStream } from 'node:fs';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { validateImageArchive, verifyDownload } from "./artifacts.js";
 import path from 'node:path';
 import { spawn, spawnSync } from 'node:child_process';
 import net from 'node:net';
-import { composeForNode, dumpYaml, serviceSecretVersions } from "./compose.js";
-import { ensureDir, errMessage, now } from "./util.js";
+import { createHash } from 'node:crypto';
+import { composeForNode, dumpYaml, isRolling, serviceSecretVersions } from "./compose.js";
+import { activeRollingStates, composeBase, hasLegacyContainers, reconcileRolling, retireAll } from "./rollout.js";
+import { ensureDir, errMessage, now, parseDuration } from "./util.js";
 import { openAgentDb } from "./db.js";
 import { createDockerRunner, createHostRunner } from "./runners.js";
 import { CADDY_CONTAINER, reconcileIngress, removeIngress } from "./ingress.js";
@@ -19,7 +25,7 @@ const INTERVAL = Number(process.env.TUG_POLL_INTERVAL_MS || 5000);
 // Opt-in WireGuard mesh: needs `wg`/`ip` in the agent image, host networking and NET_ADMIN.
 const MESH = process.env.TUG_MESH === 'true';
 export function defaultContext() {
-    return { docker: createDockerRunner(), http: fetch, nodeName: NODE_NAME, stateDir: STATE_DIR, apply: APPLY, controlUrl: CONTROL_URL, token: TOKEN, host: createHostRunner(), mesh: MESH, caddyImage: process.env.TUG_CADDY_IMAGE };
+    return { docker: createDockerRunner(), http: fetch, nodeName: NODE_NAME, stateDir: STATE_DIR, apply: APPLY, controlUrl: CONTROL_URL, token: TOKEN, host: createHostRunner(), mesh: MESH, caddyImage: process.env.TUG_CADDY_IMAGE, rolloutPorts: process.env.TUG_ROLLOUT_PORTS };
 }
 const dbsByStateDir = new Map();
 async function agentDb(stateDir) {
@@ -43,8 +49,10 @@ async function writeEnv(file, secrets) {
     const body = Object.entries(secrets).map(([k, v]) => `${k}=${String(v).replace(/\n/g, '\\n')}`).join('\n') + '\n';
     await fs.writeFile(file, body, { mode: 0o600 });
 }
-export async function containerIp(ctx, project, service, cwd) {
-    const ids = (await ctx.docker(['compose', '-p', project, 'ps', '-q', service], { cwd })).out.trim().split(/\s+/).filter(Boolean);
+/** `docker compose` prefix for a project; rolling revision projects pass their own Compose file. */
+const composeCmd = (project, cwd, composeFile) => composeFile ? composeBase(project, cwd, composeFile) : ['compose', '-p', project];
+export async function containerIp(ctx, project, service, cwd, composeFile) {
+    const ids = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out.trim().split(/\s+/).filter(Boolean);
     if (!ids.length)
         throw new Error(`${service}: no container found`);
     const inspected = JSON.parse((await ctx.docker(['inspect', ids[0]], { cwd })).out)[0];
@@ -58,7 +66,7 @@ function tcpCheck(host, port, timeoutMs = 2000) {
         s.on('error', reject);
     });
 }
-export async function waitForHealth(ctx, project, service, health, cwd) {
+export async function waitForHealth(ctx, project, service, health, cwd, composeFile) {
     if (!ctx.apply || !health)
         return;
     const retries = Number(health.retries || 10);
@@ -67,7 +75,7 @@ export async function waitForHealth(ctx, project, service, health, cwd) {
     for (let i = 0; i < retries; i++) {
         try {
             if (health.type === 'docker') {
-                const id = (await ctx.docker(['compose', '-p', project, 'ps', '-q', service], { cwd })).out.trim().split(/\s+/)[0];
+                const id = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out.trim().split(/\s+/)[0];
                 const state = JSON.parse((await ctx.docker(['inspect', id], { cwd })).out)[0]?.State;
                 if (!state?.Running)
                     throw new Error(`${service}: not running`);
@@ -75,10 +83,10 @@ export async function waitForHealth(ctx, project, service, health, cwd) {
                     throw new Error(`${service}: docker health ${state.Health.Status}`);
             }
             else if (health.type === 'tcp') {
-                await tcpCheck(await containerIp(ctx, project, service, cwd), health.port, parseDuration(health.timeout || '2s'));
+                await tcpCheck(await containerIp(ctx, project, service, cwd, composeFile), health.port, parseDuration(health.timeout || '2s'));
             }
             else if (health.type === 'http') {
-                const ip = await containerIp(ctx, project, service, cwd);
+                const ip = await containerIp(ctx, project, service, cwd, composeFile);
                 const res = await ctx.http(`http://${ip}:${health.port || 80}${health.path || '/'}`, { signal: AbortSignal.timeout(parseDuration(health.timeout || '2s')) });
                 if (!res.ok)
                     throw new Error(`${service}: http ${res.status}`);
@@ -90,17 +98,22 @@ export async function waitForHealth(ctx, project, service, health, cwd) {
             await new Promise(r => setTimeout(r, intervalMs));
         }
     }
-    throw last || new Error(`${service}: health check failed`);
+    throw new Error(`${service}: ${describeHealthCheck(health)} failed after ${retries} attempt${retries === 1 ? '' : 's'}: ${describeError(last)}`);
 }
-export function parseDuration(value) {
-    if (typeof value === 'number')
-        return value;
-    const m = String(value).match(/^(\d+)(ms|s|m)?$/);
-    if (!m)
-        return 2000;
-    const n = Number(m[1]);
-    return m[2] === 'm' ? n * 60000 : m[2] === 'ms' ? n : n * 1000;
+/** Node's fetch reports every network failure as "fetch failed"; the system error that explains it is on `cause`. */
+function describeError(error) {
+    const e = error;
+    const cause = e?.cause?.code || e?.cause?.message;
+    return cause ? `${e?.message ?? 'error'} (${cause})` : (e?.message ?? 'no response');
 }
+function describeHealthCheck(health) {
+    if (health.type === 'http')
+        return `http health check on port ${health.port || 80}${health.path || '/'}`;
+    if (health.type === 'tcp')
+        return `tcp health check on port ${health.port}`;
+    return 'docker health check';
+}
+export { parseDuration };
 export async function executeCommand(command, ctx = defaultContext()) {
     const payload = typeof command.payload_json === 'string' ? JSON.parse(command.payload_json || '{}') : command.payload || {};
     if (command.type === 'diagnostics.run') {
@@ -175,59 +188,207 @@ export async function executeCommand(command, ctx = defaultContext()) {
     }
     return { commandId: command.id, state: 'failed', result: { error: `unsupported command ${command.type}` } };
 }
+async function managedImageFile(ctx) { await ensureDir(ctx.stateDir); return path.join(ctx.stateDir, 'managed-images.json'); }
+async function managedImages(ctx) {
+    try {
+        const refs = JSON.parse(await fs.readFile(await managedImageFile(ctx), 'utf8'));
+        return Array.isArray(refs) ? refs.filter((ref) => typeof ref === 'string' && ref.startsWith('tug-build/')) : [];
+    }
+    catch {
+        return [];
+    }
+}
+export async function ensureArtifact(ctx, artifact) {
+    const identityFile = path.join(ctx.stateDir, 'image-identities.json');
+    let identities = {};
+    try {
+        identities = JSON.parse(await fs.readFile(identityFile, 'utf8'));
+    }
+    catch { }
+    const remember = async (daemonId) => {
+        identities[artifact.imageRef] = { daemonId, artifactId: artifact.imageId, sha256: artifact.sha256 };
+        await fs.writeFile(identityFile, JSON.stringify(identities), { mode: 0o600 });
+        const refs = await managedImages(ctx);
+        if (!refs.includes(artifact.imageRef))
+            await fs.writeFile(await managedImageFile(ctx), JSON.stringify([...refs, artifact.imageRef]), { mode: 0o600 });
+    };
+    const inspect = async () => JSON.parse((await ctx.docker(['image', 'inspect', artifact.imageRef])).out)[0];
+    try {
+        const image = await inspect();
+        if ((image?.Id === artifact.imageId || (identities[artifact.imageRef]?.daemonId === image?.Id && identities[artifact.imageRef]?.artifactId === artifact.imageId && identities[artifact.imageRef]?.sha256 === artifact.sha256)) && `${image.Os}/${image.Architecture}` === artifact.platform) {
+            await remember(image.Id);
+            return;
+        }
+    }
+    catch { }
+    const host = JSON.parse((await ctx.docker(['info', '--format', '{{json .}}'])).out);
+    const arch = { x86_64: 'amd64', aarch64: 'arm64' }[host.Architecture] || host.Architecture;
+    if (host.OSType && host.Architecture && `${host.OSType}/${arch}` !== artifact.platform)
+        throw new Error(`image platform ${artifact.platform} does not match node ${host.OSType}/${arch}; build for the target platform`);
+    const downloadDir = await fs.mkdtemp(path.join(ctx.stateDir, 'image-download-'));
+    const file = path.join(downloadDir, 'image.tar.gz');
+    try {
+        const response = await ctx.http(`${ctx.controlUrl}/agent/v1/images/${encodeURIComponent(artifact.buildId)}`, { headers: authHeaders(ctx), redirect: 'error', signal: AbortSignal.timeout(10 * 60 * 1000) });
+        if (!response.ok || !response.body)
+            throw new Error(`image download failed: HTTP ${response.status}`);
+        await pipeline(Readable.fromWeb(response.body), verifyDownload(artifact), createWriteStream(file, { mode: 0o600 }));
+        const archive = await validateImageArchive(file, artifact.imageRef);
+        if (archive.imageId !== artifact.imageId || archive.platform !== artifact.platform)
+            throw new Error('downloaded archive image metadata differs from desired artifact');
+        await ctx.docker(['load', '-i', file]);
+        const image = await inspect();
+        if (!image?.Id || `${image.Os}/${image.Architecture}` !== artifact.platform)
+            throw new Error('loaded image identity or platform does not match desired artifact');
+        await remember(image.Id);
+    }
+    finally {
+        await fs.rm(downloadDir, { recursive: true, force: true });
+    }
+}
+export async function pruneManagedImages(ctx, retained) {
+    if (!ctx.apply)
+        return;
+    const keep = new Set(retained);
+    const remaining = [];
+    for (const ref of await managedImages(ctx)) {
+        if (keep.has(ref)) {
+            remaining.push(ref);
+            continue;
+        }
+        try {
+            const used = (await ctx.docker(['ps', '-aq', '--filter', `ancestor=${ref}`])).out.trim();
+            if (used) {
+                remaining.push(ref);
+                continue;
+            }
+            await ctx.docker(['image', 'rm', ref]);
+        }
+        catch {
+            remaining.push(ref);
+        }
+    }
+    await fs.writeFile(await managedImageFile(ctx), JSON.stringify(remaining), { mode: 0o600 });
+}
 export async function reconcile(desiredState, ctx = defaultContext(), endpoints = {}) {
     const dir = path.join(ctx.stateDir, 'projects', desiredState.project);
     await ensureDir(dir);
     const plannedCompose = composeForNode(desiredState, ctx.nodeName, {});
     const serviceNames = Object.keys(plannedCompose.services || {});
-    if (!serviceNames.length)
-        return { project: desiredState.project, observedVersion: desiredState.version, health: 'skipped', services: [], serviceStates: {} };
+    if (!serviceNames.length) {
+        if (ctx.apply)
+            await retireAll(ctx, dir, desiredState).catch(() => { });
+        return { project: desiredState.project, observedVersion: desiredState.version, ...(desiredState.revision ? { observedRevision: desiredState.revision } : {}), health: 'skipped', services: [], serviceStates: {} };
+    }
     const serviceStates = Object.fromEntries(serviceNames.map(service => [service, { rolloutState: 'pending', image: desiredState.services[service]?.image }]));
-    const emit = (service, rolloutState, message = rolloutState) => api(ctx, '/agent/v1/events', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ projectId: desiredState.project, level: 'info', type: `service.${rolloutState}`, message, metadata: { service, version: desiredState.version, node: ctx.nodeName } }) }).catch(() => { });
+    const emit = (service, rolloutState, message = rolloutState, level = 'info') => api(ctx, '/agent/v1/events', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ projectId: desiredState.project, level, type: `service.${rolloutState}`, message, metadata: { service, version: desiredState.version, node: ctx.nodeName } }) }).catch(() => { });
     const sec = await api(ctx, `/agent/v1/secrets?project=${encodeURIComponent(desiredState.project)}&services=${encodeURIComponent(serviceNames.join(','))}`);
     const currentSecretVersions = serviceSecretVersions(desiredState.compose, serviceNames, sec.versions || {});
     const db = await agentDb(ctx.stateDir);
-    const previousRow = await db.get('SELECT version, secret_versions_json FROM applied_state WHERE project = ?', desiredState.project);
-    const previous = previousRow ? { version: previousRow.version, secretVersions: JSON.parse(previousRow.secret_versions_json) } : {};
-    const secretChangedServices = serviceNames.filter(service => JSON.stringify(previous.secretVersions?.[service] || {}) !== JSON.stringify(currentSecretVersions[service] || {}));
-    const sameDesiredVersion = previous.version === desiredState.version;
-    const nodeCompose = composeForNode(desiredState, ctx.nodeName, sec.secrets || {}, endpoints);
+    const previousRow = await db.get('SELECT version, revision, secret_versions_json FROM applied_state WHERE project = ?', desiredState.project);
+    const previous = previousRow ? { version: previousRow.version, revision: previousRow.revision, secretVersions: JSON.parse(previousRow.secret_versions_json) } : {};
+    const rolling = serviceNames.filter(service => isRolling(desiredState.compose.services[service]));
+    const rollingSet = new Set(rolling);
+    const mainServices = serviceNames.filter(service => !rollingSet.has(service));
+    const secretChangedServices = mainServices.filter(service => JSON.stringify(previous.secretVersions?.[service] || {}) !== JSON.stringify(currentSecretVersions[service] || {}));
+    const sameDesiredVersion = desiredState.revision ? previous.revision === desiredState.revision : previous.version === desiredState.version;
+    // Rolling services run in their own revision projects; the main project holds only the recreated ones.
+    const nodeCompose = composeForNode(desiredState, ctx.nodeName, sec.secrets || {}, endpoints, rollingSet);
     await fs.writeFile(path.join(dir, 'docker-compose.yml'), dumpYaml(nodeCompose));
     await writeEnv(path.join(dir, '.env'), sec.secrets || {});
+    const failedStatus = (error) => ({ project: desiredState.project, observedVersion: desiredState.version, ...(desiredState.revision ? { observedRevision: desiredState.revision } : {}), health: 'failed', services: serviceNames, serviceStates, error });
+    // Lifecycle events say what changed, not that the agent is still polling. They are sent when the desired revision,
+    // this node's rendered configuration or the secrets change, when an apply fails (once) and when it recovers (once).
+    const eventFingerprint = createHash('sha256').update(JSON.stringify([desiredState.revision || desiredState.version, dumpYaml(nodeCompose), mainServices.map(service => currentSecretVersions[service] || {})])).digest('hex').slice(0, 16);
+    const knownEvents = mainServices.length ? await db.get('SELECT fingerprint, phase FROM event_state WHERE project = ?', desiredState.project) : undefined;
+    const announce = knownEvents?.fingerprint !== eventFingerprint;
+    const wasFailed = !announce && knownEvents?.phase === 'failed';
+    const markEvents = (phase) => db.run('INSERT INTO event_state(project, fingerprint, phase) VALUES(?,?,?) ON CONFLICT(project) DO UPDATE SET fingerprint=excluded.fingerprint, phase=excluded.phase', desiredState.project, eventFingerprint, phase);
+    let failing;
     if (ctx.apply) {
-        for (const service of serviceNames) {
-            serviceStates[service].rolloutState = 'pulling';
-            await emit(service, 'pulling');
-        }
-        await ctx.docker(['compose', '-p', desiredState.project, 'pull'], { cwd: dir });
-        for (const service of serviceNames) {
-            serviceStates[service].rolloutState = 'starting';
-            await emit(service, 'starting');
+        try {
+            for (const service of mainServices) {
+                serviceStates[service].rolloutState = 'pulling';
+                if (announce)
+                    await emit(service, 'pulling');
+            }
+            for (const service of mainServices) {
+                const artifact = desiredState.services[service].artifact;
+                if (artifact)
+                    await ensureArtifact(ctx, artifact);
+            }
+            const pullServices = mainServices.filter(service => !desiredState.services[service].artifact);
+            if (pullServices.length)
+                await ctx.docker(['compose', '-p', desiredState.project, 'pull', ...(pullServices.length === mainServices.length ? [] : pullServices)], { cwd: dir });
+            for (const service of mainServices) {
+                serviceStates[service].rolloutState = 'starting';
+                if (announce)
+                    await emit(service, 'starting');
+            }
+            if (mainServices.length) {
+                // Services that just became rolling still run here until their replacement takes over; keep them.
+                const keepLegacy = rolling.length > 0 && await hasLegacyContainers(ctx, desiredState.project, rolling);
+                await ctx.docker(['compose', '-p', desiredState.project, 'up', '-d', ...(keepLegacy ? [] : ['--remove-orphans'])], { cwd: dir });
+            }
+            if (sameDesiredVersion && secretChangedServices.length) {
+                for (const service of secretChangedServices)
+                    await emit(service, 'secret_recreate', 'secret version changed; recreating service');
+                await ctx.docker(['compose', '-p', desiredState.project, 'up', '-d', '--no-deps', '--force-recreate', ...secretChangedServices], { cwd: dir });
+            }
+            for (const service of mainServices) {
+                serviceStates[service].rolloutState = 'health_checking';
+                if (announce)
+                    await emit(service, 'health_checking');
+                failing = service;
+                await waitForHealth(ctx, desiredState.project, service, desiredState.services[service]?.health, dir);
+                failing = undefined;
+                serviceStates[service].rolloutState = 'serving';
+                serviceStates[service].healthyAt = now();
+                if (announce || wasFailed)
+                    await emit(service, 'serving', wasFailed ? 'serving again after a failure' : 'serving');
+            }
+            if (mainServices.length && (announce || wasFailed))
+                await markEvents('serving');
         }
-        await ctx.docker(['compose', '-p', desiredState.project, 'up', '-d', '--remove-orphans'], { cwd: dir });
-        if (sameDesiredVersion && secretChangedServices.length) {
-            for (const service of secretChangedServices)
-                await emit(service, 'secret_recreate', 'secret version changed; recreating service');
-            await ctx.docker(['compose', '-p', desiredState.project, 'up', '-d', '--no-deps', '--force-recreate', ...secretChangedServices], { cwd: dir });
+        catch (e) {
+            if (mainServices.length && !wasFailed) {
+                // Announce a failure once; retries of the same failure stay quiet until it recovers or the inputs change.
+                await emit(failing ?? mainServices.join(', '), 'failed', errMessage(e), 'error');
+                await markEvents('failed');
+            }
+            // Rolling services are untouched by a failure here, so keep reporting their current revision as serving.
+            if (!rolling.length)
+                throw e;
+            Object.assign(serviceStates, await activeRollingStates(dir, rolling));
+            return failedStatus(errMessage(e));
         }
-        for (const service of serviceNames) {
-            serviceStates[service].rolloutState = 'health_checking';
-            await emit(service, 'health_checking');
-            await waitForHealth(ctx, desiredState.project, service, desiredState.services[service]?.health, dir);
-            serviceStates[service].rolloutState = 'serving';
-            serviceStates[service].healthyAt = now();
-            await emit(service, 'serving');
+        if (rolling.length) {
+            const rolled = await reconcileRolling(ctx, {
+                desired: desiredState, dir, services: rolling, secretVersions: currentSecretVersions,
+                full: composeForNode(desiredState, ctx.nodeName, sec.secrets || {}, endpoints),
+                deps: {
+                    waitForHealth: (project, service, composeFile) => waitForHealth(ctx, project, service, desiredState.services[service]?.health, dir, composeFile),
+                    ensureArtifact: async (service) => { const artifact = desiredState.services[service].artifact; if (artifact)
+                        await ensureArtifact(ctx, artifact); },
+                    emit
+                }
+            });
+            Object.assign(serviceStates, rolled.serviceStates);
+            const errors = Object.entries(rolled.errors);
+            if (errors.length)
+                return failedStatus(errors.map(([service, error]) => `${service}: ${error}`).join('; '));
         }
+        else
+            await retireAll(ctx, dir, desiredState);
     }
     else {
         for (const service of serviceNames)
             serviceStates[service].rolloutState = 'serving';
     }
-    await db.run(`INSERT INTO applied_state(project, version, applied_at, service_states_json, secret_versions_json)
-     VALUES(?,?,?,?,?)
-     ON CONFLICT(project) DO UPDATE SET version=excluded.version, applied_at=excluded.applied_at,
-       service_states_json=excluded.service_states_json, secret_versions_json=excluded.secret_versions_json`, desiredState.project, desiredState.version, now(), JSON.stringify(serviceStates), JSON.stringify(currentSecretVersions));
-    return { project: desiredState.project, observedVersion: desiredState.version, health: 'serving', services: serviceNames, serviceStates };
+    await db.run(`INSERT INTO applied_state(project, version, revision, applied_at, service_states_json, secret_versions_json)
+     VALUES(?,?,?,?,?,?)
+     ON CONFLICT(project) DO UPDATE SET version=excluded.version, revision=excluded.revision, applied_at=excluded.applied_at,
+       service_states_json=excluded.service_states_json, secret_versions_json=excluded.secret_versions_json`, desiredState.project, desiredState.version, desiredState.revision || null, now(), JSON.stringify(serviceStates), JSON.stringify(currentSecretVersions));
+    return { project: desiredState.project, observedVersion: desiredState.version, ...(desiredState.revision ? { observedRevision: desiredState.revision } : {}), health: 'serving', services: serviceNames, serviceStates };
 }
 /** Reports this node's WireGuard public key when control has not seen the current one yet. */
 export async function reportWgKey(ctx) {
@@ -265,7 +426,7 @@ export async function pollOnce(ctx = defaultContext()) {
             projects.push(await reconcile(desired, ctx, work.serviceEndpoints?.[desired.project]));
         }
         catch (e) {
-            projects.push({ project: desired.project, observedVersion: desired.version, health: 'failed', error: errMessage(e) });
+            projects.push({ project: desired.project, observedVersion: desired.version, ...(desired.revision ? { observedRevision: desired.revision } : {}), health: 'failed', error: errMessage(e) });
         }
     }
     let ingress;
@@ -288,6 +449,8 @@ export async function pollOnce(ctx = defaultContext()) {
     const report = { node: ctx.nodeName, projects, commandResults };
     if (ingress)
         report.ingress = ingress;
+    if (work.retainedImages && projects.every(project => project.health !== 'failed'))
+        await pruneManagedImages(ctx, work.retainedImages);
     await api(ctx, '/agent/v1/status', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(report) });
     return projects;
 }

package/dist/archive.d.ts

@@ -0,0 +1,2 @@
+import * as tar from 'tar';
+export declare function scanArchive(file: string, inspect: (entry: tar.ReadEntry) => void): Promise<void>;

package/dist/archive.js

@@ -0,0 +1,20 @@
+import { createReadStream } from 'node:fs';
+import * as tar from 'tar';
+export async function scanArchive(file, inspect) {
+    await new Promise((resolve, reject) => {
+        const input = createReadStream(file);
+        const parser = new tar.Parser({ strict: true, onReadEntry(entry) {
+                try {
+                    inspect(entry);
+                    entry.resume();
+                }
+                catch (error) {
+                    parser.abort(error);
+                }
+            } });
+        input.once('error', reject);
+        parser.once('error', error => { input.destroy(); reject(error); });
+        parser.once('end', resolve);
+        input.pipe(parser);
+    });
+}

package/dist/artifacts.d.ts

@@ -0,0 +1,23 @@
+import { Transform } from 'node:stream';
+import type { RunResult } from './build.ts';
+export type ImageRunner = (cmd: string, args: string[]) => Promise<RunResult>;
+export interface ImageInfo {
+    imageId: string;
+    platform: string;
+}
+export interface ArchiveInfo extends ImageInfo {
+    sha256: string;
+    sizeBytes: number;
+}
+export declare function inspectImage(imageRef: string, runner?: ImageRunner): Promise<ImageInfo>;
+export declare function hashArchive(file: string): Promise<{
+    sha256: string;
+    sizeBytes: number;
+}>;
+export declare function saveImageArchive(imageRef: string, file: string): Promise<ArchiveInfo>;
+/** Validate Docker save metadata before Docker can import attacker-selected repository tags. */
+export declare function validateImageArchive(file: string, expectedRef: string, maxExpandedBytes?: number, requireManagedReference?: boolean): Promise<ArchiveInfo>;
+export declare function verifyDownload(expected: {
+    sha256: string;
+    sizeBytes: number;
+}): Transform;