@wpelevator/tug 0.11.0 → 0.13.0

package/dist/db-locks.js

@@ -0,0 +1,23 @@
+/** SQL persistence only; callers own lease clocks, heartbeats and conflict responses. */
+export function lockQueries(db) {
+    return {
+        purgeExpired(at) {
+            db.run('DELETE FROM locks WHERE expires_at < ?', at);
+        },
+        insert(lock) {
+            db.run('INSERT INTO locks(name,owner,expires_at,created_at) VALUES(?,?,?,?)', lock.name, lock.owner, lock.expires_at, lock.created_at);
+        },
+        find(name) {
+            return db.get('SELECT * FROM locks WHERE name=?', name);
+        },
+        owns(name, owner, at) {
+            return Boolean(db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', name, owner, at));
+        },
+        renew(name, owner, expiresAt) {
+            db.run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', expiresAt, name, owner);
+        },
+        release(name, owner) {
+            db.run('DELETE FROM locks WHERE name=? AND owner=?', name, owner);
+        },
+    };
+}

package/dist/db-transaction.d.ts

@@ -0,0 +1,3 @@
+import type { Db } from './db-driver.ts';
+/** Run synchronous work in an immediate transaction. Nested transactions are rejected by SQLite. */
+export declare function immediateTransaction<T>(db: Db, operation: () => T & (T extends PromiseLike<unknown> ? never : unknown)): T;

package/dist/db-transaction.js

@@ -0,0 +1,24 @@
+/** Run synchronous work in an immediate transaction. Nested transactions are rejected by SQLite. */
+export function immediateTransaction(db, operation) {
+    // Keep BEGIN outside the catch: a failed BEGIN must not roll back somebody else's transaction.
+    db.exec('BEGIN IMMEDIATE');
+    try {
+        const result = operation();
+        if (result !== null &&
+            (typeof result === 'object' || typeof result === 'function') &&
+            'then' in result &&
+            typeof result.then === 'function')
+            throw new TypeError('Database transactions require synchronous callbacks');
+        db.exec('COMMIT');
+        return result;
+    }
+    catch (error) {
+        try {
+            db.exec('ROLLBACK');
+        }
+        catch {
+            /* Preserve the operation or commit failure. */
+        }
+        throw error;
+    }
+}

package/dist/docker-api.d.ts

@@ -40,6 +40,7 @@ export interface ContainerInspect {
         NetworkMode?: string;
         Binds?: string[];
         Mounts?: unknown[];
+        ExtraHosts?: string[];
     };
     State?: {
         Status?: string;

package/dist/image.d.ts

@@ -7,3 +7,12 @@ export declare function packageVersion(): string;
 export declare function defaultImage(version?: string): string;
 /** An explicit `--image` always wins; otherwise a locally built image is `tug:local` and anything else is the published image. */
 export declare function resolveImage(image: string | undefined, buildLocalImage?: boolean, version?: string): string;
+/**
+ * Fails before any host is touched when the published image for this CLI version does not exist yet. A release
+ * publishes the npm package a few minutes before its image, so a fresh CLI can briefly name a missing tag. Only a
+ * definitive "no such manifest" answer fails; no Docker, no network or a private registry carry on silently.
+ */
+export declare function assertImagePublished(image: string, inspect?: (image: string) => {
+    status: number | null;
+    stderr: string;
+}): void;

package/dist/image.js

@@ -1,3 +1,4 @@
+import { spawnSync } from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -18,3 +19,21 @@ export function defaultImage(version = packageVersion()) {
 export function resolveImage(image, buildLocalImage = false, version = packageVersion()) {
     return image || (buildLocalImage ? LOCAL_IMAGE : defaultImage(version));
 }
+/**
+ * Fails before any host is touched when the published image for this CLI version does not exist yet. A release
+ * publishes the npm package a few minutes before its image, so a fresh CLI can briefly name a missing tag. Only a
+ * definitive "no such manifest" answer fails; no Docker, no network or a private registry carry on silently.
+ */
+export function assertImagePublished(image, inspect = (ref) => {
+    const result = spawnSync('docker', ['manifest', 'inspect', ref], {
+        encoding: 'utf8',
+        timeout: 20_000,
+    });
+    return { status: result.status, stderr: String(result.stderr ?? '') };
+}) {
+    if (!image.startsWith(`${IMAGE_REPOSITORY}:`))
+        return;
+    const result = inspect(image);
+    if (result.status !== 0 && /no such manifest|manifest unknown|not found/i.test(result.stderr))
+        throw new Error(`${image} is not published yet: a release builds its image a few minutes after the npm package. Retry shortly, or pass --image ${IMAGE_REPOSITORY}:latest to use the latest release.`);
+}

package/dist/live.js

@@ -312,6 +312,7 @@ export async function startOneOff(docker, template, command) {
             NetworkMode: host.NetworkMode,
             ...(host.Binds?.length ? { Binds: host.Binds } : {}),
             ...(host.Mounts?.length ? { Mounts: host.Mounts } : {}),
+            ...(host.ExtraHosts?.length ? { ExtraHosts: host.ExtraHosts } : {}),
         },
     });
     const remove = () => docker.removeContainer(id).catch(() => { });

package/dist/util.d.ts

@@ -20,3 +20,5 @@ export declare const isDuration: (value: unknown) => boolean;
  * interface. Hostnames and other non-IP strings return false.
  */
 export declare function isPublicAddress(address: string): boolean;
+/** Node's fetch reports every network failure as "fetch failed"; the system error that explains it is on `cause`. */
+export declare function describeFetchError(error: unknown): string;

package/dist/util.js

@@ -90,3 +90,9 @@ export function isPublicAddress(address) {
     }
     return false;
 }
+/** Node's fetch reports every network failure as "fetch failed"; the system error that explains it is on `cause`. */
+export function describeFetchError(error) {
+    const e = error;
+    const cause = e?.cause?.code || e?.cause?.message;
+    return cause ? `${e?.message ?? 'error'} (${cause})` : errMessage(error);
+}

package/docs/access-control.md

@@ -0,0 +1,166 @@
+# Access control
+
+Control listens on one TLS port (default `8443`) and requires a client certificate from every peer. There is no plain HTTP listener, no unauthenticated route and no bearer token. Certificates are signed by control's private CA; clients pin that CA and still verify the server name. A missing or foreign certificate fails the TLS handshake before any HTTP is parsed. After the handshake, a gate and then every request and WebSocket upgrade check the principal in the database, so revocation and key replacement apply to the next request and close live sessions within a second.
+
+The certificate's URI SAN names the **identity class**, which decides which routes the peer can reach at all:
+
+| SAN | Class | May reach |
+| --- | --- | --- |
+| `tug:node:<id>` | node (`<id>` is the immutable node ID, not the node name) | `/agent/v1/*` (including `POST /agent/v1/renew` and the agent stream) |
+| `tug:operator:<id>` | operator or CI | `/api/v1/*` and `/api/v1/live`, subject to role and project scope |
+| `tug:enroll:<id>` | bootstrap | `POST /agent/v1/enroll` only |
+
+A principal outside its class gets `403`. Authorization follows the **public key**, not the certificate: control stores the SHA-256 of each principal's SubjectPublicKeyInfo (`key_hash`) and accepts any CA-signed certificate for that key, so renewal is just signing another certificate for the same key. Replacing a key makes every certificate for the old key unusable at once.
+
+Operator credentials come in two kinds, both with a unique name, a role and optionally allowed projects:
+
+- **`root`** is an ordinary named admin issued on the control-plane host by `tug control setup` (`control issue-credential --name root --role admin`). Upgrades never reissue it; `tug control setup --recover-root` replaces only root's key without the API.
+- **Named credentials** are for CI systems, teammates and automation. Create them with `tug credential create` (control generates the key and returns the credential once; it keeps no private-key copy) and revoke them with `tug credential revoke`. They last one year at most.
+
+Replace a credential by creating a new name, verifying it, updating consumers and then revoking the old name. Keep at least one verified admin throughout.
+
+## Roles
+
+Roles are cumulative.
+
+| Role | Can do |
+| --- | --- |
+| `read` | Status, events, project and deployment records, volume listings and secret names (never values). No mutations. |
+| `deploy` | Everything in `read`, plus create projects, deploy, roll back, cancel deployments, restart services, fetch service logs and read command results, for its allowed projects. |
+| `admin` | Everything an operator can do: secrets, volumes, nodes, enrollment, backups, restore, diagnostics and credential management. Admin credentials cannot be limited to projects. |
+
+A `deploy` credential must be trusted with its workloads' secrets: live `inspect` redacts environment values to avoid accidental disclosure, but `exec` can read them, and deployed code can disclose any secret it receives. Setting and deleting secrets is `admin` only; listing secret names is `read`. The deploy role can trigger project backups, while control backups and restores require `admin`.
+
+Project scope limits API access to project records; it does not isolate hosts or co-located projects. Current deployments can select shared nodes and request host bind mounts, external volumes, volume driver options and added capabilities. Treat deployment authors as administrators of the nodes they can select. Use separate hosts or control planes for mutually untrusted authors. See the [security audit](security.md) for the demonstrated boundaries and outstanding fixes.
+
+## Endpoint table
+
+This table is the contract. It mirrors `ROUTE_ROLES` in `src/credentials.ts`, `createApp` refuses to register a `/api/v1` route that is not listed there, and a test compares the table with the live router and with this page, so a new route cannot ship without a declared role.
+
+The project column says how a scoped credential's allowed projects are checked: the project comes from the request body, the query string (default `site`), the path, or the stored record for the given ID. Admin-only routes are control-wide.
+
+| Route | Role | Project check |
+| --- | --- | --- |
+| `GET /api/v1/projects` | `read` | Response is filtered to allowed projects |
+| `GET /api/v1/services` | `read` | Query; service names and eligible placements only |
+| `GET /api/v1/deployments` | `read` | Query; active deployments only |
+| `GET /api/v1/backup/snapshots` | `read` | Query; successful catalog backups for target |
+| `GET /api/v1/holds` | `admin` | Query; deployment move holds only |
+| `GET /api/v1/status` | `read` | Response is filtered to allowed projects |
+| `GET /api/v1/events` | `read` | `project` query, or filtered to allowed projects |
+| `GET /api/v1/projects/:project` | `read` | Path |
+| `GET /api/v1/deployments/:id` | `read` | Deployment's project |
+| `GET /api/v1/volumes` | `read` | Query; includes node name, persistence and referencing services |
+| `GET /api/v1/secrets` | `read` | Query (names and versions only) |
+| `POST /api/v1/projects` | `deploy` | Body `name` |
+| `POST /api/v1/deployments` | `deploy` | Body or upload payload `project` |
+| `POST /api/v1/deployments/:id/cancel` | `deploy` | Deployment's project |
+| `POST /api/v1/rollback` | `deploy` | Body |
+| `POST /api/v1/scale` | `deploy` | Body |
+| `POST /api/v1/restart` | `deploy` | Body |
+| `POST /api/v1/logs` | `deploy` | Body |
+| `GET /api/v1/live` | `deploy` | First WebSocket frame |
+| `GET /api/v1/commands/:id` | `deploy` | Command's project; commands without a project (diagnostics) need `admin` |
+| `POST /api/v1/backups/run` | `deploy` | Body `project` |
+| `GET /api/v1/backups` | `read` | Query `project` |
+| `POST /api/v1/restores` | `admin` | Body `project`; without `confirm` it only returns the plan |
+| `GET /api/v1/restores` | `read` | Query `project`; `state=interrupted` returns all interrupted restores |
+| `POST /api/v1/restores/:id/release` | `admin` | Drops the holds of an interrupted restore |
+| `POST /api/v1/restores/:id/resume` | `admin` | Runs an interrupted restore again |
+| `GET /api/v1/backup/destination` | `admin` | Control-wide; URL and variable names, never values |
+| `POST /api/v1/backup/destination` | `admin` | Control-wide |
+| `DELETE /api/v1/backup/destination` | `admin` | Control-wide |
+| `GET /api/v1/backup/repository-password` | `admin` | Query `project`; prints the repository password |
+| `POST /api/v1/secrets` | `admin` | Control-wide |
+| `DELETE /api/v1/secrets/:name` | `admin` | Control-wide |
+| `DELETE /api/v1/volumes/:name` | `admin` | Control-wide |
+| `GET /api/v1/nodes` | `admin` | Control-wide |
+| `POST /api/v1/nodes/:name/identity` | `admin` | Control-wide |
+| `POST /api/v1/nodes/:name/disable` | `admin` | Control-wide |
+| `POST /api/v1/nodes/:name/address` | `admin` | Control-wide |
+| `POST /api/v1/nodes/enrollment-credential` | `admin` | Control-wide |
+| `POST /api/v1/nodes/:name/ingress` | `admin` | Control-wide |
+| `DELETE /api/v1/nodes/:name` | `admin` | Control-wide |
+| `POST /api/v1/control/backups/run` | `admin` | Control-wide |
+| `POST /api/v1/control/restore` | `admin` | Control-wide |
+| `POST /api/v1/holds/release` | `admin` | Body `project`; drops the holds a deploy placed for a move |
+| `POST /api/v1/images/prune` | `admin` | Control-wide; deletes build archives older than the kept deployments |
+| `GET /api/v1/diagnostics` | `admin` | Control-wide |
+| `POST /api/v1/diagnostics/nodes` | `admin` | Control-wide |
+| `POST /api/v1/credentials` | `admin` | Control-wide |
+| `GET /api/v1/credentials` | `admin` | Control-wide |
+| `DELETE /api/v1/credentials/:name` | `admin` | Control-wide |
+
+## Failures
+
+- `401` means the certificate is unknown, revoked, expired, disabled or its key is not the stored one. A missing or foreign certificate never gets this far: the TLS handshake fails.
+- `403` means the credential is valid but its role is too low (`forbidden: this action requires the deploy role`) or the project is outside its scope (`forbidden: this credential may not access that project`). The project message is identical whether the other project exists, whether a deployment or command ID exists, or whether the project name is valid, so a scoped credential cannot probe for projects or IDs.
+
+## Projects and scoping
+
+A `read` or `deploy` credential created with `--project` may only act on those projects. Listings are filtered rather than refused: `status` returns only allowed projects, their deployments and their node status, and `events` without `--project` returns only events of allowed projects (events without a project, such as credential changes and node events, are hidden from scoped credentials). A credential without `--project` covers every project, including ones created later. `status` still lists node names and health to every credential, since placement names appear in Compose files.
+
+## Audit trail
+
+Every deployment and every control event records the acting credential name, `root` for the root credential. `tug events` and `GET /api/v1/events` return it as `actor`, and deployment records (`GET /api/v1/deployments/:id`, `tug status`) carry it as `actor`. Control actions write events of these types: `deploy`, `rollback`, `cancel`, `restart`, `secret.set`, `secret.delete`, `credential.create` and `credential.revoke`. Events never contain secret values or credential private keys. Rollbacks that control starts by itself (`x-tug: { rollback: auto }`, see [Compose extensions](compose.md#automatic-rollback)) and their `deployment.auto_rollback` events are attributed to the reserved actor `system:auto-rollback`; credential names cannot contain a colon, so no credential can use it. Events written by agents have no `actor`. Control events that belong to no project (credential changes) show up in `tug events --all`.
+
+## Managing operator credentials
+
+```bash
+tug credential create ci-site --role deploy --project site
+tug credential create ci-site --role deploy --project site --project blog --expires-in-days 90
+tug credential create alice --role read
+tug credential list
+tug credential revoke ci-site
+```
+
+- Names are 1 to 64 letters, numbers, dots, underscores or hyphens. A revoked credential keeps its name so audit entries stay unambiguous; choose a new name to replace it.
+- `credential create` prints the credential once, on stdout; a note on stderr reminds you to store it. Only the key hash is stored, so a lost credential cannot be recovered, only replaced.
+- `credential list` shows name, role, projects, creator, creation, last use, expiry and revocation time, never secrets.
+- Revocation applies to the next request. Expired credentials are refused with `401` the same way. `last_used_at` is refreshed at most once a minute in the background, so it is approximate and bookkeeping never delays or fails a request.
+- Credential records (name, role, projects, key hash, expiry, revocation) live in the control database, so they are part of `tug control backup` and come back with `tug control restore`; a restore returns the credential set (including revocations) to the backup's state. Private keys are never stored on control.
+- `root` is listed like any other admin credential. Use `--recover-root` if its credential is lost; prefer day-to-day use of named admin credentials.
+
+## Deploying from GitHub Actions
+
+Create a project-scoped `deploy` credential and store it as a repository secret (the credential already carries the control plane's default address and CA):
+
+```bash
+tug credential create github-site --role deploy --project site
+```
+
+Add `TUG_CREDENTIAL` (the printed credential) under the repository's Settings, Secrets and variables, Actions. A workflow:
+
+```yaml
+name: Deploy
+on:
+  push:
+    branches: [main]
+
+concurrency:
+  group: deploy-site
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - name: Deploy
+        env:
+          TUG_CREDENTIAL: ${{ secrets.TUG_CREDENTIAL }}
+          TUG_PROJECT: site
+        run: npx --yes @wpelevator/tug deploy -f docker-compose.yml --version "git-${GITHUB_SHA::12}"
+```
+
+`deploy` waits for the rollout and fails the job when the deployment fails or times out (add `--no-wait` to return immediately). Services with `build:` upload the checkout for a remote build on control; add `--build-local` to build on the runner, which has Docker, and upload the images instead. Pin the CLI version (`@wpelevator/tug@<version>`) to match your control plane. Rolling back from CI needs only the same credential (`npx @wpelevator/tug rollback`).
+
+## Limits
+
+- A `deploy` credential can run any image or Compose configuration on the nodes of its projects, execute commands in those containers, and read their logs and Docker inspection data. Treat it as access to those nodes: inspection redacts environment values, but `exec` and deployed code can read them, and logs can contain whatever the application prints. Scope credentials to the projects they need and expire short-lived ones.
+- Control-wide state (nodes, backups, diagnostics, credentials, secrets and volume deletion) is admin only. There is no per-project secret management role.
+- Roles and project scopes are fixed at creation; to change them, create a new credential and revoke the old one.
+- A `deploy` event is written when a deployment is published. A build that fails before publishing appears only in its deployment record (with its `actor`), not as an event.

package/docs/architecture.md

@@ -0,0 +1,254 @@
+# Architecture
+
+Tug deploys Docker Compose applications to one or more Docker hosts through a small self-hosted control plane and a pull-based agent on each node. The agents drive each node's Docker Engine API directly; Compose is used only on the operator machine to normalize the file. SSH is used only to install the control plane and nodes; normal deploys go through the control API and the agents, which share one mutual-TLS port. [Terminology](terminology.md) defines the vocabulary used here.
+
+## Components
+
+The [README](../README.md#how-it-fits-together) has an overview diagram of how these fit together.
+
+- **Control plane** (`src/control.ts`): an Express API behind a single TLS 1.3 listener that requires a client certificate ([Control plane](control-plane.md#authentication-and-pki)), backed by SQLite (`tug.db`), an install-time master key (`master.key`) and the private CA (`pki/`) in `TUG_DATA_DIR`. It stores nodes, projects, deployments, desired states, encrypted secrets, volumes, queued commands, events, operator and enrollment credentials (public-key hashes, never private keys) and locks, and builds images from uploaded source tarballs or imports operator-built archives. It always stores and serves built images as compressed artifacts and resolves published image tags to immutable registry digests before publication.
+- **Access control** (`src/credentials.ts`, `src/pki.ts`): every peer presents a certificate signed by control's CA. The certificate names one principal (`tug:operator:<id>`, `tug:node:<id>` or `tug:enroll:<id>`) and control authorizes it by comparing the certificate's public-key hash with the key stored for that principal. Every operator route is declared once in a role table (`read`, `deploy`, `admin`) and passes through one guard that checks the role and the credential's allowed projects, and attaches the acting credential name that deployments and audit events record. Agent routes derive the node from the certificate and touch only that node's objects. See [Access control](access-control.md).
+- **Agent** (`src/agent.ts`): runs on every node, fetches authoritative work from `GET /agent/v1/work` and reconciles containers through the Docker Engine API (`src/docker-api.ts`) over the local socket. It builds one container definition per placed service (`src/container-spec.ts`), substitutes released secrets into the container environment, verifies/downloads/loads artifact-backed images, pulls published images by their pinned registry digest when the desired revision is new to the node, executes health checks, and reports status. Nodes need only the Docker Engine, not the Compose plugin.
+- **Reconciler** (`src/reconciler.ts`): one loop for every service. Containers carry `tug.project`, `tug.service`, `tug.revision` and `tug.config-hash` labels; the hash covers the resolved definition, the image identity and the versions of the secrets the service uses, and a container whose hash differs is replaced. `recreate` stops the old container and starts the new one; `rolling` (`x-tug.rollout.strategy: rolling`) starts the new one on a free host port beside the old one, health-checks it, lets the ingress switch and removes the old one after a drain. Tug names its own network `tug_<project>`, volumes `tug_v<project-length>_<project>_<volume>` and containers `tug-<project>-<service>-<hash8>`; service names are network aliases. Reconciliation never removes volumes; `volume.delete` is the only way one goes away.
+- **Compose normalization** (`src/normalize.ts`): `tug deploy` runs `docker compose config` on the operator machine, which does interpolation, `extends`, profiles and `env_file`. Control secrets are interpolated as `@@tug:secret:NAME@@` sentinels that the agent replaces with released values. The CLI reads Compose's YAML rendering because its JSON rendering drops service-level `x-tug`. Control validates the result against an allowlist of keys ([Compose](compose.md#supported-keys)); agents never parse Compose YAML.
+- **CLI** (`src/cli.ts`): the `tug` command for operators. It talks to the control API and uses SSH only for `control setup` and `node add`. See [CLI reference](cli.md).
+- **Networking** (`src/ingress.ts`, `src/node-address.ts`): health-aware service endpoints, private-address port binding and the managed Caddy edge. `ingress.ts` resolves service endpoints and ingress routes from each node's `address`, renders the Caddyfile (control) and reconciles the Caddy container (agent). Tug provides no overlay or tunnel; the operator owns the inter-node network. See [Networking](networking.md).
+- **Runtime entrypoint** (`src/main.ts`, installed as the `tug-runtime` binary and `dist/main.js` in the image): one image runs both control and agent; `node src/main.ts control|agent` or `TUG_ROLE`. It also provides the `agent identity` and `agent install-cert` subcommands that node installation uses.
+- **Protocol** (`src/protocol.ts`): the control↔agent contract. It holds the types, and a few shared constants such as the active and terminal deployment states, for everything crossing the boundary: `DesiredState`, `ServiceSpec` and `HealthCheck`; the `/agent/v1/*` payloads (`WorkResponse` with its optional `serviceEndpoints`, `ports` and `ingress` blocks, `Command` / `CommandResult`, `StatusReport` / `ProjectStatus` / `IngressStatus`, `StreamMessage`, `WakeResponse`, `SecretsResponse`); and identity and enrollment (`IdentityRequest` / `IdentityResponse`, `EnrollRequest` / `EnrollResponse`, `RenewRequest` / `RenewResponse`, `EnrollmentCredentialResponse`, `JoinKey`). `control.ts`, `agent.ts`, `compose.ts` and `cli.ts` all import from it, so change the wire format there first.
+- **Injectable agent runners** (`src/agent.ts`, `src/runners.ts`): `reconcile`, `executeCommand`, `waitForHealth`, `containerIp` and `pollOnce` take an `AgentContext` (`{ docker, http, nodeName, stateDir, apply, controlUrl, credential?, caddyImage?, rolloutPorts? }`). `docker` is a `DockerApi` (the real `DockerEngine` talks to the Engine API over the local socket, `tests/fake-docker.ts` implements it in memory), and `http` is a `fetch`-compatible client pinned to the node's credential. Production code uses `defaultContext()`; the agent tests pass the in-memory Docker fake to unit-test rollout ordering, secret-triggered recreation and the docker, tcp and http health-check branches without a Docker daemon. The privileged-Docker integration test remains the end-to-end check.
+
+## Desired state and reconciliation
+
+`tug deploy` sends the Compose file to control, which validates `x-tug` placement (see [Compose extensions](compose.md)), optionally builds images for services with `build:`, and stores an immutable desired-state record for the deployment. Each agent receives the desired states that place services on its node, applies them, and reports the observed display version, immutable desired-state revision and health per project. When every assigned node reports the current revision as `serving` (or `skipped`), control also waits for every enabled managed-ingress node to report the current routing configuration as applied when the project adds, changes or removes ingress routes. The acknowledged configuration must contain each current project route with a serving upstream; a failed, stale or older ingress report cannot complete deployment. With no enabled ingress node, convergence reports service health only. This acknowledgement confirms the agent applied its Caddy configuration, not external DNS, certificate issuance or a successful public request. A failed node report still fails the deployment with its apply and rollout errors; an unavailable ingress can leave the deployment `rolling_out` until it recovers or the CLI times out.
+
+Deploys, rollbacks, secret changes and data-deleting volume removals take a per-project operation lock, so concurrent operations on one project are rejected with `409`. Long builds renew the project lease and check ownership/cancellation before publication. Deploy publication, rollback and artifact backup/restore share an operation-lifetime writer queue. Rollback creates a new deployment and revision whose desired state copies the most recent earlier `succeeded` or `rolling_out` one. With `x-tug.rollback: auto` control creates that rollback itself when a deployment turns `failed`; see [Rolling rollouts and automatic rollback](#rolling-rollouts-and-automatic-rollback).
+
+Restarts, log fetches, diagnostics and volume deletions are queued as commands for the node that runs the service; control wakes that node, and the agent fetches the command and posts its result back.
+
+### Agent wake channel
+
+After each reconciliation, the agent waits on `GET /agent/v1/stream`, a WebSocket upgrade authenticated by the same node certificate as the HTTP agent API. Control sends a `work-available` hint for durable work; desired state and queued commands remain in SQLite and the agent always fetches them through `GET /agent/v1/work`. Deployments, rollbacks, secret changes and cluster-wide network changes wake every node because they can change placement, service endpoints or ingress. Node commands and ingress-role changes wake only their target node.
+
+The same socket carries ephemeral live operations. The CLI opens `GET /api/v1/live` with its operator credential, sends one versioned request, and control resolves the project, service and optional node before binding the session to that exact connected agent. For a rolling service, the agent addresses the active rollout container instead of the Compose service container. `exec` (with a PTY when asked), `tug run` one-off containers, `logs -f`, `ps` and `inspect` stream responses directly; input, cancellation and disconnect cleanup travel in the other direction. Live requests are never persisted, queued or replayed, all three hops enforce frame, buffer and session limits, and losing any socket closes the session and kills the agent's local Docker client process. Docker does not expose the remote exec process identity through this client stream, so an already-started process inside the container may continue after cancellation or disconnect; do not treat client cancellation as remote process termination. `inspect` is deliberately a `deploy` operation and redacts environment values; it does not limit what a deploy credential can read through `exec`.
+
+If a proxy or firewall rejects WebSockets, the agent uses `GET /agent/v1/wake?after=<cursor>&timeout=<milliseconds>` for durable work only. Each node has a monotonic in-memory wake cursor, so a wake issued just before the long poll begins is returned immediately instead of being lost. Cursors and sockets are intentionally ephemeral: control restart merely loses a hint, while periodic `TUG_POLL_INTERVAL_MS` reconciliation remains the correctness and drift-repair path. Live operations require the WebSocket path because replaying them through long-poll would be unsafe.
+
+### Events
+
+Agents report lifecycle events to control, and `tug events` shows them next to control's own audit events. An agent does not report that it is still reconciling: for the recreated services of a project it keeps a fingerprint of the desired revision, the Compose file rendered for its node and the secret versions in its own database (`event_state` in `agent.db`), and sends `service.pulling`, `starting`, `health_checking` and `serving` only when the fingerprint changes. A failed apply produces one `service.failed` event (level `error`, naming the service whose check failed and why) and retries of the same failure stay quiet. When the apply later succeeds the agent sends one `service.serving` event saying it recovered. The agent still applies and health-checks the project after every wake or poll, so drift is repaired the same way; only the events are quiet. Rolling services report their own events when a revision is started, switched or rejected. Control deletes events older than `TUG_EVENT_RETENTION_DAYS` (30) and keeps at most `TUG_EVENT_MAX_ROWS` (100,000); see [Configuration](configuration.md#control-plane).
+
+## SQLite storage
+
+Control metadata and agent applied/event state use Node's built-in synchronous SQLite driver. `src/db-driver.ts` owns the runtime-neutral `Db` contract, binding values and unique/busy error categories; `src/db-node.ts` alone imports `node:sqlite`. Queries return ordinary objects, missing rows are `undefined`, `all<Row>()` returns `Row[]`, and writes expose numeric change counts. Only anonymous `?` parameters are supported, with strict arity and value validation. Integer reads use numbers and reject unsafe values. Unexpected driver failures propagate with their original identity; classified errors preserve the driver error as their cause.
+
+`src/db.ts` selects one connection factory for control, agents and restore snapshots. Writable connections enable WAL, foreign keys and a 1,000 ms busy timeout; initialization closes the connection on failure. Read-only snapshots require an existing file and run no schema creation or journal-mode change. Schemas initialize in place without migrations. Deployment publication executes synchronously from `BEGIN IMMEDIATE` through commit or rollback; writer queues and project leases coordinate workflows that also await files, Docker or network operations.
+
+`src/db-locks.ts` owns control’s project-lock queries, and `src/db-backup-destination.ts` owns destination and encrypted credential storage shared by control and the backup scheduler. Keep HTTP responses, lease timers, encryption, audit events and wake notifications in their callers. These focused modules use parameterized SQL directly; row interfaces remain manually maintained and do not validate SQL projections. Other query families remain in their existing callers for later extraction.
+
+`src/db-transaction.ts` provides `immediateTransaction()` for synchronous database-only work: it begins an immediate transaction, returns the callback result after commit and rolls back on callback or commit failure without masking the original error. Callbacks must not start asynchronous work; promise-like results are rejected as a defensive check, which cannot cancel work already started. Nested transactions are rejected, and a failed begin never rolls back the existing transaction. Destination replacement and removal commit configuration and credentials together; deployment publication uses the same helper.
+
+SQL executes on the owning process's event loop. Busy waits, snapshots and integrity checks can pause APIs, WebSocket traffic and timers; backup blocking is described in [Control plane](control-plane.md#backup-and-restore). Database-only work does not return promises. Filesystem preparation and other asynchronous workflows retain their asynchronous interfaces.
+
+A future Bun bundle that includes control or agent code can supply a `bun:sqlite` adapter through the same factory. Select it at build time so the bundle never imports the Node adapter; normalize results, bindings, error categories and connection cleanup, and run the same factory-driven database contract tests. Bun implementation and build aliases are deferred. The proposed CLI-only executable does not access SQLite and needs no adapter.
+
+## Image artifacts
+
+The `builds` table records immutable image references, config-digest identity, platform, archive SHA-256/size and a relative artifact path. Image save/download/hash operations stream bytes; saved archives are validated before import. Build archives finalize before the desired-state transaction becomes current. Failed publication leaves the previous deployment current. `GET /agent/v1/images/:buildId` authenticates the node certificate and checks the retained service placement; `WorkResponse.retainedImages` protects rollback images from agent pruning.
+
+Docker image stores can expose either a config digest or an OCI index/manifest digest as their local image ID. Artifacts use the saved configuration digest as their canonical identity. After a verified import, agents cache the daemon's actual tag ID paired with that canonical ID and archive hash; they check the tag again before reusing it. This permits classic Docker and containerd-backed Docker to exchange saved images without confusing their different IDs.
+
+Archives are validated before Docker can import them. A containerd-backed `docker save` can list BuildKit build attestations of the image next to it, either nested in the image index or as extra top-level entries. These are accepted only when they carry no image name, their in-toto contents match their digests, and their subject is this exact image; any other extra manifest, extra tag, link, absolute or parent-relative path, or oversized entry rejects the archive.
+
+Local builds use a multipart `POST /api/v1/deployments`: `payload` contains the Compose/project/version and `localImages` descriptors; each descriptor names a corresponding `image-N` file. Each archive must contain exactly its declared Tug tag and actual configuration/platform/checksum. Source contexts and local images are mutually exclusive. Control retags imported images under its own deployment identity before publishing their saved archives.
+
+## Node installation (`tug node add`)
+
+The node makes its own key and certificate request, so no bootstrap secret travels and the private key never leaves the host. The operator's admin credential authorizes the identity at that moment.
+
+```mermaid
+sequenceDiagram
+    participant Op as Operator
+    participant CLI as tug CLI
+    participant Ctrl as Control plane
+    participant DB as SQLite
+    participant Host as Node host (SSH)
+    participant Agent as Agent container
+
+    Op->>CLI: node add root@host --name app1
+    CLI->>Host: SSH preflight (docker, paths, disk, control reachable)
+    CLI->>Host: docker save | scp | docker load (only with --build-local-image)
+    CLI->>Host: ssh: docker run image agent identity (key stays in the state volume)
+    Host-->>CLI: certificate request (CSR)
+    CLI->>Ctrl: POST /api/v1/nodes/app1/identity { csr, mode: create } (admin credential, mTLS)
+    Ctrl->>DB: insert node with a new immutable ID and the CSR key hash
+    Ctrl-->>CLI: { node, certificate, ca, url }
+    CLI->>Host: ssh: agent install-cert (response on stdin)
+    CLI->>Host: ssh node-install.sh (writes /etc/tug/agent.env, docker run agent)
+    Host->>Agent: start container
+    loop every TUG_POLL_INTERVAL_MS
+        Agent->>Ctrl: GET /agent/v1/work (node certificate, mTLS)
+        Ctrl->>DB: update last_seen_at, status=online
+    end
+    CLI-->>Op: { ok, node }
+```
+
+`node replace` and `node repair` run the same flow with `mode: replace` (and, for repair, move corrupt credential documents aside first). See [Nodes](nodes.md).
+
+## Self-service enrollment (`tug node enroll`)
+
+```mermaid
+sequenceDiagram
+    participant Op as Operator
+    participant CLI as tug CLI
+    participant Ctrl as Control plane
+    participant DB as SQLite
+    participant Host as Fresh host
+    participant Agent as Agent container
+
+    Op->>CLI: node enroll --name app1
+    CLI->>Ctrl: POST /api/v1/nodes/enrollment-credential { name, labels, ttlSeconds }
+    Ctrl->>DB: insert enrollment credential (key hash, name, reserved node ID, expiry)
+    Ctrl-->>CLI: bootstrap credential
+    CLI-->>Op: docker run ... -e TUG_JOIN_KEY=<join key> ... agent
+    Op->>Host: paste docker run (or cloud-init)
+    Host->>Agent: start container
+    Agent->>Agent: no identity on the volume -> decode join key, make key and CSR
+    Agent->>Ctrl: POST /agent/v1/enroll { csr, address } (bootstrap certificate, mTLS)
+    Ctrl->>DB: validate and consume the key, insert node with the reserved ID
+    Ctrl-->>Agent: { node, certificate, ca, url }
+    Agent->>Agent: persist agent.json to the state volume
+    loop every TUG_POLL_INTERVAL_MS
+        Agent->>Ctrl: GET /agent/v1/work (node certificate, mTLS)
+        Ctrl->>DB: update last_seen_at, status=online
+    end
+```
+
+The bootstrap certificate reaches only `POST /agent/v1/enroll`. A retry with the same key signs another certificate for the same node without changing stored values.
+
+## Deploy (`tug deploy`)
+
+```mermaid
+sequenceDiagram
+    participant Op as Operator
+    participant CLI as tug CLI
+    participant Ctrl as Control plane
+    participant DB as SQLite
+    participant Agent as Agent (per node)
+
+    Op->>CLI: deploy -f docker-compose.yml
+    CLI->>CLI: read & hash compose, tar build context
+    CLI->>Ctrl: POST /api/v1/deployments (compose + context)
+    Ctrl->>DB: lock project, insert deployment (state=building)
+    Ctrl->>Ctrl: build images; save archives; pin published image digests
+    Ctrl->>DB: insert desired_state, set current_deployment_id, state=rolling_out
+    Ctrl-->>CLI: { deployment, desiredState }
+    loop poll interval
+        Agent->>Ctrl: GET /agent/v1/work
+        Ctrl-->>Agent: matching desiredStates for this node
+        Agent->>Agent: fetch secrets, pull or load images, recreate containers whose config hash changed, health checks
+        Agent->>Ctrl: POST /agent/v1/status (observedVersion, observedRevision, health)
+        Ctrl->>DB: upsert node_statuses, check convergence
+    end
+    Ctrl->>DB: all nodes serving desired revision -> deployment state=succeeded
+    CLI->>Ctrl: GET /api/v1/deployments/:id (poll until terminal state)
+    CLI-->>Op: deployment result
+```
+
+## Rollback (`tug rollback`)
+
+```mermaid
+sequenceDiagram
+    participant Op as Operator
+    participant CLI as tug CLI
+    participant Ctrl as Control plane
+    participant DB as SQLite
+    participant Agent as Agent (per node)
+
+    Op->>CLI: rollback [--force]
+    CLI->>Ctrl: POST /api/v1/rollback { project, force }
+    Ctrl->>DB: lock project, reject 409 if active deployment & not --force
+    Ctrl->>DB: find most recent prior succeeded/rolling_out desired_state
+    Ctrl->>DB: insert new deployment (version=rollback-...), insert desired_state copy, set current_deployment_id
+    Ctrl-->>CLI: { deployment, desiredState: previousState }
+    loop poll interval
+        Agent->>Ctrl: GET /agent/v1/work
+        Ctrl-->>Agent: previous desiredState
+        Agent->>Agent: reconcile (pull/apply prior images, health checks)
+        Agent->>Ctrl: POST /agent/v1/status (observedVersion=previousState.version)
+        Ctrl->>DB: convergence check -> deployment state=succeeded
+    end
+    CLI-->>Op: rollback deployment result
+```
+
+## Rolling rollouts and automatic rollback
+
+Services are `recreate` by default: a container whose config hash changed is stopped and replaced in place. A service with `x-tug.rollout.strategy: rolling` is rolled instead, one changed service at a time:
+
+```mermaid
+sequenceDiagram
+    participant Op as Operator / CI
+    participant Ctrl as Control plane
+    participant A as Agent (app node)
+    participant E as Agent (ingress node) + Caddy
+
+    Op->>Ctrl: POST /api/v1/deployments (web v2, rolling, rollback: auto)
+    Ctrl->>Ctrl: validate, store desired state, deployment rolling_out
+    A->>Ctrl: GET /agent/v1/work (desired state v2)
+    A->>A: hash the rendered service + its secret versions (new revision?)
+    A->>A: allocate a free host port, persist candidate in rollout.json
+    A->>A: create and start container tug-proj-web-HASH via the Engine API (old revision untouched)
+    A->>A: health check the new container
+    alt healthy
+        A->>A: active = new, old revision scheduled to drain
+        A->>Ctrl: POST /agent/v1/status (web serving, port 20001)
+        Ctrl->>Ctrl: endpoint port 20001, re-render Caddyfile; deployment still rolling_out
+        E->>Ctrl: GET /agent/v1/work (new Caddyfile)
+        E->>E: caddy reload (graceful, no dropped connections)
+        E->>Ctrl: report applied configuration hash
+        Ctrl->>Ctrl: matching ingress acknowledged -> deployment succeeded
+        A->>A: a later poll, after the drain: remove the old revision
+    else health check fails or no free port
+        A->>A: remove the new revision, remember the failure for this desired revision
+        A->>Ctrl: POST /agent/v1/status (health failed, web still serving, old port)
+        Ctrl->>Ctrl: deployment failed, ingress keeps the old port
+        opt x-tug.rollback is auto and a previous succeeded deployment exists
+            Ctrl->>Ctrl: createRollback(), the same code as POST /api/v1/rollback, event deployment.auto_rollback
+            A->>A: desired state equals the revision still running, nothing restarts
+            A->>Ctrl: POST /agent/v1/status (rollback revision serving)
+            Ctrl->>Ctrl: rollback deployment succeeded
+        end
+    end
+```
+
+Rules the agent follows (`src/reconciler.ts`):
+
+- A revision is identified by a hash of the rendered service (image id for built images, environment, `extra_hosts`, ports shape, volumes) and the versions of the secrets it references. The same hash means the revision is already running and nothing is restarted; a secret change rolls the service instead of force-recreating it.
+- Each revision is a container `tug-<project>-<service>-<hash8>` on the project network, with the single published port remapped to a free port from `TUG_ROLLOUT_PORTS` (default 20000-20999) and the bind address kept. A running container that predates the rollout state is adopted as the active revision.
+- State lives in `<state dir>/projects/<project>/rollout.json`: the active revision, a candidate (written before `up`, so a restarted agent resumes the same project and port instead of starting a second one), revisions waiting to drain with their deadline, and the last rejected revision per service so it is not retried on every poll. Drain deadlines are checked on every poll, so a drain never blocks the poll loop.
+- A failed health check, an exhausted port range or a failed start removes the candidate and leaves the active revision untouched. The node reports `health: failed` and the rejection reason, while the service state stays `serving` with the old port so control keeps routing to it. If a recreated service of the same project fails, rolling services keep being reported serving the same way.
+- Control resolves each endpoint's `port` from the service state, so the ingress follows takeovers without any ingress-specific agent protocol. Convergence is unchanged: `failed` still comes from a node reporting failed health for the current revision.
+- Automatic rollback runs on control right after the status update that moved a deployment from `rolling_out` to `failed`. `scheduleAutoRollback` works in the background so a busy writer queue never blocks status reports, skips rollback deployments (their versions start with the reserved `rollback-`), requires the failed deployment to still be current, and takes the writer queue and project lock like a manual rollback. On a rolling node the previous revision is still running, so applying the rollback restarts nothing; other nodes recreate as usual.
+
+## Networking
+
+`GET /agent/v1/work` returns `serviceEndpoints` by default (serving endpoints per service for injecting `extra_hosts`), `ports` bindings based on each node's `address` and, for nodes marked as ingress nodes, an `ingress` block with the rendered Caddyfile. Set `TUG_SERVICE_NAMES=false` on control to disable service endpoints. Agents report ingress state in the status report. Cross-node clients and Caddy dial the node's sole inter-node `address` over the operator's network; there is no Tug-owned overlay or workload-traffic encryption. See [Networking](networking.md) for the model and sequence diagram.
+
+## Secrets
+
+Secrets are encrypted at rest with AES-256-GCM using the control master key, and versioned per project. An agent only receives the secrets referenced by the `environment` of services placed on its node, substituted into the container environment by the agent (no file on disk). When only a referenced secret's version changes, the agent force-recreates just the affected services.
+
+## Runtime paths
+
+Agents write generated state under `TUG_STATE_DIR` (default `/var/lib/tug`):
+
+```text
+/var/lib/tug/agent.json                                # the node identity document: private key, certificate and pinned CA (0600)
+/var/lib/tug/agent-pending.json                        # a new key waiting for its certificate during enrollment or renewal
+/var/lib/tug/agent.db                                  # last applied version/revision and secret versions per project
+/var/lib/tug/managed-images.json                       # only Tug-owned image references eligible for conservative pruning
+/var/lib/tug/image-identities.json                     # canonical artifact identities paired with daemon-local tag IDs
+/var/lib/tug/ingress/                                  # ingress nodes only: applied Caddy config and its hash
+/var/lib/tug/projects/<project>/rollout.json               # rolling services: active/candidate/draining revisions and the last rejected one
+```
+
+The state directory must be a persistent volume (`-v tug-agent:/var/lib/tug`; `node add` mounts the host directory), so `agent.json` survives container recreation and the agent does not enroll again after an image update or reboot.