@wpelevator/tug 0.4.0 → 0.5.0

package/README.md

@@ -6,15 +6,15 @@ Deploy Docker Compose projects to one or more ordinary Linux VPSes through a sma
 
 ## Features
 
-- Control API with bearer-token auth, scoped `read`/`deploy`/`admin` API tokens for CI and teammates, an audit trail naming who acted, and SQLite metadata.
-- Nodes join over SSH (`node-install`) or enroll themselves with a single `docker run` (`node-enroll`), with per-node agent and registry tokens. Agents only dial out, so nodes behind NAT work.
-- Agents pull desired state from control, apply a generated per-node Compose file, run TCP/HTTP/Docker health checks and report status. An outbound WebSocket wakes them as soon as control queues work and carries live `exec`, followed logs, `ps` and `inspect` sessions, with long-poll and periodic-poll fallbacks for ordinary work.
+- Control API with bearer-token auth, scoped `read`/`deploy`/`admin` API tokens for CI and teammates, an audit trail naming who acted, and SQLite metadata through Node's built-in driver.
+- Nodes join over SSH (`node-install`) or enroll themselves with a single `docker run` (`node-enroll`), with one agent token per node. Agents only dial out, so nodes behind NAT work.
+- Agents pull desired state from control, reconcile containers through the Docker Engine API, run TCP/HTTP/Docker health checks and report status. An outbound WebSocket wakes them as soon as control queues work and carries live `exec`, followed logs, `ps` and `inspect` sessions, with long-poll and periodic-poll fallbacks for ordinary work.
 - `x-tug` placement of services and volumes on nodes, with immutable desired-state records per deployment.
-- Build application images on control from uploaded source, or locally with `deploy --build-local`. Agents download saved images directly from control by default; a bring-your-own registry is optional.
+- Build application images on control from uploaded source, or locally with `deploy --build-local`. Agents download saved images directly from control. Published images are pinned to registry digests at deployment creation.
 - Secrets encrypted at rest, released only to the nodes whose services reference them, with targeted service recreation when only a secret changes.
 - Deployment convergence, per-project operation locks, cancellation, rollback, per-service rollout events and log fetching through the agents.
 - Opt-in zero-downtime rolling rollouts for ingress-fronted services (`x-tug.rollout`) and automatic rollback of failed deployments (`x-tug.rollback: auto`). See [Compose extensions](docs/compose.md#rolling-rollouts).
-- Per-node registry pull tokens enforced by an htpasswd-backed registry you provide, with rotation and revocation.
+- Private published images use encrypted project credentials for digest resolution and Engine API pulls.
 - Optional WireGuard mesh between nodes, health-aware cross-node service names and a managed Caddy ingress with automatic HTTPS, each opt-in. See [Networking](docs/networking.md).
 - Private-by-default published ports: generated Compose binds them to the overlay IP, loopback or the node address, never every interface unless you write `0.0.0.0`. See [Networking](docs/networking.md#published-ports-private-by-default).
 - Control-plane backup and restore, and `tug doctor` diagnostics.
@@ -31,7 +31,6 @@ Deploy Docker Compose projects to one or more ordinary Linux VPSes through a sma
 |   Control API (role-checked)        |
 |   SQLite: desired states, secrets   |
 |   image builds, saved images        |
-|   registry (optional)               |
 +-------------------------------------+
        ^                       ^
        |  agents dial out: work, status,
@@ -72,7 +71,7 @@ The CLI installs the matching `ghcr.io/wpelevator/tug:<version>` image on your h
 
 - [Architecture](docs/architecture.md): components, reconciliation, workflow diagrams, secrets and runtime paths.
 - [Compose extensions](docs/compose.md): `x-tug` placement, health checks, volumes and limitations.
-- [Control plane](docs/control-plane.md): install, TLS, registry credentials, backup and restore.
+- [Control plane](docs/control-plane.md): install, TLS, image delivery, backup and restore.
 - [Backups](docs/backups.md): interim WordPress backup sidecar with restic, per-node and NFS variants, and a restore drill until `x-tug.backup` (M11).
 - [Access control](docs/access-control.md): scoped API tokens, roles, audit trail and a GitHub Actions deploy example.
 - [Networking](docs/networking.md): WireGuard mesh, service DNS, ingress and TLS, firewall notes.
@@ -80,7 +79,7 @@ The CLI installs the matching `ghcr.io/wpelevator/tug:<version>` image on your h
 - [CLI reference](docs/cli.md): every `tug` command and option.
 - [Configuration](docs/configuration.md): CLI, control and agent environment variables.
 - [Development](docs/development.md): local setup, CI, Compose stack, host mode and the integration test.
-- [Testing](docs/testing.md): test layers, the numbers rollouts are held to, and the backlog of mocked tests.
+- [Testing](docs/testing.md): test layers, rollout metrics, mocked test backlog, and the [Caddy/PHP and static-file benchmark](docs/testing.md#caddy-proxy-overhead-benchmark).
 - [Observability](docs/observability.md): what can be observed today, and the proposed logs, metrics and alerting.
 - [Releases](docs/releases.md): changesets, npm and Docker image publishing.
 - [Standalone binary](docs/standalone-binary.md): proposed Bun-compiled CLI binaries and Homebrew distribution.

package/dist/agent.d.ts

@@ -93,8 +93,6 @@ interface Creds {
     agentToken: string;
     nodeName?: string;
     controlUrl?: string;
-    registry?: string | null;
-    registryToken?: string | null;
 }
 export declare function loadOrEnroll(): Promise<Creds>;
 export declare function main(): Promise<void>;

package/dist/agent.js

@@ -1,11 +1,12 @@
 #!/usr/bin/env node
 import fs from 'node:fs/promises';
+import { imageLocation } from "./registry.js";
 import { createReadStream, createWriteStream } from 'node:fs';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { validateImageArchive, verifyDownload } from "./artifacts.js";
 import path from 'node:path';
-import { spawn, spawnSync } from 'node:child_process';
+import { spawnSync } from 'node:child_process';
 import net from 'node:net';
 import dns from 'node:dns/promises';
 import { WebSocket } from 'ws';
@@ -837,7 +838,7 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
     const sec = await api(ctx, `/agent/v1/secrets?project=${encodeURIComponent(desiredState.project)}&services=${encodeURIComponent(serviceNames.join(','))}`);
     const currentSecretVersions = serviceSecretVersions(desiredState.compose, serviceNames, sec.versions || {});
     const db = await agentDb(ctx.stateDir);
-    const previousRow = await db.get('SELECT version, revision FROM applied_state WHERE project = ?', desiredState.project);
+    const previousRow = db.get('SELECT version, revision FROM applied_state WHERE project = ?', desiredState.project);
     const rolling = serviceNames.filter((service) => isRolling(desiredState.compose.services[service]));
     const rollingSet = new Set(rolling);
     const mainServices = serviceNames.filter((service) => !rollingSet.has(service));
@@ -882,7 +883,7 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
         .digest('hex')
         .slice(0, 16);
     const knownEvents = mainServices.length
-        ? await db.get('SELECT fingerprint, phase FROM event_state WHERE project = ?', desiredState.project)
+        ? db.get('SELECT fingerprint, phase FROM event_state WHERE project = ?', desiredState.project)
         : undefined;
     const announce = knownEvents?.fingerprint !== eventFingerprint;
     const wasFailed = !announce && knownEvents?.phase === 'failed';
@@ -894,7 +895,7 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
             if (artifact)
                 await ensureArtifact(ctx, artifact);
         },
-        pullImage: (service) => ctx.docker.pullImage(plan.specs[service].body.Image),
+        pullImage: (service) => ctx.docker.pullImage(plan.specs[service].body.Image, sec.registryAuth?.[imageLocation(plan.specs[service].body.Image).host]),
         emit,
     };
     let failing;
@@ -938,13 +939,13 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
                     },
                 });
             if (mainServices.length && (announce || wasFailed))
-                await markEvents('serving');
+                markEvents('serving');
         }
         catch (e) {
             if (mainServices.length && !wasFailed) {
                 // Announce a failure once; retries of the same failure stay quiet until it recovers or the inputs change.
                 await emit(failing ?? mainServices.join(', '), 'failed', errMessage(e), 'error');
-                await markEvents('failed');
+                markEvents('failed');
             }
             // Rolling services are untouched by a failure here, so keep reporting their current revision as serving.
             if (!rolling.length)
@@ -974,7 +975,7 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
         for (const service of serviceNames)
             serviceStates[service].rolloutState = 'serving';
     }
-    await db.run(`INSERT INTO applied_state(project, version, revision, applied_at, service_states_json, secret_versions_json)
+    db.run(`INSERT INTO applied_state(project, version, revision, applied_at, service_states_json, secret_versions_json)
      VALUES(?,?,?,?,?,?)
      ON CONFLICT(project) DO UPDATE SET version=excluded.version, revision=excluded.revision, applied_at=excluded.applied_at,
        service_states_json=excluded.service_states_json, secret_versions_json=excluded.secret_versions_json`, desiredState.project, desiredState.version, desiredState.revision || null, now(), JSON.stringify(serviceStates), JSON.stringify(currentSecretVersions));
@@ -1122,19 +1123,6 @@ async function readJoinKey() {
     }
     return null;
 }
-function dockerLogin(registry, user, password) {
-    return new Promise((resolve, reject) => {
-        const p = spawn('docker', ['login', registry, '-u', user, '--password-stdin'], {
-            stdio: ['pipe', 'ignore', 'pipe'],
-        });
-        let err = '';
-        p.stderr.on('data', (d) => (err += d));
-        p.on('error', reject);
-        p.on('close', (code) => code === 0 ? resolve() : reject(new Error(err.trim() || `exit ${code}`)));
-        p.stdin.write(password);
-        p.stdin.end();
-    });
-}
 async function enroll(joinKey) {
     let decoded;
     try {
@@ -1184,8 +1172,6 @@ async function enroll(joinKey) {
         agentToken: out.agentToken,
         nodeName: out.node?.name || NODE_NAME,
         controlUrl,
-        registry: out.registry || null,
-        registryToken: out.registryToken || null,
     };
     await saveCreds(creds);
     if (wgPublicKey)
@@ -1209,14 +1195,6 @@ export async function loadOrEnroll() {
     NODE_NAME = creds.nodeName || NODE_NAME;
     if (!CONTROL_URL_FROM_ENV && creds.controlUrl)
         CONTROL_URL = creds.controlUrl;
-    if (APPLY && creds.registry && creds.registryToken) {
-        try {
-            await dockerLogin(creds.registry, NODE_NAME, creds.registryToken);
-        }
-        catch (e) {
-            console.error(`registry login failed (pulls from ${creds.registry} may fail): ${errMessage(e)}`);
-        }
-    }
     return creds;
 }
 export async function main() {

package/dist/build.d.ts

@@ -12,11 +12,10 @@ export interface BuildSpec {
 }
 export declare function run(cmd: string, args: string[], opts?: RunOptions): Promise<RunResult>;
 export declare function unpackContext(tarball: string, dest: string): Promise<void>;
-export interface BuildAndPushOptions {
+export interface BuildImageOptions {
     contextDir: string;
     service: string;
     buildSpec: string | BuildSpec;
     imageRef: string;
-    push?: boolean;
 }
-export declare function buildAndPush({ contextDir, service, buildSpec, imageRef, push, }: BuildAndPushOptions): Promise<void>;
+export declare function buildImage({ contextDir, service, buildSpec, imageRef, }: BuildImageOptions): Promise<void>;

package/dist/build.js

@@ -33,7 +33,7 @@ export async function unpackContext(tarball, dest) {
     });
     await tar.x({ file: tarball, cwd: dest, strict: true });
 }
-export async function buildAndPush({ contextDir, service, buildSpec, imageRef, push = true, }) {
+export async function buildImage({ contextDir, service, buildSpec, imageRef, }) {
     const buildPath = typeof buildSpec === 'string' ? buildSpec : buildSpec?.context || '.';
     const dockerfilePath = typeof buildSpec === 'object' && buildSpec.dockerfile
         ? path.resolve(contextDir, buildPath, buildSpec.dockerfile)
@@ -48,6 +48,4 @@ export async function buildAndPush({ contextDir, service, buildSpec, imageRef, p
     if (!(await exists(fullContext)))
         throw new Error(`build context not found for ${service}: ${buildPath}`);
     await run('docker', ['build', ...dockerfile, '-t', imageRef, fullContext]);
-    if (push)
-        await run('docker', ['push', imageRef]);
 }

package/dist/cli-build.d.ts

@@ -1,8 +1,8 @@
-import { buildAndPush } from './build.ts';
+import { buildImage } from './build.ts';
 import { saveImageArchive } from './artifacts.ts';
 import type { ComposeFile, LocalImageUpload } from './protocol.ts';
 export interface LocalBuildDependencies {
-    build: typeof buildAndPush;
+    build: typeof buildImage;
     save: typeof saveImageArchive;
     nonce: () => string;
     remove: (imageRef: string) => Promise<void>;

package/dist/cli-build.js

@@ -2,7 +2,7 @@ import path from 'node:path';
 import fs from 'node:fs/promises';
 import { randomUUID } from 'node:crypto';
 import * as tar from 'tar';
-import { buildAndPush, run } from "./build.js";
+import { buildImage, run } from "./build.js";
 import { saveImageArchive } from "./artifacts.js";
 /** Credential exclusions are mandatory and cannot be re-included by source ignore rules. */
 export function sourceArchiveFilter(entry, configPaths, ignores) {
@@ -18,7 +18,7 @@ export async function createSourceArchive(contextDir, archive, configFiles) {
     const configPaths = configFiles
         .map((file) => path.relative(contextDir, file).split(path.sep).join('/'))
         .filter((file) => file !== '..' && !file.startsWith('../') && !path.isAbsolute(file));
-    const ignores = ['node_modules', 'data', 'registry-data'];
+    const ignores = ['node_modules', 'data'];
     try {
         ignores.push(...(await fs.readFile(path.join(contextDir, '.dockerignore'), 'utf8'))
             .split(/\r?\n/)
@@ -38,7 +38,7 @@ export async function createSourceArchive(contextDir, archive, configFiles) {
 }
 /** Build only application build services; callers own temporary files and upload transport. */
 export async function prepareLocalImages(compose, contextDir, directory, dependencies = {
-    build: buildAndPush,
+    build: buildImage,
     save: saveImageArchive,
     nonce: randomUUID,
     remove: async (imageRef) => {
@@ -58,7 +58,6 @@ export async function prepareLocalImages(compose, contextDir, directory, depende
                 service,
                 buildSpec: spec.build,
                 imageRef,
-                push: false,
             });
             const archive = await dependencies.save(imageRef, path.join(directory, `${field}.tar.gz`));
             images.push({ service, imageRef, ...archive, field });

package/dist/cli.js

@@ -201,12 +201,6 @@ program
     });
     console.log(JSON.stringify(r, null, 2));
 });
-program
-    .command('node-rotate-registry-token <name>')
-    .description('issue a new registry pull token for a node, invalidating the old one')
-    .action(async (name) => {
-    console.log(JSON.stringify(await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/rotate-registry-token`, {}), null, 2));
-});
 program
     .command('node-rotate-wg-key <name>')
     .description('generate a new WireGuard keypair on the node; peers pick up the new public key on their next poll')
@@ -234,12 +228,6 @@ program
         enabled: !o.disable,
     }), null, 2));
 });
-program
-    .command('node-revoke-registry-token <name>')
-    .description("revoke a node's registry pull token, blocking future pulls until rotated")
-    .action(async (name) => {
-    console.log(JSON.stringify(await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/revoke-registry-token`, {}), null, 2));
-});
 program
     .command('scale <service> <replicas>')
     .description('scale a service to N replicas across matching nodes')
@@ -263,9 +251,6 @@ program
     .option('--token <token>', 'operator token')
     .option('--port <port>', 'published API port', '8080')
     .option('--mesh', 'enable the WireGuard mesh on control (TUG_MESH=true); install nodes with --mesh too')
-    .option('--registry <host>', 'registry host used in desired images; bring your own registry, Tug does not provision one')
-    .option('--registry-auth-path <path>', 'container path to an htpasswd file tug-control should manage credentials in, if your registry uses one')
-    .option('--registry-container <name>', "docker container name to restart when a node's registry credentials change, if your registry uses an htpasswd file")
     .action(async (sshTarget, o) => {
     const operatorToken = o.token || process.env.TUG_TOKEN || token('tug_op');
     if (o.installDocker)
@@ -274,16 +259,12 @@ program
     const image = resolveImage(o.image, o.buildLocalImage);
     if (o.buildLocalImage)
         await transferTugImage(sshTarget, image);
-    const registryHost = o.registry || '';
     const yamlValue = (value) => JSON.stringify(value);
     const compose = await loadTemplate(templatesDir, 'control-compose.yml', {
         IMAGE: yamlValue(image),
         OPERATOR_TOKEN: yamlValue(operatorToken),
-        REGISTRY_HOST: yamlValue(registryHost),
         PORT: o.port,
         MESH: yamlValue(o.mesh ? 'true' : 'false'),
-        REGISTRY_AUTH_PATH: yamlValue(o.registryAuthPath || ''),
-        REGISTRY_CONTAINER: yamlValue(o.registryContainer || ''),
     });
     const remote = await loadTemplate(templatesDir, 'control-install.sh', {
         COMPOSE: compose,
@@ -295,7 +276,6 @@ program
         sshTarget,
         operatorToken,
         url: `http://${remoteHost(sshTarget)}:${o.port}`,
-        registry: registryHost,
     }, null, 2));
 });
 program
@@ -306,7 +286,6 @@ program
     .option('--public-address <addr>')
     .option('--label <label>', 'node label (repeatable)', collect, [])
     .option('--agent-control-url <url>', 'control URL reachable by the agent (default: resolved operator control URL)')
-    .option('--registry <host>', 'registry host for docker login')
     .option('--http-only-dev', 'allow plaintext HTTP control URL for local/private development')
     .option('--mesh', 'join the WireGuard mesh (agent runs with host networking and NET_ADMIN; control needs TUG_MESH=true)')
     .option('--wg-port <port>', 'WireGuard UDP port checked by preflight', '51820')
@@ -338,17 +317,13 @@ program
         NODE_NAME: o.name,
         AGENT_TOKEN: registered.agentToken,
     })) + (o.mesh ? 'TUG_MESH=true\n' : '');
-    const dockerLogin = o.registry
-        ? `printf %s ${sh(registered.registryToken)} | docker login ${sh(o.registry)} --username ${sh(o.name)} --password-stdin`
-        : 'true';
     const remote = await loadTemplate(templatesDir, 'node-install.sh', {
         ENV: env,
-        DOCKER_LOGIN: dockerLogin,
         IMAGE_SH: sh(image),
         RUN_FLAGS: `${o.mesh ? MESH_RUN_FLAGS : HOST_NETWORK_FLAG} `,
     });
     runLocal('ssh', ['--', sshTarget, remote]);
-    console.log(JSON.stringify({ ok: true, node: registered.node, registryToken: registered.registryToken }, null, 2));
+    console.log(JSON.stringify({ ok: true, node: registered.node }, null, 2));
 });
 program
     .command('node-enroll')
@@ -463,7 +438,6 @@ program
     .option('-f, --file <file>', 'compose file', 'docker-compose.yml')
     .option('--version <version>', 'display version (default: git-<Compose hash>)')
     .option('--build-local', 'build application images locally and upload them without source')
-    .option('--no-build-upload', 'deploy published images without uploading source; conflicts with --build-local')
     .option('--no-wait', 'return after desired state is created')
     .option('--timeout <seconds>', 'wait timeout in seconds', '300')
     .action(async (o) => {
@@ -488,8 +462,7 @@ program
             await fs.rm(directory, { recursive: true, force: true });
         }
     }
-    else if (o.buildUpload &&
-        Object.values(compose.services).some((service) => service.build)) {
+    else if (Object.values(compose.services).some((service) => service.build)) {
         const tarball = await makeContext();
         try {
             const form = new FormData();
@@ -705,10 +678,6 @@ program.hook('preAction', async (_root, command) => {
         if (options[key] !== undefined)
             integerOption(String(options[key]), '--' + key.replace(/[A-Z]/g, (c) => '-' + c.toLowerCase()), min, max);
     }
-    if (options.buildLocal && options.buildUpload === false)
-        throw new Error('--build-local and --no-build-upload cannot be combined');
-    if ((options.registryAuthPath || options.registryContainer) && !options.registry)
-        throw new Error('--registry-auth-path and --registry-container require --registry');
     if (command.name() === 'control-restore' && !options.confirm)
         throw new Error('control-restore requires --confirm');
     const positional = command.name() === 'exec' ? command.args.slice(0, 1) : command.args;

package/dist/compose.d.ts

@@ -39,7 +39,6 @@ export interface BuildDesiredStateOptions {
     project: string;
     version: string;
     compose: ComposeFile;
-    registry: string;
     revision?: string;
     defaultNode?: string | null;
     nodes?: Array<{
@@ -47,7 +46,7 @@ export interface BuildDesiredStateOptions {
         labels?: string[];
     }>;
 }
-export declare function buildDesiredState({ project, version, compose, registry, revision, defaultNode, nodes, }: BuildDesiredStateOptions): DesiredState;
+export declare function buildDesiredState({ project, version, compose, revision, defaultNode, nodes, }: BuildDesiredStateOptions): DesiredState;
 /** `extra_hosts` entries (`name:ip`) for services that run on other nodes, from health-aware endpoints. */
 export declare function crossNodeHosts(project: string, endpoints: Record<string, ServiceEndpoint[]>, local: Set<string>): string[];
 /**

package/dist/compose.js

@@ -305,7 +305,7 @@ export function validateCompose(compose, nodes = []) {
     if (errors.length)
         throw new Error(errors.join('; '));
 }
-export function buildDesiredState({ project, version, compose, registry, revision, defaultNode = null, nodes, }) {
+export function buildDesiredState({ project, version, compose, revision, defaultNode = null, nodes, }) {
     const services = {};
     for (const [name, src] of Object.entries(compose.services || {})) {
         const lp = src['x-tug'] || {};
@@ -323,9 +323,7 @@ export function buildDesiredState({ project, version, compose, registry, revisio
             }
             placement = { ...placement, nodes: selected.map((node) => node.name) };
         }
-        const image = src.build
-            ? `${registry ? `${registry}/` : 'tug-build/'}${project}/${name}:${revision || version}`
-            : src.image;
+        const image = src.build ? `tug-build/${project}/${name}:${revision || version}` : src.image;
         services[name] = {
             image,
             build: src.build || null,

package/dist/control.d.ts

@@ -1,18 +1,20 @@
 #!/usr/bin/env node
 import express from 'express';
+import { resolveImage } from './registry.ts';
 import { saveImageArchive, inspectImage, validateImageArchive } from './artifacts.ts';
 import https from 'node:https';
 import http from 'node:http';
 import { openDb } from './db.ts';
 import type { NodeRow } from './db.ts';
-import { buildAndPush, run as runCommand } from './build.ts';
+import { buildImage, run as runCommand } from './build.ts';
 declare module 'express-serve-static-core' {
     interface Request {
         node?: NodeRow;
     }
 }
 export interface ControlImageOperations {
-    build: typeof buildAndPush;
+    resolve: typeof resolveImage;
+    build: typeof buildImage;
     save: typeof saveImageArchive;
     inspect: typeof inspectImage;
     validate: typeof validateImageArchive;