@wpelevator/tug 0.3.0 → 0.4.0

package/dist/cli.js

@@ -10,7 +10,8 @@ import fs from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { readCompose, composeHash } from "./compose.js";
+import { composeHash } from "./compose.js";
+import { normalizeCompose } from "./normalize.js";
 import { errMessage, token, loadTemplate } from "./util.js";
 import { PROJECT_NAME_PATTERN, ROLES, TOKEN_NAME_PATTERN } from "./tokens.js";
 import { packageVersion, resolveImage } from "./image.js";
@@ -74,14 +75,22 @@ function assertHttps(url, allowHttp) {
     if (!allowHttp && !String(url).startsWith('https://'))
         throw new Error(`${url} must use https:// outside local development (pass --http-only-dev to override)`);
 }
-function preflightRemote(target, { controlUrl, ports = [], udpPorts = [], minDiskMb = 1024, mesh = false, } = {}) {
+function preflightRemote(target, { controlUrl, ports = [], udpPorts = [], minDiskMb = 1024, mesh = false, compose = false, } = {}) {
     const checks = [
         [
             'docker.version',
             'docker version >/dev/null',
             'Install Docker Engine and ensure the SSH user can access it.',
         ],
-        ['docker.compose.version', 'docker compose version >/dev/null', 'Install Docker Compose v2.'],
+        ...(compose
+            ? [
+                [
+                    'docker.compose.version',
+                    'docker compose version >/dev/null',
+                    'Install Docker Compose v2.',
+                ],
+            ]
+            : []),
         [
             'paths',
             'mkdir -p /opt/tug-preflight /var/lib/tug-preflight /etc/tug && test -w /opt/tug-preflight && test -w /var/lib/tug-preflight && test -w /etc/tug',
@@ -147,8 +156,13 @@ async function transferTugImage(target, imageTag) {
 async function dockerBootstrap() {
     return fs.readFile(path.join(templatesDir, 'install-docker.sh'), 'utf8');
 }
-async function bootstrapDocker(target) {
-    runLocal('ssh', ['--', target, `sh -eu -c ${sh(await dockerBootstrap())}`]);
+/** Installs Docker over SSH; `compose` also installs the Compose plugin, which only the control host needs. */
+async function bootstrapDocker(target, compose = false) {
+    runLocal('ssh', [
+        '--',
+        target,
+        `${compose ? 'TUG_INSTALL_COMPOSE=1 ' : ''}sh -eu -c ${sh(await dockerBootstrap())}`,
+    ]);
 }
 async function waitForDeployment(deploymentId, timeoutSeconds = 300) {
     const deadline = Date.now() + timeoutSeconds * 1000;
@@ -255,8 +269,8 @@ program
     .action(async (sshTarget, o) => {
     const operatorToken = o.token || process.env.TUG_TOKEN || token('tug_op');
     if (o.installDocker)
-        await bootstrapDocker(sshTarget);
-    preflightRemote(sshTarget, { ports: [Number(o.port)], minDiskMb: 2048 });
+        await bootstrapDocker(sshTarget, true);
+    preflightRemote(sshTarget, { ports: [Number(o.port)], minDiskMb: 2048, compose: true });
     const image = resolveImage(o.image, o.buildLocalImage);
     if (o.buildLocalImage)
         await transferTugImage(sshTarget, image);
@@ -454,8 +468,10 @@ program
     .option('--timeout <seconds>', 'wait timeout in seconds', '300')
     .action(async (o) => {
     const cfg = await loadCfg();
-    const compose = await readCompose(o.file);
     const project = o.project || cfg.project || 'site';
+    // Control secrets are interpolated as sentinels; the agent substitutes the node's released values.
+    const { secrets } = await request('GET', `/api/v1/secrets?project=${encodeURIComponent(project)}`);
+    const compose = await normalizeCompose(o.file, secrets.map((secret) => secret.name), { project });
     const version = o.version || `git-${composeHash(compose)}`;
     let response;
     if (o.buildLocal) {

package/dist/compose.d.ts

@@ -1,5 +1,4 @@
 import type { ComposeFile, ComposeService, DesiredState, ServiceEndpoint } from './protocol.ts';
-export declare function readCompose(file?: string): Promise<ComposeFile>;
 export declare function composeHash(compose: ComposeFile): string;
 type PortEntry = NonNullable<ComposeService['ports']>[number];
 /**
@@ -33,6 +32,8 @@ export interface PortBind {
 export declare function addressBoundServices(compose: ComposeFile, names: Iterable<string>, loopback?: ReadonlySet<string>): string[];
 /** True when the service opted in to `x-tug.rollout.strategy: rolling`. */
 export declare function isRolling(svc: ComposeService | undefined): boolean;
+/** Service keys the agent can create a container from; anything else is rejected with its name. */
+export declare const SUPPORTED_SERVICE_KEYS: readonly string[];
 export declare function validateCompose(compose: ComposeFile | undefined, nodes?: string[]): asserts compose is ComposeFile;
 export interface BuildDesiredStateOptions {
     project: string;
@@ -49,24 +50,10 @@ export interface BuildDesiredStateOptions {
 export declare function buildDesiredState({ project, version, compose, registry, revision, defaultNode, nodes, }: BuildDesiredStateOptions): DesiredState;
 /** `extra_hosts` entries (`name:ip`) for services that run on other nodes, from health-aware endpoints. */
 export declare function crossNodeHosts(project: string, endpoints: Record<string, ServiceEndpoint[]>, local: Set<string>): string[];
-/** Removes `depends_on` references to `names` (services that live in another Compose project). */
-export declare function dropDependsOn(svc: ComposeService, names: ReadonlySet<string>): void;
-export interface NodeComposeOptions {
-    secrets?: Record<string, string>;
-    endpoints?: Record<string, ServiceEndpoint[]>;
-    /**
-     * Services placed here that are managed outside the node's main project (rolling services run in
-     * their own revision projects); they still count as local, so they get no cross-node `extra_hosts`
-     * entry, and `depends_on` references to them are dropped.
-     */
-    exclude?: ReadonlySet<string>;
-    nodeAddresses?: Record<string, string>;
-    /** Host IP for published ports that name none; without it `ports` stay as written. */
-    bind?: PortBind;
-}
-/** The Compose file for one node. */
-export declare function composeForNode(desiredState: DesiredState, nodeName: string, { secrets, endpoints, exclude, nodeAddresses, bind, }?: NodeComposeOptions): ComposeFile;
+/**
+ * Control secrets a service uses: the sentinels in its definition. Their versions decide whether the
+ * service is recreated, so a secret change touches exactly the services that reference it.
+ */
 export declare function serviceSecretNames(compose: ComposeFile | undefined, serviceNames: Iterable<string>): Set<string>;
 export declare function serviceSecretVersions(compose: ComposeFile | undefined, serviceNames: Iterable<string>, versions?: Record<string, number>): Record<string, Record<string, number>>;
-export declare function dumpYaml(obj: unknown): string;
 export {};

package/dist/compose.js

@@ -1,10 +1,6 @@
-import fs from 'node:fs/promises';
-import yaml from 'js-yaml';
 import crypto from 'node:crypto';
 import { isDuration } from "./util.js";
-export async function readCompose(file = 'docker-compose.yml') {
-    return yaml.load(await fs.readFile(file, 'utf8'));
-}
+import { sentinelNames } from "./normalize.js";
 export function composeHash(compose) {
     return crypto.createHash('sha1').update(JSON.stringify(compose)).digest('hex').slice(0, 12);
 }
@@ -175,16 +171,119 @@ function validateProjectExtension(compose, errors) {
     if (ext.rollback !== undefined && !['auto', 'manual'].includes(ext.rollback))
         errors.push('x-tug.rollback must be auto or manual');
 }
+/** Service keys the agent can create a container from; anything else is rejected with its name. */
+export const SUPPORTED_SERVICE_KEYS = [
+    'image',
+    'build',
+    'command',
+    'entrypoint',
+    'environment',
+    'ports',
+    'expose',
+    'volumes',
+    'tmpfs',
+    'healthcheck',
+    'restart',
+    'labels',
+    'depends_on',
+    'extra_hosts',
+    'user',
+    'working_dir',
+    'hostname',
+    'init',
+    'read_only',
+    'shm_size',
+    'mem_limit',
+    'mem_reservation',
+    'cpus',
+    'pids_limit',
+    'cap_add',
+    'cap_drop',
+    'ulimits',
+    'logging',
+    'stop_grace_period',
+    'stop_signal',
+    'x-tug',
+];
+/** Keys Compose adds or that Tug ignores on purpose: the default network, and `pull_policy` because Tug decides when images change. */
+const IGNORED_SERVICE_KEYS = ['networks', 'pull_policy'];
+const SUPPORTED_TOP_LEVEL = ['services', 'volumes', 'networks', 'name', 'x-tug'];
+const SUPPORTED_VOLUME_KEYS = ['name', 'driver', 'driver_opts', 'labels', 'external', 'x-tug'];
+const isObject = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
+const isExtension = (key) => key.startsWith('x-');
+function validateSubset(name, svc, compose, errors) {
+    for (const key of Object.keys(svc)) {
+        if (SUPPORTED_SERVICE_KEYS.includes(key) || IGNORED_SERVICE_KEYS.includes(key))
+            continue;
+        if (isExtension(key))
+            continue;
+        errors.push(key === 'deploy'
+            ? `${name}: deploy (swarm) is not supported; use x-tug placement`
+            : `${name}: ${key} is not supported`);
+    }
+    if (svc.networks !== undefined && svc.networks !== null) {
+        const nets = svc.networks;
+        const names = Array.isArray(nets) ? nets.map(String) : isObject(nets) ? Object.keys(nets) : [];
+        if (names.some((n) => n !== 'default'))
+            errors.push(`${name}: networks other than the default network are not supported`);
+    }
+    if (!svc.image && !svc.build)
+        errors.push(`${name}: image or build is required`);
+    for (const mount of svc.volumes || []) {
+        if (typeof mount === 'string') {
+            errors.push(`${name}: volumes must be normalized by docker compose config (got "${mount}")`);
+            continue;
+        }
+        const type = String(mount.type ?? '');
+        if (type === 'bind') {
+            if (!String(mount.source ?? '').startsWith('/'))
+                errors.push(`${name}: bind mount source ${JSON.stringify(mount.source)} must be an absolute path on the node`);
+        }
+        else if (type === 'volume') {
+            const source = String(mount.source ?? '');
+            if (!source)
+                errors.push(`${name}: anonymous volumes are not supported; name the volume`);
+            else if (!compose.volumes?.[source])
+                errors.push(`${name}: volume ${source} is not defined in top-level volumes`);
+        }
+        else if (type !== 'tmpfs')
+            errors.push(`${name}: volume type ${JSON.stringify(type)} is not supported`);
+    }
+    const deps = svc.depends_on;
+    if (isObject(deps))
+        for (const [dep, spec] of Object.entries(deps)) {
+            const condition = isObject(spec) ? String(spec.condition ?? 'service_started') : '';
+            if (!compose.services[dep])
+                errors.push(`${name}: depends_on ${dep} is not a service`);
+            else if (!['service_started', 'service_healthy'].includes(condition))
+                errors.push(`${name}: depends_on condition ${JSON.stringify(condition)} is not supported`);
+        }
+}
+function validateTopLevel(compose, errors) {
+    for (const key of Object.keys(compose))
+        if (!SUPPORTED_TOP_LEVEL.includes(key) && !isExtension(key))
+            errors.push(`${key} is not supported at the top level of the Compose file`);
+    const networks = compose.networks;
+    if (isObject(networks)) {
+        for (const [key, def] of Object.entries(networks))
+            if (key !== 'default' || (isObject(def) && def.external))
+                errors.push(`networks.${key}: custom networks are not supported`);
+    }
+    for (const [key, def] of Object.entries(compose.volumes || {})) {
+        if (!isObject(def))
+            continue;
+        for (const field of Object.keys(def))
+            if (!SUPPORTED_VOLUME_KEYS.includes(field) && !isExtension(field))
+                errors.push(`volumes.${key}: ${field} is not supported`);
+    }
+}
 export function validateCompose(compose, nodes = []) {
     if (!compose?.services || typeof compose.services !== 'object')
         throw new Error('compose file must define services');
     const errors = [];
     const ingressHosts = new Map();
     for (const [name, svc] of Object.entries(compose.services)) {
-        if (svc.container_name)
-            errors.push(`${name}: container_name is not supported`);
-        if (svc.deploy)
-            errors.push(`${name}: swarm deploy semantics are not supported`);
+        validateSubset(name, svc, compose, errors);
         const lp = svc['x-tug'] || {};
         const placement = lp.placement || {};
         const selected = placement.nodes || [];
@@ -201,6 +300,7 @@ export function validateCompose(compose, nodes = []) {
         validateIngress(name, svc, ingressHosts, errors);
         validateRollout(name, svc, errors);
     }
+    validateTopLevel(compose, errors);
     validateProjectExtension(compose, errors);
     if (errors.length)
         throw new Error(errors.join('; '));
@@ -247,141 +347,16 @@ export function crossNodeHosts(project, endpoints, local) {
     }
     return hosts;
 }
-/** Removes `depends_on` references to `names` (services that live in another Compose project). */
-export function dropDependsOn(svc, names) {
-    const deps = svc.depends_on;
-    if (Array.isArray(deps)) {
-        const kept = deps.filter((d) => !names.has(String(d)));
-        if (kept.length)
-            svc.depends_on = kept;
-        else
-            delete svc.depends_on;
-    }
-    else if (deps && typeof deps === 'object') {
-        const kept = Object.fromEntries(Object.entries(deps).filter(([dep]) => !names.has(dep)));
-        if (Object.keys(kept).length)
-            svc.depends_on = kept;
-        else
-            delete svc.depends_on;
-    }
-}
-function extraHostsList(value) {
-    if (!value)
-        return [];
-    return Array.isArray(value)
-        ? value.map(String)
-        : Object.entries(value).map(([k, v]) => `${k}:${v}`);
-}
-/** The Compose file for one node. */
-export function composeForNode(desiredState, nodeName, { secrets = {}, endpoints = {}, exclude = new Set(), nodeAddresses = {}, bind, } = {}) {
-    const out = {
-        services: {},
-        volumes: {},
-    };
-    const outVolumes = out.volumes;
-    const usedVolumes = new Set();
-    const placed = new Set();
-    for (const [name, src] of Object.entries(desiredState.compose.services || {})) {
-        const nodes = desiredState.services[name].placement?.nodes || [];
-        if (nodes.length && !nodes.includes(nodeName))
-            continue;
-        placed.add(name);
-        if (exclude.has(name))
-            continue;
-        const svc = { ...src, image: desiredState.services[name].image };
-        delete svc.build;
-        delete svc['x-tug'];
-        delete svc.deploy;
-        delete svc.container_name;
-        if (desiredState.services[name].artifact)
-            svc.pull_policy = 'never';
-        if (Object.keys(secrets).length)
-            svc.env_file = Array.from(new Set([
-                ...(Array.isArray(svc.env_file) ? svc.env_file : svc.env_file ? [svc.env_file] : []),
-                '.env',
-            ]));
-        for (const vol of svc.volumes || []) {
-            const source = typeof vol === 'string' ? vol.split(':')[0] : vol.source;
-            if (source && desiredState.compose?.volumes?.[source])
-                usedVolumes.add(source);
-        }
-        if (exclude.size)
-            dropDependsOn(svc, exclude);
-        if (bind && svc.ports)
-            svc.ports = bindPorts(svc.ports, bind.address, bind.loopback?.has(name) ? src['x-tug']?.ingress?.port : undefined);
-        out.services[name] = svc;
-    }
-    const hosts = crossNodeHosts(desiredState.project, endpoints, placed);
-    if (hosts.length) {
-        for (const svc of Object.values(out.services)) {
-            const existing = extraHostsList(svc.extra_hosts);
-            const taken = new Set(existing.map((h) => h.split(':')[0]));
-            svc.extra_hosts = [...existing, ...hosts.filter((h) => !taken.has(h.split(':')[0]))];
-        }
-    }
-    for (const name of usedVolumes) {
-        const vol = desiredState.compose.volumes?.[name] || {};
-        const xe = vol['x-tug'] || {};
-        const nodes = xe.placement?.nodes || [];
-        const isNfs = xe.type === 'nfs' || xe.driver === 'nfs' || (xe.type === 'shared' && xe.driver === 'nfs');
-        if (isNfs) {
-            const exporter = nodes[0];
-            const exportPath = xe.export || `/srv/tug/${name}`;
-            if (!exporter) {
-                // unplaced: nothing to export, plain local volume
-                const cleaned = { ...vol };
-                delete cleaned['x-tug'];
-                outVolumes[name] = cleaned;
-            }
-            else if (nodes.includes(nodeName)) {
-                // The exporter mounts the exported directory directly; an NFS server on this node must export it.
-                outVolumes[name] = {
-                    driver: 'local',
-                    driver_opts: { type: 'none', o: 'bind', device: exportPath },
-                };
-            }
-            else {
-                const addr = xe.address || nodeAddresses[exporter] || exporter;
-                outVolumes[name] = {
-                    driver: 'local',
-                    driver_opts: { type: 'nfs', o: `addr=${addr},rw,nolock,soft`, device: `:${exportPath}` },
-                };
-            }
-            continue;
-        }
-        if (!nodes.length || nodes.includes(nodeName))
-            outVolumes[name] = vol;
-    }
-    if (!Object.keys(outVolumes).length)
-        delete out.volumes;
-    return out;
-}
+/**
+ * Control secrets a service uses: the sentinels in its definition. Their versions decide whether the
+ * service is recreated, so a secret change touches exactly the services that reference it.
+ */
 export function serviceSecretNames(compose, serviceNames) {
-    const allowed = new Set();
-    for (const serviceName of serviceNames) {
-        const env = compose?.services?.[serviceName]?.environment || {};
-        if (Array.isArray(env)) {
-            for (const entry of env) {
-                const [key, value = ''] = String(entry).split('=');
-                if (key)
-                    allowed.add(key);
-                for (const match of String(value).matchAll(/\$\{?([A-Z_][A-Z0-9_]*)[^}]*\}?/g))
-                    allowed.add(match[1]);
-            }
-        }
-        else {
-            for (const [key, value] of Object.entries(env)) {
-                if (typeof value === 'string' && value.match(/^\$\{?[A-Z_][A-Z0-9_]*(?::-[^}]*)?\}?$/)) {
-                    const m = value.match(/^\$\{?([A-Z_][A-Z0-9_]*)/);
-                    if (m)
-                        allowed.add(m[1]);
-                }
-                else
-                    allowed.add(key);
-            }
-        }
-    }
-    return allowed;
+    const names = new Set();
+    for (const service of serviceNames)
+        for (const name of sentinelNames(compose?.services?.[service]))
+            names.add(name);
+    return names;
 }
 export function serviceSecretVersions(compose, serviceNames, versions = {}) {
     const out = {};
@@ -391,6 +366,3 @@ export function serviceSecretVersions(compose, serviceNames, versions = {}) {
     }
     return out;
 }
-export function dumpYaml(obj) {
-    return yaml.dump(obj, { noRefs: true, lineWidth: 120 });
-}

package/dist/container-spec.d.ts

@@ -0,0 +1,56 @@
+import type { CreateContainerBody } from './docker-api.ts';
+import type { DesiredState, ServiceEndpoint } from './protocol.ts';
+import type { PortBind } from './compose.ts';
+export declare const LABEL_PROJECT = "tug.project";
+export declare const LABEL_SERVICE = "tug.service";
+export declare const LABEL_REVISION = "tug.revision";
+export declare const LABEL_HASH = "tug.config-hash";
+/** The project network every container of the project joins; service names are its aliases. */
+export declare const networkName: (project: string) => string;
+/** Docker volume name for a Compose volume; `external` volumes keep their own name. */
+export declare const volumeName: (project: string, volume: string) => string;
+export declare const containerName: (project: string, service: string, hash: string) => string;
+export interface VolumeSpec {
+    /** Docker volume name. */
+    name: string;
+    driver: string;
+    opts: Record<string, string>;
+    labels: Record<string, string>;
+    /** Not created by Tug (`external: true`); it must already exist. */
+    external: boolean;
+}
+export interface ContainerSpec {
+    service: string;
+    name: string;
+    /** Identity of the definition: the body, the image identity and the versions of the secrets it uses. */
+    hash: string;
+    /** Create body with secrets substituted. Never log it. */
+    body: CreateContainerBody;
+    /** Services on this node that must be started (and healthy, for `service_healthy`) first. */
+    dependsOn: Record<string, 'service_started' | 'service_healthy'>;
+}
+export interface NodePlan {
+    network: string;
+    volumes: VolumeSpec[];
+    /** Keyed by service, only the services placed on this node. */
+    specs: Record<string, ContainerSpec>;
+}
+export interface ContainerSpecOptions {
+    /** Released secret values by name, substituted for sentinels. */
+    secrets?: Record<string, string>;
+    /** Secret versions by name; they are part of the hash instead of the values. */
+    secretVersions?: Record<string, number>;
+    endpoints?: Record<string, ServiceEndpoint[]>;
+    nodeAddresses?: Record<string, string>;
+    /** Host IP for published ports that name none; without it `ports` stay as written. */
+    bind?: PortBind;
+}
+/** Compose durations (`30s`, `1m30s`, `500ms`) or a number of seconds, in milliseconds. */
+export declare function composeDurationMs(value: unknown): number | undefined;
+/** Compose sizes: bytes as a number, or `64m` / `1g` style strings. */
+export declare function sizeBytes(value: unknown): number | undefined;
+export declare function containerSpecs(desired: DesiredState, node: string, { secrets, secretVersions, endpoints, nodeAddresses, bind, }?: ContainerSpecOptions): NodePlan;
+/** Start order: dependencies first, ties by service name; a cycle is an error. */
+export declare function startOrder(specs: Record<string, ContainerSpec>): string[];
+/** `body` with the host side of its (single) published port replaced; used for rolling revisions. */
+export declare function withHostPort(body: CreateContainerBody, port: number): CreateContainerBody;