@wpelevator/tug 0.3.0 → 0.5.0

package/dist/container-spec.d.ts

@@ -0,0 +1,56 @@
+import type { CreateContainerBody } from './docker-api.ts';
+import type { DesiredState, ServiceEndpoint } from './protocol.ts';
+import type { PortBind } from './compose.ts';
+export declare const LABEL_PROJECT = "tug.project";
+export declare const LABEL_SERVICE = "tug.service";
+export declare const LABEL_REVISION = "tug.revision";
+export declare const LABEL_HASH = "tug.config-hash";
+/** The project network every container of the project joins; service names are its aliases. */
+export declare const networkName: (project: string) => string;
+/** Docker volume name for a Compose volume; `external` volumes keep their own name. */
+export declare const volumeName: (project: string, volume: string) => string;
+export declare const containerName: (project: string, service: string, hash: string) => string;
+export interface VolumeSpec {
+    /** Docker volume name. */
+    name: string;
+    driver: string;
+    opts: Record<string, string>;
+    labels: Record<string, string>;
+    /** Not created by Tug (`external: true`); it must already exist. */
+    external: boolean;
+}
+export interface ContainerSpec {
+    service: string;
+    name: string;
+    /** Identity of the definition: the body, the image identity and the versions of the secrets it uses. */
+    hash: string;
+    /** Create body with secrets substituted. Never log it. */
+    body: CreateContainerBody;
+    /** Services on this node that must be started (and healthy, for `service_healthy`) first. */
+    dependsOn: Record<string, 'service_started' | 'service_healthy'>;
+}
+export interface NodePlan {
+    network: string;
+    volumes: VolumeSpec[];
+    /** Keyed by service, only the services placed on this node. */
+    specs: Record<string, ContainerSpec>;
+}
+export interface ContainerSpecOptions {
+    /** Released secret values by name, substituted for sentinels. */
+    secrets?: Record<string, string>;
+    /** Secret versions by name; they are part of the hash instead of the values. */
+    secretVersions?: Record<string, number>;
+    endpoints?: Record<string, ServiceEndpoint[]>;
+    nodeAddresses?: Record<string, string>;
+    /** Host IP for published ports that name none; without it `ports` stay as written. */
+    bind?: PortBind;
+}
+/** Compose durations (`30s`, `1m30s`, `500ms`) or a number of seconds, in milliseconds. */
+export declare function composeDurationMs(value: unknown): number | undefined;
+/** Compose sizes: bytes as a number, or `64m` / `1g` style strings. */
+export declare function sizeBytes(value: unknown): number | undefined;
+export declare function containerSpecs(desired: DesiredState, node: string, { secrets, secretVersions, endpoints, nodeAddresses, bind, }?: ContainerSpecOptions): NodePlan;
+/** Start order: dependencies first, ties by service name; a cycle is an error. */
+export declare function startOrder(specs: Record<string, ContainerSpec>): string[];
+/** `body` with the host side of its (single) published port replaced; used for rolling revisions. */
+export declare function withHostPort(body: CreateContainerBody, port: number): CreateContainerBody;

package/dist/container-spec.js

@@ -0,0 +1,359 @@
+// Turns the desired state for one node into Engine API container definitions. Pure: no Docker, no
+// filesystem. The config hash decides whether a running container already matches its definition.
+import crypto from 'node:crypto';
+import { bindPorts, crossNodeHosts } from "./compose.js";
+import { sentinelNames, substituteSentinels } from "./normalize.js";
+export const LABEL_PROJECT = 'tug.project';
+export const LABEL_SERVICE = 'tug.service';
+export const LABEL_REVISION = 'tug.revision';
+export const LABEL_HASH = 'tug.config-hash';
+/** The project network every container of the project joins; service names are its aliases. */
+export const networkName = (project) => `tug_${project}`;
+/** Docker volume name for a Compose volume; `external` volumes keep their own name. */
+export const volumeName = (project, volume) => `tug_${project}_${volume}`;
+export const containerName = (project, service, hash) => `tug-${project}-${service}-${hash.slice(0, 8)}`;
+const SECOND_NS = 1_000_000_000;
+const UNITS = {
+    ns: 1e-6,
+    us: 1e-3,
+    µs: 1e-3,
+    ms: 1,
+    s: 1000,
+    m: 60000,
+    h: 3600000,
+};
+/** Compose durations (`30s`, `1m30s`, `500ms`) or a number of seconds, in milliseconds. */
+export function composeDurationMs(value) {
+    if (typeof value === 'number')
+        return value * 1000;
+    if (typeof value !== 'string' || !value)
+        return undefined;
+    let total = 0;
+    let rest = value;
+    while (rest) {
+        const m = rest.match(/^(\d+(?:\.\d+)?)(ns|us|µs|ms|s|m|h)/);
+        if (!m)
+            return undefined;
+        total += Number(m[1]) * UNITS[m[2]];
+        rest = rest.slice(m[0].length);
+    }
+    return total;
+}
+const toNs = (value) => {
+    const ms = composeDurationMs(value);
+    return ms === undefined ? undefined : Math.round(ms * 1e6);
+};
+/** Compose sizes: bytes as a number, or `64m` / `1g` style strings. */
+export function sizeBytes(value) {
+    if (typeof value === 'number')
+        return value;
+    if (typeof value !== 'string')
+        return undefined;
+    const m = value
+        .trim()
+        .toLowerCase()
+        .match(/^(\d+(?:\.\d+)?)\s*([kmgt]?)b?$/);
+    if (!m)
+        return undefined;
+    const power = { '': 0, k: 1, m: 2, g: 3, t: 4 }[m[2]];
+    return Math.round(Number(m[1]) * 1024 ** power);
+}
+const asList = (value) => Array.isArray(value) ? value.map(String) : value == null ? [] : [String(value)];
+function extraHosts(value) {
+    if (!value)
+        return [];
+    return Array.isArray(value)
+        ? value.map((entry) => String(entry).replace('=', ':'))
+        : Object.entries(value).map(([name, ip]) => `${name}:${ip}`);
+}
+function restartPolicy(value) {
+    if (typeof value !== 'string' || !value)
+        return undefined;
+    const [name, max] = value.split(':');
+    return name === 'on-failure' && max
+        ? { Name: name, MaximumRetryCount: Number(max) }
+        : { Name: name === 'no' ? '' : name };
+}
+function healthcheck(value) {
+    if (!value || typeof value !== 'object')
+        return undefined;
+    const hc = value;
+    if (hc.disable === true)
+        return { Test: ['NONE'] };
+    const test = typeof hc.test === 'string' ? ['CMD-SHELL', hc.test] : hc.test;
+    return {
+        ...(test ? { Test: test } : {}),
+        ...(toNs(hc.interval) !== undefined ? { Interval: toNs(hc.interval) } : {}),
+        ...(toNs(hc.timeout) !== undefined ? { Timeout: toNs(hc.timeout) } : {}),
+        ...(toNs(hc.start_period) !== undefined ? { StartPeriod: toNs(hc.start_period) } : {}),
+        ...(toNs(hc.start_interval) !== undefined ? { StartInterval: toNs(hc.start_interval) } : {}),
+        ...(hc.retries !== undefined ? { Retries: Number(hc.retries) } : {}),
+    };
+}
+function portMaps(svc, ports) {
+    const exposed = {};
+    const bindings = {};
+    for (const entry of ports) {
+        const key = `${Number(entry.target)}/${entry.protocol || 'tcp'}`;
+        exposed[key] = {};
+        (bindings[key] ||= []).push({
+            HostIp: entry.host_ip || '',
+            HostPort: entry.published === undefined ? '' : String(entry.published),
+        });
+    }
+    for (const port of asList(svc.expose))
+        exposed[`${port.split('/')[0]}/${port.split('/')[1] || 'tcp'}`] = {};
+    return { exposed, bindings };
+}
+/** Volume definitions this node needs, with NFS volumes mounted from the exporter's directory or address. */
+function volumeSpecs(desired, node, used, nodeAddresses) {
+    const out = [];
+    for (const key of [...used].sort()) {
+        const def = desired.compose.volumes?.[key] || {};
+        const xe = def['x-tug'] || {};
+        const labels = {
+            [LABEL_PROJECT]: desired.project,
+            'tug.volume': key,
+            ...(def.labels || {}),
+        };
+        if (def.external) {
+            out.push({
+                name: String(def.name || key),
+                driver: 'local',
+                opts: {},
+                labels,
+                external: true,
+            });
+            continue;
+        }
+        const name = volumeName(desired.project, key);
+        const nodes = xe.placement?.nodes || [];
+        const isNfs = xe.type === 'nfs' || xe.driver === 'nfs' || (xe.type === 'shared' && xe.driver === 'nfs');
+        const plain = () => ({
+            name,
+            driver: String(def.driver || 'local'),
+            opts: def.driver_opts || {},
+            labels,
+            external: false,
+        });
+        if (isNfs) {
+            const exporter = nodes[0];
+            const exportPath = xe.export || `/srv/tug/${key}`;
+            if (!exporter)
+                out.push(plain());
+            else if (nodes.includes(node))
+                // The exporter mounts the exported directory directly; an NFS server on this node must export it.
+                out.push({
+                    name,
+                    driver: 'local',
+                    opts: { type: 'none', o: 'bind', device: exportPath },
+                    labels,
+                    external: false,
+                });
+            else {
+                const addr = xe.address || nodeAddresses[exporter] || exporter;
+                out.push({
+                    name,
+                    driver: 'local',
+                    opts: { type: 'nfs', o: `addr=${addr},rw,nolock,soft`, device: `:${exportPath}` },
+                    labels,
+                    external: false,
+                });
+            }
+            continue;
+        }
+        if (!nodes.length || nodes.includes(node))
+            out.push(plain());
+    }
+    return out;
+}
+function walkStrings(value, fn) {
+    if (typeof value === 'string')
+        return fn(value);
+    if (Array.isArray(value))
+        return value.map((v) => walkStrings(v, fn));
+    if (value && typeof value === 'object')
+        return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, walkStrings(v, fn)]));
+    return value;
+}
+export function containerSpecs(desired, node, { secrets = {}, secretVersions = {}, endpoints = {}, nodeAddresses = {}, bind, } = {}) {
+    const { project } = desired;
+    const network = networkName(project);
+    const placed = new Set();
+    for (const service of Object.keys(desired.compose.services || {})) {
+        const nodes = desired.services[service].placement?.nodes || [];
+        if (!nodes.length || nodes.includes(node))
+            placed.add(service);
+    }
+    const hosts = crossNodeHosts(project, endpoints, placed);
+    const usedVolumes = new Set();
+    const specs = {};
+    for (const service of [...placed].sort()) {
+        const src = desired.compose.services[service];
+        const state = desired.services[service];
+        const identity = state.artifact?.imageId ?? state.image;
+        const image = state.artifact?.imageRef ?? state.image ?? '';
+        const ports = (bind && src.ports
+            ? bindPorts(src.ports, bind.address, bind.loopback?.has(service) ? src['x-tug']?.ingress?.port : undefined)
+            : src.ports || []);
+        const { exposed, bindings } = portMaps(src, ports);
+        const binds = [];
+        const tmpfs = {};
+        for (const mount of src.volumes || []) {
+            if (typeof mount === 'string')
+                continue;
+            const m = mount;
+            const ro = m.read_only ? ':ro' : '';
+            if (m.type === 'volume') {
+                const key = String(m.source);
+                usedVolumes.add(key);
+                const def = desired.compose.volumes?.[key];
+                binds.push(`${def?.external ? (def.name ?? key) : volumeName(project, key)}:${m.target}${ro}`);
+            }
+            else if (m.type === 'bind')
+                binds.push(`${m.source}:${m.target}${ro}`);
+            else if (m.type === 'tmpfs') {
+                const size = m.tmpfs?.size;
+                tmpfs[String(m.target)] = size ? `size=${size}` : '';
+            }
+        }
+        for (const target of asList(src.tmpfs))
+            tmpfs[target] = tmpfs[target] ?? '';
+        const env = Object.entries(src.environment || {})
+            .filter(([, v]) => v !== null && v !== undefined)
+            .map(([k, v]) => `${k}=${v}`);
+        const memory = sizeBytes(src.mem_limit);
+        const reservation = sizeBytes(src.mem_reservation);
+        const shm = sizeBytes(src.shm_size);
+        const ulimits = Object.entries(src.ulimits || {}).map(([Name, v]) => typeof v === 'number'
+            ? { Name, Soft: v, Hard: v }
+            : {
+                Name,
+                Soft: Number(v.soft),
+                Hard: Number(v.hard),
+            });
+        const logging = src.logging;
+        const extra = [...extraHosts(src.extra_hosts)];
+        const taken = new Set(extra.map((h) => h.split(':')[0]));
+        for (const host of hosts)
+            if (!taken.has(host.split(':')[0]))
+                extra.push(host);
+        const stopTimeout = composeDurationMs(src.stop_grace_period);
+        const hostConfig = {
+            NetworkMode: network,
+            ...(binds.length ? { Binds: binds } : {}),
+            ...(Object.keys(tmpfs).length ? { Tmpfs: tmpfs } : {}),
+            ...(Object.keys(bindings).length ? { PortBindings: bindings } : {}),
+            ...(restartPolicy(src.restart) ? { RestartPolicy: restartPolicy(src.restart) } : {}),
+            ...(memory !== undefined ? { Memory: memory } : {}),
+            ...(reservation !== undefined ? { MemoryReservation: reservation } : {}),
+            ...(src.cpus !== undefined ? { NanoCpus: Math.round(Number(src.cpus) * SECOND_NS) } : {}),
+            ...(src.pids_limit !== undefined ? { PidsLimit: Number(src.pids_limit) } : {}),
+            ...(shm !== undefined ? { ShmSize: shm } : {}),
+            ...(src.init !== undefined ? { Init: Boolean(src.init) } : {}),
+            ...(src.read_only ? { ReadonlyRootfs: true } : {}),
+            ...(extra.length ? { ExtraHosts: extra } : {}),
+            ...(asList(src.cap_add).length ? { CapAdd: asList(src.cap_add) } : {}),
+            ...(asList(src.cap_drop).length ? { CapDrop: asList(src.cap_drop) } : {}),
+            ...(ulimits.length ? { Ulimits: ulimits } : {}),
+            ...(logging?.driver
+                ? { LogConfig: { Type: logging.driver, Config: logging.options || {} } }
+                : {}),
+        };
+        const hc = healthcheck(src.healthcheck);
+        const definition = {
+            Image: image,
+            ...(env.length ? { Env: env } : {}),
+            ...(src.command != null ? { Cmd: asList(src.command) } : {}),
+            ...(src.entrypoint != null ? { Entrypoint: asList(src.entrypoint) } : {}),
+            ...(src.user ? { User: String(src.user) } : {}),
+            ...(src.working_dir ? { WorkingDir: String(src.working_dir) } : {}),
+            ...(src.hostname ? { Hostname: String(src.hostname) } : {}),
+            Labels: Object.fromEntries(Object.entries(src.labels || {}).map(([k, v]) => [
+                k,
+                String(v),
+            ])),
+            ...(Object.keys(exposed).length ? { ExposedPorts: exposed } : {}),
+            ...(src.stop_signal ? { StopSignal: String(src.stop_signal) } : {}),
+            ...(stopTimeout !== undefined ? { StopTimeout: Math.ceil(stopTimeout / 1000) } : {}),
+            ...(hc ? { Healthcheck: hc } : {}),
+            HostConfig: hostConfig,
+            NetworkingConfig: { EndpointsConfig: { [network]: { Aliases: [service] } } },
+        };
+        const used = [...new Set(sentinelNames(definition))].sort();
+        const versions = Object.fromEntries(used
+            .filter((name) => secretVersions[name] != null)
+            .map((name) => [name, secretVersions[name]]));
+        const hash = crypto
+            .createHash('sha256')
+            .update(JSON.stringify({ definition: { ...definition, Image: identity }, versions }))
+            .digest('hex')
+            .slice(0, 16);
+        const body = walkStrings(definition, (s) => substituteSentinels(s, secrets));
+        body.Labels = {
+            ...body.Labels,
+            [LABEL_PROJECT]: project,
+            [LABEL_SERVICE]: service,
+            [LABEL_HASH]: hash,
+            ...(desired.revision || desired.version
+                ? { [LABEL_REVISION]: desired.revision || desired.version }
+                : {}),
+        };
+        const dependsOn = {};
+        const deps = src.depends_on;
+        if (Array.isArray(deps)) {
+            for (const dep of deps)
+                if (placed.has(String(dep)))
+                    dependsOn[String(dep)] = 'service_started';
+        }
+        else if (deps && typeof deps === 'object') {
+            for (const [dep, spec] of Object.entries(deps))
+                if (placed.has(dep))
+                    dependsOn[dep] =
+                        spec?.condition === 'service_healthy'
+                            ? 'service_healthy'
+                            : 'service_started';
+        }
+        specs[service] = {
+            service,
+            name: containerName(project, service, hash),
+            hash,
+            body,
+            dependsOn,
+        };
+    }
+    return { network, volumes: volumeSpecs(desired, node, usedVolumes, nodeAddresses), specs };
+}
+/** Start order: dependencies first, ties by service name; a cycle is an error. */
+export function startOrder(specs) {
+    const order = [];
+    const state = new Map();
+    const visit = (service, path) => {
+        if (state.get(service) === 'done')
+            return;
+        if (state.get(service) === 'visiting')
+            throw new Error(`depends_on cycle: ${[...path, service].join(' -> ')}`);
+        state.set(service, 'visiting');
+        for (const dep of Object.keys(specs[service].dependsOn).sort())
+            visit(dep, [...path, service]);
+        state.set(service, 'done');
+        order.push(service);
+    };
+    for (const service of Object.keys(specs).sort())
+        visit(service, []);
+    return order;
+}
+/** `body` with the host side of its (single) published port replaced; used for rolling revisions. */
+export function withHostPort(body, port) {
+    const bindings = (body.HostConfig?.PortBindings || {});
+    return {
+        ...body,
+        HostConfig: {
+            ...body.HostConfig,
+            PortBindings: Object.fromEntries(Object.entries(bindings).map(([key, list]) => [
+                key,
+                list.map((b) => ({ ...b, HostPort: String(port) })),
+            ])),
+        },
+    };
+}

package/dist/control.d.ts

@@ -1,18 +1,20 @@
 #!/usr/bin/env node
 import express from 'express';
+import { resolveImage } from './registry.ts';
 import { saveImageArchive, inspectImage, validateImageArchive } from './artifacts.ts';
 import https from 'node:https';
 import http from 'node:http';
 import { openDb } from './db.ts';
 import type { NodeRow } from './db.ts';
-import { buildAndPush, run as runCommand } from './build.ts';
+import { buildImage, run as runCommand } from './build.ts';
 declare module 'express-serve-static-core' {
     interface Request {
         node?: NodeRow;
     }
 }
 export interface ControlImageOperations {
-    build: typeof buildAndPush;
+    resolve: typeof resolveImage;
+    build: typeof buildImage;
     save: typeof saveImageArchive;
     inspect: typeof inspectImage;
     validate: typeof validateImageArchive;