@wpelevator/tug 0.2.1 → 0.4.0

package/dist/mesh.js

@@ -17,7 +17,11 @@ export const isWgKey = (value) => typeof value === 'string' && WG_KEY_RE.test(va
 const PKCS8_X25519_PREFIX = Buffer.from('302e020100300506032b656e04220420', 'hex');
 /** Public key for a base64 private key (X25519, as `wg pubkey` computes it). */
 export function publicKeyFromPrivate(privateKey) {
-    const key = crypto.createPrivateKey({ key: Buffer.concat([PKCS8_X25519_PREFIX, Buffer.from(privateKey, 'base64')]), format: 'der', type: 'pkcs8' });
+    const key = crypto.createPrivateKey({
+        key: Buffer.concat([PKCS8_X25519_PREFIX, Buffer.from(privateKey, 'base64')]),
+        format: 'der',
+        type: 'pkcs8',
+    });
     const jwk = crypto.createPublicKey(key).export({ format: 'jwk' });
     return Buffer.from(jwk.x, 'base64url').toString('base64');
 }
@@ -35,13 +39,17 @@ export async function ensureWgKeys(stateDir) {
         if (isWgKey(privateKey))
             return { privateKey, publicKey: publicKeyFromPrivate(privateKey) };
     }
-    catch { /* first boot */ }
+    catch {
+        /* first boot */
+    }
     return rotateWgKeys(stateDir);
 }
 export async function rotateWgKeys(stateDir) {
     const pair = generateWgKeyPair();
     await ensureDir(wgDir(stateDir));
-    await fs.writeFile(path.join(wgDir(stateDir), 'private.key'), `${pair.privateKey}\n`, { mode: 0o600 });
+    await fs.writeFile(path.join(wgDir(stateDir), 'private.key'), `${pair.privateKey}\n`, {
+        mode: 0o600,
+    });
     return pair;
 }
 /** The public key control last accepted, so the agent only re-reports when it changed. */
@@ -60,7 +68,7 @@ export async function writeReportedKey(stateDir, publicKey) {
 // ---- Overlay addressing (control) --------------------------------------------------------------
 export function ipToInt(ip) {
     const parts = ip.split('.').map(Number);
-    if (parts.length !== 4 || parts.some(p => !Number.isInteger(p) || p < 0 || p > 255))
+    if (parts.length !== 4 || parts.some((p) => !Number.isInteger(p) || p < 0 || p > 255))
         throw new Error(`invalid IPv4 address ${ip}`);
     return ((parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]) >>> 0;
 }
@@ -68,7 +76,11 @@ export const intToIp = (n) => [n >>> 24, (n >>> 16) & 255, (n >>> 8) & 255, n &
 export function parseCidr(cidr) {
     const [ip, bits, extra] = cidr.split('/');
     const prefix = Number(bits);
-    if (extra !== undefined || bits === undefined || !Number.isInteger(prefix) || prefix < 8 || prefix > 29)
+    if (extra !== undefined ||
+        bits === undefined ||
+        !Number.isInteger(prefix) ||
+        prefix < 8 ||
+        prefix > 29)
         throw new Error(`invalid mesh CIDR ${cidr} (IPv4, prefix /8 to /29)`);
     const size = 2 ** (32 - prefix);
     const base = (ipToInt(ip) & ~(size - 1)) >>> 0;
@@ -114,28 +126,30 @@ export function parseWgDump(dump) {
     return {
         publicKey: h[1] || '',
         listenPort: Number(h[2]) || 0,
-        peers: rest.map(line => {
+        peers: rest.map((line) => {
             const f = line.split('\t');
             return {
                 publicKey: f[0],
                 endpoint: f[2] && f[2] !== '(none)' ? f[2] : undefined,
                 allowedIps: f[3] && f[3] !== '(none)' ? f[3].split(',') : [],
-                keepalive: f[7] && f[7] !== 'off' ? Number(f[7]) : 0
+                keepalive: f[7] && f[7] !== 'off' ? Number(f[7]) : 0,
             };
-        })
+        }),
     };
 }
 /** Compares desired peers with what `wg` reports. A desired endpoint is authoritative; a peer without one keeps whatever it roamed to. */
 export function diffPeers(desired, applied) {
-    const have = new Map(applied.map(p => [p.publicKey, p]));
-    const want = new Map(desired.map(p => [p.publicKey, p]));
+    const have = new Map(applied.map((p) => [p.publicKey, p]));
+    const want = new Map(desired.map((p) => [p.publicKey, p]));
     const sameSet = (a, b) => a.length === b.length && [...a].sort().every((v, i) => v === [...b].sort()[i]);
     const diff = { add: [], remove: [], update: [] };
     for (const [key, peer] of want) {
         const cur = have.get(key);
         if (!cur)
             diff.add.push(key);
-        else if (!sameSet(peer.allowedIps, cur.allowedIps) || peer.persistentKeepalive !== cur.keepalive || (peer.endpoint && peer.endpoint !== cur.endpoint))
+        else if (!sameSet(peer.allowedIps, cur.allowedIps) ||
+            peer.persistentKeepalive !== cur.keepalive ||
+            (peer.endpoint && peer.endpoint !== cur.endpoint))
             diff.update.push(key);
     }
     for (const key of have.keys())
@@ -172,11 +186,16 @@ export async function reconcileMesh(host, stateDir, network) {
         }
     }
     const diff = diffPeers(network.peers, applied.peers);
-    const synced = created || applied.publicKey !== publicKey || applied.listenPort !== network.listenPort || diff.add.length + diff.remove.length + diff.update.length > 0;
+    const synced = created ||
+        applied.publicKey !== publicKey ||
+        applied.listenPort !== network.listenPort ||
+        diff.add.length + diff.remove.length + diff.update.length > 0;
     if (synced) {
         const file = path.join(wgDir(stateDir), `${iface}.conf`);
         await ensureDir(wgDir(stateDir));
-        await fs.writeFile(file, renderWgConfig(privateKey, network.listenPort, network.peers), { mode: 0o600 });
+        await fs.writeFile(file, renderWgConfig(privateKey, network.listenPort, network.peers), {
+            mode: 0o600,
+        });
         await host('wg', ['syncconf', iface, file]);
     }
     return { created, synced, diff };

package/dist/normalize.d.ts

@@ -0,0 +1,20 @@
+import type { RunResult } from './build.ts';
+import type { ComposeFile } from './protocol.ts';
+export type ComposeRunner = (cmd: string, args: string[], opts: {
+    env: NodeJS.ProcessEnv;
+}) => Promise<RunResult>;
+export declare const secretSentinel: (name: string) => string;
+/** Names of the control secrets that `value` references through sentinels, in order of appearance. */
+export declare function sentinelNames(value: unknown): string[];
+/** `value` with every sentinel replaced by the secret's value; a sentinel with no value becomes empty. */
+export declare function substituteSentinels(value: string, secrets: Record<string, string>): string;
+/** Arguments of the `docker compose` call that renders `file` canonically (as YAML, which keeps `x-tug`). */
+export declare function composeConfigArgs(file: string, project?: string): string[];
+/**
+ * Renders `file` with Compose and returns the canonical definition. Every name in `secretNames` is set to
+ * its sentinel in the environment, so it wins over a same-named local variable.
+ */
+export declare function normalizeCompose(file: string, secretNames: readonly string[], options?: {
+    project?: string;
+    runner?: ComposeRunner;
+}): Promise<ComposeFile>;

package/dist/normalize.js

@@ -0,0 +1,75 @@
+// Operator-side Compose normalization. `docker compose config` does the interpolation, `extends`,
+// `env_file` merging and short-syntax expansion, so control and agents only ever see canonical JSON.
+// The CLI reads Compose's YAML rendering: its `--format json` drops every service-level extension
+// (`x-tug`), while the YAML rendering keeps them.
+// Control secrets are not resolved on the operator's machine: each one is interpolated as a sentinel
+// that the agent replaces with the node's released value when it creates the container.
+import yaml from 'js-yaml';
+import { run } from "./build.js";
+const SECRET_NAME = '[A-Za-z_][A-Za-z0-9_]*';
+const SENTINEL = new RegExp(`@@tug:secret:(${SECRET_NAME})@@`, 'g');
+export const secretSentinel = (name) => `@@tug:secret:${name}@@`;
+/** Names of the control secrets that `value` references through sentinels, in order of appearance. */
+export function sentinelNames(value) {
+    const names = new Set();
+    const visit = (v) => {
+        if (typeof v === 'string')
+            for (const m of v.matchAll(SENTINEL))
+                names.add(m[1]);
+        else if (Array.isArray(v))
+            v.forEach(visit);
+        else if (v && typeof v === 'object')
+            Object.values(v).forEach(visit);
+    };
+    visit(value);
+    return [...names];
+}
+/** `value` with every sentinel replaced by the secret's value; a sentinel with no value becomes empty. */
+export function substituteSentinels(value, secrets) {
+    return value.replace(SENTINEL, (_all, name) => secrets[name] ?? '');
+}
+/** Arguments of the `docker compose` call that renders `file` canonically (as YAML, which keeps `x-tug`). */
+export function composeConfigArgs(file, project) {
+    return [
+        'compose',
+        '-f',
+        file,
+        ...(project ? ['-p', project] : []),
+        'config',
+        '--format',
+        'yaml',
+        '--no-path-resolution',
+    ];
+}
+/**
+ * Renders `file` with Compose and returns the canonical definition. Every name in `secretNames` is set to
+ * its sentinel in the environment, so it wins over a same-named local variable.
+ */
+export async function normalizeCompose(file, secretNames, options = {}) {
+    const env = { ...process.env };
+    for (const name of secretNames)
+        env[name] = secretSentinel(name);
+    const runner = options.runner ?? ((cmd, args, opts) => run(cmd, args, opts));
+    let result;
+    try {
+        result = await runner('docker', composeConfigArgs(file, options.project), { env });
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (error?.code === 'ENOENT' || /docker: .*not found/i.test(message))
+            throw new Error('tug deploy needs Docker with the Compose plugin on this machine to read the Compose file (`docker compose config`); install Docker Compose v2', { cause: error });
+        throw new Error(`docker compose config failed for ${file}: ${message}`, { cause: error });
+    }
+    let parsed;
+    try {
+        parsed = yaml.load(result.out);
+    }
+    catch (error) {
+        throw new Error(`docker compose config returned unreadable output for ${file}`, {
+            cause: error,
+        });
+    }
+    if (!parsed || typeof parsed !== 'object' || !parsed.services)
+        throw new Error(`${file} defines no services`);
+    return parsed;
+}

package/dist/protocol.d.ts

@@ -1,5 +1,7 @@
 export interface Placement {
     nodes?: string[];
+    labels?: string[];
+    global?: boolean;
 }
 export interface HealthCheck {
     type: 'docker' | 'tcp' | 'http';
@@ -33,6 +35,7 @@ export interface RolloutSpec {
 export interface ServiceExtension {
     placement?: Placement;
     replicas?: number;
+    global?: boolean;
     health?: HealthCheck;
     ingress?: IngressSpec;
     rollout?: RolloutSpec;
@@ -48,6 +51,10 @@ export interface VolumeExtension {
     driver?: string;
     persistent?: boolean;
     placement?: Placement;
+    /** NFS exporter address; control fills in the exporter's overlay IP (or node address) at deploy time. */
+    address?: string;
+    /** Directory exported by the NFS server on the exporter node (default `/srv/tug/<volume>`). */
+    export?: string;
 }
 export type BuildConfig = string | {
     context?: string;
@@ -141,12 +148,14 @@ export interface Command {
     created_at?: string;
     updated_at?: string;
 }
+/** One diagnostic result; `warning` flags something to look at that does not fail the check. */
 export interface DiagnosticCheck {
     name: string;
     ok: boolean;
     stdout?: string;
     stderr?: string;
     error?: string;
+    warning?: string;
 }
 export type CommandResultPayload = {
     checks: DiagnosticCheck[];
@@ -172,6 +181,138 @@ export interface CommandResult {
     state: 'succeeded' | 'failed';
     result: CommandResultPayload;
 }
+/**
+ * Messages sent by control over the agent-initiated WebSocket. The stream is authenticated during the HTTP
+ * upgrade, so credentials never appear in the URL or a message. A wake is only a hint: the agent fetches the
+ * authoritative state from `/agent/v1/work` and continues periodic polling if notifications are lost.
+ */
+export type StreamMessage = {
+    type: 'ready';
+    nodeId: string;
+    cursor: number;
+} | {
+    type: 'work-available';
+    cursor: number;
+    project?: string;
+};
+/** Live operations always resolve their project/service placement on control before reaching an agent. */
+export type LiveRequest = {
+    operation: 'exec';
+    project: string;
+    service: string;
+    node?: string;
+    command: string[];
+    interactive: boolean;
+    tty: boolean;
+    columns?: number;
+    rows?: number;
+} | {
+    operation: 'logs';
+    project: string;
+    service: string;
+    node?: string;
+    follow: true;
+    tail: number;
+} | {
+    operation: 'ps';
+    project: string;
+    node?: string;
+} | {
+    operation: 'inspect';
+    project: string;
+    service: string;
+    node?: string;
+};
+export declare const LIVE_MAX_FRAME_BYTES: number;
+export declare const LIVE_MAX_BUFFERED_BYTES: number;
+export declare const LIVE_MAX_SESSIONS = 64;
+export declare const LIVE_START_TIMEOUT_MS = 10000;
+export type LiveInput = {
+    v: 1;
+    type: 'live.stdin';
+    data: string;
+} | {
+    v: 1;
+    type: 'live.stdin-end';
+} | {
+    v: 1;
+    type: 'live.resize';
+    columns: number;
+    rows: number;
+} | {
+    v: 1;
+    type: 'live.cancel';
+};
+export type OperatorLiveFrame = {
+    v: 1;
+    type: 'live.open';
+    request: LiveRequest;
+} | LiveInput;
+export type AgentLiveCommand = {
+    v: 1;
+    type: 'live.open';
+    sessionId: string;
+    request: LiveRequest;
+} | (LiveInput & {
+    sessionId: string;
+});
+export type AgentLiveFrame = {
+    v: 1;
+    type: 'live.started';
+    sessionId: string;
+} | {
+    v: 1;
+    type: 'live.output';
+    sessionId: string;
+    stream: 'stdout' | 'stderr';
+    data: string;
+} | {
+    v: 1;
+    type: 'live.result';
+    sessionId: string;
+    result: unknown;
+} | {
+    v: 1;
+    type: 'live.exit';
+    sessionId: string;
+    code: number | null;
+    signal: string | null;
+    cancelled: boolean;
+    error?: string;
+};
+export type OperatorLiveMessage = {
+    v: 1;
+    type: 'live.started';
+    sessionId: string;
+    node: string;
+} | {
+    v: 1;
+    type: 'live.output';
+    sessionId: string;
+    stream: 'stdout' | 'stderr';
+    data: string;
+} | {
+    v: 1;
+    type: 'live.result';
+    sessionId: string;
+    value: unknown;
+} | {
+    v: 1;
+    type: 'live.exit';
+    sessionId: string;
+    code: number;
+} | {
+    v: 1;
+    type: 'live.error';
+    sessionId?: string;
+    message: string;
+};
+/** Response from the long-poll fallback used when a WebSocket cannot be established. */
+export type WakeResponse = {
+    type: 'work-available' | 'timeout';
+    cursor: number;
+    project?: string;
+};
 export interface WorkResponse {
     serverTime: string;
     nodeId: string;
@@ -184,6 +325,18 @@ export interface WorkResponse {
     serviceEndpoints?: ServiceEndpoints;
     /** Edge proxy config; present only for nodes marked as ingress nodes. */
     ingress?: IngressWork;
+    /** Host addresses for published ports that name no host IP. */
+    ports?: PortBinding;
+}
+/**
+ * Where a node's published ports listen when a `ports` entry names no host IP. Entries with an
+ * explicit host IP (including `0.0.0.0`) are never changed.
+ */
+export interface PortBinding {
+    /** The node's overlay IP with the mesh, otherwise its `address` (an IP or a hostname the agent resolves); null binds to `127.0.0.1`. */
+    address: string | null;
+    /** Per project, services whose ingress port only the managed Caddy on this node dials; that port binds to `127.0.0.1`. */
+    loopback: Record<string, string[]>;
 }
 export interface WgPeer {
     node: string;
@@ -285,6 +438,7 @@ export interface NodeInfo {
     address: string | null;
     public_address: string | null;
     overlay_ip?: string | null;
+    labels?: string[];
     status: string;
     created_at: string;
     updated_at: string;
@@ -303,6 +457,8 @@ export interface EnrollRequest {
     publicAddress?: string;
     /** Base64 WireGuard public key; the private key never leaves the node. */
     wgPublicKey?: string;
+    /** Node labels (tags) used for label-based placement. */
+    labels?: string[];
 }
 /** Response of `POST /agent/v1/enroll`. */
 export interface EnrollResponse extends RegisterResponse {
@@ -323,4 +479,6 @@ export interface JoinKey {
     enrollToken: string;
     /** PEM CA certificate to trust for a private/self-signed control plane. */
     ca?: string;
+    /** Node labels to register at enrollment. */
+    labels?: string[];
 }

package/dist/protocol.js

@@ -3,6 +3,20 @@
 // control.ts, agent.ts, compose.ts and cli.ts all import from here.
 // ---- Deployments -------------------------------------------------------------------------------
 /** Deployment states during which a project is considered busy. */
-export const ACTIVE_DEPLOYMENT_STATES = ['uploading', 'building', 'pending_dependencies', 'rolling_out'];
+export const ACTIVE_DEPLOYMENT_STATES = [
+    'uploading',
+    'building',
+    'pending_dependencies',
+    'rolling_out',
+];
 /** Deployment states a CLI wait loop stops on. */
-export const TERMINAL_DEPLOYMENT_STATES = ['succeeded', 'failed', 'build_failed', 'cancelled'];
+export const TERMINAL_DEPLOYMENT_STATES = [
+    'succeeded',
+    'failed',
+    'build_failed',
+    'cancelled',
+];
+export const LIVE_MAX_FRAME_BYTES = 256 * 1024;
+export const LIVE_MAX_BUFFERED_BYTES = 1024 * 1024;
+export const LIVE_MAX_SESSIONS = 64;
+export const LIVE_START_TIMEOUT_MS = 10_000;

package/dist/reconciler.d.ts

@@ -0,0 +1,89 @@
+import type { ContainerSpec, NodePlan } from './container-spec.ts';
+import type { ContainerSummary, DockerApi } from './docker-api.ts';
+import type { DesiredState, ServiceState } from './protocol.ts';
+export declare const DEFAULT_DRAIN = "30s";
+export declare const DEFAULT_PORT_RANGE = "20000-20999";
+export declare function parsePortRange(value: string | undefined): [number, number];
+/** Containers of a project (or one service of it), running or not. */
+export declare function projectContainers(docker: DockerApi, project: string, service?: string): Promise<ContainerSummary[]>;
+/** Creates the project network and its volumes; volumes are never removed by reconciliation. */
+export declare function ensureProjectResources(docker: DockerApi, project: string, plan: NodePlan): Promise<void>;
+/** Makes sure a service's container exists for `spec`, running, replacing any container with another hash. */
+export declare function ensureContainer(docker: DockerApi, spec: ContainerSpec, existing: ContainerSummary[]): Promise<string>;
+/** Removes containers of services that are not placed here any more (never volumes). */
+export declare function sweepOrphans(docker: DockerApi, project: string, keep: ReadonlySet<string>): Promise<void>;
+interface Revision {
+    hash: string;
+    /** Container name. */
+    container: string;
+    port: number;
+    image?: string;
+}
+interface Draining {
+    service: string;
+    container: string;
+    drainUntil: number;
+}
+interface ServiceRollout {
+    active?: Revision;
+    candidate?: Revision;
+    draining: Draining[];
+    failed?: {
+        key: string;
+        error: string;
+    };
+}
+export interface RolloutState {
+    services: Record<string, ServiceRollout>;
+}
+export declare function loadRolloutState(dir: string): Promise<RolloutState>;
+export interface ReconcileDeps {
+    /**
+     * Throws until the container is healthy under the service's `x-tug.health`. With `dependency` a
+     * service without one is held to its Docker `HEALTHCHECK`, which is what `service_healthy` means.
+     */
+    waitForHealth: (container: string, service: string, dependency?: boolean) => Promise<void>;
+    ensureArtifact: (service: string) => Promise<void>;
+    /** Pulls the service's published image. */
+    pullImage: (service: string) => Promise<void>;
+    emit: (service: string, event: string, message?: string) => Promise<unknown>;
+}
+export interface ReconcileInput {
+    desired: DesiredState;
+    /** The project's state directory (`rollout.json`). */
+    dir: string;
+    plan: NodePlan;
+    nodeName: string;
+    rolloutPorts?: string;
+    deps: ReconcileDeps;
+}
+/**
+ * Recreate-strategy services: pull, then create/start each in dependency order. `onStarting` and
+ * `onHealth` let the agent report per-service state; a failure throws with the service named in
+ * `failing`.
+ */
+export declare function reconcileRecreate(docker: DockerApi, input: ReconcileInput & {
+    services: string[];
+    pull: boolean;
+}, report: {
+    pulling: (service: string) => Promise<void>;
+    starting: (service: string) => Promise<void>;
+    checking: (service: string) => Promise<void>;
+    serving: (service: string) => Promise<void>;
+    failing: (service: string | undefined) => void;
+}): Promise<void>;
+export interface RollingResult {
+    /** Reported state of every rolling service; the serving revision keeps `serving` while a newer one is rejected. */
+    serviceStates: Record<string, ServiceState>;
+    /** Services whose newest revision failed (or whose current revision is unhealthy), with the reason. */
+    errors: Record<string, string>;
+}
+export declare const isRollingService: (desired: DesiredState, service: string) => boolean;
+export declare function reconcileRolling(docker: DockerApi, input: ReconcileInput & {
+    services: string[];
+}): Promise<RollingResult>;
+/** Drops rolling state of a project that has no rolling service on this node any more. */
+export declare function clearRolloutState(dir: string): Promise<void>;
+/** Reported states for rolling services whose current revision is still serving, used when another part of the reconcile failed. */
+export declare function activeRollingStates(dir: string, services: string[]): Promise<Record<string, ServiceState>>;
+export {};