@wpelevator/tug 0.3.0 → 0.5.0

package/dist/live.js

@@ -1,6 +1,4 @@
-import { spawn } from 'node:child_process';
 import { EventEmitter } from 'node:events';
-import { PassThrough } from 'node:stream';
 import { LIVE_MAX_FRAME_BYTES } from "./protocol.js";
 const identifier = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,255}$/;
 function object(value) {
@@ -46,214 +44,189 @@ function validate(value) {
             !value.projects.every((project) => typeof project === 'string' && identifier.test(project))))
         throw new Error('Invalid projects');
 }
-function argv(request) {
-    switch (request.operation) {
-        case 'exec':
-            return [
-                'exec',
-                ...(request.interactive === false ? [] : ['-i']),
-                ...(request.tty ? ['-t'] : []),
-                request.container,
-                ...request.command,
-            ];
-        case 'logs':
-            return [
-                'logs',
-                '--follow',
-                '--tail',
-                String(request.tail ?? 100),
-                ...(request.timestamps ? ['--timestamps'] : []),
-                request.container,
-            ];
-        case 'ps':
-            return [
-                'ps',
-                '--all',
-                '--no-trunc',
-                '--format',
-                '{{json .}}',
-                '--filter',
-                `label=com.docker.compose.project=${request.projects[0]}`,
-            ];
-        case 'inspect':
-            return ['inspect', '--type', 'container', request.container];
-    }
-}
-/** Execute exact project-label queries sequentially; Docker label predicates must not broaden project scope. */
-function spawnProjectQueries(spawnProcess, request) {
-    const projects = [...new Set(request.projects)];
-    const stdout = new PassThrough();
-    const stderr = new PassThrough();
-    let current;
-    let stopped = false;
-    let detachOutput = () => { };
-    const process = Object.assign(new EventEmitter(), {
-        stdin: null,
-        stdout,
-        stderr,
-        kill(signal) {
-            stopped = true;
-            detachOutput();
-            return current.kill(signal);
-        },
-    });
-    const next = () => {
-        const child = spawnProcess('docker', argv({ ...request, projects: [projects.shift()] }));
-        current = child;
-        const out = (data) => {
-            stdout.write(data);
-        };
-        const err = (data) => {
-            stderr.write(data);
-        };
-        const error = (cause) => {
-            process.emit('error', cause);
-        };
-        const close = (code, signal) => {
-            child.off('error', error).off('close', close);
-            child.stdout?.off('data', out);
-            child.stderr?.off('data', err);
-            if (!stopped && code === 0 && projects.length) {
-                stdout.write('\n');
-                if (stopped) {
-                    process.emit('close', null, null);
-                    return;
-                }
-                try {
-                    next();
-                }
-                catch (cause) {
-                    process.emit('error', cause);
-                }
-            }
-            else
-                process.emit('close', code, signal);
-        };
-        detachOutput = () => {
-            child.stdout?.off('data', out);
-            child.stderr?.off('data', err);
-        };
-        child.stdout?.on('data', out);
-        child.stderr?.on('data', err);
-        child.on('error', error).on('close', close);
-    };
-    next();
-    return process;
-}
-/** Owns Docker process lifetimes; callers own authentication, authorization and transport backpressure. */
+/** Owns Engine API streams and their lifetimes; callers own authentication, authorization and transport backpressure. */
 export class LiveOperationEngine {
     options;
     sessions = new Map();
+    /** Sessions whose container stream is still being opened; a cancel marks them so they close on arrival. */
+    starting = new Map();
     constructor(options) {
-        for (const key of [
-            'maxSessions',
-            'maxResultBytes',
-            'maxSessionMs',
-            'queryTimeoutMs',
-        ]) {
+        for (const key of ['maxSessions', 'maxSessionMs', 'queryTimeoutMs']) {
             if (options[key] !== undefined && (!Number.isSafeInteger(options[key]) || options[key] < 1))
                 throw new Error(`Invalid ${key}`);
         }
         this.options = options;
     }
     get activeSessions() {
-        return this.sessions.size;
+        return this.sessions.size + this.starting.size;
     }
-    start(value) {
+    async start(value) {
         validate(value);
-        if (this.sessions.has(value.sessionId))
+        if (this.sessions.has(value.sessionId) || this.starting.has(value.sessionId))
             throw new Error('Duplicate live session ID');
-        if (this.sessions.size >= (this.options.maxSessions ?? 16))
+        if (this.activeSessions >= (this.options.maxSessions ?? 16))
             throw new Error('Live session limit reached');
         const request = structuredClone(value);
-        const spawnProcess = request.operation === 'exec' && request.tty
-            ? this.options.spawnTty
-            : (this.options.spawn ??
-                ((file, args) => spawn(file, args, { stdio: 'pipe', shell: false })));
-        if (!spawnProcess)
-            throw new Error('TTY exec requires a PTY adapter');
-        const child = request.operation === 'ps' && request.projects.length > 1
-            ? spawnProjectQueries(spawnProcess, request)
-            : spawnProcess('docker', argv(request));
+        const id = request.sessionId;
+        const pending = { cancelled: false };
+        this.starting.set(id, pending);
         const query = request.operation === 'ps' || request.operation === 'inspect';
+        const timer = setTimeout(() => this.finish(id, null, null, false, 'Live operation timed out', true), query ? (this.options.queryTimeoutMs ?? 30000) : (this.options.maxSessionMs ?? 3600000));
+        timer.unref();
+        let child;
+        try {
+            if (query) {
+                this.starting.delete(id);
+                await this.query(request, timer);
+                return;
+            }
+            child = await this.open(request);
+        }
+        catch (error) {
+            clearTimeout(timer);
+            this.starting.delete(id);
+            throw error;
+        }
+        this.starting.delete(id);
+        if (pending.cancelled) {
+            clearTimeout(timer);
+            try {
+                child.kill('SIGKILL');
+            }
+            catch {
+                /* Already gone. */
+            }
+            this.options.emit({ sessionId: id, type: 'exit', code: null, signal: null, cancelled: true });
+            return;
+        }
+        const live = child;
         const onError = (error) => {
-            if (this.sessions.get(request.sessionId)?.child === child)
-                this.finish(request.sessionId, null, null, false, error.message, true);
+            if (this.sessions.get(id)?.child === live)
+                this.finish(id, null, null, false, error.message, true);
         };
         const stopErrors = () => {
-            child.off('error', onError).off('close', onClose);
-            child.stdin?.off('error', onError);
+            live.off('error', onError).off('close', onClose);
+            live.stdin?.off('error', onError);
         };
         const onClose = (code, signal) => {
             stopErrors();
-            if (this.sessions.get(request.sessionId)?.child === child)
-                this.finish(request.sessionId, code, signal, false);
+            if (this.sessions.get(id)?.child === live)
+                this.finish(id, code, signal, false);
         };
-        const stdout = (data) => this.output(request.sessionId, 'stdout', data);
-        const stderr = (data) => this.output(request.sessionId, 'stderr', data);
-        const timer = setTimeout(() => this.finish(request.sessionId, null, null, false, 'Live operation timed out', true), query ? (this.options.queryTimeoutMs ?? 30000) : (this.options.maxSessionMs ?? 3600000));
-        timer.unref();
-        this.sessions.set(request.sessionId, {
+        const stdout = (data) => this.output(id, 'stdout', data);
+        const stderr = (data) => this.output(id, 'stderr', data);
+        this.sessions.set(id, {
             request,
-            child,
+            child: live,
             timer,
-            output: [],
-            errorOutput: [],
-            bytes: 0,
             cleanup: (awaitClose = false) => {
                 clearTimeout(timer);
                 if (!awaitClose)
                     stopErrors();
-                child.stdout?.off('data', stdout);
-                child.stderr?.off('data', stderr);
+                live.stdout?.off('data', stdout);
+                live.stderr?.off('data', stderr);
             },
         });
-        child.on('error', onError).on('close', onClose);
-        child.stdout?.on('data', stdout);
-        child.stderr?.on('data', stderr);
-        child.stdin?.on('error', onError);
+        live.on('error', onError).on('close', onClose);
+        live.stdout?.on('data', stdout);
+        live.stderr?.on('data', stderr);
+        live.stdin?.on('error', onError);
         if (request.operation === 'exec' && request.interactive === false)
-            child.stdin?.end();
-        if (request.operation === 'exec' && request.tty) {
-            if (!child.resize) {
-                this.finish(request.sessionId, null, null, false, 'PTY adapter does not support resize', true);
-                return;
-            }
+            live.stdin?.end();
+    }
+    /** `ps` and `inspect` are one Engine API call whose JSON result is the whole session. */
+    async query(request, timer) {
+        const id = request.sessionId;
+        const session = { request, timer, cleanup: () => clearTimeout(timer) };
+        const aborted = new Promise((resolve) => {
+            session.abort = resolve;
+        });
+        this.sessions.set(id, session);
+        let error;
+        let result;
+        let settled = false;
+        const work = (async () => {
             try {
-                child.resize(request.columns ?? 80, request.rows ?? 24);
+                if (request.operation === 'ps') {
+                    const lists = await Promise.all([...new Set(request.projects)].map((project) => this.options.docker.listContainers({ label: [`tug.project=${project}`] })));
+                    result = lists.flat();
+                }
+                else
+                    result = [await this.options.docker.inspectContainer(request.container)];
+                const frame = { v: 1, type: 'live.result', sessionId: id, result };
+                if (Buffer.byteLength(JSON.stringify(frame)) > LIVE_MAX_FRAME_BYTES)
+                    throw new Error('Live result exceeds serialized frame size limit');
             }
-            catch (error) {
-                onError(error instanceof Error ? error : new Error(String(error)));
+            catch (cause) {
+                error = cause instanceof Error ? cause.message : String(cause);
             }
-        }
+            settled = true;
+        })();
+        await Promise.race([work, aborted]);
+        // A cancel or timeout already closed the session while the call was in flight.
+        if (!settled || !this.sessions.has(id))
+            return;
+        clearTimeout(timer);
+        this.sessions.delete(id);
+        if (result && !error)
+            this.options.emit({ sessionId: id, type: 'result', result });
+        this.options.emit({
+            sessionId: id,
+            type: 'exit',
+            code: error ? 1 : 0,
+            signal: null,
+            cancelled: false,
+            ...(error ? { error: error.slice(0, 4096) } : {}),
+        });
+    }
+    async open(request) {
+        const { docker } = this.options;
+        if (request.operation === 'logs')
+            return docker.logs(request.container, {
+                follow: true,
+                tail: request.tail ?? 100,
+                timestamps: request.timestamps,
+            });
+        const proc = await docker.execStart(request.container, {
+            cmd: request.command,
+            stdin: request.interactive !== false,
+            tty: Boolean(request.tty),
+            columns: request.columns,
+            rows: request.rows,
+        });
+        return proc;
     }
     /** Returns the writable backpressure signal; callers must pause transport input on false. */
     stdin(sessionId, data) {
         const session = this.requireExec(sessionId);
         if (!(data instanceof Uint8Array) || data.byteLength > 65536)
             throw new Error('Invalid stdin frame');
-        if (!session.child.stdin || session.child.stdin.destroyed || session.child.stdin.writableEnded)
+        const stdin = session.child?.stdin;
+        if (!stdin || stdin.destroyed || stdin.writableEnded)
             throw new Error('Session stdin is closed');
-        if (session.child.stdin.writableLength + data.byteLength > 262144)
+        if (stdin.writableLength + data.byteLength > 262144)
             throw new Error('Live stdin queue exceeds size limit');
-        return session.child.stdin.write(data);
+        return stdin.write(data);
     }
     endStdin(sessionId) {
-        this.requireExec(sessionId).child.stdin?.end();
+        this.requireExec(sessionId).child?.stdin?.end();
     }
     resize(sessionId, columns, rows) {
         const session = this.requireExec(sessionId);
         if (!dimension(columns) || !dimension(rows))
             throw new Error('Invalid terminal dimensions');
-        if (!session.request.tty || !session.child.resize)
+        if (!session.request.tty || !session.child?.resize)
             throw new Error('Session has no terminal');
         session.child.resize(columns, rows);
     }
     cancel(sessionId) {
+        const pending = this.starting.get(sessionId);
+        if (pending)
+            pending.cancelled = true;
         this.finish(sessionId, null, null, true, undefined, true);
     }
     dispose() {
-        for (const id of this.sessions.keys())
+        for (const id of [...this.sessions.keys(), ...this.starting.keys()])
             this.cancel(id);
     }
     requireExec(sessionId) {
@@ -263,19 +236,9 @@ export class LiveOperationEngine {
         return session;
     }
     output(id, stream, data) {
-        const session = this.sessions.get(id);
-        if (!session)
+        if (!this.sessions.has(id))
             return;
         const bytes = Buffer.isBuffer(data) ? data : Buffer.from(data);
-        if (session.request.operation === 'ps' || session.request.operation === 'inspect') {
-            session.bytes += bytes.length;
-            if (session.bytes > (this.options.maxResultBytes ?? 4194304)) {
-                this.finish(id, null, null, false, 'Live result exceeds size limit', true);
-                return;
-            }
-            (stream === 'stdout' ? session.output : session.errorOutput).push(bytes);
-            return;
-        }
         for (let offset = 0; offset < bytes.length && this.sessions.has(id); offset += 65536)
             this.options.emit({
                 sessionId: id,
@@ -290,9 +253,10 @@ export class LiveOperationEngine {
         if (!session)
             return;
         this.sessions.delete(id);
-        // Keep error handlers until close when killing: spawn errors and pending stdin errors can arrive after cancellation.
+        session.abort?.();
+        // Keep error handlers until close when killing: errors can arrive after cancellation.
         session.cleanup(kill);
-        if (kill) {
+        if (kill && session.child) {
             try {
                 session.child.kill('SIGKILL');
             }
@@ -300,34 +264,6 @@ export class LiveOperationEngine {
                 /* Already exited. */
             }
         }
-        if (!error &&
-            !cancelled &&
-            (session.request.operation === 'ps' || session.request.operation === 'inspect')) {
-            if (code !== 0)
-                error =
-                    Buffer.concat(session.errorOutput).toString('utf8').trim() ||
-                        `Docker exited with code ${code}`;
-            else {
-                try {
-                    const raw = Buffer.concat(session.output).toString('utf8').trim();
-                    const result = session.request.operation === 'ps'
-                        ? raw
-                            .split('\n')
-                            .filter(Boolean)
-                            .map((line) => JSON.parse(line))
-                        : JSON.parse(raw);
-                    if (!Array.isArray(result) || !result.every(object))
-                        throw new Error('Expected an array of container objects');
-                    const frame = { v: 1, type: 'live.result', sessionId: id, result };
-                    if (Buffer.byteLength(JSON.stringify(frame)) > LIVE_MAX_FRAME_BYTES)
-                        throw new Error('Live result exceeds serialized frame size limit');
-                    this.options.emit({ sessionId: id, type: 'result', result });
-                }
-                catch (cause) {
-                    error = `Invalid Docker result: ${cause instanceof Error ? cause.message : String(cause)}`;
-                }
-            }
-        }
         this.options.emit({
             sessionId: id,
             type: 'exit',

package/dist/normalize.d.ts

@@ -0,0 +1,20 @@
+import type { RunResult } from './build.ts';
+import type { ComposeFile } from './protocol.ts';
+export type ComposeRunner = (cmd: string, args: string[], opts: {
+    env: NodeJS.ProcessEnv;
+}) => Promise<RunResult>;
+export declare const secretSentinel: (name: string) => string;
+/** Names of the control secrets that `value` references through sentinels, in order of appearance. */
+export declare function sentinelNames(value: unknown): string[];
+/** `value` with every sentinel replaced by the secret's value; a sentinel with no value becomes empty. */
+export declare function substituteSentinels(value: string, secrets: Record<string, string>): string;
+/** Arguments of the `docker compose` call that renders `file` canonically (as YAML, which keeps `x-tug`). */
+export declare function composeConfigArgs(file: string, project?: string): string[];
+/**
+ * Renders `file` with Compose and returns the canonical definition. Every name in `secretNames` is set to
+ * its sentinel in the environment, so it wins over a same-named local variable.
+ */
+export declare function normalizeCompose(file: string, secretNames: readonly string[], options?: {
+    project?: string;
+    runner?: ComposeRunner;
+}): Promise<ComposeFile>;

package/dist/normalize.js

@@ -0,0 +1,75 @@
+// Operator-side Compose normalization. `docker compose config` does the interpolation, `extends`,
+// `env_file` merging and short-syntax expansion, so control and agents only ever see canonical JSON.
+// The CLI reads Compose's YAML rendering: its `--format json` drops every service-level extension
+// (`x-tug`), while the YAML rendering keeps them.
+// Control secrets are not resolved on the operator's machine: each one is interpolated as a sentinel
+// that the agent replaces with the node's released value when it creates the container.
+import yaml from 'js-yaml';
+import { run } from "./build.js";
+const SECRET_NAME = '[A-Za-z_][A-Za-z0-9_]*';
+const SENTINEL = new RegExp(`@@tug:secret:(${SECRET_NAME})@@`, 'g');
+export const secretSentinel = (name) => `@@tug:secret:${name}@@`;
+/** Names of the control secrets that `value` references through sentinels, in order of appearance. */
+export function sentinelNames(value) {
+    const names = new Set();
+    const visit = (v) => {
+        if (typeof v === 'string')
+            for (const m of v.matchAll(SENTINEL))
+                names.add(m[1]);
+        else if (Array.isArray(v))
+            v.forEach(visit);
+        else if (v && typeof v === 'object')
+            Object.values(v).forEach(visit);
+    };
+    visit(value);
+    return [...names];
+}
+/** `value` with every sentinel replaced by the secret's value; a sentinel with no value becomes empty. */
+export function substituteSentinels(value, secrets) {
+    return value.replace(SENTINEL, (_all, name) => secrets[name] ?? '');
+}
+/** Arguments of the `docker compose` call that renders `file` canonically (as YAML, which keeps `x-tug`). */
+export function composeConfigArgs(file, project) {
+    return [
+        'compose',
+        '-f',
+        file,
+        ...(project ? ['-p', project] : []),
+        'config',
+        '--format',
+        'yaml',
+        '--no-path-resolution',
+    ];
+}
+/**
+ * Renders `file` with Compose and returns the canonical definition. Every name in `secretNames` is set to
+ * its sentinel in the environment, so it wins over a same-named local variable.
+ */
+export async function normalizeCompose(file, secretNames, options = {}) {
+    const env = { ...process.env };
+    for (const name of secretNames)
+        env[name] = secretSentinel(name);
+    const runner = options.runner ?? ((cmd, args, opts) => run(cmd, args, opts));
+    let result;
+    try {
+        result = await runner('docker', composeConfigArgs(file, options.project), { env });
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (error?.code === 'ENOENT' || /docker: .*not found/i.test(message))
+            throw new Error('tug deploy needs Docker with the Compose plugin on this machine to read the Compose file (`docker compose config`); install Docker Compose v2', { cause: error });
+        throw new Error(`docker compose config failed for ${file}: ${message}`, { cause: error });
+    }
+    let parsed;
+    try {
+        parsed = yaml.load(result.out);
+    }
+    catch (error) {
+        throw new Error(`docker compose config returned unreadable output for ${file}`, {
+            cause: error,
+        });
+    }
+    if (!parsed || typeof parsed !== 'object' || !parsed.services)
+        throw new Error(`${file} defines no services`);
+    return parsed;
+}

package/dist/protocol.d.ts

@@ -421,6 +421,10 @@ export interface StatusReport {
     ingress?: IngressStatus;
 }
 export interface SecretsResponse {
+    registryAuth?: Record<string, {
+        username: string;
+        password: string;
+    }>;
     secrets: Record<string, string>;
     versions: Record<string, number>;
 }
@@ -447,7 +451,6 @@ export interface NodeInfo {
 export interface RegisterResponse {
     node: NodeInfo;
     agentToken: string;
-    registryToken: string;
 }
 /** Body of `POST /agent/v1/enroll`. */
 export interface EnrollRequest {
@@ -461,16 +464,13 @@ export interface EnrollRequest {
     labels?: string[];
 }
 /** Response of `POST /agent/v1/enroll`. */
-export interface EnrollResponse extends RegisterResponse {
-    registry: string;
-}
+export type EnrollResponse = RegisterResponse;
 /** Response of `POST /api/v1/nodes/enroll-token`. */
 export interface EnrollTokenResponse {
     enrollToken: string;
     name: string | null;
     usesRemaining: number;
     expiresAt: string;
-    registry: string;
 }
 /** The base64url-encoded JSON payload of TUG_JOIN_KEY. */
 export interface JoinKey {

package/dist/reconciler.d.ts

@@ -0,0 +1,89 @@
+import type { ContainerSpec, NodePlan } from './container-spec.ts';
+import type { ContainerSummary, DockerApi } from './docker-api.ts';
+import type { DesiredState, ServiceState } from './protocol.ts';
+export declare const DEFAULT_DRAIN = "30s";
+export declare const DEFAULT_PORT_RANGE = "20000-20999";
+export declare function parsePortRange(value: string | undefined): [number, number];
+/** Containers of a project (or one service of it), running or not. */
+export declare function projectContainers(docker: DockerApi, project: string, service?: string): Promise<ContainerSummary[]>;
+/** Creates the project network and its volumes; volumes are never removed by reconciliation. */
+export declare function ensureProjectResources(docker: DockerApi, project: string, plan: NodePlan): Promise<void>;
+/** Makes sure a service's container exists for `spec`, running, replacing any container with another hash. */
+export declare function ensureContainer(docker: DockerApi, spec: ContainerSpec, existing: ContainerSummary[]): Promise<string>;
+/** Removes containers of services that are not placed here any more (never volumes). */
+export declare function sweepOrphans(docker: DockerApi, project: string, keep: ReadonlySet<string>): Promise<void>;
+interface Revision {
+    hash: string;
+    /** Container name. */
+    container: string;
+    port: number;
+    image?: string;
+}
+interface Draining {
+    service: string;
+    container: string;
+    drainUntil: number;
+}
+interface ServiceRollout {
+    active?: Revision;
+    candidate?: Revision;
+    draining: Draining[];
+    failed?: {
+        key: string;
+        error: string;
+    };
+}
+export interface RolloutState {
+    services: Record<string, ServiceRollout>;
+}
+export declare function loadRolloutState(dir: string): Promise<RolloutState>;
+export interface ReconcileDeps {
+    /**
+     * Throws until the container is healthy under the service's `x-tug.health`. With `dependency` a
+     * service without one is held to its Docker `HEALTHCHECK`, which is what `service_healthy` means.
+     */
+    waitForHealth: (container: string, service: string, dependency?: boolean) => Promise<void>;
+    ensureArtifact: (service: string) => Promise<void>;
+    /** Pulls the service's published image. */
+    pullImage: (service: string) => Promise<void>;
+    emit: (service: string, event: string, message?: string) => Promise<unknown>;
+}
+export interface ReconcileInput {
+    desired: DesiredState;
+    /** The project's state directory (`rollout.json`). */
+    dir: string;
+    plan: NodePlan;
+    nodeName: string;
+    rolloutPorts?: string;
+    deps: ReconcileDeps;
+}
+/**
+ * Recreate-strategy services: pull, then create/start each in dependency order. `onStarting` and
+ * `onHealth` let the agent report per-service state; a failure throws with the service named in
+ * `failing`.
+ */
+export declare function reconcileRecreate(docker: DockerApi, input: ReconcileInput & {
+    services: string[];
+    pull: boolean;
+}, report: {
+    pulling: (service: string) => Promise<void>;
+    starting: (service: string) => Promise<void>;
+    checking: (service: string) => Promise<void>;
+    serving: (service: string) => Promise<void>;
+    failing: (service: string | undefined) => void;
+}): Promise<void>;
+export interface RollingResult {
+    /** Reported state of every rolling service; the serving revision keeps `serving` while a newer one is rejected. */
+    serviceStates: Record<string, ServiceState>;
+    /** Services whose newest revision failed (or whose current revision is unhealthy), with the reason. */
+    errors: Record<string, string>;
+}
+export declare const isRollingService: (desired: DesiredState, service: string) => boolean;
+export declare function reconcileRolling(docker: DockerApi, input: ReconcileInput & {
+    services: string[];
+}): Promise<RollingResult>;
+/** Drops rolling state of a project that has no rolling service on this node any more. */
+export declare function clearRolloutState(dir: string): Promise<void>;
+/** Reported states for rolling services whose current revision is still serving, used when another part of the reconcile failed. */
+export declare function activeRollingStates(dir: string, services: string[]): Promise<Record<string, ServiceState>>;
+export {};