@wpelevator/tug 0.2.0 → 0.2.1

package/README.md

@@ -47,6 +47,7 @@ The CLI installs the matching `ghcr.io/wpelevator/tug:<version>` image on your h
 - [Configuration](docs/configuration.md): CLI, control and agent environment variables.
 - [Development](docs/development.md): local setup, CI, Compose stack, host mode and the integration test.
 - [Testing](docs/testing.md): test layers, the numbers rollouts are held to, and the backlog of mocked tests.
+- [Observability](docs/observability.md): what can be observed today, and the proposed logs, metrics and alerting.
 - [Releases](docs/releases.md): changesets, npm and Docker image publishing.
 - [Standalone binary](docs/standalone-binary.md): proposed Bun-compiled CLI binaries and Homebrew distribution.
 - [Roadmap](docs/roadmap.md): planned milestones toward parity with and beyond uncloud.

package/dist/agent.js

@@ -7,6 +7,7 @@ import { validateImageArchive, verifyDownload } from "./artifacts.js";
 import path from 'node:path';
 import { spawn, spawnSync } from 'node:child_process';
 import net from 'node:net';
+import { createHash } from 'node:crypto';
 import { composeForNode, dumpYaml, isRolling, serviceSecretVersions } from "./compose.js";
 import { activeRollingStates, composeBase, hasLegacyContainers, reconcileRolling, retireAll } from "./rollout.js";
 import { ensureDir, errMessage, now, parseDuration } from "./util.js";
@@ -279,7 +280,7 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
         return { project: desiredState.project, observedVersion: desiredState.version, ...(desiredState.revision ? { observedRevision: desiredState.revision } : {}), health: 'skipped', services: [], serviceStates: {} };
     }
     const serviceStates = Object.fromEntries(serviceNames.map(service => [service, { rolloutState: 'pending', image: desiredState.services[service]?.image }]));
-    const emit = (service, rolloutState, message = rolloutState) => api(ctx, '/agent/v1/events', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ projectId: desiredState.project, level: 'info', type: `service.${rolloutState}`, message, metadata: { service, version: desiredState.version, node: ctx.nodeName } }) }).catch(() => { });
+    const emit = (service, rolloutState, message = rolloutState, level = 'info') => api(ctx, '/agent/v1/events', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ projectId: desiredState.project, level, type: `service.${rolloutState}`, message, metadata: { service, version: desiredState.version, node: ctx.nodeName } }) }).catch(() => { });
     const sec = await api(ctx, `/agent/v1/secrets?project=${encodeURIComponent(desiredState.project)}&services=${encodeURIComponent(serviceNames.join(','))}`);
     const currentSecretVersions = serviceSecretVersions(desiredState.compose, serviceNames, sec.versions || {});
     const db = await agentDb(ctx.stateDir);
@@ -295,11 +296,20 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
     await fs.writeFile(path.join(dir, 'docker-compose.yml'), dumpYaml(nodeCompose));
     await writeEnv(path.join(dir, '.env'), sec.secrets || {});
     const failedStatus = (error) => ({ project: desiredState.project, observedVersion: desiredState.version, ...(desiredState.revision ? { observedRevision: desiredState.revision } : {}), health: 'failed', services: serviceNames, serviceStates, error });
+    // Lifecycle events say what changed, not that the agent is still polling. They are sent when the desired revision,
+    // this node's rendered configuration or the secrets change, when an apply fails (once) and when it recovers (once).
+    const eventFingerprint = createHash('sha256').update(JSON.stringify([desiredState.revision || desiredState.version, dumpYaml(nodeCompose), mainServices.map(service => currentSecretVersions[service] || {})])).digest('hex').slice(0, 16);
+    const knownEvents = mainServices.length ? await db.get('SELECT fingerprint, phase FROM event_state WHERE project = ?', desiredState.project) : undefined;
+    const announce = knownEvents?.fingerprint !== eventFingerprint;
+    const wasFailed = !announce && knownEvents?.phase === 'failed';
+    const markEvents = (phase) => db.run('INSERT INTO event_state(project, fingerprint, phase) VALUES(?,?,?) ON CONFLICT(project) DO UPDATE SET fingerprint=excluded.fingerprint, phase=excluded.phase', desiredState.project, eventFingerprint, phase);
+    let failing;
     if (ctx.apply) {
         try {
             for (const service of mainServices) {
                 serviceStates[service].rolloutState = 'pulling';
-                await emit(service, 'pulling');
+                if (announce)
+                    await emit(service, 'pulling');
             }
             for (const service of mainServices) {
                 const artifact = desiredState.services[service].artifact;
@@ -311,7 +321,8 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
                 await ctx.docker(['compose', '-p', desiredState.project, 'pull', ...(pullServices.length === mainServices.length ? [] : pullServices)], { cwd: dir });
             for (const service of mainServices) {
                 serviceStates[service].rolloutState = 'starting';
-                await emit(service, 'starting');
+                if (announce)
+                    await emit(service, 'starting');
             }
             if (mainServices.length) {
                 // Services that just became rolling still run here until their replacement takes over; keep them.
@@ -325,14 +336,25 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
             }
             for (const service of mainServices) {
                 serviceStates[service].rolloutState = 'health_checking';
-                await emit(service, 'health_checking');
+                if (announce)
+                    await emit(service, 'health_checking');
+                failing = service;
                 await waitForHealth(ctx, desiredState.project, service, desiredState.services[service]?.health, dir);
+                failing = undefined;
                 serviceStates[service].rolloutState = 'serving';
                 serviceStates[service].healthyAt = now();
-                await emit(service, 'serving');
+                if (announce || wasFailed)
+                    await emit(service, 'serving', wasFailed ? 'serving again after a failure' : 'serving');
             }
+            if (mainServices.length && (announce || wasFailed))
+                await markEvents('serving');
         }
         catch (e) {
+            if (mainServices.length && !wasFailed) {
+                // Announce a failure once; retries of the same failure stay quiet until it recovers or the inputs change.
+                await emit(failing ?? mainServices.join(', '), 'failed', errMessage(e), 'error');
+                await markEvents('failed');
+            }
             // Rolling services are untouched by a failure here, so keep reporting their current revision as serving.
             if (!rolling.length)
                 throw e;

package/dist/cli.js

@@ -311,7 +311,7 @@ program.command('events').description('list recent events with the token that ca
     const query = `limit=${encodeURIComponent(o.limit)}${project ? `&project=${encodeURIComponent(project)}` : ''}`;
     console.log(JSON.stringify(await request('GET', `/api/v1/events?${query}`), null, 2));
 });
-program.command('doctor').description('run local and control-plane diagnostics').option('--nodes', 'also queue diagnostics on registered agents').option('--node <name>', 'specific node for agent diagnostics').action(async (o) => {
+program.command('doctor').description('run local and control-plane diagnostics').option('--nodes', 'also run diagnostics on every registered agent and wait').option('--node <name>', 'run diagnostics on one agent and wait').option('--timeout <seconds>', 'wait timeout for node diagnostics', '60').action(async (o) => {
     const local = [];
     for (const [name, cmd, args] of [['node', 'node', ['--version']], ['docker', 'docker', ['version', '--format', '{{.Server.Version}}']], ['docker_compose', 'docker', ['compose', 'version']]]) {
         try {
@@ -323,8 +323,17 @@ program.command('doctor').description('run local and control-plane diagnostics')
     }
     const controlDiagnostics = await request('GET', '/api/v1/diagnostics');
     const output = { ok: local.every(c => c.ok) && controlDiagnostics.ok, local, control: controlDiagnostics };
-    if (o.nodes || o.node)
-        output.nodeDiagnostics = await request('POST', '/api/v1/diagnostics/nodes', { node: o.node || null });
+    if (o.nodes || o.node) {
+        const queued = await request('POST', '/api/v1/diagnostics/nodes', { node: o.node || null });
+        const commands = [];
+        for (const command of queued.commands) {
+            const finished = await waitForCommand(command.commandId, Number(o.timeout));
+            commands.push({ nodeId: command.nodeId, state: finished.state, result: JSON.parse(finished.result_json || '{}') });
+        }
+        output.nodeDiagnostics = { commands };
+        if (commands.some(command => command.state !== 'succeeded'))
+            output.ok = false;
+    }
     console.log(JSON.stringify(output, null, 2));
 });
 program.command('rollback').option('--force').option('--no-wait', 'return immediately after activating rollback').option('--timeout <seconds>', 'wait timeout', '300').action(async (o) => {

package/dist/control.d.ts

@@ -19,6 +19,7 @@ export interface ControlImageOperations {
 }
 export declare function createApp(options?: {
     images?: Partial<ControlImageOperations>;
-    openDatabase?: typeof openDb;
+    openDatabase?: typeof openDb; /** Minimum time between event prunes triggered by agent reports; default one hour. */
+    eventPruneIntervalMs?: number;
 }): Promise<import("express-serve-static-core").Express>;
 export declare function listen(app: express.Express, port?: number): Promise<import('node:http').Server | https.Server>;

package/dist/control.js

@@ -21,6 +21,33 @@ import { buildIngressWork, resolveServiceEndpoints } from "./ingress.js";
 import { ACTIVE_DEPLOYMENT_STATES } from "./protocol.js";
 import { AUTO_ROLLBACK_ACTOR, ROUTE_ROLES, assertProject, bodyProject, createGuard, createToken, listTokens, parseTokenRequest, queryProject, recordEvent, revokeToken } from "./tokens.js";
 const httpError = (status, message) => Object.assign(new Error(message), { status });
+const missingProject = (project) => `project ${project} has no desired state`;
+const missingService = (project, service) => `service ${service} not found in project ${project}`;
+/** Collapse a node report into the deployment error `tug deploy` prints. Compose failures are multi-line; keep the reason, not the progress log. */
+function deploymentFailureMessage(rows) {
+    const limit = 500;
+    const oneLine = (value) => {
+        const flat = value.replace(/\s+/g, ' ').trim();
+        return flat.length > limit ? `${flat.slice(0, limit)}...` : flat;
+    };
+    const details = rows.map(row => {
+        let status = {};
+        try {
+            status = JSON.parse(row.status_json);
+        }
+        catch { /* a bad report still names the node */ }
+        const parts = [];
+        if (typeof status.error === 'string' && status.error.trim())
+            parts.push(oneLine(status.error));
+        for (const [service, state] of Object.entries(status.serviceStates || {})) {
+            if (typeof state?.rolloutError === 'string' && state.rolloutError.trim())
+                parts.push(`${service}: ${oneLine(state.rolloutError)}`);
+        }
+        const detail = parts.join('; ');
+        return detail ? `${row.node_id}: ${detail}` : row.node_id;
+    });
+    return `one or more assigned nodes reported failed health: ${details.join('; ')}`;
+}
 /** Versions of rollback deployments start with this; `tug deploy` may not use it, which is how automatic rollback tells them apart. */
 const ROLLBACK_VERSION_PREFIX = 'rollback-';
 const ACTIVE_STATES_SQL = ACTIVE_DEPLOYMENT_STATES.map(s => `'${s}'`).join(',');
@@ -90,6 +117,34 @@ export async function createApp(options = {}) {
         await reloadRegistry();
     }
     const db = await (options.openDatabase || openDb)(dbFile);
+    // Events are an operator-facing log, not a record of truth, so keep them bounded. `0` turns a limit off; anything that is
+    // not a whole number falls back to the default.
+    const envCount = (name, fallback) => {
+        const raw = process.env[name];
+        if (raw === undefined || raw === '')
+            return fallback;
+        const value = Number(raw);
+        if (Number.isInteger(value) && value >= 0)
+            return value;
+        console.warn(`${name}=${JSON.stringify(raw)} is not a whole number; using ${fallback}`);
+        return fallback;
+    };
+    const EVENT_RETENTION_DAYS = envCount('TUG_EVENT_RETENTION_DAYS', 30);
+    const EVENT_MAX_ROWS = envCount('TUG_EVENT_MAX_ROWS', 100_000);
+    const eventPruneIntervalMs = options.eventPruneIntervalMs ?? 60 * 60 * 1000;
+    let lastEventPrune = Date.now();
+    async function pruneEvents() {
+        lastEventPrune = Date.now();
+        let removed = 0;
+        if (EVENT_RETENTION_DAYS > 0)
+            removed += (await db.run('DELETE FROM events WHERE created_at < ?', new Date(Date.now() - EVENT_RETENTION_DAYS * 86_400_000).toISOString())).changes ?? 0;
+        if (EVENT_MAX_ROWS > 0)
+            removed += (await db.run('DELETE FROM events WHERE id IN (SELECT id FROM events ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?)', EVENT_MAX_ROWS)).changes ?? 0;
+        if (removed)
+            console.log(`removed ${removed} old events (kept ${EVENT_RETENTION_DAYS || 'any'} days, at most ${EVENT_MAX_ROWS || 'unlimited'} rows)`);
+        return removed;
+    }
+    await pruneEvents().catch(e => console.error(`event pruning failed: ${errMessage(e)}`));
     const masterKey = await loadOrCreateMasterKey(masterKeyFile);
     const operator = await db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash');
     if (!operator) {
@@ -184,9 +239,9 @@ export async function createApp(options = {}) {
         const statuses = await db.all('SELECT node_id,observed_version,health,status_json FROM node_statuses WHERE project_id=?', projectId);
         const byNode = new Map(statuses.map(s => [s.node_id, s]));
         const reportsRevision = (node) => state.revision ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision : byNode.get(node)?.observed_version === state.version;
-        const failed = [...required].some(node => reportsRevision(node) && byNode.get(node)?.health === 'failed');
-        if (failed) {
-            const marked = await db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=? AND state=?', 'failed', 'one or more assigned nodes reported failed health', now(), project.current_deployment_id, 'rolling_out');
+        const failedNodes = [...required].filter(node => reportsRevision(node) && byNode.get(node)?.health === 'failed').map(node => byNode.get(node));
+        if (failedNodes.length) {
+            const marked = await db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=? AND state=?', 'failed', deploymentFailureMessage(failedNodes), now(), project.current_deployment_id, 'rolling_out');
             if (marked.changes === 1)
                 scheduleAutoRollback(projectId, project.current_deployment_id, state);
         }
@@ -877,11 +932,11 @@ export async function createApp(options = {}) {
             throw httpError(400, 'service required');
         const ds = await currentDesired(project);
         if (!ds)
-            throw httpError(404, 'project has no desired state');
+            throw httpError(404, missingProject(project));
         const state = JSON.parse(ds.state_json);
         const svc = state.services?.[service];
         if (!svc)
-            throw httpError(404, 'service not found');
+            throw httpError(404, missingService(project, service));
         const nodeName = svc.placement?.nodes?.[0];
         if (!nodeName)
             throw httpError(400, 'service has no explicit node placement');
@@ -912,11 +967,11 @@ export async function createApp(options = {}) {
             return res.status(400).json({ error: 'service required' });
         const ds = await currentDesired(project);
         if (!ds)
-            return res.status(404).json({ error: 'project has no desired state' });
+            return res.status(404).json({ error: missingProject(project) });
         const state = JSON.parse(ds.state_json);
         const svc = state.services?.[service];
         if (!svc)
-            return res.status(404).json({ error: 'service not found' });
+            return res.status(404).json({ error: missingService(project, service) });
         const nodeName = svc.placement?.nodes?.[0];
         if (!nodeName)
             return res.status(400).json({ error: 'service has no explicit node placement' });
@@ -986,6 +1041,8 @@ export async function createApp(options = {}) {
     });
     api('post', '/api/v1/diagnostics/nodes', null, async (req, res) => {
         const nodes = req.body.node ? await db.all('SELECT id FROM nodes WHERE id=? OR name=?', req.body.node, req.body.node) : await db.all('SELECT id FROM nodes');
+        if (req.body.node && !nodes.length)
+            return res.status(404).json({ error: `node ${req.body.node} not found` });
         const ts = now();
         const commands = [];
         for (const node of nodes) {
@@ -1069,8 +1126,14 @@ export async function createApp(options = {}) {
         await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', ts, 'online', req.node.id);
         res.json({ ok: true });
     });
-    post('/agent/v1/events', authAgent, async (req, res) => { const evs = req.body.events || [req.body]; for (const ev of evs)
-        await db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at) VALUES(?,?,?,?,?,?,?,?,?)', id('evt'), ev.projectId || null, ev.deploymentId || null, req.node.id, ev.level || 'info', ev.type || 'event', ev.message || '', JSON.stringify(ev.metadata || {}), now()); res.json({ ok: true }); });
+    post('/agent/v1/events', authAgent, async (req, res) => {
+        const evs = req.body.events || [req.body];
+        for (const ev of evs)
+            await db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at) VALUES(?,?,?,?,?,?,?,?,?)', id('evt'), ev.projectId || null, ev.deploymentId || null, req.node.id, ev.level || 'info', ev.type || 'event', ev.message || '', JSON.stringify(ev.metadata || {}), now());
+        if (Date.now() - lastEventPrune >= eventPruneIntervalMs)
+            await pruneEvents().catch(e => console.error(`event pruning failed: ${errMessage(e)}`));
+        res.json({ ok: true });
+    });
     get('/agent/v1/secrets', authAgent, async (req, res) => {
         const project = String(req.query.project || 'site');
         let rows = await db.all('SELECT name,encrypted_value,version FROM secrets WHERE project_id=?', project);

package/dist/db.d.ts

@@ -105,6 +105,12 @@ export interface LockRow {
     expires_at: string;
     created_at: string;
 }
+/** What the agent last announced for a project's recreated services, so unchanged polls stay silent. */
+export interface EventStateRow {
+    project: string;
+    fingerprint: string;
+    phase: 'serving' | 'failed';
+}
 export interface AppliedStateRow {
     project: string;
     version: string;

package/dist/db.js

@@ -22,6 +22,11 @@ CREATE TABLE IF NOT EXISTS applied_state(
   service_states_json TEXT,
   secret_versions_json TEXT
 );
+CREATE TABLE IF NOT EXISTS event_state(
+  project TEXT PRIMARY KEY,
+  fingerprint TEXT NOT NULL,
+  phase TEXT NOT NULL
+);
 `);
     return db;
 }
@@ -43,6 +48,7 @@ CREATE TABLE IF NOT EXISTS commands(id TEXT PRIMARY KEY, node_id TEXT, project_i
 CREATE TABLE IF NOT EXISTS events(id TEXT PRIMARY KEY, project_id TEXT, deployment_id TEXT, node_id TEXT, level TEXT, type TEXT, message TEXT, metadata_json TEXT, created_at TEXT, actor TEXT);
 CREATE INDEX IF NOT EXISTS events_project_created ON events(project_id, created_at);
 CREATE TABLE IF NOT EXISTS api_tokens(id TEXT PRIMARY KEY, name TEXT UNIQUE NOT NULL, token_hash TEXT UNIQUE NOT NULL, role TEXT NOT NULL, projects_json TEXT, created_at TEXT, created_by TEXT, last_used_at TEXT, expires_at TEXT, revoked_at TEXT);
+CREATE INDEX IF NOT EXISTS events_by_time ON events(created_at);
 CREATE TABLE IF NOT EXISTS locks(name TEXT PRIMARY KEY, owner TEXT, expires_at TEXT, created_at TEXT);
 `);
     const row = await db.get('SELECT value FROM meta WHERE key = ?', 'schema_created_at');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wpelevator/tug",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Portable VPS deployments with Docker Compose",
   "repository": {
     "type": "git",