@stage-labs/metro 0.1.0-beta.13 → 0.1.0-beta.15

package/dist/trains/protocol.js

@@ -0,0 +1,99 @@
+/** Train stdin/stdout protocol: line buffering + JSON envelope + outbound call bookkeeping. */
+import { readdirSync, statSync } from 'node:fs';
+import { join, parse as parsePath } from 'node:path';
+import { errMsg, log } from '../log.js';
+export const CALL_TIMEOUT_MS = 60_000;
+export const STDOUT_LINE_MAX = 4 * 1024 * 1024; // 4 MiB safeguard per line
+/** Classify a single parsed stdout line from a train. */
+export function parseTrainLine(name, line) {
+    let msg;
+    try {
+        msg = JSON.parse(line);
+    }
+    catch (err) {
+        log.warn({ name, err: errMsg(err), line: line.slice(0, 200) }, 'train: bad JSON');
+        return null;
+    }
+    if (msg.op === 'response') {
+        if (typeof msg.id !== 'string')
+            return { op: 'ignore' };
+        return { op: 'response', id: msg.id, result: msg.result, error: msg.error };
+    }
+    if (msg.op === 'log')
+        return { op: 'log', text: msg.text };
+    /** Anything without an `op` (or with `op:"event"`) is an inbound event. */
+    /** Defensive shape check: every event needs `line` (string). Log + drop if missing. */
+    if (typeof msg.line !== 'string') {
+        log.warn({ name, line: line.slice(0, 200) }, 'train: event missing `line` (string) — dropped');
+        return { op: 'ignore' };
+    }
+    return { op: 'event', event: msg };
+}
+/** Consume complete `\n`-terminated lines from a rolling buffer; invoke `onLine` for each. */
+/** Returns the leftover (incomplete) buffer to keep accumulating. */
+export function drainLines(name, buf, onLine) {
+    if (buf.length > STDOUT_LINE_MAX && !buf.includes('\n')) {
+        log.warn({ name, bytes: buf.length }, 'train: dropping oversized stdout line');
+        return '';
+    }
+    let nl;
+    while ((nl = buf.indexOf('\n')) !== -1) {
+        const line = buf.slice(0, nl).trim();
+        buf = buf.slice(nl + 1);
+        if (!line)
+            continue;
+        onLine(line);
+    }
+    return buf;
+}
+export function encodeCall(id, action, args) {
+    return JSON.stringify({ op: 'call', id, action, args }) + '\n';
+}
+function isTrainFile(name) {
+    return /\.(ts|js|mjs)$/.test(name) && !name.startsWith('_') && !name.startsWith('.');
+}
+/** Discover trains under `dir`: regular files with allowed extensions, no `_` / `.` prefix. */
+export function listTrainFiles(dir) {
+    return readdirSync(dir).filter(isTrainFile)
+        .map(f => ({ name: parsePath(f).name, path: join(dir, f) }))
+        .filter(t => { try {
+        return statSync(t.path).isFile();
+    }
+    catch {
+        return false;
+    } });
+}
+/* ──────────── outbound call bookkeeping (per-train pending map + timeout) ──────────── */
+export function mintCallId(seq) {
+    return `req_${seq}_${Math.random().toString(36).slice(2, 8)}`;
+}
+/** Reject + clear all pending calls on a train (used on shutdown/exit). */
+export function failAllPending(pending, reason) {
+    for (const p of pending.values()) {
+        clearTimeout(p.timer);
+        p.reject(new Error(reason));
+    }
+    pending.clear();
+}
+/** Dispatch `action` to a train's stdin and register a pending entry; returns the awaitable. */
+export function sendCall(t, id, action, args) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            t.pending.delete(id);
+            reject(new Error(`train '${t.name}' call '${action}' timed out after ${CALL_TIMEOUT_MS}ms`));
+        }, CALL_TIMEOUT_MS);
+        t.pending.set(id, { resolve, reject, timer });
+        try {
+            const stdin = t.proc?.stdin;
+            if (!stdin || typeof stdin === 'number')
+                throw new Error('stdin not piped');
+            stdin.write(encodeCall(id, action, args));
+            stdin.flush();
+        }
+        catch (err) {
+            clearTimeout(timer);
+            t.pending.delete(id);
+            reject(new Error(`train '${t.name}' stdin write failed: ${errMsg(err)}`));
+        }
+    });
+}

package/dist/trains/supervisor.js

@@ -0,0 +1,210 @@
+/** Train supervisor: spawn `~/.metro/trains/*.{ts,js,mjs}` under `bun run`, multiplex their */
+/** stdout (events + call-responses), route outbound calls to their stdin. Pure transport. */
+import { mkdirSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { errMsg, log } from '../log.js';
+import { userSelf } from '../history.js';
+import { drainLines, failAllPending, listTrainFiles, mintCallId, parseTrainLine, sendCall, } from './protocol.js';
+const RESTART_BACKOFFS_MS = [1_000, 5_000, 30_000];
+const MAX_CONSECUTIVE_FAILS = 5;
+export const TRAINS_DIR = process.env.METRO_TRAINS_DIR ?? join(homedir(), '.metro', 'trains');
+export class TrainSupervisor {
+    dir;
+    trains = new Map();
+    onEvent = null;
+    nextCallId = 1;
+    constructor(dir = TRAINS_DIR) {
+        this.dir = dir;
+    }
+    onTrainEvent(handler) { this.onEvent = handler; }
+    /** Discover trains under `dir` and spawn one subprocess per file. Creates the dir if missing. */
+    start() {
+        mkdirSync(this.dir, { recursive: true });
+        for (const t of listTrainFiles(this.dir))
+            this.startTrain(t.name, t.path);
+        log.info({ dir: this.dir, count: this.trains.size }, 'train supervisor: started');
+    }
+    /** Shut everything down (graceful: send SIGTERM, then SIGKILL after grace period). */
+    async stop() {
+        const tasks = [];
+        for (const t of this.trains.values()) {
+            t.stopped = true;
+            if (t.restartTimer) {
+                clearTimeout(t.restartTimer);
+                t.restartTimer = null;
+            }
+            failAllPending(t.pending, 'train shutting down');
+            const wait = killGracefully(t.proc);
+            if (wait)
+                tasks.push(wait);
+        }
+        await Promise.all(tasks);
+    }
+    list() {
+        return [...this.trains.values()].map(t => ({
+            name: t.name, path: t.path, running: !!(t.proc && t.proc.exitCode === null),
+            pid: t.proc?.pid ?? null, startedAt: t.startedAt, failCount: t.failCount,
+        }));
+    }
+    /** Kill + respawn a named train; resets fail counter so backoff starts fresh. */
+    async restart(name) {
+        const t = this.trains.get(name);
+        if (!t)
+            throw new Error(`no train named '${name}'`);
+        if (t.restartTimer) {
+            clearTimeout(t.restartTimer);
+            t.restartTimer = null;
+        }
+        failAllPending(t.pending, `train '${name}' restarting`);
+        t.stopped = true;
+        const wait = killGracefully(t.proc);
+        if (wait)
+            await wait;
+        t.stopped = false;
+        t.failCount = 0;
+        t.proc = null;
+        this.spawn(t);
+    }
+    /** Send a call to a named train and await the matching response. */
+    async call(name, action, args) {
+        const t = this.trains.get(name);
+        if (!t)
+            throw new Error(`no train named '${name}' (have: ${[...this.trains.keys()].join(', ') || '(none)'})`);
+        if (!t.proc || t.proc.exitCode !== null)
+            throw new Error(`train '${name}' is not running`);
+        return sendCall(t, mintCallId(this.nextCallId++), action, args);
+    }
+    startTrain(name, path) {
+        if (this.trains.has(name)) {
+            log.warn({ name }, 'train supervisor: duplicate name, skipping');
+            return;
+        }
+        const state = {
+            name, path, proc: null, pending: new Map(), buf: '', errBuf: '',
+            failCount: 0, restartTimer: null, startedAt: null, stopped: false,
+        };
+        this.trains.set(name, state);
+        this.spawn(state);
+    }
+    spawn(state) {
+        if (state.stopped)
+            return;
+        try {
+            const proc = Bun.spawn(['bun', 'run', state.path], {
+                stdin: 'pipe', stdout: 'pipe', stderr: 'pipe',
+                env: { ...process.env, METRO_TRAIN_NAME: state.name, METRO_SELF_URI: userSelf() },
+            });
+            state.proc = proc;
+            state.startedAt = new Date().toISOString();
+            state.buf = '';
+            state.errBuf = '';
+            log.info({ name: state.name, pid: proc.pid }, 'train: spawned');
+            void this.pumpStdout(state);
+            void this.pumpStderr(state);
+            void proc.exited.then(code => this.onExit(state, code ?? 0));
+        }
+        catch (err) {
+            log.warn({ name: state.name, err: errMsg(err) }, 'train: spawn failed');
+            this.scheduleRestart(state);
+        }
+    }
+    pumpStdout(state) {
+        return pumpStream(state.proc?.stdout, state.name, 'stdout', chunk => {
+            state.buf += chunk;
+            state.buf = drainLines(state.name, state.buf, line => this.handleLine(state, line));
+        });
+    }
+    /** Stream train stderr line-by-line into the daemon's logger so users see crashes/warnings. */
+    pumpStderr(state) {
+        return pumpStream(state.proc?.stderr, state.name, 'stderr', chunk => {
+            state.errBuf += chunk;
+            let nl;
+            while ((nl = state.errBuf.indexOf('\n')) !== -1) {
+                const line = state.errBuf.slice(0, nl).trimEnd();
+                state.errBuf = state.errBuf.slice(nl + 1);
+                if (line)
+                    log.warn({ train: state.name }, line);
+            }
+        });
+    }
+    handleLine(state, line) {
+        const msg = parseTrainLine(state.name, line);
+        if (!msg || msg.op === 'ignore')
+            return;
+        if (msg.op === 'response') {
+            const pending = state.pending.get(msg.id);
+            if (!pending) {
+                log.debug({ name: state.name, id: msg.id }, 'train: stale response');
+                return;
+            }
+            state.pending.delete(msg.id);
+            clearTimeout(pending.timer);
+            pending.resolve({ result: msg.result, error: msg.error });
+            return;
+        }
+        if (msg.op === 'log') {
+            log.info({ name: state.name, msg: msg.text }, 'train log');
+            return;
+        }
+        this.onEvent?.(msg.event, state.name);
+    }
+    onExit(state, code) {
+        log.warn({ name: state.name, code }, 'train: exited');
+        state.proc = null;
+        state.startedAt = null;
+        failAllPending(state.pending, `train '${state.name}' exited (code=${code}) before responding`);
+        if (state.stopped)
+            return;
+        state.failCount++;
+        if (state.failCount >= MAX_CONSECUTIVE_FAILS) {
+            log.error({ name: state.name, fails: state.failCount }, 'train: too many consecutive failures, giving up (restart metro to retry)');
+            return;
+        }
+        this.scheduleRestart(state);
+    }
+    scheduleRestart(state) {
+        if (state.stopped)
+            return;
+        const idx = Math.min(state.failCount, RESTART_BACKOFFS_MS.length - 1);
+        const delay = RESTART_BACKOFFS_MS[idx];
+        log.info({ name: state.name, delay, attempt: state.failCount }, 'train: restart scheduled');
+        state.restartTimer = setTimeout(() => {
+            state.restartTimer = null;
+            /** Any subprocess that survives 30s resets its consecutive-fail counter. */
+            this.spawn(state);
+            setTimeout(() => { if (state.proc && state.proc.exitCode === null)
+                state.failCount = 0; }, 30_000);
+        }, delay);
+    }
+}
+function killGracefully(proc) {
+    if (!proc || proc.exitCode !== null)
+        return null;
+    try {
+        proc.kill('SIGTERM');
+    }
+    catch { /* ignore */ }
+    const grace = setTimeout(() => { try {
+        proc.kill('SIGKILL');
+    }
+    catch { /* ignore */ } }, 2_000);
+    return proc.exited.finally(() => clearTimeout(grace));
+}
+async function pumpStream(stream, name, kind, onChunk) {
+    if (!stream || typeof stream === 'number')
+        return;
+    const reader = stream.getReader();
+    const dec = new TextDecoder();
+    try {
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            onChunk(dec.decode(value, { stream: true }));
+        }
+    }
+    catch (err) {
+        log.debug({ name, err: errMsg(err) }, `train: ${kind} pump ended`);
+    }
+}

package/dist/tunnel.js

@@ -1,5 +1,6 @@
-/** Cloudflared tunnel manager. Prefers token-from-env so a missing local credentials JSON does not block startup. */
+/** Cloudflared tunnel manager + webhook endpoint registry. Both persist tiny JSON files in STATE_DIR. */
 import { spawn, spawnSync } from 'node:child_process';
+import { randomBytes } from 'node:crypto';
 import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { STATE_DIR } from './paths.js';
@@ -62,3 +63,40 @@ export class Tunnel {
         this.child = null;
     }
 }
+/* ──────────── webhook endpoint registry (id, label, optional secret) ──────────── */
+const WEBHOOKS_FILE = join(STATE_DIR, 'webhooks.json');
+/** Local listener port — `127.0.0.1` only; expose publicly via Cloudflare tunnel. */
+export const webhookPort = () => Number(process.env.METRO_WEBHOOK_PORT) || 8420;
+function readWebhooks() {
+    if (!existsSync(WEBHOOKS_FILE))
+        return { endpoints: [] };
+    try {
+        return JSON.parse(readFileSync(WEBHOOKS_FILE, 'utf8'));
+    }
+    catch {
+        return { endpoints: [] };
+    }
+}
+const writeWebhooks = (s) => writeFileSync(WEBHOOKS_FILE, JSON.stringify(s, null, 2));
+export const listEndpoints = () => readWebhooks().endpoints;
+export const findEndpoint = (id) => readWebhooks().endpoints.find(e => e.id === id);
+export function addEndpoint(label, secret) {
+    const s = readWebhooks();
+    /** 16-char URL-safe id (~96 bits — collision-proof for any reasonable count). */
+    const ep = {
+        id: randomBytes(12).toString('base64url'), label, createdAt: new Date().toISOString(),
+        ...(secret ? { secret } : {}),
+    };
+    s.endpoints.push(ep);
+    writeWebhooks(s);
+    return ep;
+}
+export function removeEndpoint(id) {
+    const s = readWebhooks();
+    const before = s.endpoints.length;
+    s.endpoints = s.endpoints.filter(e => e.id !== id);
+    if (s.endpoints.length === before)
+        return false;
+    writeWebhooks(s);
+    return true;
+}