@platless/axom-cli 0.1.0

package/dist/workstation/connection/encryptedChannel.js

@@ -0,0 +1,261 @@
+import { WebSocket } from 'ws';
+import { open as openFrame, seal as sealFrame } from '../crypto.js';
+class AsyncMessageQueue {
+    values = [];
+    waiters = [];
+    ended = false;
+    [Symbol.asyncIterator]() { return this; }
+    next() {
+        const value = this.values.shift();
+        if (value !== undefined)
+            return Promise.resolve({ done: false, value });
+        if (this.ended)
+            return Promise.resolve({ done: true, value: undefined });
+        return new Promise((resolve) => this.waiters.push(resolve));
+    }
+    push(value) {
+        if (this.ended)
+            return;
+        const waiter = this.waiters.shift();
+        if (waiter)
+            waiter({ done: false, value });
+        else
+            this.values.push(value);
+    }
+    end() {
+        if (this.ended)
+            return;
+        this.ended = true;
+        for (const waiter of this.waiters.splice(0))
+            waiter({ done: true, value: undefined });
+    }
+}
+const defaultTimers = {
+    now: () => Date.now(),
+    setTimeout: (callback, delayMs) => setTimeout(callback, delayMs),
+    clearTimeout: (handle) => clearTimeout(handle),
+};
+const defaultCrypto = { seal: sealFrame, open: openFrame };
+/**
+ * Opens a raw WebSocket whose payloads are encrypted with the caller-provided session key.
+ *
+ * Authentication and domain handshakes deliberately remain outside this primitive. Management
+ * derives a long-term-key session after its nonce challenge; computer and agent derive a ticket
+ * key. Once a key exists, all three use this identical framing and Client-originated liveness.
+ */
+export function connectEncryptedChannel(options) {
+    const heartbeatMs = options.heartbeatMs ?? 10_000;
+    const missedAcks = options.missedAcks ?? 3;
+    // Comfortably past a connect, an upgrade and one handshake round trip on any link this daemon is
+    // expected to work over, and well inside the supervisor's own per-attempt bound, so a channel
+    // that is merely slow is never mistaken for one that is wedged.
+    const openTimeoutMs = options.openTimeoutMs ?? 15_000;
+    if (heartbeatMs <= 0 || missedAcks <= 0)
+        throw new RangeError('heartbeat settings must be positive');
+    if (openTimeoutMs <= 0)
+        throw new RangeError('openTimeoutMs must be positive');
+    const socket = (options.createWebSocket ?? ((url) => new WebSocket(url)))(options.url);
+    const crypto = options.crypto ?? defaultCrypto;
+    const timers = options.timers ?? defaultTimers;
+    const messages = new AsyncMessageQueue();
+    const rawMessages = new AsyncMessageQueue();
+    let sessionKey = options.sessionKey;
+    let state = 'OPEN';
+    let opened = false;
+    let settled = false;
+    let heartbeatTimer;
+    let openTimer;
+    let nextSequence = 1;
+    const outstandingAcks = new Set();
+    let resolveClosed;
+    let removeAbortListener = () => { };
+    const closed = new Promise((resolve) => { resolveClosed = resolve; });
+    const disarmOpenTimer = () => {
+        if (openTimer === undefined)
+            return;
+        timers.clearTimeout(openTimer);
+        openTimer = undefined;
+    };
+    const finish = () => {
+        if (settled)
+            return;
+        settled = true;
+        state = 'CLOSED';
+        if (heartbeatTimer !== undefined)
+            timers.clearTimeout(heartbeatTimer);
+        disarmOpenTimer();
+        messages.end();
+        rawMessages.end();
+        outstandingAcks.clear();
+        removeAbortListener();
+        resolveClosed();
+    };
+    const close = () => {
+        if (settled)
+            return;
+        // Finish before calling close: test doubles and some ws error paths synchronously emit close.
+        finish();
+        socket.close();
+    };
+    const send = (frame) => {
+        if (state !== 'OPEN' || !opened)
+            throw new Error('Encrypted channel is not open');
+        socket.send(crypto.seal(sessionKey, Buffer.from(options.encode(frame))));
+    };
+    const scheduleHeartbeat = () => {
+        heartbeatTimer = timers.setTimeout(() => {
+            if (settled)
+                return;
+            const sequence = nextSequence++;
+            // Register before send: a synchronous server acknowledgement/work frame emitted from
+            // `send` must observe and satisfy this precise liveness period.
+            outstandingAcks.add(sequence);
+            try {
+                send(options.heartbeat(sequence, timers.now()));
+            }
+            catch {
+                outstandingAcks.delete(sequence);
+                close();
+                return;
+            }
+            // `ws.send` can synchronously deliver close/error through a test double or a transport
+            // implementation. Never resurrect liveness state after that terminal callback.
+            if (settled)
+                return;
+            if (outstandingAcks.size >= missedAcks) {
+                close();
+                return;
+            }
+            scheduleHeartbeat();
+        }, heartbeatMs);
+    };
+    return new Promise((resolve, reject) => {
+        let connectionSettled = false;
+        const failConnection = (error) => {
+            if (connectionSettled)
+                return;
+            connectionSettled = true;
+            finish();
+            socket.close();
+            reject(error);
+        };
+        const abort = () => {
+            if (settled)
+                return;
+            if (!connectionSettled) {
+                connectionSettled = true;
+                finish();
+                socket.close();
+                reject(new Error(`Encrypted channel aborted before opening: ${options.url}`));
+                return;
+            }
+            close();
+        };
+        removeAbortListener = () => options.signal?.removeEventListener('abort', abort);
+        if (options.signal?.aborted)
+            abort();
+        else
+            options.signal?.addEventListener('abort', abort, { once: true });
+        // Armed before the socket can report anything, and disarmed by `finish` on every terminal path
+        // as well as by the success path below -- so it bounds the opening and nothing more.
+        openTimer = timers.setTimeout(() => {
+            failConnection(new Error(`Encrypted channel timed out opening ${options.url} after ${openTimeoutMs}ms`));
+        }, openTimeoutMs);
+        socket.on('open', () => {
+            if (settled || opened || connectionSettled)
+                return;
+            void (async () => {
+                try {
+                    sessionKey = options.handshake ? await options.handshake({
+                        sendText: (text) => socket.send(text),
+                        receive: async () => {
+                            const item = await rawMessages.next();
+                            if (item.done)
+                                throw new Error('Socket closed during encrypted-channel handshake');
+                            return item.value;
+                        },
+                    }) : sessionKey;
+                    // A close/error may have happened while an asynchronous plaintext handshake was waiting
+                    // for its challenge response. The connection promise is already terminal in that case.
+                    if (settled || connectionSettled)
+                        return;
+                    if (!sessionKey)
+                        throw new Error('Encrypted channel needs a session key or handshake');
+                    opened = true;
+                    if (options.initialFrame)
+                        send(options.initialFrame());
+                    // An initial proof may synchronously trigger a socket close/error. `opened` permits the
+                    // proof frame itself, but the public connection is not published until this guard passes.
+                    if (settled || connectionSettled)
+                        return;
+                    scheduleHeartbeat();
+                    connectionSettled = true;
+                    disarmOpenTimer();
+                    resolve({
+                        send,
+                        messages,
+                        close,
+                        closed,
+                        get state() { return state; },
+                    });
+                }
+                catch (error) {
+                    failConnection(error instanceof Error ? error : new Error(String(error)));
+                }
+            })();
+        });
+        socket.on('message', (data, isBinary) => {
+            if (settled)
+                return;
+            const bytes = Array.isArray(data)
+                ? Buffer.concat(data)
+                : Buffer.isBuffer(data)
+                    ? data
+                    : Buffer.from(new Uint8Array(data));
+            if (!opened) {
+                rawMessages.push({ data: bytes, isBinary });
+                return;
+            }
+            if (!isBinary)
+                return;
+            try {
+                const frame = options.decode(crypto.open(sessionKey, bytes).toString());
+                const acknowledged = [...outstandingAcks].some((sequence) => options.isHeartbeatAck(frame, sequence));
+                // A correctly encrypted and decoded frame proves the peer is still reachable. Heartbeat
+                // acknowledgements are one such frame, but computer streaming/agent events may legitimately
+                // be the only server traffic while a task runs. Invalid frames take the catch path and never
+                // get to satisfy liveness.
+                outstandingAcks.clear();
+                if (!acknowledged)
+                    messages.push(frame);
+            }
+            catch {
+                close();
+            }
+        });
+        socket.on('close', () => {
+            if (!connectionSettled) {
+                connectionSettled = true;
+                finish();
+                reject(new Error(`Encrypted channel closed before opening: ${options.url}`));
+                return;
+            }
+            finish();
+        });
+        socket.on('error', () => {
+            if (settled)
+                return;
+            const rejectOpening = !connectionSettled;
+            // Mark this before close(): ws and deterministic fakes can emit close synchronously.
+            if (rejectOpening)
+                connectionSettled = true;
+            finish();
+            // The error event alone does not guarantee ws has released its network resources. Closing is
+            // safe and idempotent, and it also gives callers one uniform terminal path.
+            socket.close();
+            if (rejectOpening) {
+                reject(new Error(`Unable to open encrypted channel to ${options.url}`));
+            }
+        });
+    });
+}

package/dist/workstation/connection/retryPolicy.js

@@ -0,0 +1,14 @@
+/**
+ * The reconnect cadence is intentionally bounded, not exponential: an offline workstation first
+ * retries promptly, then stays observable and recoverable for days without a terminal state.
+ */
+export class RetryPolicy {
+    failures = 0;
+    recordFailure() {
+        this.failures += 1;
+        return this.failures <= 3 ? 10_000 : 30_000;
+    }
+    recordSuccess() {
+        this.failures = 0;
+    }
+}

package/dist/workstation/connection/supervisor.js

@@ -0,0 +1,353 @@
+import { RetryPolicy } from './retryPolicy.js';
+const waitFor = (milliseconds, signal) => new Promise((resolve) => {
+    if (signal?.aborted) {
+        resolve();
+        return;
+    }
+    const timer = setTimeout(done, milliseconds);
+    const abort = () => { clearTimeout(timer); done(); };
+    function done() { signal?.removeEventListener('abort', abort); resolve(); }
+    signal?.addEventListener('abort', abort, { once: true });
+});
+/**
+ * Twice the encrypted channel's own opening deadline, so this outer bound never pre-empts the inner
+ * one and turn a diagnosable "timed out opening the socket" into a vaguer attempt timeout. What it
+ * covers is everything else a connect call does around that socket -- refreshing git, waiting for
+ * `management_ready`, the readiness hook, the data channels' ticket round trip -- and, more to the
+ * point, whatever is added to that path later. The loop's guarantee is meant to be structural: it
+ * keeps retrying because no attempt can outlast this, not because every function it calls happens
+ * to have been audited.
+ */
+const DEFAULT_ATTEMPT_TIMEOUT_MS = 30_000;
+/**
+ * How long a management session has to survive before the reconnect that follows it is treated as
+ * recovery rather than churn.
+ *
+ * The distinction this draws is not "did the connection drop" -- every outage does that, and every
+ * outage must still recover as fast as it always has. It is "did the reconnect *hold*". A dropped
+ * network, a crashed server, a server down for a day: those fail at `connect`, are counted by
+ * [managementRetry], and none of this touches them. A server restart closes a session that had been
+ * up for hours, which is well past this line, so that reconnect stays immediate too.
+ *
+ * What lands under it is a session that connects and is then taken away within seconds, over and
+ * over -- the shape two daemons sharing one workstation identity make, because each new management
+ * generation evicts the other's. Reconnecting instantly is exactly wrong there: it is what turns a
+ * pair of duplicates into ~6 evictions a second (8648 of them in minutes on 2026-09-10) instead of
+ * two processes taking turns while somebody notices.
+ */
+const HEALTHY_SESSION_MS = 5_000;
+/**
+ * Delay before reconnecting after the Nth consecutive session that died younger than
+ * [HEALTHY_SESSION_MS]. The first two are still immediate, so a real flap -- a server bouncing, a
+ * laptop's Wi-Fi returning mid-handshake -- recovers exactly as promptly as it used to; only a
+ * pattern that has already repeated gets slowed, and only to five seconds. One session that
+ * survives resets the whole ladder.
+ */
+const CHURN_BACKOFF_MS = [0, 0, 250, 1_000, 2_000, 5_000];
+/** Owns one authenticated management generation and its independently reconnecting data channels. */
+export class ConnectionSupervisor {
+    options;
+    states = { management: 'DISCONNECTED', computer: 'DISCONNECTED', agent: 'DISCONNECTED' };
+    managementRetry = new RetryPolicy();
+    computerRetry = new RetryPolicy();
+    agentRetry = new RetryPolicy();
+    running = false;
+    management = null;
+    computer = null;
+    agent = null;
+    lifecycleAbort = null;
+    dataAbort = null;
+    generation = 0;
+    /** Consecutive management sessions that died younger than {@link HEALTHY_SESSION_MS}. */
+    churn = 0;
+    startPromise = null;
+    constructor(options) {
+        this.options = options;
+    }
+    get attemptTimeoutMs() { return this.options.attemptTimeoutMs ?? DEFAULT_ATTEMPT_TIMEOUT_MS; }
+    log(message) { this.options.log?.(message); }
+    get now() { return (this.options.now ?? Date.now)(); }
+    /**
+     * Closes one management session and decides how soon to open the next.
+     *
+     * A session that lived at least {@link HEALTHY_SESSION_MS} clears the ladder and reconnects with
+     * no delay -- the behaviour every outage has always had, and the one this must not slow down.
+     * Only a run of sessions that keep dying young is paced, and only up to five seconds.
+     *
+     * Closing [connected] here is the other half. The two paths that abandon a session because its
+     * data channels never attached used to leave the management socket open: `closeData()` covers
+     * computer and agent, nothing covered management, so every lap leaked a live socket that the
+     * server kept bound until a newer generation evicted it. That is why the loop could not settle --
+     * the abandoned sockets were still there to be evicted, and each eviction fed another lap.
+     */
+    async endSession(connected, connectedAt, reason, signal) {
+        connected.close?.();
+        if (this.management === connected)
+            this.management = null;
+        this.closeData();
+        if (!this.running || signal.aborted)
+            return;
+        this.states.management = 'RECONNECTING';
+        if (this.now - connectedAt >= HEALTHY_SESSION_MS) {
+            this.churn = 0;
+            this.log(`[connection] ${reason}; reconnecting immediately`);
+            return;
+        }
+        const delayMs = CHURN_BACKOFF_MS[Math.min(this.churn, CHURN_BACKOFF_MS.length - 1)];
+        this.churn += 1;
+        if (delayMs === 0) {
+            this.log(`[connection] ${reason}; reconnecting immediately`);
+            return;
+        }
+        this.log(`[connection] ${reason}; reconnecting in ${delayMs}ms (${this.churn} short-lived sessions in a row -- another daemon may hold this workstation)`);
+        await this.wait(delayMs, signal);
+    }
+    /**
+     * Runs one connect attempt under a deadline, and hands back the cleanup that unlinks it from the
+     * supervisor's lifecycle.
+     *
+     * The attempt gets its own AbortController rather than the lifecycle signal directly, because the
+     * deadline has to be able to cancel *this* attempt without tearing down the supervisor. The
+     * controller still cascades from the lifecycle signal, so stopping the supervisor closes an
+     * in-flight opening exactly as it did before.
+     *
+     * A timed-out attempt is aborted, not merely dropped: the socket underneath it would otherwise
+     * stay open, and a peer that is silent rather than gone would cost one file descriptor per
+     * reconnect for as long as it stayed that way.
+     */
+    async attempt(lifecycle, connect) {
+        const attempt = new AbortController();
+        const cascade = () => attempt.abort();
+        if (lifecycle.aborted)
+            attempt.abort();
+        else
+            lifecycle.addEventListener('abort', cascade, { once: true });
+        const release = () => lifecycle.removeEventListener('abort', cascade);
+        const armed = new AbortController();
+        try {
+            const connection = await Promise.race([
+                connect(attempt.signal),
+                // Disarming (the `finally` below) also ends this wait, so this can throw after the race is
+                // already decided. That is harmless and deliberately not guarded against: `Promise.race`
+                // subscribes to both inputs, so it absorbs the late rejection instead of leaving it
+                // unhandled. What the disarm is actually for is the timer itself -- an attempt that
+                // succeeded in a second must not hold a 30-second timer for the other 29.
+                this.deadline(this.attemptTimeoutMs, armed.signal).then(() => {
+                    throw new Error(`Connect attempt exceeded ${this.attemptTimeoutMs}ms`);
+                }),
+            ]);
+            return { connection, release };
+        }
+        catch (error) {
+            attempt.abort();
+            release();
+            throw error;
+        }
+        finally {
+            armed.abort();
+        }
+    }
+    channelSnapshot() { return { ...this.states }; }
+    async start() {
+        if (this.startPromise)
+            return this.startPromise;
+        this.running = true;
+        const lifecycleAbort = new AbortController();
+        this.lifecycleAbort = lifecycleAbort;
+        let ready;
+        this.startPromise = new Promise((resolve) => { ready = resolve; });
+        void this.managementLoop(ready, lifecycleAbort.signal);
+        return this.startPromise;
+    }
+    async run(signal) {
+        const abortError = new Error('Connection supervisor stopped');
+        if (signal.aborted) {
+            this.stop();
+            throw abortError;
+        }
+        let rejectAbort;
+        const aborted = new Promise((_resolve, reject) => { rejectAbort = reject; });
+        const abort = () => {
+            this.stop();
+            rejectAbort(abortError);
+        };
+        signal.addEventListener('abort', abort, { once: true });
+        try {
+            await Promise.race([this.start(), aborted]);
+            await aborted;
+        }
+        finally {
+            signal.removeEventListener('abort', abort);
+            this.stop();
+        }
+        throw new Error('Connection supervisor stopped');
+    }
+    stop() {
+        if (!this.running)
+            return;
+        this.running = false;
+        this.startPromise = null;
+        this.lifecycleAbort?.abort();
+        this.lifecycleAbort = null;
+        this.generation += 1;
+        this.management?.close?.();
+        this.management = null;
+        this.closeData();
+        this.states.management = 'DISCONNECTED';
+    }
+    async managementLoop(onFirstReady, signal) {
+        let firstReady = true;
+        while (this.running && !signal.aborted) {
+            this.states.management = firstReady ? 'CONNECTING' : 'RECONNECTING';
+            let failure = 'unknown error';
+            try {
+                const attempt = await this.attempt(signal, (attemptSignal) => this.options.management.connect(attemptSignal));
+                const connected = attempt.connection;
+                try {
+                    if (!this.running) {
+                        connected.close?.();
+                        return;
+                    }
+                    this.managementRetry.recordSuccess();
+                    this.management = connected;
+                    const connectedAt = this.now;
+                    this.states.management = 'CONNECTED';
+                    this.log(`[connection] management connected (generation ${connected.generation})`);
+                    const epoch = ++this.generation;
+                    this.closeData();
+                    const dataAbort = new AbortController();
+                    this.dataAbort = dataAbort;
+                    const computerLoop = this.dataLoop('COMPUTER', connected, epoch, dataAbort.signal);
+                    // Initial attachment is deliberately ordered: only an authenticated management generation
+                    // can mint tickets, and computer finishes its first join before agent asks for its ticket.
+                    if (!await this.waitForDataOrManagementClose('computer', connected)) {
+                        void computerLoop;
+                        dataAbort.abort();
+                        await this.endSession(connected, connectedAt, 'management closed before the computer channel attached', signal);
+                        continue;
+                    }
+                    const agentLoop = this.dataLoop('AGENT', connected, epoch, dataAbort.signal);
+                    if (firstReady) {
+                        // The first successful attachment is the useful completion point for daemon startup.
+                        if (!await this.waitForDataOrManagementClose('agent', connected)) {
+                            void computerLoop;
+                            void agentLoop;
+                            dataAbort.abort();
+                            await this.endSession(connected, connectedAt, 'management closed before the agent channel attached', signal);
+                            continue;
+                        }
+                        firstReady = false;
+                        onFirstReady();
+                    }
+                    await connected.closed;
+                    void computerLoop;
+                    void agentLoop;
+                    if (this.running) {
+                        // Immediate reconnect after a clean closure is still the rule: a server that explicitly
+                        // closed the channel (restart, heartbeat timeout) should be rejoined at once, and a
+                        // session that had been up for any real length of time takes that path unchanged.
+                        // `endSession` only paces a *run* of sessions that die within seconds of connecting,
+                        // which is the duplicate-daemon shape rather than an outage.
+                        await this.endSession(connected, connectedAt, 'management channel closed', signal);
+                        continue;
+                    }
+                    if (this.management === connected)
+                        this.management = null;
+                }
+                finally {
+                    attempt.release();
+                }
+            }
+            catch (error) {
+                if (!this.running)
+                    return;
+                failure = error instanceof Error ? error.message : String(error);
+                // Network errors (failed connect, handshake error) get backoff via retryPolicy.
+            }
+            if (!this.running || signal.aborted)
+                return;
+            const delayMs = this.managementRetry.recordFailure();
+            // The one line that made the 2026-09-03 outage a forensic exercise instead of a `tail`: every
+            // failure on this path used to be swallowed silently, so a loop that had stopped trying and a
+            // loop that was trying and failing looked identical from the log.
+            this.log(`[connection] management connect failed: ${failure}; retrying in ${delayMs}ms`);
+            await this.wait(delayMs, signal);
+        }
+    }
+    async dataLoop(kind, management, epoch, signal) {
+        const name = kind === 'COMPUTER' ? 'computer' : 'agent';
+        const retry = kind === 'COMPUTER' ? this.computerRetry : this.agentRetry;
+        let connectedOnce = false;
+        while (this.running && this.generation === epoch && this.management === management) {
+            this.states[name] = connectedOnce ? 'RECONNECTING' : 'CONNECTING';
+            let failure = 'unknown error';
+            let release = () => { };
+            try {
+                // Ticket and join are bounded as one attempt: the ticket is a request/response over the
+                // management channel, so it is one more place a silent peer can park this loop.
+                const attempt = await this.attempt(signal, async (attemptSignal) => {
+                    const ticket = await management.ticket(kind);
+                    return (kind === 'COMPUTER' ? this.options.computer : this.options.agent).connect(ticket, attemptSignal);
+                });
+                release = attempt.release;
+                const connected = attempt.connection;
+                if (!this.running || this.generation !== epoch || this.management !== management) {
+                    connected.close?.();
+                    return;
+                }
+                retry.recordSuccess();
+                if (kind === 'COMPUTER')
+                    this.computer = connected;
+                else
+                    this.agent = connected;
+                this.states[name] = 'CONNECTED';
+                connectedOnce = true;
+                await connected.closed;
+                // Assigned here, not left at the initial value: reaching this line means the channel opened
+                // and was then closed, which is a different event from the open having thrown. Both used to
+                // print "unknown error", because `failure` was only ever written in the `catch` -- so the
+                // log said the same thing whether the peer had rejected the ticket or had simply hung up,
+                // and a duplicate-daemon eviction (a perfectly clean close) read as a transport fault. Two
+                // hours of 2026-09-10 went into a reverse proxy that was never involved.
+                failure = 'closed by the peer';
+                if (kind === 'COMPUTER' && this.computer === connected)
+                    this.computer = null;
+                if (kind === 'AGENT' && this.agent === connected)
+                    this.agent = null;
+            }
+            catch (error) {
+                failure = error instanceof Error ? error.message : String(error);
+                // The persistent retry policy below owns all failed opens and unexpected closes.
+            }
+            finally {
+                release();
+            }
+            if (!this.running || this.generation !== epoch || this.management !== management)
+                return;
+            this.states[name] = 'RECONNECTING';
+            const delayMs = retry.recordFailure();
+            this.log(`[connection] ${name} channel dropped: ${failure}; retrying in ${delayMs}ms`);
+            await this.wait(delayMs, signal);
+        }
+    }
+    closeData() {
+        this.dataAbort?.abort();
+        this.dataAbort = null;
+        this.computer?.close?.();
+        this.agent?.close?.();
+        this.computer = this.agent = null;
+        this.states.computer = this.states.agent = 'DISCONNECTED';
+    }
+    async waitForDataState(name) {
+        while (this.running && this.states[name] !== 'CONNECTED')
+            await waitFor(1);
+    }
+    async waitForDataOrManagementClose(name, management) {
+        return Promise.race([
+            this.waitForDataState(name).then(() => this.running && this.management === management),
+            management.closed.then(() => false),
+        ]);
+    }
+    wait(milliseconds, signal) { return (this.options.wait ?? waitFor)(milliseconds, signal); }
+    deadline(milliseconds, signal) { return (this.options.deadline ?? waitFor)(milliseconds, signal); }
+}