@platless/axom-cli 0.3.13 → 0.5.9

package/dist/workstation/keepers/legacy.js

@@ -0,0 +1,296 @@
+// Daemon-side: the legacy tools keeper (`axom-tools-keeper-<id>`, tools/keeperProcess.ts from CLIs before keeper-terminal) that
+// may still hold live PTYs on an upgraded machine. Spec §9, analysis C §2.3. The daemon serves those PTYs until they end, then
+// stops the keeper; nothing here can start one.
+//
+// Read-only discovery: the old socket and pid manifest (tools/client.ts `keeperPaths(…, 'terminal')`), `keeper.ping` with an
+// identity check, `terminal.list`. It speaks the old one-request-per-connection protocol through `keeperRpc` only and NEVER
+// through `ToolsClient('terminal').request`, which lazily starts that keeper. It never unlinks anything: a keeper that does not
+// answer a ping while its manifest pid is alive is busy, and is asked again later.
+import { execFile, spawn } from 'node:child_process';
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { keeperPaths, keeperRpc } from '../tools/client.js';
+import { pidAlive } from './kit/lock.js';
+/** A legacy keeper with no open PTY is stopped once nothing was routed to it for this long (the user may still read an exited one). */
+export const LEGACY_IDLE_MS = 10 * 60_000;
+/**
+ * Bound on one router-side lookup (probe + `terminal.list`), well under the COMPUTER channel's 7.5 s tool budget: a hung legacy
+ * keeper must not time out terminal.list / create / unknown-id requests, which would leave the terminal tab unusable.
+ */
+export const LEGACY_LOOKUP_MS = 1500;
+/** Today's wording when no keeper holds a session id. */
+export const TERMINAL_NOT_FOUND = 'Terminal session not found; keeper restart loses live PTYs';
+function runSystemctl(args, timeoutMs) {
+    return new Promise((resolve) => {
+        const child = spawn('systemctl', ['--user', ...args], { stdio: 'ignore' });
+        let done = false;
+        const finish = (ok) => { if (done)
+            return; done = true; clearTimeout(timer); resolve(ok); };
+        const timer = setTimeout(() => { child.kill(); finish(false); }, timeoutMs);
+        child.on('error', () => finish(false));
+        child.on('exit', (code) => finish(code === 0));
+    });
+}
+export const userSystemd = {
+    available: () => runSystemctl(['show-environment'], 1000),
+    isActive: (unit) => runSystemctl(['is-active', '--quiet', unit], 2000),
+    stop: (unit) => runSystemctl(['stop', unit], 15_000),
+};
+function psCommandLine(pid) {
+    return new Promise((resolve) => {
+        execFile('ps', ['-ww', '-o', 'command=', '-p', String(pid)], { timeout: 2000 }, (error, stdout) => resolve(error ? null : stdout.trim() || null));
+    });
+}
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+export class LegacyToolsKeeper {
+    options;
+    paths;
+    current = 'unknown';
+    pid;
+    open = null;
+    lastTrafficAt = 0;
+    probing;
+    ids = null;
+    listed = [];
+    retiring;
+    now;
+    rpc;
+    constructor(options) {
+        this.options = options;
+        this.paths = options.paths ?? keeperPaths({ ...options.tools }, undefined, undefined, 'terminal');
+        this.now = options.now ?? Date.now;
+        this.rpc = options.rpc ?? keeperRpc;
+    }
+    get state() { return this.current; }
+    /** Present or busy: it may hold sessions, so the router asks it. */
+    get live() { return this.current === 'present' || this.current === 'busy'; }
+    /**
+     * Ids of the last successful list (null until one succeeded). The legacy keeper gains no session after the upgrade (every create
+     * goes to keeper-terminal except a same-id retry), so this answers "is it one of its sessions?" even while it is busy or hung.
+     */
+    get knownIds() { return this.ids; }
+    /** The sessions of the last successful list (for the heartbeat's items; no IO). Empty once it is gone or retired. */
+    lastSessions() { return this.live ? this.listed : []; }
+    /** For the heartbeat: state, pid and the number of non-exited sessions at the last successful list (null = unknown). */
+    status() {
+        return { state: this.current, pid: this.pid ?? null, open: this.live ? this.open : 0 };
+    }
+    log(line) { this.options.log?.(`[legacy-tools-keeper] ${line}`); }
+    async ping(timeoutMs = this.options.probeTimeoutMs ?? 1000) {
+        try {
+            const reply = await this.rpc(this.paths.socket, 'keeper.ping', {}, timeoutMs);
+            return reply.identity === this.paths.unit && Number.isSafeInteger(reply.pid) && reply.pid > 0 ? reply.pid : null;
+        }
+        catch {
+            return null;
+        }
+    }
+    /** The pid of a live process the manifest names under our identity, else null. Reads only. */
+    async manifestPid() {
+        try {
+            const stat = await fs.lstat(this.paths.pidFile);
+            if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 4096)
+                return null;
+            const value = JSON.parse(await fs.readFile(this.paths.pidFile, 'utf8'));
+            if (value.identity !== this.paths.unit || !Number.isSafeInteger(value.pid))
+                return null;
+            return pidAlive(value.pid) ? value.pid : null;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Probes (single-flight) unless the answer is final. A keeper found for the first time starts its idle window now, so a
+     * daemon restart never shortens the time a user has to look at an exited session's output.
+     */
+    detect(timeoutMs) {
+        if (this.current === 'absent' || this.current === 'retired' || this.current === 'stopping')
+            return Promise.resolve(this.current);
+        this.probing ??= (async () => {
+            const pid = await this.ping(timeoutMs);
+            if (this.current === 'stopping' || this.current === 'retired')
+                return this.current;
+            if (pid !== null) {
+                if (this.current !== 'present' && this.current !== 'busy') {
+                    this.lastTrafficAt = this.now();
+                    this.log(`found pid ${pid}`);
+                }
+                this.pid = pid;
+                this.current = 'present';
+            }
+            else if (await this.manifestPid() !== null) {
+                if (this.current === 'unknown')
+                    this.lastTrafficAt = this.now();
+                this.current = 'busy';
+            }
+            else {
+                if (this.current !== 'unknown')
+                    this.log('gone');
+                this.current = 'absent';
+                this.open = 0;
+            }
+            return this.current;
+        })().finally(() => { this.probing = undefined; });
+        return this.probing;
+    }
+    /**
+     * `terminal.list` of the legacy keeper, or null when it is not there or did not answer now. Not routed traffic. Bounded by
+     * `budgetMs` in total (probe included). Only a keeper that answered its last probe is listed: a busy one gives null at once
+     * (the periodic tick re-probes it), and a list that fails or times out sends it back to probing in the background.
+     */
+    async sessions(budgetMs = LEGACY_LOOKUP_MS) {
+        const deadline = Date.now() + budgetMs;
+        if (this.current === 'unknown')
+            await this.detect(Math.min(this.options.probeTimeoutMs ?? 1000, Math.floor(budgetMs / 2)));
+        if (this.current !== 'present')
+            return null;
+        try {
+            const reply = await this.rpc(this.paths.socket, 'terminal.list', {}, Math.max(1, deadline - Date.now()));
+            const sessions = Array.isArray(reply.sessions) ? reply.sessions : [];
+            this.open = sessions.filter((session) => !session.exited).length;
+            this.ids = new Set(sessions.map((session) => session.id));
+            this.listed = sessions;
+            return sessions;
+        }
+        catch {
+            void this.detect();
+            return null;
+        }
+    }
+    /** One routed `terminal.*` request, sent exactly once (no retry: input may have been delivered even if the reply was lost). */
+    async request(method, params) {
+        if (!this.live)
+            throw new Error(TERMINAL_NOT_FOUND);
+        this.lastTrafficAt = this.now();
+        try {
+            const result = await this.rpc(this.paths.socket, method, params, this.options.requestTimeoutMs ?? 7000);
+            if (method === 'terminal.close' && typeof params.id === 'string')
+                this.ids?.delete(params.id);
+            return result;
+        }
+        catch (error) {
+            void this.detect(); // a failed connection (or a keeper that went away mid-request) shows up on the next probe
+            throw error;
+        }
+    }
+    /**
+     * Retirement check (call periodically): once no session is open (only exited ones, or none) and nothing was routed to it for
+     * the idle window, stop it. Also re-probes a busy keeper.
+     */
+    async tick() {
+        if (this.retiring)
+            return this.retiring;
+        if (await this.detect() !== 'present')
+            return;
+        const sessions = await this.sessions(this.options.requestTimeoutMs ?? 7000);
+        if (!sessions)
+            return;
+        const idle = () => this.open === 0 && this.now() - this.lastTrafficAt >= (this.options.idleMs ?? LEGACY_IDLE_MS);
+        if (idle())
+            await this.retire('idle, no open terminal', idle);
+    }
+    /** Stops it now whatever is open (the management page's "replace now"). Returns the open sessions that end with it. */
+    async replaceNow() {
+        if (this.retiring) {
+            await this.retiring;
+            return [];
+        }
+        const state = await this.detect();
+        if (state !== 'present' && state !== 'busy')
+            return [];
+        const affected = (await this.sessions() ?? []).filter((session) => !session.exited);
+        await this.retire(`replaced now with ${affected.length} open terminal(s)`);
+        return affected;
+    }
+    retire(reason, stillWanted) {
+        this.retiring ??= this.stop(reason, stillWanted).finally(() => { this.retiring = undefined; });
+        return this.retiring;
+    }
+    async stop(reason, stillWanted) {
+        const previous = this.current;
+        this.current = 'stopping'; // from here on nothing more is routed to it
+        // A request routed between the check and now (it stamped the traffic time before `stopping`) cancels an idle stop.
+        if (stillWanted && !stillWanted()) {
+            this.current = previous;
+            return;
+        }
+        let pid = this.pid ?? null;
+        let stopped = false;
+        try {
+            const systemd = this.options.systemd === undefined ? (process.platform === 'linux' ? userSystemd : null) : this.options.systemd;
+            if (systemd && await systemd.available() && await systemd.isActive(this.paths.unit)) {
+                // SIGTERM through the unit: the old keeper exits 0, so `Restart=on-failure` does not bring it back.
+                this.log(`stopping unit ${this.paths.unit} (${reason})`);
+                stopped = await systemd.stop(this.paths.unit) && await this.gone(pid);
+            }
+            if (!stopped) {
+                // Kill only a process that is provably the keeper: the manifest names it under our identity AND it answers the ping
+                // as that same pid. Anything less (pid reuse, a stale manifest, a keeper too busy to answer) is left alone for now.
+                // The manifest may be gone while the keeper lives: macOS's daily /tmp cleaner deletes regular files not accessed for
+                // 3 days and leaves the socket. Only then is the process's own launch argument (socket + identity) the proof instead;
+                // a manifest that exists but names another pid or identity still refuses.
+                const [manifest, answered] = [await this.manifestPid(), await this.ping()];
+                if (answered === null || !(manifest !== null ? manifest === answered
+                    : await this.manifestAbsent() && await this.launchedAsUs(answered))) {
+                    this.log(`cannot verify the keeper process (manifest ${manifest}, ping ${answered}); retrying later`);
+                    this.current = previous === 'busy' ? 'busy' : 'present';
+                    return;
+                }
+                pid = answered;
+                this.log(`SIGTERM pid ${pid} (${reason})`);
+                (this.options.signal ?? ((target, signal) => process.kill(target, signal)))(pid, 'SIGTERM');
+                stopped = await this.gone(pid);
+            }
+            if (!stopped) {
+                this.log(`pid ${pid} did not exit; retrying later`);
+                this.current = 'present';
+                return;
+            }
+            this.current = 'retired';
+            this.open = 0;
+            this.log('retired');
+        }
+        catch (error) {
+            this.log(`stop failed: ${error instanceof Error ? error.message : String(error)}`);
+            if (this.current === 'stopping')
+                this.current = 'present';
+            return;
+        }
+        if (this.options.recoverUploads) {
+            // Its own stop() already cleaned the uploads it knew about; this catches what an earlier crash left behind.
+            await this.options.recoverUploads(path.join(this.paths.dir, 'uploads')).catch((error) => this.log(`uploads recover failed: ${error instanceof Error ? error.message : String(error)}`));
+        }
+    }
+    manifestAbsent() {
+        return fs.lstat(this.paths.pidFile).then(() => false, (error) => error.code === 'ENOENT');
+    }
+    /**
+     * True when `pid` runs the old keeper with our launch config: the old ToolsClient passed it as the last argument, base64 JSON
+     * naming this socket and identity. Reads only.
+     */
+    async launchedAsUs(pid) {
+        const command = await (this.options.commandLine ?? psCommandLine)(pid);
+        const last = command?.trim().split(/\s+/).at(-1);
+        if (!last)
+            return false;
+        try {
+            const config = JSON.parse(Buffer.from(last, 'base64').toString('utf8'));
+            return config.socket === this.paths.socket && config.identity === this.paths.unit;
+        }
+        catch {
+            return false;
+        }
+    }
+    /** Waits (bounded, wall clock) until `pid` is gone and the socket no longer answers as the keeper. */
+    async gone(pid) {
+        const deadline = Date.now() + (this.options.stopTimeoutMs ?? 10_000);
+        for (;;) {
+            if ((pid === null || !pidAlive(pid)) && await this.ping() === null)
+                return true;
+            if (Date.now() >= deadline)
+                return false;
+            await delay(50);
+        }
+    }
+}