@platless/axom-cli 0.1.0 → 0.5.9

package/dist/workstation/files/client.js

@@ -0,0 +1,455 @@
+import { fork } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import path from 'node:path';
+import { createInterface } from 'node:readline';
+import { fileURLToPath } from 'node:url';
+import { privateDirectory } from '../tools/client.js';
+import { cancelledFilePayload, fileErrorFrame } from './adapters/computerFileFrames.js';
+import { FILES_MUTATING_METHODS } from './adapters/toolRequests.js';
+import { FILES_HOST_MAX_IN_FLIGHT } from './hostProtocol.js';
+/*
+ * Daemon side of the files host (files/host.ts): spawns it, multiplexes requests over its IPC channel, cancels, samples
+ * its activity, and keeps it alive. A crash (or the hung-IO watchdog's kill) fails every request already sent to it —
+ * mutations as outcome-unknown, everything else as offline — and the host is respawned with 1 s -> 30 s exponential
+ * backoff; the new host's start runs uploads recover(). Requests that arrive while no host is ready wait for one,
+ * bounded. A configuration change restarts the host once it is idle.
+ */
+export const TOOL_DEADLINE_MS = 7_500;
+export const DEFAULT_FILES_HOST_TIMINGS = {
+    backoffInitialMs: 1_000, backoffMaxMs: 30_000, backoffResetAfterMs: 30_000, hungPinMs: 60_000, readyWaitMs: 10_000,
+    cancelGraceMs: 2_000, stopBudgetMs: 6_000, restartUploadGraceMs: 60_000, readyDeadlineMs: 60_000, spawnPrepareMs: 10_000,
+    killExitGraceMs: 5_000, watchdogIntervalMs: 1_000,
+};
+function bounded(work, ms, message) {
+    let timer;
+    const timeout = new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(message)), ms); });
+    return Promise.race([work, timeout]).finally(() => clearTimeout(timer));
+}
+const defaultScript = () => fileURLToPath(new URL('./host.js', import.meta.url));
+export class FilesHostClient {
+    options;
+    hostDir;
+    timings;
+    child;
+    /** Requests may be sent to the current host. */
+    ready = false;
+    /** The current host reported ready (stays true while a pending restart drains it). */
+    hostUp = false;
+    readyAt = 0;
+    optionsReadAt = 0;
+    spawning = false;
+    stopped = false;
+    failures = 0;
+    respawnTimer;
+    watchdog;
+    lastInfo;
+    lastStatus;
+    lastStatusAt = 0;
+    lastActivity = 0;
+    /** Requests sent to the current host and not yet answered. */
+    sent = new Map();
+    /** Requests waiting for a ready host (not yet sent). */
+    waiting = 0;
+    readyWaiters = new Set();
+    restartRequestedAt = null;
+    plannedExit = false;
+    /** The current host is being shut down or killed: its status and ready messages no longer describe a live host. */
+    retiring = false;
+    spawnedAt = 0;
+    exited;
+    constructor(options) {
+        this.options = options;
+        this.hostDir = path.join(path.resolve(options.configDir), 'files-host');
+        this.timings = { ...DEFAULT_FILES_HOST_TIMINGS, ...options.timings };
+        this.watchdog = setInterval(() => this.tick(), this.timings.watchdogIntervalMs);
+        this.watchdog.unref();
+    }
+    /** Starts the host (idempotent). Failures are retried with backoff; requests wait for readiness, bounded. */
+    start() {
+        if (this.stopped || this.child || this.spawning || this.respawnTimer)
+            return;
+        void this.spawn();
+    }
+    /** The `tools.info` file fields of the last host that became ready (undefined before the first one). */
+    info() { return this.lastInfo; }
+    /** Live sample for TaskRegistry: in-progress uploads + running media encodes of the current host. */
+    activity() {
+        if (!this.child || !this.hostUp || !this.lastStatus)
+            return 0;
+        return this.lastStatus.uploads + this.lastStatus.encodes;
+    }
+    /** Process id of the current host (diagnostics and tests). */
+    get pid() { return this.child?.pid; }
+    get isReady() { return this.ready; }
+    /** The configured roots changed: restart the host once it is idle (or drained, after a grace). */
+    reconfigure() {
+        if (this.stopped)
+            return;
+        this.restartRequestedAt ??= Date.now();
+        this.tick();
+    }
+    /** One `file_*_request`; never rejects — every failure is a result frame with one of the wire codes. */
+    async runFrame(request, signal) {
+        const mutation = request.t === 'file_write_request' || request.t === 'file_mkdir_request';
+        const outcome = await this.call({ op: 'frame', params: request, deadlineAt: null }, mutation, null, signal);
+        if (outcome.ok)
+            return outcome.result;
+        if (outcome.reason === 'cancelled')
+            return fileErrorFrame(request, cancelledFilePayload(request.t));
+        if (outcome.reason === 'lost' && mutation)
+            return fileErrorFrame(request, { code: 'outcome_unknown', message: outcome.error });
+        return fileErrorFrame(request, { code: 'workstation_offline', message: outcome.error });
+    }
+    /** One `files.*` tool request; rejects with the `tool_result.error` text. */
+    async tool(method, params) {
+        const deadlineAt = Date.now() + TOOL_DEADLINE_MS;
+        const outcome = await this.call({ op: 'tool', params: { method, params }, deadlineAt }, FILES_MUTATING_METHODS.has(method), deadlineAt);
+        if (outcome.ok)
+            return outcome.result;
+        throw new Error(outcome.error);
+    }
+    /** `terminal.create`'s cwd (absent = home) resolved in the configured-roots scope, as the legacy keeper did; rejects with its error text. */
+    async terminalCwd(cwd) {
+        const deadlineAt = Date.now() + TOOL_DEADLINE_MS;
+        const outcome = await this.call({ op: 'terminalCwd', params: { cwd }, deadlineAt }, false, deadlineAt);
+        if (outcome.ok)
+            return outcome.result.path;
+        throw new Error(outcome.error);
+    }
+    /** Cleans orphaned upload temporaries recorded in a retired process's manifest directory (`recoverUploads` in the host). */
+    async recoverUploads(dir) {
+        const deadlineAt = Date.now() + 60_000;
+        const outcome = await this.call({ op: 'recoverUploads', params: { dir }, deadlineAt }, true, deadlineAt);
+        if (!outcome.ok)
+            throw new Error(outcome.error);
+    }
+    /** Stops the host for good: graceful shutdown within the stop budget, then SIGKILL. Bounded. */
+    async stop() {
+        this.stopped = true;
+        clearInterval(this.watchdog);
+        if (this.respawnTimer) {
+            clearTimeout(this.respawnTimer);
+            this.respawnTimer = undefined;
+        }
+        for (const wake of [...this.readyWaiters])
+            wake();
+        await this.shutdownChild();
+    }
+    // -------------------------------------------------------------------------------------------------------------------
+    call(body, mutation, deadlineAt, signal) {
+        return new Promise(resolve => {
+            let done = false;
+            let grace;
+            let deadline;
+            const id = randomUUID();
+            const message = { ...body, id };
+            const pending = { message, mutation, settle: outcome => {
+                    if (done)
+                        return;
+                    done = true;
+                    if (grace)
+                        clearTimeout(grace);
+                    if (deadline)
+                        clearTimeout(deadline);
+                    signal?.removeEventListener('abort', onAbort);
+                    if (this.sent.get(id) === pending)
+                        this.sent.delete(id);
+                    resolve(outcome);
+                } };
+            const onAbort = () => {
+                if (!this.sent.has(id)) {
+                    pending.settle({ ok: false, error: 'File operation was cancelled', reason: 'cancelled' });
+                    return;
+                }
+                this.post({ cancel: id });
+                // The host answers a cancelled frame itself at its next checkpoint; a hung one does not, so answer for it.
+                grace ??= setTimeout(() => pending.settle({ ok: false, error: 'File operation was cancelled', reason: 'cancelled' }), this.timings.cancelGraceMs);
+            };
+            if (signal?.aborted) {
+                pending.settle({ ok: false, error: 'File operation was cancelled', reason: 'cancelled' });
+                return;
+            }
+            signal?.addEventListener('abort', onAbort, { once: true });
+            if (deadlineAt !== null) {
+                deadline = setTimeout(() => {
+                    if (this.sent.has(id))
+                        this.post({ cancel: id });
+                    pending.settle({ ok: false, error: 'Workstation tool timed out; mutation outcome may be unknown', reason: 'timeout' });
+                }, Math.max(0, deadlineAt - Date.now()));
+            }
+            if (this.sent.size + this.waiting >= FILES_HOST_MAX_IN_FLIGHT) {
+                pending.settle({ ok: false, error: 'Workstation file service is busy; try again shortly', reason: 'unavailable' });
+                return;
+            }
+            void (async () => {
+                this.waiting++;
+                let ready;
+                try {
+                    ready = await this.whenReady(deadlineAt === null ? this.timings.readyWaitMs : Math.min(this.timings.readyWaitMs, deadlineAt - Date.now()), () => done);
+                }
+                finally {
+                    this.waiting--;
+                }
+                if (done)
+                    return;
+                if (!ready || !this.child) {
+                    pending.settle({ ok: false, error: 'Workstation file service is unavailable; try again shortly', reason: 'unavailable' });
+                    return;
+                }
+                this.sent.set(id, pending);
+                if (!this.post(message)) {
+                    // Nothing reached the host: not run, so offline even for a mutation.
+                    this.sent.delete(id);
+                    pending.settle({ ok: false, error: 'Workstation file service is unavailable; try again shortly', reason: 'unavailable' });
+                }
+            })();
+        });
+    }
+    post(message) {
+        const child = this.child;
+        if (!child?.connected)
+            return false;
+        try {
+            child.send(message);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    whenReady(timeoutMs, abandoned) {
+        if (this.ready && this.child)
+            return Promise.resolve(true);
+        if (this.stopped || timeoutMs <= 0)
+            return Promise.resolve(false);
+        this.start();
+        return new Promise(resolve => {
+            const finish = () => {
+                clearTimeout(timer);
+                clearInterval(poll);
+                this.readyWaiters.delete(wake);
+                resolve(this.ready && !!this.child && !this.stopped);
+            };
+            const wake = () => finish();
+            const timer = setTimeout(finish, timeoutMs);
+            // A caller that was settled meanwhile (cancel, deadline) stops waiting.
+            const poll = setInterval(() => { if (abandoned())
+                finish(); }, 50);
+            this.readyWaiters.add(wake);
+        });
+    }
+    async spawn() {
+        if (this.stopped || this.child || this.spawning)
+            return;
+        this.spawning = true;
+        try {
+            const readAt = Date.now();
+            const base = await bounded((async () => { await privateDirectory(this.hostDir); return this.options.options(); })(), this.timings.spawnPrepareMs, 'preparing the files host timed out');
+            this.optionsReadAt = readAt;
+            const init = { ...base, uploadStateDir: path.join(this.hostDir, 'uploads') };
+            if (this.stopped)
+                return;
+            const child = fork(this.options.script ?? defaultScript(), [], {
+                cwd: this.hostDir, stdio: ['ignore', 'ignore', 'pipe', 'ipc'], execArgv: [], serialization: 'json',
+            });
+            this.child = child;
+            this.ready = false;
+            this.hostUp = false;
+            this.lastStatus = undefined;
+            this.lastStatusAt = Date.now();
+            this.plannedExit = false;
+            this.retiring = false;
+            this.spawnedAt = Date.now();
+            let resolveExited;
+            this.exited = new Promise(resolve => { resolveExited = resolve; });
+            if (child.stderr) {
+                const lines = createInterface({ input: child.stderr });
+                lines.on('line', line => this.log(`[files-host] ${line.slice(0, 2048)}`));
+            }
+            child.on('message', (message) => { if (this.child === child)
+                this.onMessage(message); });
+            let gone = false;
+            const onGone = (detail) => {
+                if (gone)
+                    return;
+                gone = true;
+                resolveExited();
+                if (this.child === child)
+                    this.onExit(detail);
+            };
+            child.on('exit', (code, signal) => onGone(signal ? `signal ${signal}` : `code ${code}`));
+            child.on('error', error => { this.log(`[files-host] ${error.message}`); if (child.exitCode === null && child.signalCode === null && !child.pid)
+                onGone('spawn failed'); });
+            child.send({ t: 'init', options: init });
+        }
+        catch (error) {
+            this.log(`[files-host] start failed: ${error instanceof Error ? error.message : String(error)}`);
+            this.scheduleRespawn();
+        }
+        finally {
+            this.spawning = false;
+        }
+    }
+    onMessage(message) {
+        if (!message || typeof message !== 'object')
+            return;
+        if ('id' in message) {
+            const pending = this.sent.get(message.id);
+            if (!pending)
+                return;
+            pending.settle(message.ok ? { ok: true, result: message.result } : { ok: false, error: message.error, reason: 'host' });
+            return;
+        }
+        // Only replies are still meaningful from a host being retired (in-flight requests may finish during a graceful stop).
+        if (this.retiring)
+            return;
+        if (message.t === 'ready') {
+            this.lastInfo = message.info;
+            this.ready = true;
+            this.hostUp = true;
+            this.readyAt = Date.now();
+            this.lastStatusAt = this.readyAt;
+            for (const wake of [...this.readyWaiters])
+                wake();
+        }
+        else if (message.t === 'init_failed') {
+            this.log(`[files-host] init failed: ${message.error}`);
+        }
+        else if (message.t === 'status') {
+            this.lastStatus = message;
+            this.lastStatusAt = Date.now();
+            this.activityMayHaveChanged();
+        }
+    }
+    onExit(detail) {
+        const planned = this.plannedExit;
+        const upFor = this.readyAt ? Date.now() - this.readyAt : 0;
+        this.child = undefined;
+        this.ready = false;
+        this.hostUp = false;
+        this.readyAt = 0;
+        this.lastStatus = undefined;
+        this.retiring = false;
+        for (const pending of [...this.sent.values()]) {
+            pending.settle({
+                ok: false, reason: 'lost',
+                error: pending.mutation
+                    ? (pending.message.op === 'frame' ? 'Workstation file service restarted; the outcome is unknown' : 'Workstation file service restarted; mutation outcome may be unknown')
+                    : 'Workstation file service restarted; try again',
+            });
+        }
+        this.sent.clear();
+        this.activityMayHaveChanged();
+        if (this.stopped)
+            return;
+        if (planned) {
+            this.failures = 0;
+            void this.spawn();
+            return;
+        }
+        this.log(`[files-host] exited (${detail}); restarting`);
+        if (upFor >= this.timings.backoffResetAfterMs)
+            this.failures = 0;
+        this.scheduleRespawn();
+    }
+    scheduleRespawn() {
+        if (this.stopped || this.respawnTimer)
+            return;
+        const delay = Math.min(this.timings.backoffMaxMs, this.timings.backoffInitialMs * 2 ** this.failures);
+        this.failures++;
+        this.respawnTimer = setTimeout(() => { this.respawnTimer = undefined; void this.spawn(); }, delay);
+        this.respawnTimer.unref();
+    }
+    tick() {
+        const child = this.child;
+        if (!child || this.stopped || this.retiring)
+            return;
+        const now = Date.now();
+        if (!this.hostUp) {
+            if (now - this.spawnedAt > this.timings.readyDeadlineMs) {
+                this.log(`[files-host] not ready ${now - this.spawnedAt} ms after start; killing pid ${child.pid}`);
+                this.kill(child);
+            }
+            return;
+        }
+        const since = now - this.lastStatusAt;
+        const pinned = this.lastStatus?.pinnedMs ? this.lastStatus.pinnedMs + since : 0;
+        if (pinned > this.timings.hungPinMs || since > this.timings.hungPinMs) {
+            // Last resort for a hung IO (NFS): nothing else releases the shared pinned() mutex. In-flight requests fail as
+            // lost (mutations outcome-unknown) when the exit arrives; the respawn runs uploads recover().
+            this.log(`[files-host] ${pinned > this.timings.hungPinMs ? `pinned section held ${pinned} ms` : `no status for ${since} ms`}; killing pid ${child.pid}`);
+            this.kill(child);
+            return;
+        }
+        if (this.restartRequestedAt === null || this.plannedExit)
+            return;
+        // The current host read its options after the change was requested: it is already current.
+        if (this.restartRequestedAt <= this.optionsReadAt) {
+            this.restartRequestedAt = null;
+            return;
+        }
+        const overdue = now - this.restartRequestedAt > this.timings.restartUploadGraceMs;
+        // Past the grace, stop feeding it: new requests wait for the restarted host while the ones in flight finish.
+        if (overdue)
+            this.ready = false;
+        if (this.sent.size === 0 && (overdue || (this.lastStatus?.uploads ?? 0) === 0)) {
+            this.restartRequestedAt = null;
+            this.log('[files-host] workspaces changed; restarting');
+            void this.shutdownChild(true);
+        }
+    }
+    /** Graceful shutdown of the current host (bounded by SIGKILL). `planned` respawns right after it exits. */
+    async shutdownChild(planned = false) {
+        const child = this.child;
+        if (!child)
+            return;
+        this.ready = false;
+        this.plannedExit = planned;
+        this.retiring = true;
+        this.lastStatus = undefined;
+        this.activityMayHaveChanged();
+        const exited = this.exited ?? Promise.resolve();
+        if (!this.post({ t: 'shutdown' }))
+            this.kill(child);
+        let timer;
+        const budget = new Promise(resolve => { timer = setTimeout(() => resolve('timeout'), this.timings.stopBudgetMs); });
+        if (await Promise.race([exited.then(() => 'exited'), budget]) === 'timeout') {
+            this.kill(child);
+            await Promise.race([exited, new Promise(resolve => setTimeout(resolve, this.timings.killExitGraceMs + 100))]);
+        }
+        if (timer)
+            clearTimeout(timer);
+    }
+    /**
+     * SIGKILL the host. Its 'exit' normally follows and fails what was sent to it; if it does not arrive within the grace
+     * (a zombie the OS will not reap, an exit event lost), the child is treated as gone anyway so requests settle and a
+     * successor starts.
+     */
+    kill(child) {
+        if (this.child !== child)
+            return;
+        this.ready = false;
+        this.hostUp = false;
+        this.retiring = true;
+        this.lastStatus = undefined;
+        this.activityMayHaveChanged();
+        try {
+            child.kill('SIGKILL');
+        }
+        catch { /* already gone */ }
+        const timer = setTimeout(() => { if (this.child === child)
+            this.onExit('killed; no exit event'); }, this.timings.killExitGraceMs);
+        timer.unref();
+    }
+    activityMayHaveChanged() {
+        const now = this.activity();
+        if (now === this.lastActivity)
+            return;
+        this.lastActivity = now;
+        try {
+            this.options.onActivityChanged?.();
+        }
+        catch { /* a listener failure must not break the host client */ }
+    }
+    log(line) { (this.options.log ?? console.error)(line); }
+}

package/dist/workstation/files/errors.js

@@ -0,0 +1,28 @@
+export class FileError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = 'FileError';
+    }
+}
+/** Maps an fs errno (or anything else) to a coded error; a FileError passes through unchanged. */
+export function toFileError(error) {
+    if (error instanceof FileError)
+        return error;
+    const code = error?.code;
+    if (code === 'ENOENT')
+        return new FileError('not_found', 'File or parent directory not found');
+    if (code === 'EACCES' || code === 'EPERM')
+        return new FileError('permission_denied', 'File access denied');
+    if (code === 'EEXIST')
+        return new FileError('conflict', 'Target already exists');
+    if (code === 'ELOOP')
+        return new FileError('invalid_path', 'Symbolic links are not allowed');
+    return new FileError('io_error', 'File operation failed');
+}
+export function cancelledError() { return new FileError('workstation_offline', 'File operation was cancelled'); }
+export function throwIfAborted(signal) {
+    if (signal?.aborted)
+        throw cancelledError();
+}