@sigrun/sigrun 0.1.6 → 0.2.0

package/dist/sandbox/pool.js

@@ -0,0 +1,548 @@
+/** Per-user worker lifecycle and the client side of the stdio protocol. */
+import { spawn } from "node:child_process";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { ExecutionError, FileError } from "@earendil-works/pi-durable/env";
+import { decodeError, decodeMessage, decodeValue, encodeMessage, encodeValue, } from "./protocol.js";
+import { launchInJail } from "./launch.js";
+function crashError(op, user, detail) {
+    const message = `sandbox worker for ${user} exited (${detail})`;
+    // The M1 protocol names these transport failures "crashed" and "io".
+    // pi-durable 1.3.0 has not added those codes to its public unions yet;
+    // keep the protocol codes visible to callers rather than hiding the crash
+    // behind a generic success or an unrelated spawn error.
+    return op === "exec"
+        ? new ExecutionError("crashed", message)
+        : new FileError("io", message);
+}
+class WorkerProcess {
+    client;
+    child;
+    exit;
+    capabilities;
+    buffer = "";
+    nextRequest = 1;
+    pending = new Map();
+    watchEvents = new Map();
+    watchHandles = new Map();
+    commandGroups = new Set();
+    drainedResolve;
+    drained;
+    stopPromise;
+    exited = false;
+    retiring = false;
+    constructor(client, launch) {
+        this.client = client;
+        this.child = spawn(launch.command, launch.args, {
+            env: launch.env,
+            // Own a process group: pi-durable's commands are detached too, so
+            // stopping only the worker can orphan them. Group cleanup reaches
+            // the command groups below this worker, including after SIGKILL.
+            detached: process.platform !== "win32",
+            // Exactly three descriptors: pi-durable gives every command its own
+            // pipes, so neither side of this IPC channel reaches a command.
+            stdio: ["pipe", "pipe", "pipe"],
+            windowsHide: true,
+        });
+        this.child.stdout.setEncoding("utf8");
+        this.child.stdout.on("data", (chunk) => this.receive(chunk));
+        this.child.stderr.setEncoding("utf8");
+        let stderr = "";
+        this.child.stderr.on("data", (chunk) => {
+            stderr += chunk;
+            for (;;) {
+                const newline = stderr.indexOf("\n");
+                if (newline < 0)
+                    break;
+                const line = stderr.slice(0, newline).trimEnd();
+                stderr = stderr.slice(newline + 1);
+                if (line !== "")
+                    this.client.log(`sandbox[${this.client.user}] ${line}`);
+            }
+        });
+        this.exit = new Promise((resolve) => {
+            this.child.once("error", () => { this.exited = true; });
+            this.child.once("exit", (code, signal) => {
+                this.exited = true;
+                // A direct crash cannot run the worker's stdin-close cleanup.
+                // The negative pid addresses the worker's whole process group;
+                // ESRCH simply means every member is already gone.
+                this.killGroup("SIGKILL");
+                this.failAll(code, signal);
+                this.client.processExited(this);
+                resolve();
+            });
+        });
+        this.capabilities = this.request("hello", []).then((value) => {
+            // The canonical hello shape is {capabilities: {listFiles}}.
+            return { listFiles: value.capabilities.listFiles === true };
+        });
+        // A caller will observe this through its first operation; attaching a
+        // rejection sink here prevents a failed boot becoming unhandled.
+        void this.capabilities.catch(() => { });
+    }
+    get running() {
+        return !this.exited && this.child.exitCode === null && this.child.signalCode === null;
+    }
+    get inFlight() {
+        return this.pending.size;
+    }
+    commandGroupsSnapshot() {
+        return [...this.commandGroups];
+    }
+    request(op, args, onEvent, context, failureOp = op) {
+        if (!this.running)
+            return Promise.reject(crashError(op, this.client.user, "not running"));
+        if (context?.abortSignal?.aborted) {
+            return Promise.reject(op === "exec"
+                ? new ExecutionError("aborted", "aborted")
+                : new FileError("aborted", "aborted"));
+        }
+        const id = this.nextRequest++;
+        return new Promise((resolve, reject) => {
+            if (op === "handle.close" && typeof args[0] === "number") {
+                const watchRequest = this.watchHandles.get(args[0]);
+                if (watchRequest !== undefined) {
+                    this.watchEvents.delete(watchRequest);
+                    this.watchHandles.delete(args[0]);
+                }
+            }
+            const pending = {
+                op: failureOp,
+                watch: op === "result" && args[0] === "watch",
+                resolve: (value) => resolve(value),
+                reject,
+                onEvent,
+                context,
+            };
+            this.pending.set(id, pending);
+            if (context?.abortSignal !== undefined) {
+                const signal = context.abortSignal;
+                const abort = () => {
+                    if (!this.pending.has(id) || !this.running)
+                        return;
+                    this.child.stdin.write(encodeMessage({ id: 0, op: "abort", args: [id] }));
+                };
+                pending.onAbort = abort;
+                signal.addEventListener("abort", abort, { once: true });
+            }
+            this.child.stdin.write(encodeMessage({ id, op, args }), (error) => {
+                if (error !== null && error !== undefined)
+                    this.settleError(id, error);
+            });
+        });
+    }
+    markRetiring() {
+        this.retiring = true;
+    }
+    async drainAndStop() {
+        if (!this.running)
+            return;
+        // A stop during a policy change drains calls already handed to the
+        // worker; it never races their still-running commands.
+        if (this.inFlight > 0) {
+            this.drained ??= new Promise((resolve) => { this.drainedResolve = resolve; });
+            await this.drained;
+        }
+        await this.stop();
+    }
+    stop() {
+        if (this.stopPromise !== undefined)
+            return this.stopPromise;
+        this.stopPromise = (async () => {
+            if (!this.running) {
+                this.killGroup("SIGKILL");
+                await this.exit;
+                return;
+            }
+            const killTimer = setTimeout(() => {
+                // The worker may be wedged and unable to run its own cleanup;
+                // kill its group, never just the Node process.
+                this.killGroup("SIGKILL");
+            }, 2_000);
+            killTimer.unref();
+            try {
+                // Let the worker's stdin-close hook run pi-durable cleanup for
+                // tracked command groups before the forced group-kill fallback.
+                this.child.stdin.end();
+                await this.exit;
+            }
+            finally {
+                clearTimeout(killTimer);
+                this.killGroup("SIGKILL");
+            }
+        })();
+        return this.stopPromise;
+    }
+    killGroup(signal) {
+        const pid = this.child.pid;
+        if (pid === undefined || process.platform === "win32") {
+            this.child.kill(signal);
+            return;
+        }
+        // pi-durable puts each command in a NEW process group, so killing only
+        // -workerPid cannot reach it. The worker reports actual spawned pids;
+        // kill each command's group as well as the worker's group. ESRCH means
+        // a group already exited.
+        const groups = new Set();
+        for (const commandPid of this.commandGroups)
+            groups.add(commandPid);
+        this.commandGroups.clear();
+        groups.add(pid);
+        for (const group of groups) {
+            try {
+                process.kill(-group, signal);
+            }
+            catch (error) {
+                if (error.code !== "ESRCH")
+                    throw error;
+            }
+        }
+    }
+    receive(chunk) {
+        this.buffer += chunk;
+        for (;;) {
+            const newline = this.buffer.indexOf("\n");
+            if (newline < 0)
+                break;
+            const line = this.buffer.slice(0, newline);
+            this.buffer = this.buffer.slice(newline + 1);
+            if (line === "")
+                continue;
+            let message;
+            try {
+                message = decodeMessage(line);
+            }
+            catch (error) {
+                this.client.log(`sandbox[${this.client.user}] invalid worker message: ${error instanceof Error ? error.message : String(error)}`);
+                continue;
+            }
+            const pending = this.pending.get(message.id);
+            if ("event" in message) {
+                if (message.event === "command") {
+                    if (message.phase === "start")
+                        this.commandGroups.add(message.pid);
+                    else
+                        this.commandGroups.delete(message.pid);
+                    continue;
+                }
+                const listener = pending?.onEvent ?? this.watchEvents.get(message.id);
+                if (listener === undefined)
+                    continue;
+                try {
+                    listener(message);
+                }
+                catch (error) {
+                    if (pending !== undefined)
+                        this.settleError(message.id, error);
+                    else
+                        this.client.log(`sandbox[${this.client.user}] watcher callback failed: ${error instanceof Error ? error.message : String(error)}`);
+                }
+                continue;
+            }
+            if (pending === undefined)
+                continue;
+            this.pending.delete(message.id);
+            this.removeAbort(pending);
+            if (message.ok) {
+                try {
+                    const value = decodeValue(message.value);
+                    this.client.valueDecoded(value);
+                    if (pending.watch && pending.onEvent !== undefined) {
+                        this.watchEvents.set(message.id, pending.onEvent);
+                        const handle = this.findHandle(value);
+                        if (handle !== undefined)
+                            this.watchHandles.set(handle, message.id);
+                    }
+                    pending.resolve(value);
+                }
+                catch (error) {
+                    pending.reject(error instanceof Error ? error : new Error(String(error)));
+                }
+            }
+            else {
+                pending.reject(decodeError(message.error));
+            }
+            this.maybeDrained();
+        }
+    }
+    findHandle(value) {
+        if (Array.isArray(value)) {
+            for (const item of value) {
+                const found = this.findHandle(item);
+                if (found !== undefined)
+                    return found;
+            }
+            return undefined;
+        }
+        if (typeof value !== "object" || value === null || value instanceof Uint8Array)
+            return undefined;
+        if ("$handle" in value)
+            return typeof value.$handle === "number" ? value.$handle : undefined;
+        for (const item of Object.values(value)) {
+            const found = this.findHandle(item);
+            if (found !== undefined)
+                return found;
+        }
+        return undefined;
+    }
+    settleError(id, error) {
+        const pending = this.pending.get(id);
+        if (pending === undefined)
+            return;
+        this.pending.delete(id);
+        this.removeAbort(pending);
+        pending.reject(error instanceof Error ? error : new Error(String(error)));
+        this.maybeDrained();
+    }
+    removeAbort(pending) {
+        if (pending.onAbort !== undefined) {
+            pending.context?.abortSignal?.removeEventListener("abort", pending.onAbort);
+        }
+    }
+    failAll(code, signal) {
+        const detail = signal ?? `code ${code ?? "unknown"}`;
+        for (const [id, pending] of this.pending) {
+            this.removeAbort(pending);
+            pending.reject(crashError(pending.op, this.client.user, detail));
+            this.pending.delete(id);
+        }
+        this.drainedResolve?.();
+        this.drainedResolve = undefined;
+        this.watchEvents.clear();
+        this.watchHandles.clear();
+        this.client.clearHandles();
+    }
+    maybeDrained() {
+        if (this.inFlight === 0) {
+            this.drainedResolve?.();
+            this.drainedResolve = undefined;
+            this.client.callSettled();
+        }
+    }
+}
+export class WorkerClient {
+    pool;
+    user;
+    policy;
+    policyJson;
+    process;
+    replacement;
+    idleTimer;
+    handles = new Set();
+    closed = false;
+    constructor(policy, pool) {
+        this.pool = pool;
+        this.user = policy.user;
+        this.policy = policy;
+        this.policyJson = JSON.stringify(policy);
+    }
+    /** @internal Pool-only policy change notification. */
+    updatePolicy(policy) {
+        const json = JSON.stringify(policy);
+        if (json === this.policyJson)
+            return;
+        this.policy = policy;
+        this.policyJson = json;
+        this.process?.markRetiring();
+        // Start the handoff now, but do not spawn until the old process has
+        // drained. The next call joins this same promise.
+        if (this.process !== undefined)
+            void this.ensureProcess().catch(() => { });
+        // Retiring workers are owned by the drain path, never the idle clock.
+        this.clearIdle();
+    }
+    log(line) {
+        this.pool.log(line);
+    }
+    async call(op, args, onEvent, context) {
+        const crashOp = op === "result" && args[0] === "exec" ? "exec" : op;
+        if (this.closed)
+            throw crashError(crashOp, this.user, "client stopped");
+        this.clearIdle();
+        const process = await this.ensureProcess();
+        try {
+            const result = await process.request(op, args, onEvent, context, crashOp);
+            return encodeValue(result);
+        }
+        finally {
+            this.callSettled();
+        }
+    }
+    /** Decode a wire failure into pi-durable's typed error (test/tool seam). */
+    async decodeError(reply) {
+        if (reply.ok)
+            throw new Error("not an error reply");
+        throw decodeError(reply.error);
+    }
+    capabilities() {
+        return this.ensureProcess().then((process) => process.capabilities);
+    }
+    openHandles() {
+        return this.handles.size;
+    }
+    /** Read-only snapshot of command groups currently tracked for crash cleanup. */
+    trackedCommandGroups() {
+        return [...(this.process?.commandGroupsSnapshot() ?? [])];
+    }
+    /** Current process id without starting a replacement (lifecycle probe). */
+    get pid() {
+        return this.process?.running ? this.process.child.pid : undefined;
+    }
+    valueDecoded(value) {
+        this.collectHandles(value);
+        this.armIdle();
+    }
+    collectHandles(value) {
+        if (Array.isArray(value)) {
+            for (const item of value)
+                this.collectHandles(item);
+            return;
+        }
+        if (typeof value !== "object" ||
+            value === null ||
+            value instanceof Uint8Array)
+            return;
+        if ("$handle" in value) {
+            const id = value.$handle;
+            if (typeof id === "number")
+                this.handles.add(id);
+            return;
+        }
+        for (const item of Object.values(value))
+            this.collectHandles(item);
+    }
+    hasHandle(id) {
+        return this.handles.has(id);
+    }
+    releaseHandle(id) {
+        this.handles.delete(id);
+        this.armIdle();
+    }
+    clearHandles() {
+        this.handles.clear();
+    }
+    processExited(process) {
+        if (this.process === process)
+            this.armIdle();
+    }
+    callSettled() {
+        this.armIdle();
+    }
+    close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        this.clearIdle();
+        this.clearHandles();
+        const process = this.process;
+        if (process !== undefined)
+            void process.stop();
+    }
+    async stop() {
+        this.close();
+        await this.replacement?.catch(() => { });
+        await this.process?.stop();
+    }
+    async ensureProcess() {
+        if (this.closed)
+            throw new Error(`sandbox client for ${this.user} is stopped`);
+        if (this.process?.running && !this.process.retiring)
+            return this.process;
+        if (this.replacement !== undefined)
+            return this.replacement;
+        this.replacement = (async () => {
+            const previous = this.process;
+            if (previous !== undefined)
+                await previous.drainAndStop();
+            if (this.closed)
+                throw new Error(`sandbox client for ${this.user} is stopped`);
+            const launch = this.pool.launcher(this.policy);
+            const process = new WorkerProcess(this, launch);
+            this.process = process;
+            await process.capabilities;
+            return process;
+        })().finally(() => {
+            this.replacement = undefined;
+        });
+        return this.replacement;
+    }
+    clearIdle() {
+        if (this.idleTimer !== undefined)
+            clearTimeout(this.idleTimer);
+        this.idleTimer = undefined;
+    }
+    armIdle() {
+        this.clearIdle();
+        if (this.closed ||
+            this.handles.size > 0 ||
+            this.process === undefined ||
+            !this.process.running ||
+            this.process.retiring ||
+            this.process.inFlight > 0)
+            return;
+        const minutes = this.pool.idleMinutes();
+        if (!Number.isFinite(minutes) || minutes <= 0)
+            return;
+        this.idleTimer = setTimeout(() => {
+            const process = this.process;
+            if (process !== undefined && process.inFlight === 0 && this.handles.size === 0) {
+                process.markRetiring();
+                void process.stop();
+            }
+        }, minutes * 60_000);
+        this.idleTimer.unref();
+    }
+}
+export class WorkerPool {
+    opts;
+    clients = new Map();
+    constructor(opts) {
+        this.opts = opts;
+    }
+    client(policy) {
+        let client = this.clients.get(policy.user);
+        if (client === undefined) {
+            client = new WorkerClient(policy, this.opts);
+            this.clients.set(policy.user, client);
+        }
+        else {
+            client.updatePolicy(policy);
+        }
+        return client;
+    }
+    async stopUser(user) {
+        const client = this.clients.get(user);
+        if (client === undefined)
+            return;
+        this.clients.delete(user);
+        await client.stop();
+    }
+    async stopAll() {
+        const clients = [...this.clients.values()];
+        this.clients.clear();
+        await Promise.all(clients.map((client) => client.stop()));
+    }
+}
+/** Prefer the built sibling; source checkouts run the TypeScript entry via tsx. */
+export function resolveWorkerEntry(coreRoot) {
+    const built = join(import.meta.dirname, "worker.js");
+    if (existsSync(built))
+        return { command: process.execPath, args: [built] };
+    return {
+        command: process.execPath,
+        args: ["--import", "tsx", join(coreRoot, "src", "sandbox", "worker.ts")],
+    };
+}
+/** The OS-independent test launcher: no helper, only the worker entry. */
+export function testLauncher(workerEntry) {
+    return (policy) => ({
+        command: workerEntry.command,
+        args: [...workerEntry.args],
+        env: { ...policy.env },
+    });
+}
+/** The production launcher applies Landlock before exec'ing the Node worker. */
+export function sandboxLauncher(helper, workerEntry) {
+    return (policy) => launchInJail(helper, policy, workerEntry.command, workerEntry.args, policy.env);
+}