firedrill-run 2.2.4 → 3.0.0

package/dist/cli/src/native/drivers/jsonl-process.js

@@ -0,0 +1,538 @@
+/**
+ * jsonl-process.v1 driver and the shared JSON-lines channel (Plan20 spec 3.3).
+ *
+ * Protocol (one JSON object per line, UTF-8, `\n` terminated):
+ *   request  -> stdin : {"id": string, "session": string, "kind": "message"|"task", "input": <payload>}
+ *   response <- stdout: {"id": string, "output": <any>}  or  {"id": string, "error": <any>}
+ * `input` is the step's target-visible payload only (message text or task input).
+ *
+ * Process ownership: the protocol needs the service's stdin/stdout, which an
+ * already-running (attach) process cannot provide, so the plan service named by
+ * `service_alias` must be a `launch` service. The supervisor deliberately
+ * skips this service, so the driver owns it: it starts it in prepare() (spawn,
+ * shell:false, approved argv, cwd inside the approved root, env = ctx.env
+ * exactly) and stops its process group in dispose(). Readiness supports `none`
+ * and `stdout` patterns. A service handle from any other owner is refused.
+ * One channel per driver instance (driverFor() returns a new instance per drill).
+ *
+ * Requests are serialized per process (one in flight) so every malformed or
+ * oversized line is attributed to the in-flight step, which fails explicitly.
+ * The process is never killed by name or port; only the group this driver
+ * created is signalled (POSIX), or the child pid (Windows, no process groups).
+ */
+import { spawn } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { CliError } from '../../http.js';
+import { NATIVE_ERRORS, nativeError } from '../contracts.js';
+import { sha256Hex } from '../canonical.js';
+import { insideRoot } from '../roots.js';
+import { ownGroup, sweepGroup } from '../owned-groups.js';
+import { identityHeaders } from '../target-auth.js';
+import { DRIVER_ERRORS, OutputTail, StepRecorder, driverBounds, driverPlanFor, emit, planError, precheck, rootPath, stepAbort, stepInput, stepTimeoutMs, unattested, } from './types.js';
+export class ChannelFailure extends Error {
+    kind;
+    written;
+    constructor(kind, written) {
+        super(`The JSON-lines exchange did not complete (${kind}).`);
+        this.kind = kind;
+        this.written = written;
+    }
+}
+const MAX_ABANDONED = 256;
+export class JsonLinesProcess {
+    ctx;
+    label;
+    bounds;
+    io;
+    /** Present only when this channel spawned the process (driver-owned). */
+    owned;
+    stderr;
+    preReady;
+    partial = [];
+    partialSize = 0;
+    discarding = false;
+    ready = false;
+    /** SHA-256 of the UTF-8 stdout line that carried the approved readiness text; null without stdout readiness. */
+    readyLineSha256 = null;
+    readyPattern = null;
+    readyWaiter = null;
+    pending = null;
+    abandoned = new Set();
+    queue = Promise.resolve();
+    exitState = null;
+    exitPromise;
+    stopping = false;
+    onData = (chunk) => this.onStdout(chunk);
+    constructor(ctx, label, io, exited, owned) {
+        this.ctx = ctx;
+        this.label = label;
+        this.io = io;
+        this.owned = owned;
+        this.bounds = driverBounds(ctx);
+        this.stderr = new OutputTail(this.bounds.stderrTailBytes);
+        this.preReady = new OutputTail(4096);
+        io.stdin.on('error', () => undefined);
+        io.stdout.on('data', this.onData);
+        io.stdout.once('close', () => this.failAfterExit());
+        io.stderr?.on('data', (chunk) => this.stderr.push(chunk));
+        this.exitPromise = exited.then((info) => {
+            const record = (info && typeof info === 'object' ? info : {});
+            this.exitState = {
+                code: typeof record.code === 'number' ? record.code : null,
+                signal: typeof record.signal === 'string' ? record.signal : null,
+            };
+            // Give stdout a moment to drain a final response line, then fail what is left.
+            setTimeout(() => this.failAfterExit(), 500).unref();
+        }, () => { this.exitState = { code: null, signal: null }; this.failAfterExit(); });
+    }
+    /** Spawn the approved command (argv only, shell:false) and wait for readiness. */
+    static async start(ctx, launch) {
+        // Literal substring, matching the supervisor and the approval text ("output contains").
+        const pattern = launch.readiness.kind === 'stdout' ? launch.readiness.pattern : null;
+        if (launch.readiness.kind === 'stdout') {
+            if (!pattern)
+                throw planError(`The readiness text for ${launch.label} is empty, so the process was not started.`);
+        }
+        else if (launch.readiness.kind !== 'none') {
+            throw planError(`A driver-owned JSON-lines process (${launch.label}) supports only "none" or "stdout" readiness, so it was not started.`);
+        }
+        const posix = process.platform !== 'win32';
+        let child;
+        try {
+            child = spawn(launch.executable, launch.args, {
+                cwd: launch.cwd, env: { ...launch.env }, shell: false, detached: posix,
+                stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true,
+            });
+        }
+        catch {
+            throw nativeError('process_exited', `The ${launch.label} process could not be started, so no step was sent to it.`);
+        }
+        child.once('spawn', () => { if (posix)
+            ownGroup(child.pid); });
+        child.once('exit', () => { if (posix)
+            sweepGroup(child.pid); });
+        if (!child.stdin || !child.stdout) {
+            child.on('error', () => undefined);
+            child.once('spawn', () => {
+                if (posix)
+                    sweepGroup(child.pid);
+                else {
+                    try {
+                        child.kill('SIGKILL');
+                    }
+                    catch { /* already gone */ }
+                }
+            });
+            throw nativeError('process_exited', `The ${launch.label} process has no protocol pipes, so no step was sent to it.`);
+        }
+        const exited = new Promise((resolve) => {
+            child.once('exit', (code, signal) => resolve({ code, signal }));
+        });
+        const channel = new JsonLinesProcess(ctx, launch.label, { stdin: child.stdin, stdout: child.stdout, stderr: child.stderr }, exited, { child, ownsGroup: posix, stopTimeoutMs: launch.stopTimeoutMs });
+        try {
+            await channel.waitSpawned(child);
+            launch.onSpawned?.(channel);
+            if (pattern)
+                await channel.waitReady(pattern, launch.readinessTimeoutMs);
+            else
+                channel.ready = true;
+        }
+        catch (error) {
+            await channel.stop();
+            throw error;
+        }
+        emit(ctx, 'action.observed', { type: 'process.started', label: launch.label, pid: child.pid ?? null, process_group: posix });
+        return channel;
+    }
+    get exited() { return this.exitState !== null; }
+    /** Unique per launch, reported in the drill's signed close. */
+    launchId = randomUUID();
+    /** How the owned process ended, once it has; null while running (or when this channel does not own it). */
+    get exit() { return this.owned ? this.exitState : null; }
+    waitSpawned(child) {
+        return new Promise((resolve, reject) => {
+            const onError = () => {
+                child.off('spawn', onSpawn);
+                this.exitState ??= { code: null, signal: null };
+                reject(nativeError('process_exited', `The ${this.label} process could not be started, so no step was sent to it.`));
+            };
+            const onSpawn = () => { child.off('error', onError); child.on('error', () => undefined); resolve(); };
+            child.once('error', onError);
+            child.once('spawn', onSpawn);
+        });
+    }
+    async waitReady(pattern, timeoutMs) {
+        this.readyPattern = pattern;
+        const ready = new Promise((resolve, reject) => { this.readyWaiter = { resolve, reject }; });
+        const timer = setTimeout(() => this.readyWaiter?.reject(nativeError('readiness_timeout', `The ${this.label} process did not report readiness in time, so no step was sent to it.`)), Math.max(1, timeoutMs));
+        try {
+            await ready;
+        }
+        catch (error) {
+            emit(this.ctx, 'driver.failed', { type: 'process.not_ready', label: this.label,
+                stdout_tail: this.preReady.text(this.ctx.secrets), stderr_tail: this.stderr.text(this.ctx.secrets) });
+            await this.stop();
+            throw error;
+        }
+        finally {
+            clearTimeout(timer);
+            this.readyWaiter = null;
+        }
+    }
+    // -- stdout framing -------------------------------------------------------
+    onStdout(chunk) {
+        let start = 0;
+        while (start < chunk.length) {
+            const newline = chunk.indexOf(10, start);
+            if (newline === -1) {
+                this.appendPartial(chunk.subarray(start));
+                return;
+            }
+            this.appendPartial(chunk.subarray(start, newline));
+            start = newline + 1;
+            if (this.discarding) {
+                this.discarding = false;
+                continue;
+            }
+            const line = Buffer.concat(this.partial).toString('utf8');
+            this.partial = [];
+            this.partialSize = 0;
+            this.onLine(line);
+        }
+    }
+    appendPartial(piece) {
+        if (this.discarding || !piece.length)
+            return;
+        if (this.partialSize + piece.length > this.bounds.maxLineBytes) {
+            this.partial = [];
+            this.partialSize = 0;
+            this.discarding = true;
+            this.lineFault('oversized');
+            return;
+        }
+        this.partial.push(piece);
+        this.partialSize += piece.length;
+    }
+    onLine(line) {
+        if (!this.ready) {
+            this.preReady.push(Buffer.from(`${line}\n`, 'utf8'));
+            if (this.readyPattern !== null && line.includes(this.readyPattern)) {
+                // The exact line, UTF-8, without its terminator (a CRLF's carriage return included).
+                this.ready = true;
+                this.readyLineSha256 = sha256Hex(Buffer.from(line.endsWith('\r') ? line.slice(0, -1) : line, 'utf8'));
+                this.readyWaiter?.resolve();
+            }
+            return;
+        }
+        const trimmed = line.trim();
+        if (!trimmed)
+            return;
+        let message;
+        try {
+            message = JSON.parse(trimmed);
+        }
+        catch {
+            this.lineFault('malformed');
+            return;
+        }
+        if (!message || typeof message !== 'object' || Array.isArray(message) || typeof message.id !== 'string') {
+            this.lineFault('malformed');
+            return;
+        }
+        const record = message;
+        const id = record.id;
+        const hasOutput = Object.prototype.hasOwnProperty.call(record, 'output');
+        const hasError = Object.prototype.hasOwnProperty.call(record, 'error');
+        if (this.pending && this.pending.id === id) {
+            const pending = this.pending;
+            if (hasOutput === hasError)
+                pending.reject(new ChannelFailure('malformed', true));
+            else if (hasOutput)
+                pending.resolve({ ok: true, output: record.output });
+            else
+                pending.resolve({ ok: false, error: record.error });
+            return;
+        }
+        if (this.abandoned.delete(id)) {
+            emit(this.ctx, 'action.observed', { type: 'protocol.late_response', label: this.label, request_id: id,
+                output: hasOutput ? record.output : null, error: hasError ? record.error : null });
+            return;
+        }
+        emit(this.ctx, 'action.observed', { type: 'protocol.unsolicited_line', label: this.label, content: record });
+    }
+    lineFault(kind) {
+        if (!this.ready)
+            return;
+        // A faulty line carries no usable id. While an abandoned request may still be
+        // answering late, it could belong to that request, so it is not blamed on the
+        // in-flight step (which then ends by its own reply or its timeout).
+        if (this.pending && this.abandoned.size === 0) {
+            this.pending.reject(new ChannelFailure(kind, true));
+            return;
+        }
+        emit(this.ctx, 'action.observed', { type: 'protocol.fault', label: this.label, fault: kind,
+            attribution: this.pending ? 'ambiguous_after_abandoned_request' : 'no_request_in_flight' });
+    }
+    failAfterExit() {
+        if (!this.exitState)
+            return;
+        if (this.pending)
+            this.pending.reject(new ChannelFailure('exited', true));
+        if (this.readyWaiter)
+            this.readyWaiter.reject(nativeError('process_exited', `The ${this.label} process exited before it was ready, so no step was sent to it.`));
+        if (!this.stopping && this.exitState) {
+            const state = this.exitState;
+            this.stopping = true; // report once
+            emit(this.ctx, 'driver.failed', { type: 'process.exited', label: this.label, code: state.code, signal: state.signal,
+                stderr_tail: this.stderr.text(this.ctx.secrets) });
+        }
+    }
+    // -- requests -------------------------------------------------------------
+    /** Send one serialized request line; one request is in flight at a time. */
+    request(id, line, signal) {
+        const run = () => this.send(id, line, signal);
+        const result = this.queue.then(run, run);
+        this.queue = result.catch(() => undefined);
+        return result;
+    }
+    send(id, line, signal) {
+        return new Promise((resolve, reject) => {
+            if (signal.aborted) {
+                reject(new ChannelFailure('aborted', false));
+                return;
+            }
+            const stdin = this.io.stdin;
+            if (this.exitState || this.stopping || stdin.destroyed || !stdin.writable) {
+                reject(new ChannelFailure('exited', false));
+                return;
+            }
+            const cleanup = () => {
+                signal.removeEventListener('abort', onAbort);
+                if (this.pending?.id === id)
+                    this.pending = null;
+            };
+            const onAbort = () => {
+                cleanup();
+                this.abandoned.add(id);
+                if (this.abandoned.size > MAX_ABANDONED)
+                    this.abandoned.delete(this.abandoned.values().next().value);
+                reject(new ChannelFailure('aborted', true));
+            };
+            this.pending = {
+                id,
+                resolve: (reply) => { cleanup(); resolve(reply); },
+                reject: (failure) => { cleanup(); reject(failure); },
+            };
+            signal.addEventListener('abort', onAbort, { once: true });
+            // From here on the request may have reached the target.
+            stdin.write(line);
+        });
+    }
+    /**
+     * Driver-owned: signal only the group this channel created, then escalate after
+     * the grace period. Supervisor-owned: stop using the pipes; the supervisor stops
+     * the process.
+     */
+    async stop() {
+        this.stopping = true;
+        const owned = this.owned;
+        if (!owned) {
+            this.pending?.reject(new ChannelFailure('aborted', true));
+            this.io.stdout.off('data', this.onData);
+            return;
+        }
+        if (!this.exitState) {
+            try {
+                this.io.stdin.end();
+            }
+            catch { /* already closed */ }
+            this.signal('SIGTERM');
+            if (!(await this.waitExit(Math.max(1, owned.stopTimeoutMs))) && !this.exitState) {
+                this.signal('SIGKILL');
+                await this.waitExit(2_000);
+            }
+        }
+        // The exit handler swept the group; never let a descendant holding a pipe
+        // keep the companion alive.
+        this.pending?.reject(new ChannelFailure('exited', true));
+        for (const stream of [this.io.stdin, this.io.stdout, this.io.stderr]) {
+            try {
+                stream?.destroy();
+            }
+            catch { /* already closed */ }
+        }
+    }
+    /** Resolve true if the owned leader exits within `ms`; the timer never outlives the wait. */
+    async waitExit(ms) {
+        let timer;
+        try {
+            return await Promise.race([this.exitPromise.then(() => true),
+                new Promise((resolve) => { timer = setTimeout(() => resolve(false), ms); })]);
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
+    }
+    signal(name) {
+        const owned = this.owned;
+        const pid = owned?.child.pid;
+        if (!owned || this.exitState || pid === undefined || owned.child.exitCode !== null || owned.child.signalCode !== null)
+            return;
+        try {
+            if (owned.ownsGroup)
+                process.kill(-pid, name);
+            else
+                owned.child.kill(name);
+        }
+        catch { /* already gone */ }
+    }
+}
+// ---------------------------------------------------------------------------
+// Session shared by jsonl-process.v1 and function-wrapper.v1.
+export class JsonLinesSession {
+    sessionKey;
+    ctx;
+    channel;
+    onClose;
+    counter = 0;
+    closed = false;
+    constructor(ctx, sessionKey, channel, onClose) {
+        this.ctx = ctx;
+        this.sessionKey = sessionKey;
+        this.channel = channel;
+        this.onClose = onClose;
+    }
+    async invoke(step, signal) {
+        const { ctx } = this;
+        const bounds = driverBounds(ctx);
+        const recorder = new StepRecorder(ctx, step);
+        if (this.closed)
+            return recorder.finish('failed', null, DRIVER_ERRORS.session_closed);
+        const refused = precheck(step, this.sessionKey, ['message', 'task']);
+        if (refused)
+            return recorder.finish('failed', null, refused);
+        const input = stepInput(step);
+        this.counter += 1;
+        const id = `${step.step_id}.${this.counter}`;
+        let line;
+        try {
+            // A synthetic-session grant travels in the line's `headers` object under its
+            // declared header name (the line is never recorded; only `input` is observed).
+            const headers = identityHeaders(ctx.identity);
+            line = JSON.stringify({ id, session: this.sessionKey, kind: input.kind, input: input.input,
+                ...(Object.keys(headers).length ? { headers } : {}) });
+        }
+        catch (error) {
+            return recorder.finish('failed', null, error instanceof CliError ? error.code : DRIVER_ERRORS.step_unsupported);
+        }
+        if (Buffer.byteLength(line, 'utf8') + 1 > bounds.maxLineBytes)
+            return recorder.finish('failed', null, DRIVER_ERRORS.step_unsupported);
+        let channel;
+        try {
+            channel = await this.channel();
+        }
+        catch (error) {
+            return recorder.finish('failed', null, error instanceof CliError ? error.code : NATIVE_ERRORS.process_exited);
+        }
+        recorder.record('action.observed', { type: 'target.exchange',
+            direction: 'to_target', kind: input.kind, session_key: this.sessionKey, request_id: id, content: input.input,
+        });
+        const abort = stepAbort(ctx, signal, stepTimeoutMs(step, bounds));
+        try {
+            const reply = await channel.request(id, `${line}\n`, abort.signal);
+            if (reply.ok) {
+                recorder.record('action.observed', { type: 'target.exchange', direction: 'from_target', request_id: id, content: reply.output });
+                return recorder.finish('completed', reply.output, null);
+            }
+            recorder.record('action.observed', { type: 'target.exchange', direction: 'from_target', request_id: id, error: reply.error });
+            return recorder.finish('failed', { error: reply.error }, DRIVER_ERRORS.target_error);
+        }
+        catch (error) {
+            if (error instanceof ChannelFailure)
+                return recorder.interrupted(error.kind === 'aborted' ? (abort.cause() ?? 'cancelled') : error.kind, error.written);
+            return recorder.interrupted('thrown', true);
+        }
+        finally {
+            abort.dispose();
+        }
+    }
+    async close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        this.onClose();
+    }
+}
+// ---------------------------------------------------------------------------
+// jsonl-process.v1 driver.
+export class JsonlProcessDriver {
+    profile = 'jsonl-process.v1';
+    process = null;
+    /** The process this driver launched for the drill, kept after dispose so the close can report how it ended. */
+    launched = null;
+    sessions = new Set();
+    async describe() {
+        return { available: true, reason: null };
+    }
+    async prepare(ctx) {
+        const plan = driverPlanFor(ctx, 'jsonl-process.v1');
+        if (this.process)
+            throw planError('The JSON-lines driver is already prepared; a second process was not started.');
+        const service = ctx.plan.services.find((entry) => entry.alias === plan.service_alias);
+        if (!service)
+            throw planError(`The approved plan has no service "${plan.service_alias}", so the JSON-lines driver was not prepared.`);
+        if (service.mode !== 'launch' || !service.command)
+            throw planError(`The JSON-lines service "${service.alias}" must be a launch service: the driver owns its standard input and output, which an already running process cannot provide. Nothing was started.`);
+        const handle = ctx.services.get(service.alias);
+        // The supervisor skips this service (see supervisor.ts); a handle here means
+        // something else started it, and its stdio is not ours to speak over.
+        if (handle)
+            throw planError(`The JSON-lines service "${service.alias}" was started by another owner, so the driver cannot use its standard input and output. Nothing was sent.`);
+        const cwd = await insideRoot(rootPath(ctx, service.root_alias), service.cwd);
+        this.process = await JsonLinesProcess.start(ctx, {
+            label: `service "${service.alias}"`,
+            executable: service.command.executable, args: [...service.command.args], cwd, env: ctx.env,
+            readiness: service.readiness, readinessTimeoutMs: service.readiness_timeout_ms,
+            stopTimeoutMs: service.stop_timeout_ms || driverBounds(ctx).stopTimeoutMs,
+            // Recorded on spawn, so a process that failed readiness is still reported as launched and reaped.
+            onSpawned: (channel) => { this.launched = { alias: service.alias, channel }; },
+        });
+    }
+    /** SHA-256 of the stdout line in which the driver-owned process announced readiness. */
+    async readinessProbe(ctx) {
+        const digest = this.process?.readyLineSha256 ?? null;
+        if (!digest) {
+            const plan = driverPlanFor(ctx, 'jsonl-process.v1');
+            throw unattested(`The JSON-lines service "${plan.service_alias}" needs stdout readiness text in the approved plan before the service can admit the drill. Nothing was claimed.`);
+        }
+        return { kind: 'jsonl_handshake', response_sha256: digest };
+    }
+    ownedProcesses() {
+        const launched = this.launched;
+        return launched ? [{ alias: launched.alias, launch_id: launched.channel.launchId, exit: launched.channel.exit }] : [];
+    }
+    async openSession(ctx, sessionKey) {
+        if (!this.process)
+            throw new CliError(DRIVER_ERRORS.session_failed, 'The JSON-lines driver was not prepared, so no session was opened.');
+        if (!sessionKey)
+            throw new CliError(DRIVER_ERRORS.session_failed, 'The session key is empty, so no session was opened.');
+        const channel = async () => {
+            const current = this.process;
+            if (!current || current.exited)
+                throw nativeError('process_exited', 'The JSON-lines service is no longer running, so the step was not sent. It is not restarted automatically.');
+            return current;
+        };
+        const session = new JsonLinesSession(ctx, sessionKey, channel, () => this.sessions.delete(session));
+        this.sessions.add(session);
+        emit(ctx, 'action.observed', { type: 'session.opened', profile: this.profile, session_key: sessionKey });
+        return session;
+    }
+    async dispose() {
+        for (const session of [...this.sessions])
+            await session.close();
+        const current = this.process;
+        this.process = null;
+        await current?.stop();
+    }
+}