@platless/axom-cli 0.3.13 → 0.5.9

package/dist/workstation/exec/durable.js

@@ -0,0 +1,517 @@
+import { ExecErrorCode, } from '../keepers/terminal/protocol.js';
+import { ExecShaper } from './shaping.js';
+/** TERM → KILL grace of a cancel (spec §14). */
+export const EXEC_KILL_GRACE_MS = 5000;
+const CHECKPOINT_VERSION = 1;
+function parseBlob(blob) {
+    if (!blob)
+        return null;
+    try {
+        const value = JSON.parse(blob);
+        const stream = (s) => !!s && Number.isSafeInteger(s.total) && Number.isSafeInteger(s.headLen)
+            && typeof s.tail === 'string' && typeof s.pending === 'string' && Number.isSafeInteger(s.lost);
+        if (value?.v !== CHECKPOINT_VERSION || !stream(value.stdout) || !stream(value.stderr))
+            return null;
+        const wire = value.wire && typeof value.wire === 'object' && typeof value.wire.streaming === 'boolean' ? { streaming: value.wire.streaming } : undefined;
+        return {
+            ...value, cancel: value.cancel && Number.isFinite(value.cancel.termAtMs) ? value.cancel : null, outlivesChannel: value.outlivesChannel === true,
+            wire, gapBytes: Number.isSafeInteger(value.gapBytes) && value.gapBytes > 0 ? value.gapBytes : 0,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+function errorCode(error) {
+    return typeof error === 'object' && error !== null && typeof error.code === 'string' ? error.code : undefined;
+}
+const messageOf = (error) => (error instanceof Error ? error.message : String(error));
+/**
+ * The head bytes a checkpoint did not carry, re-read from the keeper: records from offset 0 (below the commit only the kept head
+ * reserve and the spawn record remain) up to the checkpoint's offset, until each stream has `want` contiguous bytes from 0.
+ */
+async function readHeads(keeper, execId, want, through) {
+    const heads = { stdout: Buffer.alloc(want.stdout), stderr: Buffer.alloc(want.stderr) };
+    const filled = { stdout: 0, stderr: 0 };
+    let from = 0;
+    while ((filled.stdout < want.stdout || filled.stderr < want.stderr) && from <= through) {
+        const page = await keeper.execRead({ execId, from, limit: 1024 });
+        if (page.records.length === 0)
+            break;
+        for (const record of page.records) {
+            if (record.o > through) {
+                from = through + 1;
+                break;
+            }
+            if (record.k !== 'out' && record.k !== 'err')
+                continue;
+            const name = record.k === 'out' ? 'stdout' : 'stderr';
+            const so = record.so;
+            if (so !== filled[name] || so >= want[name])
+                continue; // contiguous from 0 only
+            const bytes = Buffer.from(record.b, 'base64');
+            filled[name] += bytes.copy(heads[name], so, 0, Math.min(bytes.length, want[name] - so));
+        }
+        if (from <= through)
+            from = page.next;
+    }
+    return { stdout: heads.stdout.subarray(0, filled.stdout), stderr: heads.stderr.subarray(0, filled.stderr) };
+}
+export class DurableExec {
+    keeper;
+    execId;
+    shaper;
+    outlivesChannel;
+    wire;
+    options;
+    now;
+    killGraceMs;
+    log;
+    next = 0;
+    processed = -1;
+    committed = -1;
+    checkpointed = -1;
+    bytesSinceCheckpoint = 0;
+    lastCheckpointMs;
+    lastRecordMs;
+    lastContactMs;
+    cancelState = null;
+    sink;
+    sub;
+    subscribing = false;
+    done = false;
+    exitOffset;
+    released = false;
+    processing = Promise.resolve();
+    persisting = Promise.resolve();
+    killTimer;
+    resubscribeTimer;
+    watchdog;
+    settle;
+    /** Highest offset the server acknowledged (wire v5); the keeper's commit watermark after a restart. */
+    acked = -1;
+    /** Offsets of written checkpoints whose decoders held no bytes, ascending (ack-gated commits land only on these). */
+    cleanCheckpoints = [];
+    holds = new Set();
+    progressListeners = new Set();
+    gapBytesSeen = 0;
+    /** Target of the commit in flight (or the last one sent), so a new reader never starts below what it may delete. */
+    committing = -1;
+    /** Resolves once with the exec's outcome. */
+    outcome;
+    /** Process id of the command (from the start answer or the spawn record), when known. */
+    pid = null;
+    /** When keeper-terminal created the exec (its clock). */
+    startedAtMs;
+    /**
+     * Settles once the exec has a checkpoint (its first write was attempted; at once for a restored exec). Until then a restarted
+     * daemon finds no checkpoint and has to guess what the exec is (exec/policy.ts `load`).
+     */
+    firstCheckpoint = Promise.resolve();
+    constructor(keeper, execId, shaper, 
+    /** See {@link CheckpointBlob.outlivesChannel}. */
+    outlivesChannel, 
+    /** See {@link CheckpointBlob.wire}; null for an exec no server can re-attach to (wire ≤ 4). */
+    wire, options) {
+        this.keeper = keeper;
+        this.execId = execId;
+        this.shaper = shaper;
+        this.outlivesChannel = outlivesChannel;
+        this.wire = wire;
+        this.options = options;
+        this.now = options.now ?? Date.now;
+        this.killGraceMs = options.killGraceMs ?? EXEC_KILL_GRACE_MS;
+        this.log = options.log ?? (() => { });
+        this.lastCheckpointMs = this.lastRecordMs = this.lastContactMs = this.startedAtMs = this.now();
+        this.outcome = new Promise((resolve) => { this.settle = resolve; });
+    }
+    /** Follows an exec the keeper just started (`exec.start` answered `descriptor`). Writes the first checkpoint at once. */
+    static follow(keeper, descriptor, init, options = {}) {
+        const exec = new DurableExec(keeper, descriptor.execId, new ExecShaper(), init.outlivesChannel, init.wire ?? null, options);
+        exec.pid = descriptor.pid;
+        exec.startedAtMs = descriptor.createdAtMs;
+        exec.begin();
+        // Settles once the checkpoint WRITE was answered (or failed), not after the commit behind it: a keeper that is slow to commit
+        // must not hold the caller's shell slot (and a drain or a force update) for as long as the commit takes (Task 26).
+        exec.firstCheckpoint = new Promise((resolve) => {
+            exec.persist(resolve).catch((error) => exec.log(`exec ${exec.execId}: checkpoint failed: ${messageOf(error)}`)).finally(resolve);
+        });
+        return exec;
+    }
+    /**
+     * Restores an exec a previous daemon followed (after a crash, an upgrade, a kill -9): its checkpoint's shaping and cancel state,
+     * the head re-read from the keeper, then the records after the checkpoint's offset. A pending cancel continues its grace.
+     */
+    static async restore(keeper, status, options = {}) {
+        const checkpoint = status.checkpoint;
+        const blob = parseBlob(checkpoint?.blob);
+        let shaper = new ExecShaper();
+        if (blob && checkpoint && !status.logReclaimed) {
+            const heads = await readHeads(keeper, status.execId, { stdout: blob.stdout.headLen, stderr: blob.stderr.headLen }, checkpoint.offset);
+            shaper = new ExecShaper(blob, heads);
+        }
+        // Without a checkpoint (it was never written) the caller says whether it outlives its channel; else it does not (wire ≤ 4).
+        const wire = blob ? blob.wire ?? null : options.wire ?? null;
+        const exec = new DurableExec(keeper, status.execId, shaper, blob?.outlivesChannel ?? options.outlivesChannel ?? false, wire, options);
+        exec.pid = status.pid;
+        exec.startedAtMs = status.createdAtMs;
+        if (blob && checkpoint) {
+            exec.next = checkpoint.offset + 1;
+            exec.processed = exec.checkpointed = checkpoint.offset;
+            exec.cancelState = blob.cancel;
+            exec.gapBytesSeen = blob.gapBytes ?? 0;
+            if (shaper.clean)
+                exec.cleanCheckpoints.push(checkpoint.offset);
+        }
+        // Under an ack-gated commit the keeper's commit watermark never passed an acknowledgement: it is the server's lower bound.
+        exec.committed = exec.acked = status.committed;
+        if (status.logReclaimed) {
+            exec.exitOffset = status.head;
+            exec.finishUnknown('keeper-terminal reclaimed the command output');
+            return exec;
+        }
+        exec.begin();
+        if (exec.cancelState && status.state !== 'exited')
+            exec.armKill();
+        return exec;
+    }
+    /** Stops following without an outcome (another follower owns the exec). Touches nothing on the keeper. */
+    abandon() { if (!this.done)
+        this.stop(); }
+    /** Streaming output goes to `sink` from now on (undefined: nobody is listening). */
+    attach(sink) { this.sink = sink; }
+    get finished() { return this.done; }
+    get cancelling() { return this.cancelState !== null; }
+    /** The keeper's commit watermark as this daemon last set or saw it (records up to here may be gone). */
+    get committedOffset() { return this.committed; }
+    /**
+     * Where a reader may start decoding from nothing (exec/attach.ts): the highest commit sent, whether or not it was answered yet.
+     * Every ack-gated commit lands on a clean checkpoint, so decoding from the record after it gives the shaper's text.
+     */
+    get commitFloor() { return Math.max(this.committed, this.committing); }
+    /** Offset of the exit record, once processed (or the head of a reclaimed log). */
+    get exitRecordOffset() { return this.exitOffset; }
+    /**
+     * The offset an `exec_ack` must reach to release the ended exec: its exit record, or (an outcome that is unknown without an exit
+     * record) one past the last record this daemon processed.
+     */
+    get finalHead() { return this.exitOffset ?? this.processed + 1; }
+    /** Output bytes the keeper's quota dropped (so far). */
+    get gapBytes() { return this.gapBytesSeen; }
+    get ackGated() { return this.wire?.streaming === true; }
+    /** Calls `listener` whenever more records were processed or the exec ended. Returns the unsubscribe. */
+    onProgress(listener) {
+        this.progressListeners.add(listener);
+        return () => { this.progressListeners.delete(listener); };
+    }
+    progressed() {
+        for (const listener of [...this.progressListeners]) {
+            try {
+                listener();
+            }
+            catch { /* a listener never breaks processing */ }
+        }
+    }
+    /** A page of the exec's records from keeper-terminal (exec/attach.ts reads its own way through the log). */
+    read(from, limit) { return this.keeper.execRead({ execId: this.execId, from, limit }); }
+    /** Keeps every record from `at` on readable (an ack-gated commit stays below it) until {@link unhold}. */
+    hold(at) { const hold = { at }; this.holds.add(hold); return hold; }
+    unhold(hold) { if (this.holds.delete(hold))
+        this.commitSoon(); }
+    /**
+     * The server persisted everything through `through` (`exec_ack`): an ack-gated exec may commit up to there (on a clean
+     * checkpoint, below every hold). Never moves backwards.
+     */
+    acknowledge(through) {
+        if (!this.wire || through <= this.acked)
+            return;
+        this.acked = through;
+        this.commitSoon();
+    }
+    /** Commits what the rules allow now, after any checkpoint or commit in flight. */
+    commitSoon() {
+        if (this.done || !this.ackGated)
+            return;
+        this.persisting = this.persisting.catch(() => { }).then(() => this.commitDue())
+            .catch((error) => this.log(`exec ${this.execId}: commit failed: ${messageOf(error)}`));
+    }
+    /** `exec_status` of this exec: from keeper-terminal when it answers, else from what this daemon knows. */
+    async wireStatus() {
+        if (this.done)
+            return { state: 'exited', headOffset: this.finalHead, committedOffset: this.committed };
+        try {
+            const status = await this.keeper.execStatus(this.execId);
+            return { state: status.state === 'exited' ? 'exited' : 'running', headOffset: status.head, committedOffset: status.committed };
+        }
+        catch {
+            return { state: 'running', headOffset: this.processed, committedOffset: this.committed };
+        }
+    }
+    begin() {
+        void this.subscribe();
+        this.watchdog = setInterval(() => { void this.check(); }, this.options.watchdogMs ?? 10_000);
+        this.watchdog.unref();
+    }
+    async subscribe() {
+        if (this.done || this.subscribing)
+            return;
+        this.subscribing = true;
+        try {
+            // Unsubscribe first: an unsubscribe sent after a new subscription would end the new one on the keeper.
+            const old = this.sub;
+            this.sub = undefined;
+            await old?.unsubscribe().catch(() => { });
+            if (this.done)
+                return;
+            this.sub = await this.keeper.execSubscribe(this.execId, (records) => this.enqueue(records), {
+                from: this.next,
+                onClosed: (error) => { this.log(`exec ${this.execId}: stream closed: ${messageOf(error)}`); this.sub = undefined; this.resubscribeSoon(); },
+            });
+            this.lastContactMs = this.now();
+        }
+        catch (error) {
+            this.log(`exec ${this.execId}: subscribe failed: ${messageOf(error)}`);
+            this.resubscribeSoon(1000);
+        }
+        finally {
+            this.subscribing = false;
+        }
+    }
+    resubscribeSoon(ms = 500) {
+        if (this.done || this.resubscribeTimer)
+            return;
+        this.resubscribeTimer = setTimeout(() => { this.resubscribeTimer = undefined; void this.check(); }, ms);
+        this.resubscribeTimer.unref();
+    }
+    /** Records are processed strictly in order, one batch after the other (also across a resubscription). */
+    enqueue(records) {
+        this.processing = this.processing.then(() => this.process(records)).catch((error) => { this.log(`exec ${this.execId}: ${messageOf(error)}`); });
+        return this.processing;
+    }
+    async process(records) {
+        if (this.done)
+            return;
+        this.lastRecordMs = this.lastContactMs = this.now();
+        for (const record of records) {
+            if (record.o < this.next)
+                continue;
+            this.next = record.o + 1;
+            if (record.k === 'out' || record.k === 'err') {
+                const name = record.k === 'out' ? 'stdout' : 'stderr';
+                const bytes = Buffer.from(record.b, 'base64');
+                const text = this.shaper.stream(name).push(record.so, bytes);
+                this.bytesSinceCheckpoint += bytes.length;
+                this.processed = record.o;
+                if (text)
+                    this.emit(name, text);
+            }
+            else if (record.k === 'exit') {
+                this.processed = record.o;
+                await this.finish(record);
+                return;
+            }
+            else {
+                // spawn, gap (bytes the quota dropped show up as a jump in the next chunk's stream offset)
+                if (record.k === 'spawn' && Number.isSafeInteger(record.pid))
+                    this.pid = record.pid;
+                if (record.k === 'gap' && Number.isSafeInteger(record.droppedBytes) && record.droppedBytes > 0)
+                    this.gapBytesSeen += record.droppedBytes;
+                this.processed = record.o;
+            }
+        }
+        this.progressed();
+        const due = this.bytesSinceCheckpoint >= (this.options.checkpointBytes ?? 1024 * 1024)
+            || (this.processed > this.checkpointed && this.now() - this.lastCheckpointMs >= (this.options.checkpointMs ?? 2000));
+        if (due)
+            await this.persist().catch((error) => this.log(`exec ${this.execId}: checkpoint failed: ${messageOf(error)}`));
+    }
+    emit(stream, text) {
+        try {
+            this.sink?.(stream, text);
+        }
+        catch (error) {
+            this.log(`exec ${this.execId}: output sink failed: ${messageOf(error)}`);
+        }
+    }
+    blob() {
+        const blob = {
+            v: CHECKPOINT_VERSION, ...this.shaper.state(), cancel: this.cancelState, outlivesChannel: this.outlivesChannel,
+            ...(this.wire ? { wire: this.wire } : {}), ...(this.gapBytesSeen > 0 ? { gapBytes: this.gapBytesSeen } : {}),
+        };
+        return JSON.stringify(blob);
+    }
+    /**
+     * Writes a checkpoint of the state as of the last processed record, then commits through that same offset (never further;
+     * an ack-gated exec possibly less, see the file comment). Checkpoints are serialised: each one covers at least what the previous
+     * one did.
+     */
+    persist(onWritten) {
+        const run = async () => {
+            if (this.done)
+                return;
+            const offset = this.processed;
+            const blob = this.blob();
+            const clean = this.shaper.clean;
+            this.bytesSinceCheckpoint = 0;
+            this.lastCheckpointMs = this.now();
+            await this.keeper.execCheckpoint({ execId: this.execId, blob, offset });
+            onWritten?.();
+            this.checkpointed = Math.max(this.checkpointed, offset);
+            if (clean && offset > (this.cleanCheckpoints.at(-1) ?? -2))
+                this.cleanCheckpoints.push(offset);
+            await this.commitDue();
+        };
+        this.persisting = this.persisting.catch(() => { }).then(run);
+        return this.persisting;
+    }
+    /** Where a commit may go now: the checkpoint; ack-gated, the last clean checkpoint at or below the ack and every hold. */
+    commitTarget() {
+        if (!this.ackGated)
+            return this.checkpointed;
+        let limit = Math.min(this.acked, this.checkpointed);
+        for (const hold of this.holds)
+            limit = Math.min(limit, hold.at - 1);
+        let target = this.committed;
+        for (const offset of this.cleanCheckpoints)
+            if (offset <= limit)
+                target = Math.max(target, offset);
+        return target;
+    }
+    async commitDue() {
+        if (this.done)
+            return;
+        const target = this.commitTarget();
+        if (target <= this.committed)
+            return;
+        this.committing = Math.max(this.committing, target);
+        const result = await this.keeper.execCommit({ execId: this.execId, through: target });
+        this.committed = Math.max(this.committed, result.committed);
+        // Clean checkpoints below the commit are of no use any more (keep the one at it).
+        const keepFrom = this.cleanCheckpoints.findIndex((offset) => offset >= this.committed);
+        this.cleanCheckpoints = keepFrom < 0 ? [] : this.cleanCheckpoints.slice(keepFrom);
+    }
+    /**
+     * Cancels: the decision is checkpointed, then `signal` (default SIGTERM) goes to the process group and SIGKILL follows the grace
+     * (also from a restarted daemon, which finds the decision in the checkpoint). Idempotent: a second cancel changes nothing.
+     */
+    async cancel(signal = 'SIGTERM') {
+        if (this.done || this.cancelState)
+            return;
+        this.cancelState = { termAtMs: this.now() };
+        await this.persist().catch((error) => this.log(`exec ${this.execId}: cancel checkpoint failed: ${messageOf(error)}`));
+        if (this.done)
+            return;
+        await this.keeper.execKill({ execId: this.execId, signal }).catch((error) => this.log(`exec ${this.execId}: ${signal} failed: ${messageOf(error)}`));
+        this.armKill();
+    }
+    armKill() {
+        if (this.done || !this.cancelState || this.killTimer)
+            return;
+        const delay = Math.max(0, this.cancelState.termAtMs + this.killGraceMs - this.now());
+        this.killTimer = setTimeout(() => { void this.sendKill(); }, delay);
+        this.killTimer.unref();
+    }
+    async sendKill() {
+        if (this.done)
+            return;
+        await this.keeper.execKill({ execId: this.execId, signal: 'SIGKILL' }).catch((error) => this.log(`exec ${this.execId}: SIGKILL failed: ${messageOf(error)}`));
+    }
+    /** Watchdog: a deleted exec, a stalled or lost subscription, an overdue SIGKILL, an unreachable keeper, a quiet exec's checkpoint. */
+    async check() {
+        if (this.done)
+            return;
+        let status;
+        try {
+            status = await this.keeper.execStatus(this.execId);
+            this.lastContactMs = this.now();
+        }
+        catch (error) {
+            if (errorCode(error) === ExecErrorCode.notFound) {
+                this.finishUnknown('keeper-terminal no longer knows the command');
+                return;
+            }
+            if (this.now() - this.lastContactMs >= (this.options.giveUpMs ?? 120_000)) {
+                this.finishUnknown('keeper-terminal could not be reached');
+                return;
+            }
+            this.resubscribeSoon(2000);
+            return;
+        }
+        if (this.done)
+            return;
+        if (status.logReclaimed) {
+            this.exitOffset = status.head;
+            this.finishUnknown('keeper-terminal reclaimed the command output');
+            return;
+        }
+        // A command that went quiet still gets its last records checkpointed (and so committed, or committable on the server's ack).
+        if (this.processed > this.checkpointed && this.now() - this.lastCheckpointMs >= (this.options.checkpointMs ?? 2000)) {
+            void this.persist().catch((error) => this.log(`exec ${this.execId}: checkpoint failed: ${messageOf(error)}`));
+        }
+        if (this.cancelState && status.state !== 'exited' && this.now() >= this.cancelState.termAtMs + this.killGraceMs)
+            void this.sendKill();
+        const stalled = status.head >= this.next && this.now() - this.lastRecordMs >= (this.options.watchdogMs ?? 10_000);
+        if (!this.sub || stalled)
+            void this.subscribe();
+    }
+    stop() {
+        this.done = true;
+        clearTimeout(this.killTimer);
+        clearTimeout(this.resubscribeTimer);
+        clearInterval(this.watchdog);
+        const sub = this.sub;
+        this.sub = undefined;
+        void sub?.unsubscribe().catch(() => { });
+    }
+    flushStreaming() {
+        for (const name of ['stdout', 'stderr']) {
+            const text = this.shaper.stream(name).flushStreaming();
+            if (text)
+                this.emit(name, text);
+        }
+    }
+    async finish(record) {
+        this.stop();
+        this.exitOffset = record.o;
+        this.flushStreaming();
+        await this.persisting.catch(() => { }); // a checkpoint in flight lands before anything else touches the exec
+        const exit = {
+            code: record.code ?? null, signal: record.signal ?? null, timedOut: record.timedOut === true,
+            ...(record.reason ? { reason: record.reason } : {}), ...(record.error ? { error: String(record.error) } : {}),
+            atMs: record.t,
+        };
+        this.settle({ state: 'exited', exit, cancelled: this.cancelState !== null, ...this.shaper.result() });
+        this.progressed();
+    }
+    finishUnknown(reason) {
+        this.stop();
+        this.flushStreaming();
+        this.settle({ state: 'unknown', reason, cancelled: this.cancelState !== null, ...this.shaper.result() });
+        this.progressed();
+    }
+    /**
+     * The outcome was delivered (or nobody will ever want it): the keeper may delete the exec (`exec.commit{final}`). Only for an
+     * ended exec; a no-op otherwise and when already released.
+     */
+    async release() {
+        if (!this.done || this.released || this.exitOffset === undefined)
+            return;
+        this.released = true;
+        try {
+            await this.keeper.execCommit({ execId: this.execId, through: this.exitOffset, final: true });
+        }
+        catch (error) {
+            if (errorCode(error) === ExecErrorCode.notFound)
+                return;
+            try { // the head moved (it cannot after an exit, but the keeper decides): cover its head
+                const status = await this.keeper.execStatus(this.execId);
+                if (status.state === 'exited')
+                    await this.keeper.execCommit({ execId: this.execId, through: status.head, final: true });
+            }
+            catch (retry) {
+                if (errorCode(retry) !== ExecErrorCode.notFound)
+                    this.log(`exec ${this.execId}: final commit failed: ${messageOf(retry)}`);
+            }
+        }
+    }
+}

package/dist/workstation/exec/fallback.js

@@ -0,0 +1,106 @@
+// The in-process exec (spec §4.2 "fallback"): the daemon spawns the command itself, as it always did, when keeper-terminal is not
+// available or not compatible. Such an exec is NOT durable: it dies with the daemon and with its channel (today's semantics).
+// Result shaping and argv/env are the same as the durable path's (exec/shaping.ts, exec/command.ts); cancellation and the timeout
+// keep today's immediate SIGKILL of the process group.
+import { spawn } from 'node:child_process';
+import { execArgv, execDeadline, execEnvironment } from './command.js';
+import { ExecShaper } from './shaping.js';
+export function runExec(req, onOutput, signal, options = {}) {
+    const running = startExec(req, onOutput, options);
+    const onAbort = () => { void running.cancel(); };
+    if (signal?.aborted)
+        onAbort();
+    else
+        signal?.addEventListener('abort', onAbort, { once: true });
+    void running.result.finally(() => signal?.removeEventListener('abort', onAbort));
+    return running.result;
+}
+/** Starts an exec in its own process group and exposes the same cancellation used by force update. */
+export function startExec(req, onOutput, options = {}) {
+    const now = options.now ?? Date.now;
+    const deadline = options.deadlineEpochMs ?? execDeadline(req, now());
+    let cancel = async () => { };
+    let pid;
+    if (deadline !== undefined && deadline <= now()) { // already past its deadline: never spawned
+        const result = Promise.resolve({ t: 'exec_exit', id: req.id, exitCode: null, stdout: '', stderr: '', timedOut: true, truncated: false });
+        return { result, cancel: async () => { }, pid: undefined };
+    }
+    const result = new Promise((resolve) => {
+        const env = options.env ?? execEnvironment(process.env, req);
+        const [file, ...args] = execArgv(req);
+        // macOS and Linux create a new session/process group for a detached child. Every command it
+        // launches (pipelines and background jobs included) stays in that group, so cancellation does
+        // not leave descendants running after the shell parent is gone.
+        const child = spawn(file, args, { cwd: req.cwd, env, detached: process.platform !== 'win32' });
+        pid = child.pid;
+        const shaper = new ExecShaper();
+        const offsets = { stdout: 0, stderr: 0 };
+        let timedOut = false;
+        let cancelled = false;
+        let settled = false;
+        const capture = (buf, stream) => {
+            const text = shaper.stream(stream).push(offsets[stream], buf);
+            offsets[stream] += buf.length;
+            if (req.streaming && onOutput && text)
+                onOutput(stream, text);
+        };
+        child.stdout.on('data', (b) => capture(b, 'stdout'));
+        child.stderr.on('data', (b) => capture(b, 'stderr'));
+        const terminateCommandGroup = () => {
+            if (child.pid && process.platform !== 'win32') {
+                try {
+                    process.kill(-child.pid, 'SIGKILL');
+                    return;
+                }
+                catch {
+                    // The group may have already exited between pid observation and signal delivery.
+                }
+            }
+            child.kill('SIGKILL');
+        };
+        let timer;
+        if (deadline !== undefined)
+            timer = setTimeout(() => { timedOut = true; terminateCommandGroup(); }, Math.max(0, deadline - now()));
+        const abort = () => { cancelled = true; terminateCommandGroup(); };
+        cancel = async () => { abort(); };
+        const flushStreaming = () => {
+            for (const stream of ['stdout', 'stderr']) {
+                const text = shaper.stream(stream).flushStreaming();
+                if (req.streaming && onOutput && text)
+                    onOutput(stream, text);
+            }
+        };
+        child.on('close', (code) => {
+            if (settled)
+                return;
+            settled = true;
+            if (timer)
+                clearTimeout(timer);
+            flushStreaming();
+            const shaped = shaper.result();
+            resolve({
+                t: 'exec_exit', id: req.id,
+                exitCode: timedOut || cancelled ? null : code,
+                stdout: shaped.stdout, stderr: shaped.stderr, timedOut, truncated: shaped.truncated,
+            });
+        });
+        child.on('error', (err) => {
+            if (settled)
+                return;
+            settled = true;
+            if (timer)
+                clearTimeout(timer);
+            flushStreaming();
+            const shaped = shaper.result();
+            resolve({
+                t: 'exec_exit', id: req.id,
+                exitCode: null,
+                stdout: shaped.stdout,
+                stderr: shaped.stderr + String(err?.message ?? err),
+                timedOut: false,
+                truncated: shaped.truncated,
+            });
+        });
+    });
+    return { result, cancel: async () => cancel(), get pid() { return pid; } };
+}