llm-relay 0.85.0 → 0.86.0

package/dist/mcp/lane-runner.js

@@ -1,9 +1,10 @@
-import { exec, execFile, execSync } from "node:child_process";
+import { exec, execFile, execSync, spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
 import { existsSync, statSync } from "node:fs";
 import { resolve as resolvePath } from "node:path";
-import { quoteCmdArg } from "../lane-probe.js";
-import { nullJobJournal } from "./job-journal.js";
-import { jobSeqOf, nullJobArchive } from "./job-archive.js";
+import { resolveWindowsNpmShim } from "./windows-npm-shim.js";
+import { nullJobJournal, } from "./job-journal.js";
+import { nullJobArchive } from "./job-archive.js";
 import { classifyLaneProbeOutput } from "../lane-quota-probe.js";
 import { laneOfRung } from "../lane-manifest.js";
 export const DEPTH_ENV = "LLM_RELAY_DISPATCH_DEPTH";
@@ -227,12 +228,120 @@ function observeOutput(child, onOutput) {
     watch("stdout");
     watch("stderr");
 }
+function spawnPosixLane(processApi, command, args, opts) {
+    const spawnProcess = processApi.spawn;
+    if (!spawnProcess) {
+        return {
+            result: Promise.resolve({
+                code: null,
+                stdout: "",
+                stderr: "POSIX lane spawning requires the injected spawn boundary",
+                timedOut: false,
+            }),
+            kill: () => { },
+            pids: () => [],
+        };
+    }
+    let child;
+    let timedOut = false;
+    let bufferExceeded = null;
+    let timeoutTimer;
+    const stdoutChunks = [];
+    const stderrChunks = [];
+    let stdoutBytes = 0;
+    let stderrBytes = 0;
+    const killOwned = () => {
+        if (child?.pid)
+            terminateProcessTree(child.pid, processApi.platform);
+        child?.kill();
+    };
+    const clearTimer = () => {
+        if (timeoutTimer !== undefined)
+            clearTimeout(timeoutTimer);
+        timeoutTimer = undefined;
+    };
+    const append = (stream, chunk) => {
+        const data = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
+        const isStdout = stream === "stdout";
+        const used = isStdout ? stdoutBytes : stderrBytes;
+        const remaining = Math.max(0, MAX_OUTPUT_BYTES - used);
+        if (remaining > 0) {
+            const kept = data.length <= remaining ? data : data.subarray(0, remaining);
+            (isStdout ? stdoutChunks : stderrChunks).push(kept);
+            if (isStdout)
+                stdoutBytes += kept.length;
+            else
+                stderrBytes += kept.length;
+        }
+        if (data.length > remaining && bufferExceeded === null) {
+            bufferExceeded = stream;
+            killOwned();
+        }
+    };
+    const result = new Promise((resolve) => {
+        let settled = false;
+        const settle = (code, error) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimer();
+            const stdout = Buffer.concat(stdoutChunks).toString("utf8");
+            let stderr = Buffer.concat(stderrChunks).toString("utf8");
+            if (!stderr && error)
+                stderr = error.message;
+            if (bufferExceeded !== null) {
+                const msg = `${bufferExceeded} exceeded ${MAX_OUTPUT_BYTES} byte lane output limit`;
+                stderr = stderr ? `${stderr}\n${msg}` : msg;
+            }
+            resolve({
+                code: bufferExceeded === null ? code : null,
+                stdout,
+                stderr,
+                timedOut,
+            });
+        };
+        try {
+            child = spawnProcess(command, args, {
+                cwd: opts.cwd,
+                env: opts.env,
+                windowsHide: true,
+                detached: true,
+            });
+        }
+        catch (e) {
+            settle(null, e);
+            return;
+        }
+        child.stdout?.on("data", (chunk) => append("stdout", chunk));
+        child.stderr?.on("data", (chunk) => append("stderr", chunk));
+        observeOutput(child, opts.onOutput);
+        child.on("error", (err) => settle(null, err));
+        child.on("close", (code) => settle(code));
+        child.stdin?.end();
+        if (opts.timeoutMs > 0) {
+            timeoutTimer = setTimeout(() => {
+                timedOut = true;
+                killOwned();
+            }, opts.timeoutMs);
+            timeoutTimer.unref?.();
+        }
+    });
+    return {
+        result,
+        kill: () => {
+            clearTimer();
+            killOwned();
+        },
+        pids: () => (child?.pid === undefined ? [] : [child.pid]),
+    };
+}
 const nodeProcessApi = {
     platform: process.platform,
     execFile: (command, args, opts, callback) => execFile(command, args, opts, callback),
     exec: (command, opts, callback) => exec(command, opts, callback),
+    spawn: (command, args, opts) => spawn(command, args, opts),
 };
-export function createLaneSpawner(processApi, hostEnv = process.env) {
+export function createLaneSpawner(processApi, hostEnv = process.env, resolveNpmShim = resolveWindowsNpmShim) {
     return (command, args, opts) => {
         if (hostEnv["VITEST"]) {
             return {
@@ -245,6 +354,8 @@ export function createLaneSpawner(processApi, hostEnv = process.env) {
                 kill: () => { },
             };
         }
+        if (processApi.platform !== "win32")
+            return spawnPosixLane(processApi, command, args, opts);
         const execOpts = {
             encoding: "utf8",
             maxBuffer: MAX_OUTPUT_BYTES,
@@ -269,15 +380,17 @@ export function createLaneSpawner(processApi, hostEnv = process.env) {
                 });
             };
             child = processApi.execFile(command, args, execOpts, (err, stdout, stderr) => {
-                if (!err) {
-                    resolve({ code: 0, stdout, stderr, timedOut: false });
-                    return;
-                }
-                if (processApi.platform === "win32" && err.code === "ENOENT" && !killed) {
-                    const line = `${quoteCmdArg(command)} ${args.map(quoteCmdArg).join(" ")}`;
-                    const fallback = processApi.exec(line, execOpts, (err2, stdout2, stderr2) => {
-                        settle(err2, stdout2 ?? "", stderr2 ?? "");
+                if (err && processApi.platform === "win32" && err.code === "ENOENT" && !killed) {
+                    const resolved = resolveNpmShim(command, args, {
+                        env: opts.env,
+                        cwd: opts.cwd,
+                        nodeExecutable: process.execPath,
                     });
+                    if (!resolved.ok) {
+                        settle(Object.assign(new Error(resolved.error), { code: "ENOENT" }), stdout ?? "", stderr ?? "");
+                        return;
+                    }
+                    const fallback = processApi.execFile(resolved.command, resolved.args, execOpts, (err2, stdout2, stderr2) => settle(err2, stdout2 ?? "", stderr2 ?? ""));
                     child = fallback;
                     fallback.stdin?.end();
                     observeOutput(fallback, opts.onOutput);
@@ -292,9 +405,8 @@ export function createLaneSpawner(processApi, hostEnv = process.env) {
             result,
             kill: () => {
                 killed = true;
-                if (child?.pid) {
+                if (child?.pid)
                     terminateProcessTree(child.pid, processApi.platform);
-                }
                 child?.kill();
             },
             pids: () => (child?.pid === undefined ? [] : [child.pid]),
@@ -355,17 +467,20 @@ function recentOf(job, elsewhere) {
     };
 }
 function newestFirst(a, b) {
-    return b.startedAt - a.startedAt || (jobSeqOf(b.id) ?? 0) - (jobSeqOf(a.id) ?? 0);
-}
-let jobCounter = 0;
-function nextJobId() {
-    jobCounter += 1;
-    return `job-${String(jobCounter).padStart(4, "0")}`;
+    return b.startedAt - a.startedAt || b.id.localeCompare(a.id);
 }
-export function seedJobCounter(seq) {
-    if (Number.isSafeInteger(seq) && seq > jobCounter)
-        jobCounter = seq;
+export function createJobIdFactory(entropy = randomBytes(16)) {
+    const bytes = Buffer.from(entropy);
+    if (bytes.length !== 16)
+        throw new Error("job id entropy must be exactly 16 bytes");
+    const instance = BigInt(`0x${bytes.toString("hex")}`).toString(10).padStart(39, "0");
+    let local = 0n;
+    return () => {
+        local += 1n;
+        return `job-9${instance}${local.toString(10).padStart(8, "0")}`;
+    };
 }
+const defaultJobIdFactory = createJobIdFactory();
 export class LaneJobStore {
     jobs = new Map();
     kills = new Map();
@@ -381,22 +496,23 @@ export class LaneJobStore {
     };
     journal;
     archive;
-    constructor(journal = nullJobJournal, archive = nullJobArchive) {
+    adoptedStartingTrees = new Map();
+    brokerOrphans = new Map();
+    brokerExecutions = new Map();
+    nextJobId;
+    constructor(journal = nullJobJournal, archive = nullJobArchive, nextJobId = defaultJobIdFactory) {
         this.journal = journal;
         this.archive = archive;
+        this.nextJobId = nextJobId;
         this.restoreArchive();
         this.adoptOrphans();
     }
     restoreArchive() {
-        const { jobs, lastSeq } = this.archive.restore();
-        seedJobCounter(lastSeq);
+        const { jobs } = this.archive.restore();
         for (const row of jobs) {
             if (this.jobs.has(row.id))
                 continue;
             this.jobs.set(row.id, { ...row, restored: true });
-            const seq = jobSeqOf(row.id);
-            if (seq !== null)
-                seedJobCounter(seq);
         }
     }
     flush() {
@@ -406,9 +522,12 @@ export class LaneJobStore {
         for (const row of this.journal.orphans()) {
             if (this.jobs.has(row.jobId))
                 continue;
-            const seq = jobSeqOf(row.jobId);
-            if (seq !== null)
-                seedJobCounter(seq);
+            if (row.brokerExecution !== undefined) {
+                const claimed = this.journal.claimBrokerOrphan?.(row.jobId);
+                if (claimed !== undefined)
+                    this.brokerOrphans.set(row.jobId, claimed);
+                continue;
+            }
             const killed = {
                 id: row.jobId,
                 status: "killed",
@@ -428,14 +547,16 @@ export class LaneJobStore {
                 process: { pids: [], survivors: [], terminated: false },
             };
             this.jobs.set(row.jobId, killed);
-            this.archive.record(killed);
+            if (row.startingTree !== undefined) {
+                this.adoptedStartingTrees.set(row.jobId, { cwd: row.cwd, startingTree: row.startingTree });
+            }
+            if (this.archive.record(killed))
+                this.journal.clearOrphan?.(row);
         }
     }
     create(laneId, spec, cwd, dispatchSource, label) {
-        seedJobCounter(this.archive.lastSeqOnDisk());
-        seedJobCounter(this.journal.maxSeqOnDisk());
         const job = {
-            id: nextJobId(),
+            id: this.nextJobId(),
             status: "running",
             laneId,
             spec,
@@ -452,9 +573,6 @@ export class LaneJobStore {
             ...(label ? { label } : {}),
         };
         this.jobs.set(job.id, job);
-        const seq = jobSeqOf(job.id);
-        if (seq !== null)
-            this.archive.noteSeq(seq);
         this.journal.note({
             jobId: job.id,
             laneId: job.laneId,
@@ -469,6 +587,12 @@ export class LaneJobStore {
         this.kills.set(id, handle.kill);
         this.owned.set(id, handle);
     }
+    runningPids(id) {
+        const job = this.jobs.get(id);
+        if (!job || job.status !== "running")
+            return [];
+        return readOwnedPids(this.owned.get(id));
+    }
     registerKill(id, kill) {
         this.kills.set(id, kill);
         this.owned.set(id, { kill });
@@ -479,14 +603,25 @@ export class LaneJobStore {
         const kill = this.kills.get(id);
         this.kills.delete(id);
         this.owned.delete(id);
-        this.journal.clear(id);
-        if (!job)
+        const brokerOwned = this.brokerExecutions.has(id);
+        this.brokerExecutions.delete(id);
+        if (!job) {
+            this.journal.clear(id);
             return;
+        }
         delete job.activity;
+        delete job.liveness;
         const pids = readOwnedPids(handle);
+        if (brokerOwned) {
+            job.process = { pids: [], survivors: [], terminated: false };
+            if (this.archive.record(job))
+                this.journal.clear(id);
+            return;
+        }
         if (kill === undefined) {
             job.process = { pids, survivors: [], terminated: false };
-            this.archive.record(job);
+            if (this.archive.record(job))
+                this.journal.clear(id);
             return;
         }
         try {
@@ -495,7 +630,8 @@ export class LaneJobStore {
         catch {
         }
         job.process = { pids, survivors: pids.filter((pid) => this.isAlive(pid)), terminated: true };
-        this.archive.record(job);
+        if (this.archive.record(job))
+            this.journal.clear(id);
     }
     ownedProcesses() {
         const rows = [];
@@ -524,7 +660,17 @@ export class LaneJobStore {
         delete job.activity;
         delete job.readOnly;
         delete job.launch;
-        delete job.lastActive;
+        delete job.liveness;
+        this.brokerExecutions.delete(id);
+        delete job.executionOwner;
+        this.journal.note({
+            jobId: job.id,
+            laneId: job.laneId,
+            ...(job.spec === undefined ? {} : { spec: job.spec }),
+            cwd: job.cwd,
+            startedAt: job.startedAt,
+            ...(job.label === undefined ? {} : { label: job.label }),
+        });
     }
     beginAttemptActivity(id, now) {
         const job = this.jobs.get(id);
@@ -542,6 +688,167 @@ export class LaneJobStore {
         else
             activity.stderrBytes += bytes;
     }
+    noteStartingTree(id, tree, scope) {
+        const job = this.jobs.get(id);
+        if (!job || job.status !== "running")
+            return;
+        this.journal.noteStartingTree?.(id, tree, scope);
+    }
+    takeBrokerOrphans() {
+        const rows = [...this.brokerOrphans.values()].map((row) => ({ ...row }));
+        this.brokerOrphans.clear();
+        return rows;
+    }
+    brokerExecution(id) {
+        return this.brokerExecutions.get(id);
+    }
+    noteExecutionOwner(id, owner) {
+        const job = this.jobs.get(id);
+        if (job === undefined || job.status !== "running")
+            return;
+        if (owner === undefined)
+            delete job.executionOwner;
+        else
+            job.executionOwner = owner;
+    }
+    clearBrokerExecution(id) {
+        const job = this.jobs.get(id);
+        if (job === undefined || job.status !== "running")
+            return;
+        this.brokerExecutions.delete(id);
+        if (job.executionOwner === "relay-daemon")
+            delete job.executionOwner;
+        this.journal.note({
+            jobId: job.id,
+            laneId: job.laneId,
+            ...(job.spec === undefined ? {} : { spec: job.spec }),
+            cwd: job.cwd,
+            startedAt: job.startedAt,
+            ...(job.label === undefined ? {} : { label: job.label }),
+        });
+    }
+    noteBrokerExecution(id, executionId) {
+        const job = this.jobs.get(id);
+        if (job === undefined || job.status !== "running")
+            return;
+        const execution = { kind: "daemon-v1", executionId };
+        this.brokerExecutions.set(id, executionId);
+        this.journal.noteBrokerExecution?.(id, execution);
+    }
+    adoptBrokerRunning(row, snapshot) {
+        if (row.brokerExecution === undefined)
+            return undefined;
+        const existing = this.jobs.get(row.jobId);
+        if (existing !== undefined)
+            return existing;
+        const job = {
+            id: row.jobId,
+            status: "running",
+            laneId: row.laneId,
+            spec: row.spec,
+            startedAt: row.startedAt,
+            endedAt: undefined,
+            exitCode: null,
+            stdout: "",
+            stderr: "",
+            timedOut: false,
+            cwd: row.cwd,
+            error: undefined,
+            attempts: [],
+            restored: true,
+            executionOwner: "relay-daemon",
+            ...(row.label === undefined ? {} : { label: row.label }),
+        };
+        this.jobs.set(job.id, job);
+        this.brokerExecutions.set(job.id, row.brokerExecution.executionId);
+        if (snapshot !== undefined)
+            this.applyBrokerSnapshot(job.id, snapshot);
+        return job;
+    }
+    applyBrokerSnapshot(id, snapshot) {
+        const job = this.jobs.get(id);
+        const executionId = this.brokerExecutions.get(id);
+        if (job === undefined ||
+            executionId === undefined ||
+            snapshot.executionId !== executionId ||
+            snapshot.jobId !== id ||
+            snapshot.laneId !== job.laneId) {
+            return false;
+        }
+        if (snapshot.launchNotes !== undefined)
+            job.launch = [...snapshot.launchNotes];
+        job.activity = {
+            attemptStartedAt: snapshot.startedAt,
+            lastOutputAt: snapshot.lastOutputAt,
+            stdoutBytes: snapshot.stdoutBytes,
+            stderrBytes: snapshot.stderrBytes,
+        };
+        if (snapshot.status === "running")
+            return true;
+        if (snapshot.status === "completed") {
+            if (snapshot.code !== 0 || snapshot.timedOut !== false || snapshot.stdout === undefined)
+                return false;
+            if (isContentEmpty(snapshot.stdout)) {
+                job.status = "failed";
+                job.error = EMPTY_OUTPUT_REASON;
+            }
+            else {
+                job.status = "completed";
+            }
+        }
+        else if (snapshot.status === "failed") {
+            if (snapshot.timedOut !== false || snapshot.code === 0)
+                return false;
+            job.status = "failed";
+        }
+        else if (snapshot.status === "timed_out") {
+            if (snapshot.timedOut !== true)
+                return false;
+            job.status = "timed_out";
+            job.error = "the daemon-owned lane exceeded its configured timeout and was stopped before it finished";
+        }
+        else if (snapshot.status === "cancelled") {
+            job.status = "cancelled";
+        }
+        else {
+            const _never = snapshot.status;
+            return _never;
+        }
+        if (snapshot.code !== undefined)
+            job.exitCode = snapshot.code;
+        if (snapshot.stdout !== undefined)
+            job.stdout = snapshot.stdout;
+        if (snapshot.stderr !== undefined)
+            job.stderr = snapshot.stderr;
+        if (snapshot.timedOut !== undefined)
+            job.timedOut = snapshot.timedOut;
+        job.endedAt = snapshot.endedAt ?? Date.now();
+        delete job.activity;
+        delete job.liveness;
+        this.brokerExecutions.delete(id);
+        if (this.archive.record(job))
+            this.journal.clear(id);
+        return true;
+    }
+    markBrokerKilled(id, message) {
+        const job = this.jobs.get(id);
+        if (job === undefined || !this.brokerExecutions.has(id) || job.status !== "running")
+            return false;
+        job.status = "killed";
+        job.endedAt = Date.now();
+        job.error = message;
+        delete job.activity;
+        delete job.liveness;
+        this.brokerExecutions.delete(id);
+        if (this.archive.record(job))
+            this.journal.clear(id);
+        return true;
+    }
+    takeAdoptedStartingTrees() {
+        const adopted = [...this.adoptedStartingTrees].map(([jobId, value]) => ({ jobId, ...value }));
+        this.adoptedStartingTrees.clear();
+        return adopted;
+    }
     noteReadOnly(id, laneId, binding) {
         const job = this.jobs.get(id);
         if (!job || job.status !== "running")
@@ -553,14 +860,15 @@ export class LaneJobStore {
         if (!job)
             return;
         job.treeDelta = text;
-        if (job.status !== "running")
-            this.archive.record(job);
+        if (job.status !== "running" && this.archive.record(job)) {
+            this.journal.clear(id);
+        }
     }
-    noteLastActivity(id, at, source) {
+    noteLiveness(id, liveness) {
         const job = this.jobs.get(id);
         if (!job || job.status !== "running")
             return;
-        job.lastActive = { at, source };
+        job.liveness = { ...liveness };
     }
     noteLaunch(id, notes) {
         const job = this.jobs.get(id);
@@ -687,6 +995,8 @@ export class LaneJobStore {
             return false;
         if (job.status !== "running")
             return false;
+        if (this.brokerExecutions.has(id))
+            return false;
         job.status = "cancelled";
         job.endedAt = Date.now();
         this.reap(id);
@@ -694,6 +1004,8 @@ export class LaneJobStore {
     }
     cancelAll() {
         for (const id of [...this.jobs.values()].filter((j) => j.status === "running").map((j) => j.id)) {
+            if (this.brokerExecutions.has(id))
+                continue;
             this.cancel(id);
         }
     }