next-issue 2.0.0 → 2.1.0

package/README.md

@@ -356,6 +356,7 @@ that a label still holds back stays as it was.
 | Argument | Effect |
 | --- | --- |
 | `init` | Write the config file with the defaults |
+| `serve` | Run the worker that pi extensions talk to, see [Worker](#worker) |
 | `--force` | Replace an existing config file, with `init` |
 | `--once` | Stop after the first handled issue |
 | `--max <n>` | Handle at most n issues |
@@ -397,6 +398,33 @@ The watch stops:
   after `--merge`;
 - when the open issues cannot be read.
 
+## Worker
+
+`next-issue serve` runs one worker for the machine. The
+[`@ravshansbox/pi-next-issue`](../pi-next-issue) extension starts it and talks
+to it. The worker:
+
+- listens with WebSocket on the socket file `~/.pi/agent/next-issue.sock`, or
+  in the folder that `PI_CODING_AGENT_DIR` names. Only your user can open the
+  file. It works on macOS and Linux;
+- holds `next-issue.lock` in the same folder, with its process id, so only one
+  worker runs. A lock whose process is no longer alive is taken over;
+- runs at most one run per repository, and runs in several repositories at
+  once. Each run is like `next-issue` without `--watch`, and the client says
+  whether it merges;
+- sends every client the progress of every run when it connects and on every
+  change: the issue, the step, the tool count, the last tool and the handled
+  issues;
+- stops a run, when a client asks, like a second `Ctrl-C`: after the current
+  round, with the state kept;
+- stays when all clients close;
+- closes once no run is going when a client with a newer version connects, so
+  that the client can start the newer worker;
+- stops every agent and command and exits on `SIGTERM` or `SIGINT`, and
+  removes its socket and lock files.
+
+Stop it by hand with `kill $(cat ~/.pi/agent/next-issue.lock)`.
+
 ## Development
 
 ```bash

package/dist/args.mjs

@@ -2,6 +2,7 @@ import { CONFIG_FILE } from "./config.mjs";
 export const USAGE = `next-issue [command] [options]
 
   init         write ${CONFIG_FILE} with the defaults
+  serve        run the worker that pi extensions talk to
 
   --once       stop after the first handled issue
   --max <n>    handle at most n issues
@@ -29,10 +30,10 @@ export function parseArgs(argv) {
     for (let index = 0; index < argv.length; index += 1) {
         const arg = argv[index];
         if (index === 0 && !arg.startsWith("-")) {
-            if (arg !== "init") {
+            if (arg !== "init" && arg !== "serve") {
                 throw new Error(`Unknown command: ${arg}`);
             }
-            args.command = "init";
+            args.command = arg;
         }
         else if (arg === "--force") {
             args.force = true;

package/dist/cli.mjs

@@ -1,18 +1,13 @@
 #!/usr/bin/env node
-import { writeFile } from "node:fs/promises";
-import { join } from "node:path";
 import { parseArgs, USAGE } from "./args.mjs";
-import { loadConfig, writeDefaultConfig } from "./config.mjs";
-import { setCommandObserver, setDefaultTimeout } from "./exec.mjs";
-import { detectRepo, ensureIgnored, repoRoot, WORKTREE_DIR } from "./git.mjs";
-import { defaultBranch } from "./github.mjs";
+import { writeDefaultConfig } from "./config.mjs";
+import { ensureIgnored, repoRoot, WORKTREE_DIR } from "./git.mjs";
 import { listenForInterrupts } from "./interrupt.mjs";
-import { exitCode, runIssues, watchIssues } from "./loop.mjs";
-import { message, pruneRuns, Recorder } from "./observe.mjs";
-import { pendingIssues, PORTS, processIssue } from "./pipeline.mjs";
+import { message } from "./observe.mjs";
+import { runRepo } from "./run.mjs";
+import { serve } from "./serve.mjs";
 import { STATE_DIR } from "./state.mjs";
 import { formatSummary } from "./summary.mjs";
-import { packageVersion } from "./version.mjs";
 function write(stream, text) {
     return new Promise((resolve, reject) => {
         stream.write(text, (error) => (error === null || error === undefined ? resolve() : reject(error)));
@@ -24,33 +19,18 @@ async function main() {
         await write(process.stdout, USAGE);
         return 0;
     }
+    if (args.command === "serve") {
+        return serve();
+    }
     const root = await repoRoot(process.cwd());
     if (args.command === "init") {
         return init(root, args.force);
     }
-    const config = await loadConfig(root);
-    setDefaultTimeout(config.commandTimeoutMinutes * 60_000);
-    const repo = await detectRepo(root, config.remote);
-    const recorder = await Recorder.create(repo.root, args.level);
-    const pruned = await pruneRuns(repo.root, config.keepRunDays);
-    if (pruned > 0) {
-        recorder.event("runs.pruned", { files: pruned, keepDays: config.keepRunDays }, "verbose");
-    }
-    setCommandObserver((record) => {
-        recorder.event(record.code === 0 ? "command" : "command.fail", {
-            cmd: `${record.command} ${record.args.join(" ")}`,
-            code: record.code,
-            ms: record.ms,
-            stderr: record.stderr.length > 0 ? record.stderr : undefined,
-            timedOut: record.timedOut ? true : undefined,
-        }, record.code === 0 ? "verbose" : "quiet");
+    const interrupts = listenForInterrupts((text) => void write(process.stderr, `${text}\n`));
+    return runRepo(root, args, {
+        interrupts,
+        print: (summary) => write(process.stdout, args.json ? `${JSON.stringify(summary, null, 2)}\n` : formatSummary(summary)),
     });
-    try {
-        return await handle(args, config, repo, recorder);
-    }
-    finally {
-        await recorder.close();
-    }
 }
 async function init(root, force) {
     const path = await writeDefaultConfig(root, force);
@@ -63,71 +43,6 @@ async function init(root, force) {
     }
     return 0;
 }
-async function handle(args, config, repo, recorder) {
-    const interrupts = listenForInterrupts((text) => void write(process.stderr, `${text}\n`));
-    const base = await defaultBranch(repo);
-    const context = {
-        repo,
-        config,
-        base,
-        reset: args.reset,
-        merge: args.merge,
-        stopAfterRound: interrupts.afterRound,
-        recorder,
-        ports: PORTS,
-    };
-    recorder.event("run.start", {
-        version: await packageVersion(),
-        repo: `${repo.owner}/${repo.name}`,
-        base,
-        log: recorder.logFile,
-    }, "quiet");
-    const loop = {
-        issues: async () => {
-            const issues = await pendingIssues(context);
-            recorder.event("issues", { open: issues.length }, "quiet");
-            return issues;
-        },
-        process: (issue) => processIssue(context, issue),
-        stop: interrupts.afterIssue,
-    };
-    const print = (summary) => write(process.stdout, args.json ? `${JSON.stringify(summary, null, 2)}\n` : formatSummary(summary));
-    let passStart = recorder.summary();
-    const { reports, stoppedAt, error } = args.watch
-        ? await watchIssues(args, {
-            ...loop,
-            wait: (minutes) => {
-                recorder.event("watch.wait", { minutes }, "quiet");
-                passStart = recorder.summary();
-                return interrupts.sleep(minutes * 60_000);
-            },
-            passEnd: (issues) => print({ ...recorder.summary(passStart), issues }),
-        })
-        : await runIssues(args, loop);
-    if (stoppedAt !== undefined) {
-        recorder.event("run.stop", { issue: stoppedAt }, "quiet");
-    }
-    if (error !== undefined) {
-        recorder.event("issues.fail", { error: message(error) }, "quiet");
-    }
-    recorder.event("run.end", {
-        done: count(reports, "done"),
-        needsHuman: count(reports, "needs-human"),
-        failed: count(reports, "error"),
-        skipped: count(reports, "skipped"),
-        tokens: recorder.summary().total.total,
-    }, "quiet");
-    const summary = { ...recorder.summary(), issues: reports };
-    const summaryFile = join(repo.root, STATE_DIR, "runs", `${recorder.runId}.summary.json`);
-    await writeFile(summaryFile, `${JSON.stringify(summary, null, 2)}\n`);
-    if (!args.watch) {
-        await print(summary);
-    }
-    return error === undefined ? exitCode(reports) : 1;
-}
-function count(reports, outcome) {
-    return reports.filter((report) => report.outcome === outcome).length;
-}
 main().then((code) => process.exit(code), async (error) => {
     await write(process.stderr, `next-issue failed: ${message(error)}\n`);
     process.exit(1);

package/dist/observe.mjs

@@ -28,13 +28,16 @@ class Sink {
     stream;
     level;
     file;
-    constructor(stream, level, file) {
+    listener;
+    constructor(stream, level, file, listener) {
         this.stream = stream;
         this.level = level;
         this.file = file;
+        this.listener = listener;
     }
     write(record) {
         this.stream.write(`${JSON.stringify(record)}\n`);
+        this.listener?.(record);
     }
     count(kind) {
         this.counts.set(kind, (this.counts.get(kind) ?? 0) + 1);
@@ -59,12 +62,12 @@ export class Recorder {
         this.base = base;
         this.runId = runId;
     }
-    static async create(root, level) {
+    static async create(root, level, listener) {
         const runId = new Date().toISOString().replace(/[:.]/g, "-");
         const dir = join(root, STATE_DIR, "runs");
         await mkdir(dir, { recursive: true });
         const file = join(dir, `${runId}.jsonl`);
-        const sink = new Sink(createWriteStream(file, { flags: "a" }), level, file);
+        const sink = new Sink(createWriteStream(file, { flags: "a" }), level, file, listener);
         return new Recorder(sink, {}, runId);
     }
     get logFile() {

package/dist/progress.mjs

@@ -0,0 +1,48 @@
+const TOOL_LINE_MAX = 80;
+const NOT_STAGES = new Set(["issue.start", "agent.start", "run.start"]);
+export function startProgress() {
+    return { status: "running", tools: 0, handled: [] };
+}
+function idle(progress) {
+    const { issue: _issue, title: _title, stage: _stage, lastTool: _lastTool, issueStartedAt: _at, ...rest } = progress;
+    return { ...rest, tools: 0 };
+}
+function toolLine(record) {
+    const tool = String(record["tool"]);
+    const command = record["command"];
+    const line = typeof command === "string" ? `${tool} ${command.split("\n")[0]}` : tool;
+    return line.length > TOOL_LINE_MAX ? `${line.slice(0, TOOL_LINE_MAX - 1)}…` : line;
+}
+export function applyEvent(progress, record) {
+    const kind = String(record["kind"]);
+    if (kind === "issue.start") {
+        const next = {
+            ...idle(progress),
+            issue: Number(record["issue"]),
+            issueStartedAt: Date.parse(String(record["ts"])),
+        };
+        if (typeof record["title"] === "string") {
+            next.title = record["title"];
+        }
+        return next;
+    }
+    if (kind === "tool") {
+        return { ...progress, tools: progress.tools + 1, lastTool: toolLine(record) };
+    }
+    if (kind === "issue.end" || kind === "issue.error") {
+        const outcome = kind === "issue.error" ? "error" : record["outcome"];
+        const handled = outcome === "done" || outcome === "needs-human" || outcome === "error"
+            ? [...progress.handled, { issue: Number(record["issue"]), outcome }]
+            : progress.handled;
+        return { ...idle(progress), handled };
+    }
+    if (kind === "issues.fail") {
+        return { ...progress, error: String(record["error"]) };
+    }
+    if (kind.endsWith(".start") && !NOT_STAGES.has(kind) && progress.issue !== undefined) {
+        const stage = kind.slice(0, -".start".length);
+        const round = record["round"];
+        return { ...progress, stage: typeof round === "number" ? `${stage} ${round}` : stage };
+    }
+    return progress;
+}

package/dist/run.mjs

@@ -0,0 +1,109 @@
+import { writeFile } from "node:fs/promises";
+import { join, sep } from "node:path";
+import { loadConfig } from "./config.mjs";
+import { setCommandObserver, setDefaultTimeout } from "./exec.mjs";
+import { detectRepo } from "./git.mjs";
+import { defaultBranch } from "./github.mjs";
+import { exitCode, runIssues, watchIssues } from "./loop.mjs";
+import { message, pruneRuns, Recorder } from "./observe.mjs";
+import { pendingIssues, PORTS, processIssue } from "./pipeline.mjs";
+import { STATE_DIR } from "./state.mjs";
+import { packageVersion } from "./version.mjs";
+const recorders = new Map();
+setCommandObserver((record) => {
+    for (const [root, recorder] of recorders) {
+        if (record.cwd !== undefined && record.cwd !== root && !record.cwd.startsWith(root + sep)) {
+            continue;
+        }
+        recorder.event(record.code === 0 ? "command" : "command.fail", {
+            cmd: `${record.command} ${record.args.join(" ")}`,
+            code: record.code,
+            ms: record.ms,
+            stderr: record.stderr.length > 0 ? record.stderr : undefined,
+            timedOut: record.timedOut ? true : undefined,
+        }, record.code === 0 ? "verbose" : "quiet");
+    }
+});
+export async function runRepo(root, args, controls) {
+    const config = await loadConfig(root);
+    setDefaultTimeout(config.commandTimeoutMinutes * 60_000);
+    const repo = await detectRepo(root, config.remote);
+    const recorder = await Recorder.create(repo.root, args.level, controls.onEvent);
+    const pruned = await pruneRuns(repo.root, config.keepRunDays);
+    if (pruned > 0) {
+        recorder.event("runs.pruned", { files: pruned, keepDays: config.keepRunDays }, "verbose");
+    }
+    recorders.set(repo.root, recorder);
+    try {
+        return await handle(args, config, repo, recorder, controls);
+    }
+    finally {
+        recorders.delete(repo.root);
+        await recorder.close();
+    }
+}
+async function handle(args, config, repo, recorder, controls) {
+    const { interrupts } = controls;
+    const base = await defaultBranch(repo);
+    const context = {
+        repo,
+        config,
+        base,
+        reset: args.reset,
+        merge: args.merge,
+        stopAfterRound: interrupts.afterRound,
+        recorder,
+        ports: PORTS,
+    };
+    recorder.event("run.start", {
+        version: await packageVersion(),
+        repo: `${repo.owner}/${repo.name}`,
+        base,
+        log: recorder.logFile,
+    }, "quiet");
+    const loop = {
+        issues: async () => {
+            const issues = await pendingIssues(context);
+            recorder.event("issues", { open: issues.length }, "quiet");
+            return issues;
+        },
+        process: (issue) => processIssue(context, issue),
+        stop: interrupts.afterIssue,
+    };
+    const print = controls.print ?? (async () => undefined);
+    let passStart = recorder.summary();
+    const { reports, stoppedAt, error } = args.watch
+        ? await watchIssues(args, {
+            ...loop,
+            wait: (minutes) => {
+                recorder.event("watch.wait", { minutes }, "quiet");
+                passStart = recorder.summary();
+                return interrupts.sleep(minutes * 60_000);
+            },
+            passEnd: (issues) => print({ ...recorder.summary(passStart), issues }),
+        })
+        : await runIssues(args, loop);
+    if (stoppedAt !== undefined) {
+        recorder.event("run.stop", { issue: stoppedAt }, "quiet");
+    }
+    if (error !== undefined) {
+        recorder.event("issues.fail", { error: message(error) }, "quiet");
+    }
+    recorder.event("run.end", {
+        done: count(reports, "done"),
+        needsHuman: count(reports, "needs-human"),
+        failed: count(reports, "error"),
+        skipped: count(reports, "skipped"),
+        tokens: recorder.summary().total.total,
+    }, "quiet");
+    const summary = { ...recorder.summary(), issues: reports };
+    const summaryFile = join(repo.root, STATE_DIR, "runs", `${recorder.runId}.summary.json`);
+    await writeFile(summaryFile, `${JSON.stringify(summary, null, 2)}\n`);
+    if (!args.watch) {
+        await print(summary);
+    }
+    return error === undefined ? exitCode(reports) : 1;
+}
+function count(reports, outcome) {
+    return reports.filter((report) => report.outcome === outcome).length;
+}

package/dist/serve.mjs

@@ -0,0 +1,213 @@
+import { rmSync } from "node:fs";
+import { chmod, readFile, rm, writeFile } from "node:fs/promises";
+import { createServer } from "node:http";
+import { join } from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import { WebSocketServer } from "ws";
+import { parseArgs } from "./args.mjs";
+import { stopChildren } from "./exec.mjs";
+import { message } from "./observe.mjs";
+import { applyEvent, startProgress } from "./progress.mjs";
+import { runRepo } from "./run.mjs";
+import { packageVersion } from "./version.mjs";
+export function socketPaths(agentDir = getAgentDir()) {
+    return {
+        socketPath: join(agentDir, "next-issue.sock"),
+        lockPath: join(agentDir, "next-issue.lock"),
+    };
+}
+export function isNewer(candidate, current) {
+    const left = candidate.split(".").map(Number);
+    const right = current.split(".").map(Number);
+    for (let index = 0; index < Math.max(left.length, right.length); index += 1) {
+        const difference = (left[index] ?? 0) - (right[index] ?? 0);
+        if (difference !== 0) {
+            return difference > 0;
+        }
+    }
+    return false;
+}
+function alive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === "EPERM";
+    }
+}
+async function lock(path) {
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+        try {
+            await writeFile(path, `${process.pid}\n`, { flag: "wx", mode: 0o600 });
+            return true;
+        }
+        catch (error) {
+            if (error.code !== "EEXIST") {
+                throw error;
+            }
+            const pid = Number((await readFile(path, "utf8").catch(() => "")).trim());
+            if (pid > 0 && alive(pid)) {
+                return false;
+            }
+            await rm(path, { force: true });
+        }
+    }
+    return false;
+}
+export async function startWorker(options) {
+    if (!(await lock(options.lockPath))) {
+        return undefined;
+    }
+    await rm(options.socketPath, { force: true });
+    const runs = new Map();
+    let restart = false;
+    let finish = () => undefined;
+    const closed = new Promise((resolve) => {
+        finish = resolve;
+    });
+    const server = createServer();
+    const sockets = new WebSocketServer({ server });
+    const snapshot = () => ({
+        type: "runs",
+        runs: Object.fromEntries([...runs].map(([root, run]) => [root, run.progress])),
+    });
+    const send = (socket, value) => {
+        socket.send(JSON.stringify(value));
+    };
+    const broadcast = () => {
+        const value = snapshot();
+        for (const socket of sockets.clients) {
+            send(socket, value);
+        }
+    };
+    const busy = () => [...runs.values()].some((run) => run.progress.status === "running" || run.progress.status === "stopping");
+    const close = async () => {
+        for (const socket of sockets.clients) {
+            socket.terminate();
+        }
+        await new Promise((resolve) => sockets.close(() => resolve()));
+        await new Promise((resolve) => server.close(() => resolve()));
+        await rm(options.socketPath, { force: true });
+        await rm(options.lockPath, { force: true });
+        finish();
+    };
+    const restartWhenIdle = () => {
+        if (restart && !busy()) {
+            void close();
+        }
+    };
+    const start = (socket, root, merge) => {
+        const current = runs.get(root);
+        if (current?.progress.status === "running" || current?.progress.status === "stopping") {
+            send(socket, { type: "error", root, message: "a run is going in this repository already" });
+            return;
+        }
+        if (restart) {
+            send(socket, { type: "error", root, message: "the worker is restarting, try again soon" });
+            return;
+        }
+        const run = { progress: startProgress(), stop: false };
+        runs.set(root, run);
+        const interrupts = {
+            afterIssue: () => run.stop,
+            afterRound: () => run.stop,
+            sleep: (ms) => new Promise((resolve) => {
+                if (run.stop) {
+                    resolve(false);
+                    return;
+                }
+                const timer = setTimeout(() => resolve(true), ms);
+                run.wake = () => {
+                    clearTimeout(timer);
+                    resolve(false);
+                };
+            }),
+        };
+        const update = (next) => {
+            run.progress = next;
+            broadcast();
+        };
+        broadcast();
+        options
+            .runRepo(root, { ...parseArgs([]), merge }, {
+            interrupts,
+            onEvent: (record) => update(applyEvent(run.progress, record)),
+        })
+            .then(() => {
+            const status = run.stop ? "stopped" : run.progress.error === undefined ? "finished" : "failed";
+            update({ ...run.progress, status });
+        }, (error) => update({ ...run.progress, status: "failed", error: message(error) }))
+            .finally(restartWhenIdle);
+    };
+    const stop = (root) => {
+        const run = runs.get(root);
+        if (run === undefined || run.progress.status !== "running") {
+            return;
+        }
+        run.stop = true;
+        run.wake?.();
+        run.progress = { ...run.progress, status: "stopping" };
+        broadcast();
+    };
+    sockets.on("connection", (socket) => {
+        send(socket, { type: "hello", version: options.version });
+        send(socket, snapshot());
+        socket.on("message", (data) => {
+            let value;
+            try {
+                value = JSON.parse(String(data));
+            }
+            catch {
+                return;
+            }
+            if (value.type === "hello") {
+                if (isNewer(value.version, options.version)) {
+                    restart = true;
+                    restartWhenIdle();
+                }
+            }
+            else if (value.type === "start") {
+                start(socket, value.root, value.merge === true);
+            }
+            else if (value.type === "stop") {
+                stop(value.root);
+            }
+        });
+    });
+    await new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(options.socketPath, () => {
+            server.off("error", reject);
+            resolve();
+        });
+    });
+    await chmod(options.socketPath, 0o600);
+    return { closed, close };
+}
+export async function serve() {
+    const paths = socketPaths();
+    const worker = await startWorker({
+        ...paths,
+        version: await packageVersion(),
+        runRepo,
+    });
+    if (worker === undefined) {
+        process.stderr.write("next-issue: another worker is running\n");
+        return 1;
+    }
+    const end = (signal) => {
+        process.off("SIGTERM", onTerminate);
+        process.off("SIGINT", onInterrupt);
+        rmSync(paths.socketPath, { force: true });
+        rmSync(paths.lockPath, { force: true });
+        stopChildren(signal);
+    };
+    const onTerminate = () => end("SIGTERM");
+    const onInterrupt = () => end("SIGINT");
+    process.on("SIGTERM", onTerminate);
+    process.on("SIGINT", onInterrupt);
+    process.stderr.write(`next-issue: worker listening on ${paths.socketPath}\n`);
+    await worker.closed;
+    return 0;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "next-issue",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Autonomous issue harness driven by the pi SDK",
   "keywords": [
     "github",
@@ -39,10 +39,12 @@
   "devDependencies": {
     "@earendil-works/pi-ai": "^1.5.0",
     "@types/node": "^26.6.6",
+    "@types/ws": "^8.18.2",
     "typescript": "^7.0.2"
   },
   "dependencies": {
     "@earendil-works/pi-coding-agent": "^1.5.0",
-    "typebox": "^1.4.3"
+    "typebox": "^1.4.3",
+    "ws": "^8.22.1"
   }
 }