agent-dealer 0.2.0 → 1.0.0

package/bundle/server/dist/direct-start-liveness.integration.test.js

@@ -0,0 +1,350 @@
+// packages/server/src/direct-start-liveness.integration.test.ts
+//
+// Reproduces the exact reviewer-reported gap: `npm run dev` / `npm run start` launch
+// packages/server/src/index.ts directly, bypassing the CLI daemon supervisor that writes
+// run.json — so a liveness check that only looked at run.json silently reported "not
+// running" for the two most common ways this app is actually launched. This spawns that
+// real entrypoint (not a fake), the same way `npm run dev` does, and drives
+// migrate-to-issues.ts's isServiceRunning() against it end to end.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { spawn, execFileSync } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import net from "node:net";
+import { fileURLToPath } from "node:url";
+import { resolveTsxBin } from "./resolve-tsx-bin.js";
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.resolve(__dirname, "..", "..", "..");
+const serverEntry = path.join(repoRoot, "packages", "server", "src", "index.ts");
+const tsxBin = resolveTsxBin(repoRoot);
+const { isServiceRunning, runMigration } = await import("./db/migrate-to-issues.js");
+const Database = (await import("better-sqlite3")).default;
+async function getEphemeralPort() {
+    return new Promise((resolve, reject) => {
+        const srv = net.createServer();
+        srv.listen(0, "127.0.0.1", () => {
+            const addr = srv.address();
+            if (addr && typeof addr === "object") {
+                const { port } = addr;
+                srv.close(() => resolve(port));
+            }
+            else {
+                srv.close(() => reject(new Error("could not allocate an ephemeral port")));
+            }
+        });
+        srv.on("error", reject);
+    });
+}
+async function waitUntil(check, timeoutMs, stepMs = 150) {
+    const deadline = Date.now() + timeoutMs;
+    for (;;) {
+        if (await check())
+            return true;
+        if (Date.now() >= deadline)
+            return false;
+        await new Promise((r) => setTimeout(r, stepMs));
+    }
+}
+/** A temp AGENT_DEALER_HOME that is deleted when the test ends, however it ends. Registered
+ * as a test hook rather than dropped in a `finally` on purpose: teardown here asserts (that
+ * the server it stopped is really dead), and an assertion that throws must not be able to
+ * skip the cleanup and leave a directory behind — which is precisely how $TMPDIR filled up
+ * with hundreds of these. Runs after the body's own `finally`, so servers are stopped by
+ * then. */
+function makeHome(t, prefix) {
+    const home = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+    t.after(() => fs.rmSync(home, { recursive: true, force: true }));
+    return home;
+}
+// Every server spawned by this file that stopServer() has not yet confirmed dead. A
+// detached child sits in its own session, so it no longer receives the terminal's SIGINT
+// along with the test runner — the usual "Ctrl-C reaps the whole foreground group" safety
+// net does not cover these, and an interrupted run would leak exactly what this file was
+// leaking before. This registry is that net, re-implemented explicitly.
+const liveServers = new Set();
+function killAllServerGroups() {
+    for (const child of liveServers)
+        signalGroup(child, "SIGKILL");
+    liveServers.clear();
+}
+process.on("exit", killAllServerGroups);
+for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
+    // Registering a listener suppresses the default terminate, so exit explicitly.
+    process.on(signal, () => {
+        killAllServerGroups();
+        process.exit(1);
+    });
+}
+function spawnServer(home, port) {
+    const child = spawn(tsxBin, [serverEntry], {
+        cwd: repoRoot,
+        env: {
+            ...process.env,
+            AGENT_DEALER_HOME: home,
+            AGENT_DEALER_ENV: "development",
+            PORT: String(port),
+        },
+        stdio: "ignore",
+        // The launcher and the child it forks get their own process group (pgid == launcher
+        // pid), so stopServer() can signal both at once.
+        detached: true,
+    });
+    // Every test awaits its own teardown, so this handle is never what keeps the run
+    // correct — but while it is ref'd it *does* keep the runner's event loop alive, so a
+    // teardown that throws before stopping a server wedges the whole file at 100% "waiting
+    // for a child that will never exit" instead of failing. unref() + killAllServerGroups()
+    // on exit is the pair that makes the failure path terminate and still reap.
+    child.unref();
+    liveServers.add(child);
+    return child;
+}
+function hasExited(child) {
+    return child.exitCode !== null || child.signalCode !== null;
+}
+function isPidAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** The process group `pid` belongs to, or undefined once it is gone. */
+function pgidOf(pid) {
+    try {
+        const pgid = Number.parseInt(execFileSync("ps", ["-o", "pgid=", "-p", String(pid)], { encoding: "utf8" }).trim(), 10);
+        return Number.isFinite(pgid) ? pgid : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * The pid recorded in `<home>/server.pid`, but only when that pid is part of `child`'s
+ * process group — i.e. only when this particular server is the marker's current owner.
+ *
+ * These tests deliberately point two servers at one AGENT_DEALER_HOME, and the loser never
+ * writes the marker, so "the pid in server.pid" is emphatically *not* "the pid of the
+ * server I am holding a handle to". Signalling or asserting on it unconditionally would
+ * reach across to the healthy server the test is still using.
+ */
+function ownedRecordedPid(child, home) {
+    if (child.pid === undefined)
+        return undefined;
+    let recorded;
+    try {
+        recorded = JSON.parse(fs.readFileSync(path.join(home, "server.pid"), "utf8")).pid;
+    }
+    catch {
+        return undefined; // no marker (never claimed, or already released on a clean exit)
+    }
+    if (typeof recorded !== "number" || !Number.isFinite(recorded))
+        return undefined;
+    return pgidOf(recorded) === child.pid ? recorded : undefined;
+}
+function signalGroup(child, signal) {
+    if (child.pid === undefined)
+        return;
+    try {
+        process.kill(-child.pid, signal);
+    }
+    catch {
+        // ESRCH — the whole group is already gone, which is exactly what we were after.
+    }
+}
+/**
+ * The one place in this file that knows how to terminate a server spawned by
+ * spawnServer() — every case goes through it, so this knowledge is not re-derived per test.
+ *
+ * spawnServer() launches `node_modules/.bin/tsx`, not node, and that bin *forks a child* to
+ * actually run the TypeScript: `child.pid` is the launcher, and the real server is its
+ * grandchild. tsx forwards a catchable SIGTERM but cannot forward an unblockable SIGKILL,
+ * so `child.kill("SIGKILL")` reaps the launcher and leaves the server alive, reparented to
+ * init, still bound to its PORT — forever. (pid-marker-hold-arbiter-child.ts documents the
+ * same wrapper behaviour for the same reason.) Hence: signal the whole process group,
+ * SIGTERM first so the server can release its own marker, SIGKILL only as a backstop, and
+ * then prove the pid the server actually recorded is dead rather than trusting the launcher
+ * handle's exit code to speak for it.
+ */
+async function stopServer(child, home) {
+    // Read while the server is still alive: a clean shutdown deletes the marker.
+    const recordedPid = ownedRecordedPid(child, home);
+    const allDead = () => hasExited(child) && (recordedPid === undefined || !isPidAlive(recordedPid));
+    signalGroup(child, "SIGTERM");
+    const exitedOnTerm = await waitUntil(allDead, 10000);
+    if (!exitedOnTerm) {
+        signalGroup(child, "SIGKILL");
+        await waitUntil(allDead, 5000);
+    }
+    if (recordedPid !== undefined) {
+        assert.equal(isPidAlive(recordedPid), false, `the server pid recorded in server.pid (${recordedPid}) survived teardown — the tsx launcher (${child.pid}) was reaped but its server child leaked`);
+    }
+    liveServers.delete(child);
+    return { exitedOnTerm, recordedPid };
+}
+async function waitForHealth(port, timeoutMs) {
+    return waitUntil(async () => {
+        try {
+            const res = await fetch(`http://127.0.0.1:${port}/health`);
+            return res.ok;
+        }
+        catch {
+            return false;
+        }
+    }, timeoutMs);
+}
+test("a directly-launched server (npm run dev's exact entrypoint) is detected as running, and stops being detected once it exits", { timeout: 45000 }, async (t) => {
+    const home = makeHome(t, "dealer-direct-start-");
+    const dbPath = path.join(home, "dealer.db");
+    const port = await getEphemeralPort();
+    const child = spawnServer(home, port);
+    try {
+        const healthy = await waitForHealth(port, 20000);
+        assert.ok(healthy, "the directly-launched server should become healthy within 20s");
+        const running = isServiceRunning(dbPath);
+        assert.equal(running.running, true, "a directly-launched server should be detected as running");
+        assert.ok(running.detail?.includes("server.pid"), `expected server.pid evidence, got: ${running.detail}`);
+        const { exitedOnTerm } = await stopServer(child, home);
+        assert.ok(exitedOnTerm, "the server should exit within 10s of SIGTERM");
+        const stopped = await waitUntil(() => !isServiceRunning(dbPath).running, 5000);
+        assert.ok(stopped, "the server should no longer be detected as running once it has exited");
+    }
+    finally {
+        await stopServer(child, home);
+    }
+});
+test("a second server losing the port race does not corrupt or delete the first server's liveness marker", { timeout: 45000 }, async (t) => {
+    // Reproduces the reviewer's exact second-round finding: two real index.ts launches
+    // sharing one AGENT_DEALER_HOME and port. Before the ownership-aware fix, B's write
+    // clobbered A's marker with B's own (dead-after-exit) pid, and B's unconditional exit
+    // cleanup then deleted it — leaving a healthy A with no liveness marker at all.
+    const home = makeHome(t, "dealer-direct-start-race-");
+    const dbPath = path.join(home, "dealer.db");
+    const port = await getEphemeralPort();
+    const serverA = spawnServer(home, port);
+    let serverB;
+    let serverAActualPid;
+    try {
+        const aHealthy = await waitForHealth(port, 20000);
+        assert.ok(aHealthy, "server A should become healthy within 20s");
+        // The marker's own recorded pid is the real process.pid inside index.ts, which is
+        // *not* necessarily serverA.pid: the tsx bin wrapper spawn() returns forks a child
+        // to actually run the TypeScript, so serverA.pid is the launcher, not the server.
+        // Compare against this recorded value, not serverA.pid, for that reason — and, just
+        // as importantly, signal that value too (stopServer()), not serverA.pid.
+        const stateBeforeB = JSON.parse(fs.readFileSync(path.join(home, "server.pid"), "utf8"));
+        serverAActualPid = stateBeforeB.pid;
+        assert.ok(Number.isFinite(serverAActualPid));
+        // Same home, same port — B can only ever lose the app.listen() race.
+        serverB = spawnServer(home, port);
+        const bExited = await waitUntil(() => serverB.exitCode !== null || serverB.signalCode !== null, 20000);
+        assert.ok(bExited, "server B should exit (losing the port bind) within 20s");
+        assert.notEqual(serverB.exitCode, 0, "server B should exit non-zero, having failed to bind the port");
+        // A must still be healthy, and the marker must still be A's — never overwritten by
+        // B, and never deleted by B's own exit cleanup.
+        const aStillHealthy = await waitForHealth(port, 5000);
+        assert.ok(aStillHealthy, "server A should remain healthy after B's failed launch and exit");
+        const running = isServiceRunning(dbPath);
+        assert.equal(running.running, true, "the migration guard must still see server A as running");
+        const stateAfterB = JSON.parse(fs.readFileSync(path.join(home, "server.pid"), "utf8"));
+        assert.equal(stateAfterB.pid, serverAActualPid, "the marker must still name server A, not server B");
+    }
+    finally {
+        if (serverB)
+            await stopServer(serverB, home);
+        // A is healthy and unsignalled at this point, so this teardown is the *only* thing
+        // between it and outliving the test run: it must reach the real server, not the
+        // launcher (see stopServer()).
+        const { recordedPid } = await stopServer(serverA, home);
+        if (serverAActualPid !== undefined) {
+            assert.equal(recordedPid, serverAActualPid, "teardown must have signalled server A's own recorded pid, not the launcher's");
+        }
+    }
+});
+test("a second server on a different port is refused at startup rather than running unmonitored", { timeout: 45000 }, async (t) => {
+    // Reproduces the reviewer's exact third-round finding: with a *different* port, B is
+    // no longer stopped by app.listen()'s EADDRINUSE. Before treating a failed claim as
+    // fatal, B ran to completion fully healthy but untracked (A owns server.pid), so once
+    // A stopped — correctly removing its own marker — B kept running with no liveness
+    // marker at all, and isServiceRunning() returned false despite B being very much alive.
+    const home = makeHome(t, "dealer-direct-start-diffport-");
+    const dbPath = path.join(home, "dealer.db");
+    const portA = await getEphemeralPort();
+    const portB = await getEphemeralPort();
+    const serverA = spawnServer(home, portA);
+    let serverB;
+    try {
+        const aHealthy = await waitForHealth(portA, 20000);
+        assert.ok(aHealthy, "server A should become healthy within 20s");
+        const stateBeforeB = JSON.parse(fs.readFileSync(path.join(home, "server.pid"), "utf8"));
+        serverB = spawnServer(home, portB);
+        const bExited = await waitUntil(() => serverB.exitCode !== null || serverB.signalCode !== null, 20000);
+        assert.ok(bExited, "server B should exit within 20s of failing to claim the liveness marker");
+        assert.notEqual(serverB.exitCode, 0, "server B should exit non-zero, having refused to start");
+        // B must never have become healthy on its own port — it should have aborted before
+        // ever reaching app.listen().
+        const bEverHealthy = await waitUntil(async () => {
+            try {
+                const res = await fetch(`http://127.0.0.1:${portB}/health`);
+                return res.ok;
+            }
+            catch {
+                return false;
+            }
+        }, 1500);
+        assert.equal(bEverHealthy, false, "server B must never have started serving on its own port");
+        // A is unaffected, and the marker still names A.
+        const aStillHealthy = await waitForHealth(portA, 5000);
+        assert.ok(aStillHealthy);
+        const stateAfterB = JSON.parse(fs.readFileSync(path.join(home, "server.pid"), "utf8"));
+        assert.equal(stateAfterB.pid, stateBeforeB.pid);
+        assert.equal(isServiceRunning(dbPath).running, true);
+        // And once A stops (removing its own marker as intended), isServiceRunning correctly
+        // reports nothing running — B never having claimed the marker means there is no
+        // now-invisible second server left behind for this check to miss.
+        const { exitedOnTerm } = await stopServer(serverA, home);
+        assert.ok(exitedOnTerm, "server A should exit within 10s of SIGTERM");
+        assert.equal(isServiceRunning(dbPath).running, false, "nothing should be left running or tracked after A stops");
+    }
+    finally {
+        if (serverB)
+            await stopServer(serverB, home);
+        await stopServer(serverA, home);
+    }
+});
+test("a real running server blocks the migration end to end, and the migration succeeds once the server is stopped", { timeout: 45000 }, async (t) => {
+    const home = makeHome(t, "dealer-direct-start-migration-");
+    const dbPath = path.join(home, "dealer.db");
+    const port = await getEphemeralPort();
+    const server = spawnServer(home, port);
+    try {
+        const healthy = await waitForHealth(port, 20000);
+        assert.ok(healthy, "the server should become healthy within 20s (this also runs migrate(), creating the schema)");
+        // Seed one legacy lineage directly — a separate connection to the same file is fine
+        // alongside the server's own WAL-mode connection.
+        const now = new Date().toISOString();
+        const seedDb = new Database(dbPath);
+        seedDb
+            .prepare(`INSERT INTO runs (id, source, external_id, task_category, title, repo, agent_id, status,
+            lineage_id, created_at, updated_at)
+           VALUES ('run-live-block', 'manual', 'run-live-block', 'code', 'Blocked task', '/repo', NULL, 'done', NULL, ?, ?)`)
+            .run(now, now);
+        seedDb.close();
+        const refused = runMigration(dbPath);
+        assert.ok(refused.mismatches.length > 0);
+        assert.match(refused.mismatches[0], /service is running/);
+        assert.equal(fs.existsSync(`${dbPath}.pre-issue-migration-backup`), false);
+        const { exitedOnTerm } = await stopServer(server, home);
+        assert.ok(exitedOnTerm, "the server should exit within 10s of SIGTERM");
+        await waitUntil(() => !isServiceRunning(dbPath).running, 5000);
+        const succeeded = runMigration(dbPath);
+        assert.deepStrictEqual(succeeded.mismatches, []);
+        assert.equal(succeeded.issuesCreated, 1);
+    }
+    finally {
+        await stopServer(server, home);
+    }
+});

package/bundle/server/dist/index.js

@@ -3,24 +3,77 @@ import Fastify from "fastify";
 import cors from "@fastify/cors";
 import { enrichPathForCliTools } from "./cli-env.js";
 import { migrate } from "./db/index.js";
+import { writeServerPidFile, removeServerPidFile } from "./server-liveness.js";
 import { registerRoutes } from "./routes/index.js";
-import { startQueue, recoverOrphanedRuns } from "./queue/dispatcher.js";
+import { registerIssueRoutes } from "./routes/issues.js";
+import { registerQueueRoutes } from "./routes/queue.js";
+import { registerHumanActionRoutes } from "./routes/human-actions.js";
+import { recoverCoordinator } from "./coordinator/recovery.js";
+import { startCoordinatorLoop } from "./coordinator/worker-loop.js";
+import { registerEffectHandler } from "./coordinator/effect-registry.js";
+import { runDeveloperEffect } from "./coordinator/developer-effect.js";
+import { runReviewerEffect } from "./coordinator/reviewer-effect.js";
 import { registerStaticUi } from "./static-ui.js";
+import { cleanupOrphanedWorkerMcpConfig } from "./paths.js";
+import net from "node:net";
 const { mode, envFile } = loadAgentDealerEnv();
 console.log(formatEnvStartupLine(mode, envFile));
 const port = Number(process.env.PORT ?? 2221);
+// Hosts that publish AAAA records but have no usable IPv6 route make Node's
+// happy-eyeballs burn its whole connect budget on unreachable addresses, so
+// outbound fetch (Linear GraphQL) fails with UND_ERR_CONNECT_TIMEOUT.
+net.setDefaultAutoSelectFamily(false);
 async function main() {
     enrichPathForCliTools();
+    // Written before migrate() touches the database, and independent of how this process
+    // was launched — see server-liveness.ts for why this exists alongside the CLI's own
+    // run.json. Cleaned up on a normal shutdown; a stale file from a crash is harmless since
+    // every reader checks the pid is actually alive, not just that the file exists.
+    //
+    // A false return means a different, still-alive process already owns this
+    // AGENT_DEALER_HOME — this must be fatal, not merely logged: a server that continued
+    // unmonitored (e.g. on a different port than the owner, so app.listen() below would
+    // have succeeded) would run invisibly to isServiceRunning()/the migration guard, and to
+    // the owner's own eventual shutdown, which only ever removes a marker it still owns —
+    // so a second, unclaimed server surviving past the first one's clean exit would leave
+    // nothing recording that it is still running against this same database.
+    if (!writeServerPidFile(port)) {
+        console.error("[startup] another live process already owns this AGENT_DEALER_HOME's liveness marker — refusing to start. " +
+            "Stop it first, or point AGENT_DEALER_HOME at a different directory.");
+        process.exit(1);
+    }
+    process.on("SIGINT", () => {
+        removeServerPidFile();
+        process.exit(0);
+    });
+    process.on("SIGTERM", () => {
+        removeServerPidFile();
+        process.exit(0);
+    });
+    process.on("exit", removeServerPidFile);
     migrate();
+    cleanupOrphanedWorkerMcpConfig();
     const app = Fastify({ logger: true });
     await app.register(cors, { origin: true });
     await registerRoutes(app);
+    await registerIssueRoutes(app);
+    await registerQueueRoutes(app);
+    await registerHumanActionRoutes(app);
     const uiDist = await registerStaticUi(app);
-    const orphans = recoverOrphanedRuns();
-    if (orphans > 0) {
-        console.warn(`[startup] recovered ${orphans} orphaned running run(s) → failed`);
+    // Without this, getEffectHandler() falls back to the always-session_failed placeholders
+    // (effect-registry.ts) and every started issue would burn its infra retries and
+    // escalate without ever launching an agent, creating a worktree, or opening a PR.
+    registerEffectHandler("developer", (ctx) => runDeveloperEffect(ctx));
+    registerEffectHandler("reviewer", (ctx) => runReviewerEffect(ctx));
+    const coordinatorRecovery = await recoverCoordinator();
+    if (coordinatorRecovery.reclaimed.length ||
+        coordinatorRecovery.republished.length ||
+        coordinatorRecovery.deadLettered.length ||
+        coordinatorRecovery.deferredForCap.length ||
+        coordinatorRecovery.autoMergesFinalized.length) {
+        console.warn(`[startup] coordinator recovery: reclaimed ${coordinatorRecovery.reclaimed.length}, republished ${coordinatorRecovery.republished.length}, dead-lettered ${coordinatorRecovery.deadLettered.length}, deferred for cap ${coordinatorRecovery.deferredForCap.length}, auto-merges finalized ${coordinatorRecovery.autoMergesFinalized.length}`);
     }
-    startQueue();
+    startCoordinatorLoop();
     await app.listen({ port, host: "127.0.0.1" });
     const base = `http://127.0.0.1:${port}`;
     console.log(`agent-dealer API ${base}`);

package/bundle/server/dist/paths.js

@@ -4,13 +4,49 @@ import { getDataDir } from "./db/index.js";
 export function getTemporalDir() {
     return path.join(getDataDir(), ".temporal");
 }
-export function getTemporalOutputDir() {
-    const dir = path.join(getTemporalDir(), "output");
+export function getTemporalLogsDir() {
+    const dir = path.join(getTemporalDir(), "logs");
     fs.mkdirSync(dir, { recursive: true });
     return dir;
 }
-export function getTemporalLogsDir() {
-    const dir = path.join(getTemporalDir(), "logs");
+/**
+ * Per-attempt Agent Deck MCP config files for worker CLIs (claude `--mcp-config`,
+ * codex `CODEX_HOME`) — never the worktree (a generated checkout must not carry a
+ * persistent-looking config) and never `.temporal/logs` (those are retained; this
+ * directory is scrubbed per-attempt by the caller and at coordinator startup).
+ * Mode 0700 — configs are process-local and short-lived.
+ */
+export function getWorkerMcpConfigDir() {
+    const dir = path.join(getTemporalDir(), "worker-mcp-config");
+    fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+    return dir;
+}
+/**
+ * Startup-only: removes every per-attempt entry left under `getWorkerMcpConfigDir()`
+ * by a crash (a clean shutdown never reaches here — `releaseWorkerDeckConnection` already
+ * removed its own entry). Safe unconditionally: every entry is scoped to exactly one
+ * attempt's spawn, and that attempt cannot still be running (this *is* the coordinator
+ * restarting). Claude and codex still write per-attempt MCP configs outside the worktree;
+ * cursor's config lives inside the worktree and is not cleaned here.
+ */
+export function cleanupOrphanedWorkerMcpConfig() {
+    const dir = getWorkerMcpConfigDir();
+    for (const entry of fs.readdirSync(dir)) {
+        fs.rmSync(path.join(dir, entry), { recursive: true, force: true });
+    }
+}
+/**
+ * Root for Dealer-managed repository clones and session worktrees (NOT-149).
+ * Defaults under AGENT_DEALER_HOME; override with AGENT_DEALER_EXECUTION_ROOT for a
+ * mounted disk / cloud volume.
+ */
+export function getExecutionRoot() {
+    const override = process.env.AGENT_DEALER_EXECUTION_ROOT?.trim();
+    if (override) {
+        fs.mkdirSync(override, { recursive: true });
+        return override;
+    }
+    const dir = path.join(getDataDir(), "execution");
     fs.mkdirSync(dir, { recursive: true });
     return dir;
 }

package/bundle/server/dist/paths.test.js

@@ -0,0 +1,42 @@
+// packages/server/src/paths.test.ts
+//
+// cleanupOrphanedWorkerMcpConfig: a coordinator crash mid-attempt skips
+// releaseWorkerDeckConnection's own cleanup, leaving leftover per-attempt MCP
+// configs (claude JSON files, codex CODEX_HOME dirs) under worker-mcp-config.
+// Every entry there is scoped to one already-unrecoverable attempt, so a startup
+// sweep can remove all of them unconditionally.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+process.env.AGENT_DEALER_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-paths-"));
+const { getWorkerMcpConfigDir, cleanupOrphanedWorkerMcpConfig } = await import("./paths.js");
+test("cleanupOrphanedWorkerMcpConfig removes leftover per-attempt files and directories", () => {
+    const dir = getWorkerMcpConfigDir();
+    fs.writeFileSync(path.join(dir, "deck-abc.json"), JSON.stringify({ mcpServers: {} }));
+    const codexHome = path.join(dir, "codex-home-deck-def");
+    fs.mkdirSync(codexHome);
+    fs.writeFileSync(path.join(codexHome, "config.toml"), "");
+    cleanupOrphanedWorkerMcpConfig();
+    assert.deepEqual(fs.readdirSync(dir), []);
+});
+test("cleanupOrphanedWorkerMcpConfig removes a leftover auth.json symlink without touching its target", () => {
+    const dir = getWorkerMcpConfigDir();
+    const ambientAuthPath = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-ambient-auth-"));
+    const realAuth = path.join(ambientAuthPath, "auth.json");
+    fs.writeFileSync(realAuth, "real-credential");
+    const codexHome = path.join(dir, "codex-home-deck-ghi");
+    fs.mkdirSync(codexHome);
+    fs.symlinkSync(realAuth, path.join(codexHome, "auth.json"));
+    cleanupOrphanedWorkerMcpConfig();
+    assert.equal(fs.existsSync(codexHome), false);
+    assert.ok(fs.existsSync(realAuth));
+    assert.equal(fs.readFileSync(realAuth, "utf8"), "real-credential");
+    fs.rmSync(ambientAuthPath, { recursive: true, force: true });
+});
+test("cleanupOrphanedWorkerMcpConfig is a no-op on an already-empty directory", () => {
+    const dir = getWorkerMcpConfigDir();
+    assert.deepEqual(fs.readdirSync(dir), []);
+    assert.doesNotThrow(() => cleanupOrphanedWorkerMcpConfig());
+});

package/bundle/server/dist/pid-marker-claim-child.js

@@ -0,0 +1,18 @@
+// packages/server/src/pid-marker-claim-child.ts
+//
+// Test helper only (spawned by pid-marker.test.ts as a real child process) — attempts a
+// single claimPidMarker() call against the path given as argv[2] and prints the outcome
+// as JSON, so a test can spawn several of these at once to exercise a genuine multi-process
+// race, not just an in-process simulation of one.
+import { claimPidMarker } from "./pid-marker.js";
+const filePath = process.argv[2];
+const claimed = claimPidMarker(filePath, { role: "test-child" });
+console.log(JSON.stringify({ pid: process.pid, claimed }));
+if (claimed) {
+    // Stay alive so any other racing child sees a genuinely *live* owner when it checks —
+    // matching how a real long-running server holds this marker for its whole lifetime, not
+    // a one-shot script that claims and immediately exits (which would make every later
+    // racer see a "stale" marker regardless of who legitimately won). The test kills this
+    // process once it has read the result.
+    setInterval(() => { }, 1000);
+}

package/bundle/server/dist/pid-marker-hold-arbiter-child.js

@@ -0,0 +1,24 @@
+// packages/server/src/pid-marker-hold-arbiter-child.ts
+//
+// Test helper only (spawned by pid-marker.test.ts) — opens the same `.arbiter` SQLite file
+// claimPidMarker() uses, takes its BEGIN EXCLUSIVE lock, prints "held" once acquired, then
+// blocks forever (until killed). Used to prove that a process crashing while it holds the
+// arbiter lock does not wedge it: the OS releases the advisory lock the instant this
+// process dies, for any reason — there is no lease/timeout heuristic to get right, unlike
+// a hand-rolled lock *file* (which is exactly the bug this design replaced).
+import Database from "better-sqlite3";
+const filePath = process.argv[2];
+const db = new Database(`${filePath}.arbiter`);
+db.pragma("busy_timeout = 3000");
+db.transaction(() => {
+    // Print our *real* pid, not the tsx launcher's: the test kills this exact pid directly
+    // with SIGKILL to simulate a crash — killing the tsx wrapper process alone does not
+    // reach this actual node process running the script (the wrapper forks a child to run
+    // it and does not forward an unblockable SIGKILL, only a catchable SIGTERM).
+    console.log(JSON.stringify({ event: "held", pid: process.pid }));
+    // Block forever while still inside the transaction (still holding the exclusive lock) —
+    // this process is only ever meant to be terminated externally (SIGKILL), simulating a
+    // crash mid-lock.
+    const SharedArrayBufferBlock = new Int32Array(new SharedArrayBuffer(4));
+    Atomics.wait(SharedArrayBufferBlock, 0, 0);
+}).exclusive();