@ryanfw/prompt-orchestration-pipeline 1.2.7 → 1.2.9

package/src/ui/server/__tests__/job-control-endpoints.test.ts

@@ -1,12 +1,19 @@
-import { mkdir, writeFile } from "node:fs/promises";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import type { Subprocess } from "bun";
 
-import { afterEach, describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 
 import { initPATHS, resetPATHS } from "../config-bridge-node";
-import { handleJobRestart, handleJobStop } from "../endpoints/job-control-endpoints";
+import { handleJobRestart, handleJobStop, handleTaskStart } from "../endpoints/job-control-endpoints";
+import { resetConfig } from "../../../core/config";
+import {
+  getConcurrencyRuntimePaths,
+  releaseJobSlot,
+  tryAcquireJobSlot,
+  type JobSlotLease,
+} from "../../../core/job-concurrency";
 import { readJobStatus } from "../../../core/status-writer";
 
 const tempRoots: string[] = [];
@@ -34,6 +41,25 @@ async function setupJob(
   return jobDir;
 }
 
+async function setupPipelineConfig(root: string, slug: string, tasks: string[]): Promise<void> {
+  const configDir = path.join(root, "pipeline-config", slug);
+  await mkdir(configDir, { recursive: true });
+  await writeFile(path.join(configDir, "pipeline.json"), JSON.stringify({ name: slug, tasks }));
+  await writeFile(path.join(root, "pipeline-config", "registry.json"), JSON.stringify({
+    pipelines: {
+      [slug]: {},
+    },
+  }));
+}
+
+function mockRunnerSpawn(pid = 424242) {
+  const proc = {
+    pid,
+    unref: vi.fn(),
+  } as unknown as Subprocess;
+  return vi.spyOn(Bun, "spawn").mockReturnValue(proc);
+}
+
 function spawnSleeper(): Subprocess {
   const proc = Bun.spawn(["sleep", "60"], {
     stdout: "ignore",
@@ -45,7 +71,10 @@ function spawnSleeper(): Subprocess {
 }
 
 afterEach(async () => {
+  vi.restoreAllMocks();
   resetPATHS();
+  delete process.env["PO_MAX_RUNNING_JOBS"];
+  resetConfig();
   for (const proc of childProcs.splice(0)) {
     try { proc.kill(); } catch {}
   }
@@ -201,6 +230,73 @@ describe("handleJobStop", () => {
     expect(analysis.tokenUsage).toEqual([]);
   });
 
+  it("resets restartCount on the running task that gets reset to pending", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    const jobDir = await setupJob(root, "job-restart-count", {
+      id: "job-restart-count",
+      state: "running",
+      current: "analysis",
+      currentStage: "stage-2",
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: {
+          state: "running",
+          currentStage: "stage-2",
+          attempts: 1,
+          restartCount: 2,
+        },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/job-restart-count/stop", { method: "POST" });
+    const res = await handleJobStop(req, "job-restart-count", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body["resetTask"]).toBe("analysis");
+
+    const snapshot = await readJobStatus(jobDir);
+    const analysis = snapshot!.tasks["analysis"]!;
+    expect(analysis.state).toBe("pending");
+    expect(analysis.restartCount).toBe(0);
+  });
+
+  it("does not modify restartCount on tasks that are not reset", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    const jobDir = await setupJob(root, "job-restart-count-untouched", {
+      id: "job-restart-count-untouched",
+      state: "running",
+      current: "analysis",
+      currentStage: "stage-2",
+      tasks: {
+        research: { state: "done", currentStage: null, restartCount: 4 },
+        analysis: {
+          state: "running",
+          currentStage: "stage-2",
+          attempts: 1,
+          restartCount: 1,
+        },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/job-restart-count-untouched/stop", { method: "POST" });
+    const res = await handleJobStop(req, "job-restart-count-untouched", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body["resetTask"]).toBe("analysis");
+
+    const snapshot = await readJobStatus(jobDir);
+    expect(snapshot!.tasks["research"]!.restartCount).toBe(4);
+    expect(snapshot!.tasks["analysis"]!.restartCount).toBe(0);
+  });
+
   it("recovers a stale job by resetting the first non-terminal task after partial progress", async () => {
     const root = await makeTempRoot();
     initPATHS(root);
@@ -340,6 +436,8 @@ describe("handleJobRestart", () => {
       files: { artifacts: [], logs: [], tmp: [] },
     });
 
+    mockRunnerSpawn();
+
     const req = new Request("http://localhost/api/jobs/restart-3/restart", { method: "POST" });
     const res = await handleJobRestart(req, "restart-3", root);
     const body = await res.json() as Record<string, unknown>;
@@ -350,6 +448,297 @@ describe("handleJobRestart", () => {
   });
 });
 
+describe("handleTaskStart", () => {
+  it("returns 404 JOB_NOT_FOUND when job exists in neither current/ nor complete/", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    const req = new Request("http://localhost/api/jobs/missing/tasks/research/start", { method: "POST" });
+    const res = await handleTaskStart(req, "missing", "research", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(404);
+    expect(body["code"]).toBe("JOB_NOT_FOUND");
+  });
+
+  it("returns 409 unsupported_lifecycle when job exists only in complete/, leaving job directory in place", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    const completeDir = path.join(root, "pipeline-data", "complete", "task-start-complete");
+    await mkdir(completeDir, { recursive: true });
+    await writeFile(path.join(completeDir, "tasks-status.json"), JSON.stringify({
+      id: "task-start-complete",
+      state: "complete",
+      current: null,
+      currentStage: null,
+      tasks: {
+        research: { state: "done", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }));
+
+    const req = new Request("http://localhost/api/jobs/task-start-complete/tasks/research/start", { method: "POST" });
+    const res = await handleTaskStart(req, "task-start-complete", "research", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(409);
+    expect(body["code"]).toBe("unsupported_lifecycle");
+
+    const stillThere = await Bun.file(path.join(completeDir, "tasks-status.json")).exists();
+    expect(stillThere).toBe(true);
+
+    const movedToCurrent = await Bun.file(path.join(root, "pipeline-data", "current", "task-start-complete", "tasks-status.json")).exists();
+    expect(movedToCurrent).toBe(false);
+  });
+
+  it("returns 409 job_running when runner.pid points to a live process", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    const proc = spawnSleeper();
+
+    await setupJob(root, "task-start-live", {
+      id: "task-start-live",
+      state: "running",
+      current: "research",
+      currentStage: "prompt",
+      tasks: {
+        research: { state: "running", currentStage: "prompt" },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }, proc.pid);
+
+    const req = new Request("http://localhost/api/jobs/task-start-live/tasks/research/start", { method: "POST" });
+    const res = await handleTaskStart(req, "task-start-live", "research", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(409);
+    expect(body["code"]).toBe("job_running");
+  });
+
+  it("returns 409 job_running when snapshot has a task in \"running\" state and PID is dead", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    await setupJob(root, "task-start-stale", {
+      id: "task-start-stale",
+      state: "running",
+      current: "research",
+      currentStage: "prompt",
+      tasks: {
+        research: { state: "running", currentStage: "prompt" },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }, 999999);
+
+    const req = new Request("http://localhost/api/jobs/task-start-stale/tasks/analysis/start", { method: "POST" });
+    const res = await handleTaskStart(req, "task-start-stale", "analysis", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(409);
+    expect(body["code"]).toBe("job_running");
+  });
+
+  it("returns 404 task_not_found for an unknown taskId", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    await setupJob(root, "task-start-unknown", {
+      id: "task-start-unknown",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/task-start-unknown/tasks/synthesis/start", { method: "POST" });
+    const res = await handleTaskStart(req, "task-start-unknown", "synthesis", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(404);
+    expect(body["code"]).toBe("task_not_found");
+  });
+
+  it("returns 422 task_not_pending when target task is \"done\"", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    await setupJob(root, "task-start-done", {
+      id: "task-start-done",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/task-start-done/tasks/research/start", { method: "POST" });
+    const res = await handleTaskStart(req, "task-start-done", "research", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(422);
+    expect(body["code"]).toBe("task_not_pending");
+  });
+
+  it("returns 202 and spawns runner when target task is pending and dependencies are done", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "task-start-ok-pipeline", ["research", "analysis"]);
+    const spawnSpy = mockRunnerSpawn(12345);
+
+    const jobDir = await setupJob(root, "task-start-ok", {
+      id: "task-start-ok",
+      pipeline: "task-start-ok-pipeline",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {
+        analysis: { state: "pending", currentStage: null },
+        research: { state: "done", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const statusPath = path.join(jobDir, "tasks-status.json");
+    const snapshotBefore = await Bun.file(statusPath).text();
+
+    const req = new Request("http://localhost/api/jobs/task-start-ok/tasks/analysis/start", { method: "POST" });
+    const originalRunSingleTask = process.env["PO_RUN_SINGLE_TASK"];
+    process.env["PO_RUN_SINGLE_TASK"] = "true";
+    let res: Response;
+    try {
+      res = await handleTaskStart(req, "task-start-ok", "analysis", root);
+    } finally {
+      if (originalRunSingleTask === undefined) {
+        delete process.env["PO_RUN_SINGLE_TASK"];
+      } else {
+        process.env["PO_RUN_SINGLE_TASK"] = originalRunSingleTask;
+      }
+    }
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body).toEqual({
+      ok: true,
+      jobId: "task-start-ok",
+      taskId: "analysis",
+      action: "start",
+      lifecycle: "current",
+      spawned: true,
+    });
+
+    const snapshotAfter = await Bun.file(statusPath).text();
+    expect(snapshotAfter).toBe(snapshotBefore);
+
+    expect(spawnSpy).toHaveBeenCalledTimes(1);
+    const spawnOptions = spawnSpy.mock.calls[0]![0] as unknown as {
+      cmd: string[];
+      env: Record<string, string | undefined>;
+    };
+    expect(spawnOptions.cmd[0]).toBe("bun");
+    expect(spawnOptions.cmd[1]).toBe("run");
+    expect(spawnOptions.cmd[2]).toContain("pipeline-runner.ts");
+    expect(spawnOptions.cmd[3]).toBe("task-start-ok");
+    expect(spawnOptions.env["PO_ROOT"]).toBe(root);
+    expect(spawnOptions.env["PO_START_FROM_TASK"]).toBe("analysis");
+    expect(spawnOptions.env["PO_RUN_SINGLE_TASK"]).toBeUndefined();
+    await expect(Bun.file(path.join(jobDir, "runner.pid")).text()).resolves.toBe("12345\n");
+  });
+
+  it("returns 409 job_running on an immediate second start after writing runner.pid", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "task-start-duplicate-pipeline", ["research", "analysis"]);
+    const spawnSpy = mockRunnerSpawn(process.pid);
+
+    await setupJob(root, "task-start-duplicate", {
+      id: "task-start-duplicate",
+      pipeline: "task-start-duplicate-pipeline",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const firstReq = new Request("http://localhost/api/jobs/task-start-duplicate/tasks/analysis/start", { method: "POST" });
+    const firstRes = await handleTaskStart(firstReq, "task-start-duplicate", "analysis", root);
+    expect(firstRes.status).toBe(202);
+
+    const secondReq = new Request("http://localhost/api/jobs/task-start-duplicate/tasks/analysis/start", { method: "POST" });
+    const secondRes = await handleTaskStart(secondReq, "task-start-duplicate", "analysis", root);
+    const secondBody = await secondRes.json() as Record<string, unknown>;
+
+    expect(secondRes.status).toBe(409);
+    expect(secondBody["code"]).toBe("job_running");
+    expect(spawnSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it("returns 412 dependencies_not_satisfied when an earlier task is \"pending\"", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "task-start-deps-pipeline", ["research", "analysis"]);
+
+    await setupJob(root, "task-start-deps", {
+      id: "task-start-deps",
+      pipeline: "task-start-deps-pipeline",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {
+        analysis: { state: "pending", currentStage: null },
+        research: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/task-start-deps/tasks/analysis/start", { method: "POST" });
+    const res = await handleTaskStart(req, "task-start-deps", "analysis", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(412);
+    expect(body["code"]).toBe("dependencies_not_satisfied");
+  });
+
+  it("returns 412 dependencies_not_satisfied when an earlier task is \"failed\"", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "task-start-failed-deps-pipeline", ["research", "analysis"]);
+
+    await setupJob(root, "task-start-failed-deps", {
+      id: "task-start-failed-deps",
+      pipeline: "task-start-failed-deps-pipeline",
+      state: "failed",
+      current: null,
+      currentStage: null,
+      tasks: {
+        analysis: { state: "pending", currentStage: null },
+        research: { state: "failed", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/task-start-failed-deps/tasks/analysis/start", { method: "POST" });
+    const res = await handleTaskStart(req, "task-start-failed-deps", "analysis", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(412);
+    expect(body["code"]).toBe("dependencies_not_satisfied");
+  });
+});
+
 describe("stop → restart integration", () => {
   it("stop moves running to pending, then restart proceeds without 409", async () => {
     const root = await makeTempRoot();
@@ -379,6 +768,7 @@ describe("stop → restart integration", () => {
     expect(snapshot!.tasks["research"]!.state).toBe("pending");
 
     // Restart should now succeed (no 409 deadlock)
+    mockRunnerSpawn();
     const restartReq = new Request("http://localhost/api/jobs/integ-1/restart", { method: "POST" });
     const restartRes = await handleJobRestart(restartReq, "integ-1", root);
     const restartBody = await restartRes.json() as Record<string, unknown>;
@@ -405,6 +795,8 @@ describe("stop → restart integration", () => {
       files: { artifacts: [], logs: [], tmp: [] },
     }, 999999);
 
+    mockRunnerSpawn();
+
     const req = new Request("http://localhost/api/jobs/integ-2/restart", { method: "POST" });
     const res = await handleJobRestart(req, "integ-2", root);
     const body = await res.json() as Record<string, unknown>;
@@ -464,6 +856,94 @@ describe("stop → restart integration", () => {
     expect(body["code"]).toBe("job_running");
   });
 
+  it("restart returns 409 job_running while runner is mid-retry-sleep (alive PID, task running, restartCount > 0)", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    // Simulate the runner being asleep between automatic retries: the runner
+    // process is alive, the task it owns is still in "running" state, and
+    // restartCount has already been incremented by the prior failed attempt.
+    const proc = spawnSleeper();
+
+    await setupJob(root, "retry-sleep-restart", {
+      id: "retry-sleep-restart",
+      state: "running",
+      current: "research",
+      currentStage: null,
+      tasks: {
+        research: {
+          state: "running",
+          currentStage: null,
+          attempts: 2,
+          restartCount: 1,
+        },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }, proc.pid);
+
+    const req = new Request("http://localhost/api/jobs/retry-sleep-restart/restart", { method: "POST" });
+    const res = await handleJobRestart(req, "retry-sleep-restart", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    // The PID-liveness check fires before any task-state inspection, so the
+    // in-flight Bun.sleep cannot race with a manual restart.
+    expect(res.status).toBe(409);
+    expect(body["code"]).toBe("job_running");
+  });
+
+  it("stop terminates a runner mid-retry-sleep within KILL_GRACE_MS + KILL_POLL_MS via SIGTERM", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    // Same scenario as above: runner alive, task running, retry counter set.
+    // The stop path must signal the runner via the existing PID/SIGTERM contract.
+    const proc = spawnSleeper();
+    const pid = proc.pid;
+
+    const jobDir = await setupJob(root, "retry-sleep-stop", {
+      id: "retry-sleep-stop",
+      state: "running",
+      current: "research",
+      currentStage: null,
+      tasks: {
+        research: {
+          state: "running",
+          currentStage: null,
+          attempts: 2,
+          restartCount: 1,
+        },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }, pid);
+
+    // KILL_GRACE_MS = 1500, KILL_POLL_MS = 100 in job-control-endpoints.ts.
+    // Allow a generous margin for filesystem and signal-delivery jitter while
+    // still asserting the bound holds.
+    const KILL_BUDGET_MS = 1500 + 100 + 500;
+
+    const start = Date.now();
+    const req = new Request("http://localhost/api/jobs/retry-sleep-stop/stop", { method: "POST" });
+    const res = await handleJobStop(req, "retry-sleep-stop", root);
+    const elapsed = Date.now() - start;
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body["ok"]).toBe(true);
+    expect(body["stopped"]).toBe(true);
+    expect(body["signal"]).toBe("SIGTERM");
+    expect(body["resetTask"]).toBe("research");
+    expect(elapsed).toBeLessThan(KILL_BUDGET_MS);
+
+    // PID file is removed by cleanupRunnerPid in handleJobStop.
+    const pidFileExists = await Bun.file(path.join(jobDir, "runner.pid")).exists();
+    expect(pidFileExists).toBe(false);
+
+    // Reset restored the task to pending and zeroed restartCount (criterion 10).
+    const snapshot = await readJobStatus(jobDir);
+    expect(snapshot!.tasks["research"]!.state).toBe("pending");
+    expect(snapshot!.tasks["research"]!.restartCount).toBe(0);
+  });
+
   it("full cycle: stop clears running task, restart resets all tasks to pending", async () => {
     const root = await makeTempRoot();
     initPATHS(root);
@@ -498,6 +978,7 @@ describe("stop → restart integration", () => {
     expect(afterStop!.tasks["synthesis"]!.state).toBe("pending");
 
     // Restart with clean-slate resets everything
+    mockRunnerSpawn();
     const restartReq = new Request("http://localhost/api/jobs/integ-5/restart", { method: "POST" });
     const restartRes = await handleJobRestart(restartReq, "integ-5", root);
     const restartBody = await restartRes.json() as Record<string, unknown>;
@@ -512,3 +993,179 @@ describe("stop → restart integration", () => {
     expect(afterRestart!.tasks["synthesis"]!.state).toBe("pending");
   });
 });
+
+describe("concurrency enforcement", () => {
+  function setMaxConcurrentJobs(value: number): void {
+    process.env["PO_MAX_RUNNING_JOBS"] = String(value);
+    resetConfig();
+  }
+
+  async function readLeaseFile(root: string, jobId: string): Promise<JobSlotLease> {
+    const { runningJobsDir } = getConcurrencyRuntimePaths(path.join(root, "pipeline-data"));
+    const raw = await readFile(path.join(runningJobsDir, `${jobId}.json`), "utf-8");
+    return JSON.parse(raw) as JobSlotLease;
+  }
+
+  it("returns 409 concurrency_limit_reached on restart when capacity is full", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    setMaxConcurrentJobs(1);
+
+    await setupJob(root, "blocking-job", {
+      id: "blocking-job",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {},
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+    const blocking = await tryAcquireJobSlot({
+      dataDir: path.join(root, "pipeline-data"),
+      jobId: "blocking-job",
+      maxConcurrentJobs: 1,
+      source: "orchestrator",
+      pid: process.pid,
+    });
+    expect(blocking.ok).toBe(true);
+
+    await setupJob(root, "restart-full", {
+      id: "restart-full",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const spawnSpy = vi.spyOn(Bun, "spawn");
+    const req = new Request("http://localhost/api/jobs/restart-full/restart", { method: "POST" });
+    const res = await handleJobRestart(req, "restart-full", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(409);
+    expect(body["code"]).toBe("concurrency_limit_reached");
+    expect(body["ok"]).toBe(false);
+    expect(spawnSpy).not.toHaveBeenCalled();
+  });
+
+  it("returns 409 concurrency_limit_reached on task-start when capacity is full", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "ts-full-pipeline", ["research", "analysis"]);
+    setMaxConcurrentJobs(1);
+
+    await setupJob(root, "blocking-job", {
+      id: "blocking-job",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {},
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+    const blocking = await tryAcquireJobSlot({
+      dataDir: path.join(root, "pipeline-data"),
+      jobId: "blocking-job",
+      maxConcurrentJobs: 1,
+      source: "orchestrator",
+      pid: process.pid,
+    });
+    expect(blocking.ok).toBe(true);
+
+    await setupJob(root, "task-start-full", {
+      id: "task-start-full",
+      pipeline: "ts-full-pipeline",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const spawnSpy = vi.spyOn(Bun, "spawn");
+    const req = new Request("http://localhost/api/jobs/task-start-full/tasks/analysis/start", { method: "POST" });
+    const res = await handleTaskStart(req, "task-start-full", "analysis", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(409);
+    expect(body["code"]).toBe("concurrency_limit_reached");
+    expect(body["ok"]).toBe(false);
+    expect(spawnSpy).not.toHaveBeenCalled();
+  });
+
+  it("releases the slot when the runner spawn throws after acquisition", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "spawn-throw-pipeline", ["research", "analysis"]);
+    setMaxConcurrentJobs(2);
+
+    await setupJob(root, "spawn-throw", {
+      id: "spawn-throw",
+      pipeline: "spawn-throw-pipeline",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    vi.spyOn(Bun, "spawn").mockImplementation(() => {
+      throw new Error("simulated spawn failure");
+    });
+
+    const req = new Request("http://localhost/api/jobs/spawn-throw/tasks/analysis/start", { method: "POST" });
+    await expect(handleTaskStart(req, "spawn-throw", "analysis", root)).rejects.toThrow(/spawn failure/);
+
+    const { runningJobsDir } = getConcurrencyRuntimePaths(path.join(root, "pipeline-data"));
+    const leasePath = path.join(runningJobsDir, "spawn-throw.json");
+    await expect(readFile(leasePath, "utf-8")).rejects.toMatchObject({ code: "ENOENT" });
+
+    // Slot is freed: a fresh acquisition for the same job succeeds.
+    const reacquired = await tryAcquireJobSlot({
+      dataDir: path.join(root, "pipeline-data"),
+      jobId: "spawn-throw",
+      maxConcurrentJobs: 2,
+      source: "orchestrator",
+    });
+    expect(reacquired.ok).toBe(true);
+    await releaseJobSlot(path.join(root, "pipeline-data"), "spawn-throw");
+  });
+
+  it("records the spawned PID in the lease via updateJobSlotPid on successful task-start", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "lease-pid-pipeline", ["research", "analysis"]);
+    setMaxConcurrentJobs(2);
+    mockRunnerSpawn(54321);
+
+    await setupJob(root, "lease-pid", {
+      id: "lease-pid",
+      pipeline: "lease-pid-pipeline",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/lease-pid/tasks/analysis/start", { method: "POST" });
+    const res = await handleTaskStart(req, "lease-pid", "analysis", root);
+    expect(res.status).toBe(202);
+
+    const lease = await readLeaseFile(root, "lease-pid");
+    expect(lease.pid).toBe(54321);
+    expect(lease.source).toBe("task-start");
+    expect(lease.jobId).toBe("lease-pid");
+  });
+});