@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

package/src/ui/server/__tests__/job-control-endpoints.test.ts

@@ -7,11 +7,11 @@ import { afterEach, describe, expect, it, vi } from "vitest";
 
 import { initPATHS, resetPATHS } from "../config-bridge-node";
 import {
-  classifyStaleRunningStatus,
   handleJobRestart,
   handleJobStop,
   handleTaskStart,
 } from "../endpoints/job-control-endpoints";
+import { classifyStaleRunningStatus } from "../../../core/runner-liveness";
 import { resetConfig } from "../../../core/config";
 import {
   getConcurrencyRuntimePaths,
@@ -56,8 +56,9 @@ async function setupPipelineConfig(
   await mkdir(configDir, { recursive: true });
   await writeFile(path.join(configDir, "pipeline.json"), JSON.stringify({ name: slug, tasks }));
   await writeFile(path.join(root, "pipeline-config", "registry.json"), JSON.stringify({
+    version: 1,
     pipelines: {
-      [slug]: {},
+      [slug]: { name: slug, description: "test pipeline" },
     },
   }));
 }
@@ -122,6 +123,8 @@ afterEach(async () => {
   vi.restoreAllMocks();
   resetPATHS();
   delete process.env["PO_MAX_RUNNING_JOBS"];
+  delete process.env["PO_STALE_RUNNING_GRACE_MS"];
+  delete process.env["PO_STALE_LEASE_TIMEOUT_MS"];
   resetConfig();
   for (const proc of childProcs.splice(0)) {
     try { proc.kill(); } catch {}
@@ -514,6 +517,120 @@ describe("handleJobStop", () => {
   });
 });
 
+describe("handleJobStop slot release (AC-1, AC-9)", () => {
+  it("releases the active slot lease after stopping (AC-1)", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    await setupJob(root, "stop-release-active", {
+      id: "stop-release-active",
+      state: "running",
+      current: "research",
+      currentStage: null,
+      tasks: { research: { state: "running", currentStage: null } },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }, 999999);
+
+    const acquired = await tryAcquireJobSlot({
+      dataDir: path.join(root, "pipeline-data"),
+      jobId: "stop-release-active",
+      maxConcurrentJobs: 2,
+      source: "task-start",
+      pid: 999999,
+    });
+    expect(acquired.ok).toBe(true);
+
+    const { runningJobsDir } = getConcurrencyRuntimePaths(path.join(root, "pipeline-data"));
+    const leasePath = path.join(runningJobsDir, "stop-release-active.json");
+    expect(await Bun.file(leasePath).exists()).toBe(true);
+
+    const req = new Request("http://localhost/api/jobs/stop-release-active/stop", { method: "POST" });
+    const res = await handleJobStop(req, "stop-release-active", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body["ok"]).toBe(true);
+    expect(body["stopped"]).toBe(true);
+
+    expect(await Bun.file(leasePath).exists()).toBe(false);
+  });
+
+  it("does not throw and returns 202 when no lease exists (AC-9)", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    await setupJob(root, "stop-release-no-lease", {
+      id: "stop-release-no-lease",
+      state: "running",
+      current: "research",
+      currentStage: null,
+      tasks: { research: { state: "running", currentStage: null } },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/stop-release-no-lease/stop", { method: "POST" });
+    const res = await handleJobStop(req, "stop-release-no-lease", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body["ok"]).toBe(true);
+    expect(body["stopped"]).toBe(false);
+  });
+
+  it("does not throw and returns 202 when stopping a complete/ job (AC-9)", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    const completeDir = path.join(root, "pipeline-data", "complete", "stop-release-complete");
+    await mkdir(completeDir, { recursive: true });
+    await writeFile(path.join(completeDir, "tasks-status.json"), JSON.stringify({
+      id: "stop-release-complete",
+      state: "complete",
+      current: null,
+      currentStage: null,
+      tasks: { research: { state: "done", currentStage: null } },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }));
+
+    const req = new Request("http://localhost/api/jobs/stop-release-complete/stop", { method: "POST" });
+    const res = await handleJobStop(req, "stop-release-complete", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body["ok"]).toBe(true);
+  });
+
+  it("still releases the slot when the status is corrupt (AC-1)", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    const jobDir = path.join(root, "pipeline-data", "current", "stop-release-corrupt");
+    await mkdir(jobDir, { recursive: true });
+    await writeFile(path.join(jobDir, "tasks-status.json"), "not-valid-json{{{");
+
+    const acquired = await tryAcquireJobSlot({
+      dataDir: path.join(root, "pipeline-data"),
+      jobId: "stop-release-corrupt",
+      maxConcurrentJobs: 2,
+      source: "task-start",
+    });
+    expect(acquired.ok).toBe(true);
+
+    const { runningJobsDir } = getConcurrencyRuntimePaths(path.join(root, "pipeline-data"));
+    const leasePath = path.join(runningJobsDir, "stop-release-corrupt.json");
+    expect(await Bun.file(leasePath).exists()).toBe(true);
+
+    const req = new Request("http://localhost/api/jobs/stop-release-corrupt/stop", { method: "POST" });
+    const res = await handleJobStop(req, "stop-release-corrupt", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(409);
+    expect(body["code"]).toBe("STATUS_CORRUPT");
+
+    expect(await Bun.file(leasePath).exists()).toBe(false);
+  });
+});
+
 describe("handleJobRestart", () => {
   it("returns 409 when runner PID is alive", async () => {
     const root = await makeTempRoot();
@@ -566,6 +683,33 @@ describe("handleJobRestart", () => {
     expect(body["code"]).toBe("job_running");
   });
 
+  it("uses configured staleRunningGraceMs when deciding whether restart can recover a running task", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    process.env["PO_STALE_RUNNING_GRACE_MS"] = "60000";
+    resetConfig();
+
+    vi.spyOn(Date, "now").mockReturnValue(Date.parse("2026-04-01T10:00:45.000Z"));
+    await setupJob(root, "restart-configured-grace", {
+      id: "restart-configured-grace",
+      state: "running",
+      current: "research",
+      currentStage: "prompt",
+      lastUpdated: "2026-04-01T10:00:00.000Z",
+      tasks: {
+        research: { state: "running", currentStage: "prompt" },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }, 999999);
+
+    const req = new Request("http://localhost/api/jobs/restart-configured-grace/restart", { method: "POST" });
+    const res = await handleJobRestart(req, "restart-configured-grace", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(409);
+    expect(body["code"]).toBe("job_running");
+  });
+
   it("proceeds when task-derived running status is stale and the runner is gone", async () => {
     const root = await makeTempRoot();
     initPATHS(root);
@@ -813,6 +957,265 @@ describe("handleJobRestart", () => {
   });
 });
 
+describe("handleJobRestart dependency gate (AC-5 restart, AC-8, AC-9)", () => {
+  it("returns 412 and leaves status/lease untouched when a from-task predecessor is pending", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "restart-deps-pending-pipeline", ["research", "analysis"]);
+
+    const statusObj = {
+      id: "restart-deps-pending",
+      pipeline: "restart-deps-pending-pipeline",
+      state: "failed",
+      current: "analysis",
+      currentStage: null,
+      lastUpdated: "2026-04-01T10:00:00.000Z",
+      tasks: {
+        research: { state: "pending", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    };
+    const jobDir = await setupJob(root, "restart-deps-pending", statusObj);
+
+    const statusPath = path.join(jobDir, "tasks-status.json");
+    const snapshotBefore = await Bun.file(statusPath).text();
+    const spawnSpy = vi.spyOn(Bun, "spawn");
+
+    const req = new Request("http://localhost/api/jobs/restart-deps-pending/restart", {
+      method: "POST",
+      body: JSON.stringify({ fromTask: "analysis" }),
+    });
+    const res = await handleJobRestart(req, "restart-deps-pending", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(412);
+    expect(body["code"]).toBe("dependencies_not_satisfied");
+    expect(body["message"]).toContain("research");
+
+    expect(await Bun.file(statusPath).text()).toBe(snapshotBefore);
+    expect(spawnSpy).not.toHaveBeenCalled();
+
+    const { runningJobsDir } = getConcurrencyRuntimePaths(path.join(root, "pipeline-data"));
+    expect(await Bun.file(path.join(runningJobsDir, "restart-deps-pending.json")).exists()).toBe(false);
+  });
+
+  it("proceeds when from-task predecessor is done", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "restart-deps-done-pipeline", ["research", "analysis"]);
+    mockRunnerSpawn();
+
+    await setupJob(root, "restart-deps-done", {
+      id: "restart-deps-done",
+      pipeline: "restart-deps-done-pipeline",
+      state: "failed",
+      current: "analysis",
+      currentStage: null,
+      lastUpdated: "2026-04-01T10:00:00.000Z",
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/restart-deps-done/restart", {
+      method: "POST",
+      body: JSON.stringify({ fromTask: "analysis" }),
+    });
+    const res = await handleJobRestart(req, "restart-deps-done", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body["ok"]).toBe(true);
+    expect(body["mode"]).toBe("from-task");
+  });
+
+  it("proceeds when from-task predecessor is skipped", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "restart-deps-skipped-pipeline", ["research", "analysis"]);
+    mockRunnerSpawn();
+
+    await setupJob(root, "restart-deps-skipped", {
+      id: "restart-deps-skipped",
+      pipeline: "restart-deps-skipped-pipeline",
+      state: "failed",
+      current: "analysis",
+      currentStage: null,
+      lastUpdated: "2026-04-01T10:00:00.000Z",
+      tasks: {
+        research: { state: "skipped", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/restart-deps-skipped/restart", {
+      method: "POST",
+      body: JSON.stringify({ fromTask: "analysis" }),
+    });
+    const res = await handleJobRestart(req, "restart-deps-skipped", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body["ok"]).toBe(true);
+    expect(body["mode"]).toBe("from-task");
+  });
+
+  it("gates single-task restart identically when a predecessor is pending", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "restart-single-deps-pending-pipeline", ["review", "deploy"]);
+
+    const statusObj = {
+      id: "restart-single-deps-pending",
+      pipeline: "restart-single-deps-pending-pipeline",
+      state: "failed",
+      current: "deploy",
+      currentStage: null,
+      lastUpdated: "2026-04-01T10:00:00.000Z",
+      tasks: {
+        review: { state: "pending", currentStage: null },
+        deploy: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    };
+    const jobDir = await setupJob(root, "restart-single-deps-pending", statusObj);
+
+    const statusPath = path.join(jobDir, "tasks-status.json");
+    const snapshotBefore = await Bun.file(statusPath).text();
+    const spawnSpy = vi.spyOn(Bun, "spawn");
+
+    const req = new Request("http://localhost/api/jobs/restart-single-deps-pending/restart", {
+      method: "POST",
+      body: JSON.stringify({ fromTask: "deploy", singleTask: true }),
+    });
+    const res = await handleJobRestart(req, "restart-single-deps-pending", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(412);
+    expect(body["code"]).toBe("dependencies_not_satisfied");
+    expect(body["message"]).toContain("review");
+
+    expect(await Bun.file(statusPath).text()).toBe(snapshotBefore);
+    expect(spawnSpy).not.toHaveBeenCalled();
+
+    const { runningJobsDir } = getConcurrencyRuntimePaths(path.join(root, "pipeline-data"));
+    expect(await Bun.file(path.join(runningJobsDir, "restart-single-deps-pending.json")).exists()).toBe(false);
+  });
+
+  it("does not gate clean-slate restart (no fromTask) even when a predecessor is pending", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "restart-clean-ungated-pipeline", ["research", "analysis"]);
+    mockRunnerSpawn();
+
+    await setupJob(root, "restart-clean-ungated", {
+      id: "restart-clean-ungated",
+      pipeline: "restart-clean-ungated-pipeline",
+      state: "failed",
+      current: "analysis",
+      currentStage: null,
+      lastUpdated: "2026-04-01T10:00:00.000Z",
+      tasks: {
+        research: { state: "pending", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/restart-clean-ungated/restart", { method: "POST" });
+    const res = await handleJobRestart(req, "restart-clean-ungated", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body["ok"]).toBe(true);
+    expect(body["mode"]).toBe("clean-slate");
+  });
+
+  it("returns 500 status_unavailable when the pipeline task order cannot be loaded", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+
+    const statusObj = {
+      id: "restart-no-order",
+      state: "failed",
+      current: "analysis",
+      currentStage: null,
+      lastUpdated: "2026-04-01T10:00:00.000Z",
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    };
+    const jobDir = await setupJob(root, "restart-no-order", statusObj);
+
+    const statusPath = path.join(jobDir, "tasks-status.json");
+    const snapshotBefore = await Bun.file(statusPath).text();
+    const spawnSpy = vi.spyOn(Bun, "spawn");
+
+    const req = new Request("http://localhost/api/jobs/restart-no-order/restart", {
+      method: "POST",
+      body: JSON.stringify({ fromTask: "analysis" }),
+    });
+    const res = await handleJobRestart(req, "restart-no-order", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(500);
+    expect(body["code"]).toBe("status_unavailable");
+
+    expect(await Bun.file(statusPath).text()).toBe(snapshotBefore);
+    expect(spawnSpy).not.toHaveBeenCalled();
+
+    const { runningJobsDir } = getConcurrencyRuntimePaths(path.join(root, "pipeline-data"));
+    expect(await Bun.file(path.join(runningJobsDir, "restart-no-order.json")).exists()).toBe(false);
+  });
+
+  it("returns 404 task_not_found when fromTask is absent from the pipeline order", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "restart-unknown-task-pipeline", ["research", "analysis"]);
+
+    const statusObj = {
+      id: "restart-unknown-task",
+      pipeline: "restart-unknown-task-pipeline",
+      state: "failed",
+      current: "analysis",
+      currentStage: null,
+      lastUpdated: "2026-04-01T10:00:00.000Z",
+      tasks: {
+        research: { state: "done", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    };
+    const jobDir = await setupJob(root, "restart-unknown-task", statusObj);
+
+    const statusPath = path.join(jobDir, "tasks-status.json");
+    const snapshotBefore = await Bun.file(statusPath).text();
+    const spawnSpy = vi.spyOn(Bun, "spawn");
+
+    const req = new Request("http://localhost/api/jobs/restart-unknown-task/restart", {
+      method: "POST",
+      body: JSON.stringify({ fromTask: "synthesis" }),
+    });
+    const res = await handleJobRestart(req, "restart-unknown-task", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(404);
+    expect(body["code"]).toBe("task_not_found");
+
+    expect(await Bun.file(statusPath).text()).toBe(snapshotBefore);
+    expect(spawnSpy).not.toHaveBeenCalled();
+
+    const { runningJobsDir } = getConcurrencyRuntimePaths(path.join(root, "pipeline-data"));
+    expect(await Bun.file(path.join(runningJobsDir, "restart-unknown-task.json")).exists()).toBe(false);
+  });
+});
+
 describe("handleTaskStart", () => {
   it("allows restart/start when snapshot has no valid lastUpdated and no live PID (crashed runner scenario)", async () => {
     // A runner that crashed before writing a valid lastUpdated should not permanently block
@@ -939,6 +1342,34 @@ describe("handleTaskStart", () => {
     expect(body["code"]).toBe("job_running");
   });
 
+  it("uses configured staleRunningGraceMs when deciding whether task start can recover a running task", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    process.env["PO_STALE_RUNNING_GRACE_MS"] = "60000";
+    resetConfig();
+
+    vi.spyOn(Date, "now").mockReturnValue(Date.parse("2026-04-01T10:00:45.000Z"));
+    await setupJob(root, "task-start-configured-grace", {
+      id: "task-start-configured-grace",
+      state: "running",
+      current: "research",
+      currentStage: "prompt",
+      lastUpdated: "2026-04-01T10:00:00.000Z",
+      tasks: {
+        research: { state: "running", currentStage: "prompt" },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }, 999999);
+
+    const req = new Request("http://localhost/api/jobs/task-start-configured-grace/tasks/analysis/start", { method: "POST" });
+    const res = await handleTaskStart(req, "task-start-configured-grace", "analysis", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(409);
+    expect(body["code"]).toBe("job_running");
+  });
+
   it("returns 202 when stale-running status is the only blocker for an eligible pending task", async () => {
     const root = await makeTempRoot();
     initPATHS(root);
@@ -1239,6 +1670,7 @@ describe("handleTaskStart", () => {
 
     expect(res.status).toBe(412);
     expect(body["code"]).toBe("dependencies_not_satisfied");
+    expect(body["message"]).toContain("research");
   });
 
   it("returns 412 dependencies_not_satisfied when an earlier task is \"failed\"", async () => {
@@ -1265,6 +1697,80 @@ describe("handleTaskStart", () => {
 
     expect(res.status).toBe(412);
     expect(body["code"]).toBe("dependencies_not_satisfied");
+    expect(body["message"]).toContain("research");
+  });
+
+  it("returns 202 when an earlier task is \"skipped\" (done-or-skipped dependency rule)", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    await setupPipelineConfig(root, "task-start-skipped-deps-pipeline", ["research", "analysis"]);
+    const spawnSpy = mockRunnerSpawn(88888);
+
+    await setupJob(root, "task-start-skipped-deps", {
+      id: "task-start-skipped-deps",
+      pipeline: "task-start-skipped-deps-pipeline",
+      state: "pending",
+      current: null,
+      currentStage: null,
+      tasks: {
+        research: { state: "skipped", currentStage: null },
+        analysis: { state: "pending", currentStage: null },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    });
+
+    const req = new Request("http://localhost/api/jobs/task-start-skipped-deps/tasks/analysis/start", { method: "POST" });
+    const res = await handleTaskStart(req, "task-start-skipped-deps", "analysis", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(202);
+    expect(body["ok"]).toBe(true);
+    expect(body["taskId"]).toBe("analysis");
+    expect(spawnSpy).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe("acquireConcurrencySlot stale-lease age (AC-5)", () => {
+  it("does not prune a fresh null-pid lease when lockFileTimeout is short relative to PO_STALE_LEASE_TIMEOUT_MS", async () => {
+    const root = await makeTempRoot();
+    initPATHS(root);
+    process.env["PO_STALE_LEASE_TIMEOUT_MS"] = "60000";
+    resetConfig();
+
+    const now = Date.parse("2026-04-01T10:01:00.000Z");
+    vi.spyOn(Date, "now").mockReturnValue(now);
+
+    const jobDir = await setupJob(root, "ac-5-held", {
+      id: "ac-5-held",
+      state: "running",
+      current: "research",
+      currentStage: "prompt",
+      lastUpdated: "2026-04-01T10:00:15.000Z",
+      tasks: {
+        research: { state: "running", currentStage: "prompt" },
+      },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }, 999999);
+
+    const runningJobsDir = path.join(root, "pipeline-data", "runtime", "running-jobs");
+    await mkdir(runningJobsDir, { recursive: true });
+    const slotPath = path.join(runningJobsDir, "ac-5-held.json");
+    await writeFile(slotPath, JSON.stringify({
+      jobId: "ac-5-held",
+      pid: null,
+      acquiredAt: new Date(now - 40_000).toISOString(),
+      source: "restart",
+      slotPath,
+    }));
+
+    const req = new Request("http://localhost/api/jobs/ac-5-held/restart", { method: "POST" });
+    const res = await handleJobRestart(req, "ac-5-held", root);
+    const body = await res.json() as Record<string, unknown>;
+
+    expect(res.status).toBe(409);
+    expect(body["code"]).toBe("job_running");
+
+    expect(await Bun.file(slotPath).text()).toBeDefined();
   });
 });
 
@@ -1811,6 +2317,10 @@ describe("corrupt status handling", () => {
       tasks: { research: { state: "pending" } },
       files: { artifacts: [], logs: [], tmp: [] },
     });
+    await writeFile(path.join(jobDir, "pipeline.json"), JSON.stringify({
+      name: "write-corrupt-restart-pipeline",
+      tasks: [{ name: "research" }],
+    }));
     const statusPath = path.join(jobDir, "tasks-status.json");
     corruptStatusOnTextRead(statusPath, 3);