@ryanfw/prompt-orchestration-pipeline 1.2.7 → 1.2.9

package/src/core/__tests__/pipeline-runner.test.ts

@@ -1,7 +1,12 @@
 import { describe, test, expect, mock, spyOn, beforeEach, afterEach } from "bun:test";
-import { mkdtemp, writeFile, mkdir, readFile, rm } from "node:fs/promises";
+import { mkdtemp, writeFile, mkdir, readFile, rm, access } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import {
+  getConcurrencyRuntimePaths,
+  releaseJobSlot,
+  tryAcquireJobSlot,
+} from "../job-concurrency";
 
 // ─── Mocks: NON-target externals only ─────────────────────────────────────────
 // We leave status-writer and lifecycle-policy REAL so the stale-snapshot bug
@@ -9,12 +14,14 @@ import { join } from "node:path";
 // deterministic no-ops; config/validation/module-loader are replaced so we
 // don't need a real pipelines directory on disk.
 
+const mockGetConfig = mock(() => ({ taskRunner: { maxAttempts: 3 } }));
+
 mock.module("../config", () => ({
   getPipelineConfig: mock((_slug: string) => ({
     pipelineJsonPath: "/mock/pipeline.json",
     tasksDir: "/mock/tasks",
   })),
-  getConfig: mock(() => ({})),
+  getConfig: mockGetConfig,
   loadConfig: mock(async () => ({})),
   resetConfig: mock(() => {}),
 }));
@@ -95,6 +102,7 @@ const PO_ENV_KEYS = [
   "PO_TASK_REGISTRY",
   "PO_START_FROM_TASK",
   "PO_RUN_SINGLE_TASK",
+  "PO_TASK_MAX_ATTEMPTS",
 ] as const;
 
 interface MultiTaskFixture {
@@ -359,3 +367,423 @@ describe("runPipelineJob — outer-catch failure surfacing", () => {
     expect(stderrContainsMessage).toBe(true);
   });
 });
+
+describe("runPipelineJob — bounded retry loop", () => {
+  const savedEnv: Record<string, string | undefined> = {};
+  const cleanupDirs: string[] = [];
+  let originalSleep: typeof Bun.sleep;
+  let sleepDelays: number[];
+
+  beforeEach(() => {
+    for (const key of Object.keys(savedEnv)) delete savedEnv[key];
+    for (const key of PO_ENV_KEYS) {
+      savedEnv[key] = process.env[key];
+      delete process.env[key];
+    }
+
+    mockRunPipeline.mockClear();
+    mockEnsureTaskSymlinkBridge.mockClear();
+    mockValidateTaskSymlinks.mockClear();
+    mockRepairTaskSymlinks.mockClear();
+    mockCleanupTaskSymlinks.mockClear();
+    mockLoadFreshModule.mockClear();
+    mockGetConfig.mockReset();
+    mockGetConfig.mockImplementation(() => ({ taskRunner: { maxAttempts: 3 } }));
+
+    sleepDelays = [];
+    originalSleep = Bun.sleep;
+    (Bun as unknown as { sleep: (ms: number) => Promise<void> }).sleep = async (ms: number) => {
+      sleepDelays.push(ms);
+    };
+  });
+
+  afterEach(async () => {
+    (Bun as unknown as { sleep: typeof Bun.sleep }).sleep = originalSleep;
+
+    for (const key of PO_ENV_KEYS) {
+      if (savedEnv[key] === undefined) {
+        delete process.env[key];
+      } else {
+        process.env[key] = savedEnv[key];
+      }
+    }
+    process.exitCode = 0;
+
+    await Promise.all(cleanupDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
+  });
+
+  function makeFailureResult() {
+    return {
+      ok: false as const,
+      failedStage: "generate",
+      error: {
+        name: "TaskFailure",
+        message: "stub failure",
+        stack: "stack",
+        debug: { stage: "generate", logPath: "/tmp/log" },
+      },
+      logs: [{ stage: "generate", ok: false as const, ms: 5, error: "stub" }],
+      context: {} as Record<string, unknown>,
+    };
+  }
+
+  function makeSuccessResult() {
+    return {
+      ok: true as const,
+      logs: [{ stage: "generate", ok: true as const, ms: 5 }],
+      context: {} as Record<string, unknown>,
+      llmMetrics: [],
+    };
+  }
+
+  test("maxAttempts: 1 — failing task runs once and exits non-zero", async () => {
+    mockGetConfig.mockImplementation(() => ({ taskRunner: { maxAttempts: 1 } }));
+    const fixture = await setupMultiTaskFixture(["task-a"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    mockRunPipeline.mockImplementation(async () => makeFailureResult() as never);
+
+    const exitCalls: Array<number | undefined> = [];
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      exitCalls.push(code);
+      throw new Error(`__test_exit__:${String(code)}`);
+    }) as typeof process.exit);
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } catch (e) {
+      if (!(e instanceof Error) || !/^__test_exit__:/.test(e.message)) throw e;
+    } finally {
+      exitSpy.mockRestore();
+    }
+
+    expect(mockRunPipeline.mock.calls.length).toBe(1);
+    expect(sleepDelays).toEqual([]);
+    expect(exitCalls).toContain(1);
+
+    const statusText = await readFile(join(fixture.jobDir, "tasks-status.json"), "utf-8");
+    const status = JSON.parse(statusText) as {
+      tasks: Record<string, { state?: string; attempts?: number; restartCount?: number }>;
+    };
+    expect(status.tasks["task-a"]?.state).toBe("failed");
+    expect(status.tasks["task-a"]?.restartCount).toBeUndefined();
+    expect(status.tasks["task-a"]?.attempts).toBe(1);
+  });
+
+  test("maxAttempts: 3 — fails twice then succeeds: three calls, restartCount=2, exits zero", async () => {
+    mockGetConfig.mockImplementation(() => ({ taskRunner: { maxAttempts: 3 } }));
+    const fixture = await setupMultiTaskFixture(["task-a"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    let call = 0;
+    mockRunPipeline.mockImplementation(async () => {
+      call += 1;
+      return (call <= 2 ? makeFailureResult() : makeSuccessResult()) as never;
+    });
+
+    const exitCalls: Array<number | undefined> = [];
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      exitCalls.push(code);
+      throw new Error(`__test_exit__:${String(code)}`);
+    }) as typeof process.exit);
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } catch (e) {
+      if (!(e instanceof Error) || !/^__test_exit__:/.test(e.message)) throw e;
+    } finally {
+      exitSpy.mockRestore();
+    }
+
+    expect(mockRunPipeline.mock.calls.length).toBe(3);
+    expect(sleepDelays).toEqual([2000, 4000]);
+    expect(exitCalls).toEqual([]);
+
+    const statusText = await readFile(fixture.statusPath, "utf-8");
+    const status = JSON.parse(statusText) as {
+      tasks: Record<string, { state?: string; attempts?: number; restartCount?: number }>;
+    };
+    expect(status.tasks["task-a"]?.state).toBe("done");
+    expect(status.tasks["task-a"]?.attempts).toBe(3);
+    expect(status.tasks["task-a"]?.restartCount).toBe(2);
+  });
+
+  test("maxAttempts: 3 — always fails: three calls, restartCount=2, exits non-zero", async () => {
+    mockGetConfig.mockImplementation(() => ({ taskRunner: { maxAttempts: 3 } }));
+    const fixture = await setupMultiTaskFixture(["task-a"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    mockRunPipeline.mockImplementation(async () => makeFailureResult() as never);
+
+    const exitCalls: Array<number | undefined> = [];
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      exitCalls.push(code);
+      throw new Error(`__test_exit__:${String(code)}`);
+    }) as typeof process.exit);
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } catch (e) {
+      if (!(e instanceof Error) || !/^__test_exit__:/.test(e.message)) throw e;
+    } finally {
+      exitSpy.mockRestore();
+    }
+
+    expect(mockRunPipeline.mock.calls.length).toBe(3);
+    expect(sleepDelays).toEqual([2000, 4000]);
+    expect(exitCalls).toContain(1);
+
+    const statusText = await readFile(join(fixture.jobDir, "tasks-status.json"), "utf-8");
+    const status = JSON.parse(statusText) as {
+      tasks: Record<string, { state?: string; attempts?: number; restartCount?: number }>;
+    };
+    expect(status.tasks["task-a"]?.state).toBe("failed");
+    expect(status.tasks["task-a"]?.attempts).toBe(3);
+    expect(status.tasks["task-a"]?.restartCount).toBe(2);
+  });
+
+  test("interim status between attempts: state=running, no failedStage/error, restartCount incremented", async () => {
+    mockGetConfig.mockImplementation(() => ({ taskRunner: { maxAttempts: 3 } }));
+    const fixture = await setupMultiTaskFixture(["task-a"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    let call = 0;
+    let interimSnapshot: { state?: string; attempts?: number; failedStage?: unknown; error?: unknown; restartCount?: number } | undefined;
+
+    // Capture the snapshot from disk *during* the second call (after the first failure
+    // and the interim writeJobStatus). At call #2 we read tasks-status.json, then
+    // return success so the test ends cleanly.
+    mockRunPipeline.mockImplementation(async () => {
+      call += 1;
+      if (call === 2) {
+        const text = await readFile(join(fixture.jobDir, "tasks-status.json"), "utf-8");
+        const parsed = JSON.parse(text) as {
+          tasks: Record<string, { state?: string; attempts?: number; failedStage?: unknown; error?: unknown; restartCount?: number }>;
+        };
+        interimSnapshot = parsed.tasks["task-a"];
+        return makeSuccessResult() as never;
+      }
+      return makeFailureResult() as never;
+    });
+
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`__test_exit__:${String(code)}`);
+    }) as typeof process.exit);
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } catch (e) {
+      if (!(e instanceof Error) || !/^__test_exit__:/.test(e.message)) throw e;
+    } finally {
+      exitSpy.mockRestore();
+    }
+
+    expect(interimSnapshot).toBeDefined();
+    expect(interimSnapshot?.state).toBe("running");
+    expect(interimSnapshot?.failedStage).toBeUndefined();
+    expect(interimSnapshot?.error).toBeUndefined();
+    expect(interimSnapshot?.attempts).toBe(2);
+    expect(interimSnapshot?.restartCount).toBe(1);
+  });
+
+  test("missing taskRunner config falls back to the default retry cap", async () => {
+    mockGetConfig.mockImplementation(() => ({} as never));
+    const fixture = await setupMultiTaskFixture(["task-a"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    let call = 0;
+    mockRunPipeline.mockImplementation(async () => {
+      call += 1;
+      return (call === 1 ? makeFailureResult() : makeSuccessResult()) as never;
+    });
+
+    await runPipelineJob(fixture.jobId);
+
+    expect(mockRunPipeline.mock.calls.length).toBe(2);
+    expect(sleepDelays).toEqual([2000]);
+  });
+
+  test("exceptions from runPipeline bypass result retries and surface through outer failure handling", async () => {
+    mockGetConfig.mockImplementation(() => ({ taskRunner: { maxAttempts: 3 } }));
+    const fixture = await setupMultiTaskFixture(["task-a"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    mockRunPipeline.mockImplementation(async () => {
+      throw new Error("task module exploded");
+    });
+
+    const exitCalls: Array<number | undefined> = [];
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      exitCalls.push(code);
+      throw new Error(`__test_exit__:${String(code)}`);
+    }) as typeof process.exit);
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } catch (e) {
+      if (!(e instanceof Error) || !/^__test_exit__:/.test(e.message)) throw e;
+    } finally {
+      exitSpy.mockRestore();
+    }
+
+    expect(mockRunPipeline.mock.calls.length).toBe(1);
+    expect(sleepDelays).toEqual([]);
+    expect(exitCalls).toContain(1);
+  });
+});
+
+describe("runPipelineJob — releases job slot", () => {
+  const savedEnv: Record<string, string | undefined> = {};
+  const cleanupDirs: string[] = [];
+
+  beforeEach(() => {
+    for (const key of Object.keys(savedEnv)) delete savedEnv[key];
+    for (const key of PO_ENV_KEYS) {
+      savedEnv[key] = process.env[key];
+      delete process.env[key];
+    }
+
+    mockRunPipeline.mockClear();
+    mockRunPipeline.mockImplementation(async (_modulePath: string, _ctx: unknown) => ({
+      ok: true as const,
+      logs: [{ stage: "generate", ok: true as const, ms: 10 }],
+      context: {} as Record<string, unknown>,
+      llmMetrics: [],
+    }));
+    mockEnsureTaskSymlinkBridge.mockClear();
+    mockValidateTaskSymlinks.mockClear();
+    mockRepairTaskSymlinks.mockClear();
+    mockCleanupTaskSymlinks.mockClear();
+    mockLoadFreshModule.mockClear();
+  });
+
+  afterEach(async () => {
+    for (const key of PO_ENV_KEYS) {
+      if (savedEnv[key] === undefined) {
+        delete process.env[key];
+      } else {
+        process.env[key] = savedEnv[key];
+      }
+    }
+    process.exitCode = 0;
+
+    await Promise.all(cleanupDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
+  });
+
+  async function fileExists(path: string): Promise<boolean> {
+    try {
+      await access(path);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  test("successful completion releases the lease", async () => {
+    const fixture = await setupMultiTaskFixture(["task-a"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    const acquire = await tryAcquireJobSlot({
+      dataDir: fixture.tmpDir,
+      jobId: fixture.jobId,
+      maxConcurrentJobs: 1,
+      source: "orchestrator",
+      pid: process.pid,
+    });
+    expect(acquire.ok).toBe(true);
+
+    const { runningJobsDir } = getConcurrencyRuntimePaths(fixture.tmpDir);
+    const slotPath = join(runningJobsDir, `${fixture.jobId}.json`);
+    expect(await fileExists(slotPath)).toBe(true);
+
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`process.exit called with ${String(code)}`);
+    }) as typeof process.exit);
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } finally {
+      exitSpy.mockRestore();
+    }
+
+    expect(await fileExists(slotPath)).toBe(false);
+  });
+
+  test("catch-path failure releases the lease and surfaces the original error", async () => {
+    const fixture = await setupMultiTaskFixture(["task-a"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    const injectedMessage = "release-on-catch-injected";
+    mockLoadFreshModule.mockImplementation(async () => {
+      throw new Error(injectedMessage);
+    });
+
+    const acquire = await tryAcquireJobSlot({
+      dataDir: fixture.tmpDir,
+      jobId: fixture.jobId,
+      maxConcurrentJobs: 1,
+      source: "orchestrator",
+      pid: process.pid,
+    });
+    expect(acquire.ok).toBe(true);
+
+    const { runningJobsDir } = getConcurrencyRuntimePaths(fixture.tmpDir);
+    const slotPath = join(runningJobsDir, `${fixture.jobId}.json`);
+    expect(await fileExists(slotPath)).toBe(true);
+
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`__test_exit__:${String(code)}`);
+    }) as typeof process.exit);
+    const fakeTimer = { unref: () => fakeTimer, ref: () => fakeTimer };
+    const setTimeoutSpy = spyOn(globalThis, "setTimeout").mockImplementation(
+      ((() => fakeTimer as unknown as ReturnType<typeof setTimeout>) as unknown) as typeof setTimeout,
+    );
+    const consoleErrorMessages: unknown[][] = [];
+    const consoleErrorSpy = spyOn(console, "error").mockImplementation((...args: unknown[]) => {
+      consoleErrorMessages.push(args);
+    });
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } catch (e) {
+      if (!(e instanceof Error) || !/^__test_exit__:/.test(e.message)) throw e;
+    } finally {
+      exitSpy.mockRestore();
+      setTimeoutSpy.mockRestore();
+      consoleErrorSpy.mockRestore();
+    }
+
+    expect(await fileExists(slotPath)).toBe(false);
+
+    const stderrContainsMessage = consoleErrorMessages.some((args) =>
+      args.some((a) => typeof a === "string" && a.includes(injectedMessage)),
+    );
+    expect(stderrContainsMessage).toBe(true);
+  });
+
+  test("a second release after runPipelineJob is a no-op", async () => {
+    const fixture = await setupMultiTaskFixture(["task-a"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    const acquire = await tryAcquireJobSlot({
+      dataDir: fixture.tmpDir,
+      jobId: fixture.jobId,
+      maxConcurrentJobs: 1,
+      source: "orchestrator",
+      pid: process.pid,
+    });
+    expect(acquire.ok).toBe(true);
+
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`process.exit called with ${String(code)}`);
+    }) as typeof process.exit);
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } finally {
+      exitSpy.mockRestore();
+    }
+
+    await releaseJobSlot(fixture.tmpDir, fixture.jobId);
+  });
+});

package/src/core/__tests__/task-runner.test.ts

@@ -91,11 +91,10 @@ describe("task-runner does not write job-level status fields", () => {
 
     const status = JSON.parse(await readFile(path.join(workDir, "tasks-status.json"), "utf8")) as StatusSnapshot;
 
-    // Job-level lifecycle fields must remain untouched by task-runner.
+    // Lifecycle fields are owned by pipeline-runner; task-runner persists progress only.
     expect(status.state).toBe("pending");
     expect(status.current).toBeNull();
     expect(status.currentStage).toBeNull();
-    expect(status.progress).toBe(100);
   });
 
   it("does not set snapshot.state on task failure", async () => {

package/src/core/config.ts

@@ -10,12 +10,14 @@ export interface OrchestratorConfig {
   watchDebounce: number;
   watchStabilityThreshold: number;
   watchPollInterval: number;
+  maxConcurrentJobs: number;
 }
 
 export interface TaskRunnerConfig {
   maxRefinementAttempts: number;
   stageTimeout: number;
   llmRequestTimeout: number;
+  maxAttempts: number;
 }
 
 export interface LLMConfig {
@@ -90,11 +92,13 @@ export const defaultConfig = {
     watchDebounce: 500,
     watchStabilityThreshold: 1000,
     watchPollInterval: 100,
+    maxConcurrentJobs: 3,
   },
   taskRunner: {
     maxRefinementAttempts: 3,
     stageTimeout: 300000,
     llmRequestTimeout: 3600000,
+    maxAttempts: 3,
   },
   llm: {
     defaultProvider: "openai",
@@ -184,9 +188,38 @@ function loadFromEnvironment(config: AppConfig): AppConfig {
     overrides["llm"] = { maxConcurrency: parseInt(maxConcurrencyRaw, 10) };
   }
 
+  const maxAttemptsRaw = process.env["PO_TASK_MAX_ATTEMPTS"];
+  if (maxAttemptsRaw) {
+    const maxAttempts = Number(maxAttemptsRaw);
+    if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
+      throw new Error(`PO_TASK_MAX_ATTEMPTS must be an integer >= 1, got ${maxAttemptsRaw}`);
+    }
+    overrides["taskRunner"] = {
+      ...((overrides["taskRunner"] as PlainObject | undefined) ?? {}),
+      maxAttempts,
+    };
+  }
+
   const shutdownTimeoutRaw = process.env["PO_SHUTDOWN_TIMEOUT"];
   if (shutdownTimeoutRaw) {
-    overrides["orchestrator"] = { shutdownTimeout: parseInt(shutdownTimeoutRaw, 10) };
+    overrides["orchestrator"] = {
+      ...((overrides["orchestrator"] as PlainObject | undefined) ?? {}),
+      shutdownTimeout: parseInt(shutdownTimeoutRaw, 10),
+    };
+  }
+
+  const maxRunningJobsRaw = process.env["PO_MAX_RUNNING_JOBS"];
+  if (maxRunningJobsRaw !== undefined) {
+    const maxConcurrentJobs = Number(maxRunningJobsRaw);
+    if (!Number.isInteger(maxConcurrentJobs) || maxConcurrentJobs < 1) {
+      throw new Error(
+        `orchestrator.maxConcurrentJobs must be a positive integer (PO_MAX_RUNNING_JOBS), got ${maxRunningJobsRaw}`
+      );
+    }
+    overrides["orchestrator"] = {
+      ...((overrides["orchestrator"] as PlainObject | undefined) ?? {}),
+      maxConcurrentJobs,
+    };
   }
 
   const logLevel = process.env["PO_LOG_LEVEL"];
@@ -227,6 +260,12 @@ function validateConfig(config: AppConfig): void {
   if (config.taskRunner.maxRefinementAttempts < 1) {
     errors.push(`taskRunner.maxRefinementAttempts must be >= 1, got ${config.taskRunner.maxRefinementAttempts}`);
   }
+  if (!Number.isInteger(config.taskRunner.maxAttempts) || config.taskRunner.maxAttempts < 1) {
+    errors.push(`taskRunner.maxAttempts must be an integer >= 1, got ${config.taskRunner.maxAttempts}`);
+  }
+  if (!Number.isInteger(config.orchestrator.maxConcurrentJobs) || config.orchestrator.maxConcurrentJobs < 1) {
+    errors.push(`orchestrator.maxConcurrentJobs must be a positive integer, got ${config.orchestrator.maxConcurrentJobs}`);
+  }
   if (!(VALID_LOG_LEVELS as readonly string[]).includes(config.logging.level)) {
     errors.push(`logging.level must be one of ${VALID_LOG_LEVELS.join(", ")}, got ${config.logging.level}`);
   }
@@ -375,6 +414,14 @@ export function getConfig(): AppConfig {
   return cachedConfig;
 }
 
+export function getOrchestratorConfig(): OrchestratorConfig {
+  const config = getConfig();
+  return {
+    ...defaultConfig.orchestrator,
+    ...(config.orchestrator ?? {}),
+  };
+}
+
 export function getConfigValue(dotPath: string, defaultValue?: unknown): unknown {
   const config = getConfig();
   const parts = dotPath.split(".");