@ryanfw/prompt-orchestration-pipeline 1.2.6 → 1.2.7

package/src/core/__tests__/pipeline-runner.test.ts

@@ -0,0 +1,361 @@
+import { describe, test, expect, mock, spyOn, beforeEach, afterEach } from "bun:test";
+import { mkdtemp, writeFile, mkdir, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+// ─── Mocks: NON-target externals only ─────────────────────────────────────────
+// We leave status-writer and lifecycle-policy REAL so the stale-snapshot bug
+// manifests end-to-end. runPipeline and symlink helpers are replaced with
+// deterministic no-ops; config/validation/module-loader are replaced so we
+// don't need a real pipelines directory on disk.
+
+mock.module("../config", () => ({
+  getPipelineConfig: mock((_slug: string) => ({
+    pipelineJsonPath: "/mock/pipeline.json",
+    tasksDir: "/mock/tasks",
+  })),
+  getConfig: mock(() => ({})),
+  loadConfig: mock(async () => ({})),
+  resetConfig: mock(() => {}),
+}));
+
+mock.module("../validation", () => ({
+  validatePipelineOrThrow: mock((_pipeline: unknown, _pathHint?: string) => {}),
+}));
+
+const mockLoadFreshModule = mock(async (_path: string) => ({
+  default: {} as Record<string, string>,
+}));
+
+mock.module("../module-loader", () => ({
+  loadFreshModule: mockLoadFreshModule,
+}));
+
+const mockRunPipeline = mock(async (_modulePath: string, _ctx: unknown) => ({
+  ok: true as const,
+  logs: [{ stage: "generate", ok: true as const, ms: 10 }],
+  context: {} as Record<string, unknown>,
+  llmMetrics: [],
+}));
+
+mock.module("../task-runner", () => ({
+  runPipeline: mockRunPipeline,
+}));
+
+const mockEnsureTaskSymlinkBridge = mock(
+  async (_workDir: string, _taskName: string, _registryDir: string, modulePath: string) => ({
+    relocatedEntryPath: modulePath,
+  }),
+);
+
+mock.module("../symlink-bridge", () => ({
+  ensureTaskSymlinkBridge: mockEnsureTaskSymlinkBridge,
+}));
+
+const mockValidateTaskSymlinks = mock(async () => true);
+const mockRepairTaskSymlinks = mock(async () => {});
+const mockCleanupTaskSymlinks = mock(async () => {});
+
+mock.module("../symlink-utils", () => ({
+  validateTaskSymlinks: mockValidateTaskSymlinks,
+  repairTaskSymlinks: mockRepairTaskSymlinks,
+  cleanupTaskSymlinks: mockCleanupTaskSymlinks,
+}));
+
+// Silence the job-level logger so test output stays readable.
+const quietLogger = {
+  debug: mock(() => {}),
+  log: mock(() => {}),
+  warn: mock(() => {}),
+  error: mock(() => {}),
+  group: mock(() => {}),
+  groupEnd: mock(() => {}),
+  sse: mock(() => {}),
+};
+
+mock.module("../logger", () => ({
+  createJobLogger: mock(() => quietLogger),
+  createLogger: mock(() => quietLogger),
+  createTaskLogger: mock(() => quietLogger),
+}));
+
+// ─── Import the module under test (after all mock.module calls) ───────────────
+
+import { runPipelineJob } from "../pipeline-runner";
+
+// ─── Test fixtures and helpers ────────────────────────────────────────────────
+
+const PO_ENV_KEYS = [
+  "PO_ROOT",
+  "PO_DATA_DIR",
+  "PO_CURRENT_DIR",
+  "PO_COMPLETE_DIR",
+  "PO_PIPELINE_PATH",
+  "PO_PIPELINE_SLUG",
+  "PO_TASK_REGISTRY",
+  "PO_START_FROM_TASK",
+  "PO_RUN_SINGLE_TASK",
+] as const;
+
+interface MultiTaskFixture {
+  tmpDir: string;
+  jobId: string;
+  jobDir: string;
+  completeDir: string;
+  statusPath: string;
+}
+
+async function setupMultiTaskFixture(taskNames: string[]): Promise<MultiTaskFixture> {
+  const tmpDir = await mkdtemp(join(tmpdir(), "pipeline-runner-regression-"));
+  const jobId = "job-regression";
+  const currentDir = join(tmpDir, "current");
+  const completeDir = join(tmpDir, "complete");
+  const jobDir = join(currentDir, jobId);
+  const pipelineDir = join(tmpDir, "pipeline");
+
+  await mkdir(jobDir, { recursive: true });
+  await mkdir(completeDir, { recursive: true });
+  await mkdir(pipelineDir, { recursive: true });
+
+  await writeFile(join(jobDir, "seed.json"), JSON.stringify({ pipeline: "test-pipeline" }));
+
+  const tasks: Record<string, { state: string }> = {};
+  for (const name of taskNames) {
+    tasks[name] = { state: "pending" };
+  }
+  await writeFile(
+    join(jobDir, "tasks-status.json"),
+    JSON.stringify({
+      id: jobId,
+      state: "pending",
+      current: null,
+      currentStage: null,
+      lastUpdated: new Date().toISOString(),
+      tasks,
+      files: { artifacts: [], logs: [], tmp: [] },
+    }),
+  );
+
+  await writeFile(join(pipelineDir, "pipeline.json"), JSON.stringify({ tasks: taskNames }));
+
+  process.env["PO_ROOT"] = tmpDir;
+  process.env["PO_DATA_DIR"] = ".";
+  process.env["PO_CURRENT_DIR"] = currentDir;
+  process.env["PO_COMPLETE_DIR"] = completeDir;
+  process.env["PO_PIPELINE_PATH"] = join(pipelineDir, "pipeline.json");
+  process.env["PO_TASK_REGISTRY"] = join(pipelineDir, "tasks", "index.js");
+
+  mockLoadFreshModule.mockImplementation(async (_path: string) => ({
+    default: Object.fromEntries(taskNames.map((n) => [n, `./${n}.js`])),
+  }));
+
+  return { tmpDir, jobId, jobDir, completeDir, statusPath: join(completeDir, jobId, "tasks-status.json") };
+}
+
+describe("runPipelineJob — multi-task success regression", () => {
+  const savedEnv: Record<string, string | undefined> = {};
+  const cleanupDirs: string[] = [];
+
+  beforeEach(() => {
+    for (const key of PO_ENV_KEYS) {
+      savedEnv[key] = process.env[key];
+      delete process.env[key];
+    }
+
+    mockRunPipeline.mockClear();
+    mockRunPipeline.mockImplementation(async (_modulePath: string, _ctx: unknown) => ({
+      ok: true as const,
+      logs: [{ stage: "generate", ok: true as const, ms: 10 }],
+      context: {} as Record<string, unknown>,
+      llmMetrics: [],
+    }));
+    mockEnsureTaskSymlinkBridge.mockClear();
+    mockValidateTaskSymlinks.mockClear();
+    mockRepairTaskSymlinks.mockClear();
+    mockCleanupTaskSymlinks.mockClear();
+    mockLoadFreshModule.mockClear();
+  });
+
+  afterEach(async () => {
+    for (const key of PO_ENV_KEYS) {
+      if (savedEnv[key] === undefined) {
+        delete process.env[key];
+      } else {
+        process.env[key] = savedEnv[key];
+      }
+    }
+    process.exitCode = 0;
+
+    await Promise.all(cleanupDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
+  });
+
+  test("two-task successful run: both tasks end in state 'done'", async () => {
+    const fixture = await setupMultiTaskFixture(["task-a", "task-b"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`process.exit called with ${String(code)}`);
+    }) as typeof process.exit);
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } finally {
+      exitSpy.mockRestore();
+    }
+
+    const statusText = await readFile(fixture.statusPath, "utf-8");
+    const status = JSON.parse(statusText) as {
+      tasks: Record<string, { state?: string }>;
+    };
+    expect(status.tasks["task-a"]?.state).toBe("done");
+    expect(status.tasks["task-b"]?.state).toBe("done");
+  });
+
+  test("three-task successful run: all three tasks end in state 'done'", async () => {
+    const fixture = await setupMultiTaskFixture(["task-a", "task-b", "task-c"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`process.exit called with ${String(code)}`);
+    }) as typeof process.exit);
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } finally {
+      exitSpy.mockRestore();
+    }
+
+    const statusText = await readFile(fixture.statusPath, "utf-8");
+    const status = JSON.parse(statusText) as {
+      tasks: Record<string, { state?: string }>;
+    };
+    expect(status.tasks["task-a"]?.state).toBe("done");
+    expect(status.tasks["task-b"]?.state).toBe("done");
+    expect(status.tasks["task-c"]?.state).toBe("done");
+  });
+
+  test("multi-task success does not throw a lifecycle-policy block error", async () => {
+    const fixture = await setupMultiTaskFixture(["task-a", "task-b", "task-c"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    const exitCalls: Array<number | undefined> = [];
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      exitCalls.push(code);
+      throw new Error(`process.exit called with ${String(code)}`);
+    }) as typeof process.exit);
+
+    const consoleErrorMessages: unknown[][] = [];
+    const consoleErrorSpy = spyOn(console, "error").mockImplementation((...args: unknown[]) => {
+      consoleErrorMessages.push(args);
+    });
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } catch {
+      // If the bug throws, runPipelineJob's outer catch will call process.exit,
+      // which our spy converts to a throw. Swallow here so we can inspect the
+      // captured console.error output below.
+    } finally {
+      exitSpy.mockRestore();
+      consoleErrorSpy.mockRestore();
+    }
+
+    const anyLifecycleBlock = consoleErrorMessages.some((args) =>
+      args.some((a) => {
+        if (a instanceof Error) return /Lifecycle policy blocked task start/.test(a.message);
+        if (typeof a === "string") return /Lifecycle policy blocked task start/.test(a);
+        return false;
+      }),
+    );
+    expect(anyLifecycleBlock).toBe(false);
+    expect(exitCalls).toEqual([]);
+  });
+});
+
+describe("runPipelineJob — outer-catch failure surfacing", () => {
+  const savedEnv: Record<string, string | undefined> = {};
+  const cleanupDirs: string[] = [];
+
+  beforeEach(() => {
+    for (const key of Object.keys(savedEnv)) delete savedEnv[key];
+    for (const key of PO_ENV_KEYS) {
+      savedEnv[key] = process.env[key];
+      delete process.env[key];
+    }
+
+    mockRunPipeline.mockClear();
+    mockEnsureTaskSymlinkBridge.mockClear();
+    mockValidateTaskSymlinks.mockClear();
+    mockRepairTaskSymlinks.mockClear();
+    mockCleanupTaskSymlinks.mockClear();
+    mockLoadFreshModule.mockClear();
+  });
+
+  afterEach(async () => {
+    for (const key of PO_ENV_KEYS) {
+      if (savedEnv[key] === undefined) {
+        delete process.env[key];
+      } else {
+        process.env[key] = savedEnv[key];
+      }
+    }
+    process.exitCode = 0;
+
+    await Promise.all(cleanupDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
+  });
+
+  test("unhandled error: sets exitCode=1, writes orchestrator failure log, stderr includes message", async () => {
+    const fixture = await setupMultiTaskFixture(["task-a"]);
+    cleanupDirs.push(fixture.tmpDir);
+
+    const injectedMessage = "injected-outer-catch-failure";
+    mockLoadFreshModule.mockImplementation(async (_path: string) => {
+      throw new Error(injectedMessage);
+    });
+
+    const exitCalls: Array<number | undefined> = [];
+    const exitSpy = spyOn(process, "exit").mockImplementation(((code?: number) => {
+      exitCalls.push(code);
+      throw new Error(`__test_exit__:${String(code)}`);
+    }) as typeof process.exit);
+
+    const fakeTimer = { unref: () => fakeTimer, ref: () => fakeTimer };
+    const setTimeoutSpy = spyOn(globalThis, "setTimeout").mockImplementation(
+      ((() => fakeTimer as unknown as ReturnType<typeof setTimeout>) as unknown) as typeof setTimeout,
+    );
+
+    const consoleErrorMessages: unknown[][] = [];
+    const consoleErrorSpy = spyOn(console, "error").mockImplementation((...args: unknown[]) => {
+      consoleErrorMessages.push(args);
+    });
+
+    try {
+      await runPipelineJob(fixture.jobId);
+    } catch (e) {
+      if (!(e instanceof Error) || !/^__test_exit__:/.test(e.message)) throw e;
+    } finally {
+      exitSpy.mockRestore();
+      setTimeoutSpy.mockRestore();
+      consoleErrorSpy.mockRestore();
+    }
+
+    expect(process.exitCode).toBe(1);
+    expect(exitCalls).toContain(1);
+
+    const failurePath = join(
+      fixture.jobDir,
+      "files",
+      "logs",
+      "orchestrator-runPipelineJob-failure-details.json",
+    );
+    const failureText = await readFile(failurePath, "utf-8");
+    const failure = JSON.parse(failureText) as { message?: unknown };
+    expect(typeof failure.message).toBe("string");
+    expect(failure.message).toContain(injectedMessage);
+
+    const stderrContainsMessage = consoleErrorMessages.some((args) =>
+      args.some((a) => typeof a === "string" && a.includes(injectedMessage)),
+    );
+    expect(stderrContainsMessage).toBe(true);
+  });
+});

package/src/core/__tests__/task-runner.test.ts

@@ -60,7 +60,7 @@ describe("runPipeline log tracking", () => {
 });
 
 describe("task-runner does not write job-level status fields", () => {
-  it("does not set snapshot.state, snapshot.current, snapshot.currentStage, or snapshot.progress on success", async () => {
+  it("does not set snapshot.state, snapshot.current, or snapshot.currentStage on success", async () => {
     const root = await makeTempRoot();
     const workDir = path.join(root, "job-1");
     await mkdir(workDir, { recursive: true });
@@ -91,11 +91,11 @@ describe("task-runner does not write job-level status fields", () => {
 
     const status = JSON.parse(await readFile(path.join(workDir, "tasks-status.json"), "utf8")) as StatusSnapshot;
 
-    // Job-level fields must remain untouched by task-runner
+    // Job-level lifecycle fields must remain untouched by task-runner.
     expect(status.state).toBe("pending");
     expect(status.current).toBeNull();
     expect(status.currentStage).toBeNull();
-    expect(status.progress).toBeUndefined();
+    expect(status.progress).toBe(100);
   });
 
   it("does not set snapshot.state on task failure", async () => {

package/src/core/pipeline-runner.ts

@@ -268,9 +268,6 @@ export async function runPipelineJob(jobId: string): Promise<void> {
   const pipeline = await loadPipeline(config.pipelineJsonPath);
   const taskRegistry = await loadTaskRegistry(config.taskRegistryPath);
 
-  const statusText = await Bun.file(config.statusPath).text();
-  const status = JSON.parse(statusText) as { tasks: Record<string, { state?: string }> };
-
   const { startFromTask, runSingleTask } = config;
 
   // ─── Validate startFromTask / runSingleTask config ───────────────────────
@@ -294,6 +291,9 @@ export async function runPipelineJob(jobId: string): Promise<void> {
   for (const task of pipeline.tasks) {
     const taskName = getTaskName(task);
 
+    const statusText = await Bun.file(config.statusPath).text();
+    const status = JSON.parse(statusText) as { tasks: Record<string, { state?: string }> };
+
     // Skip tasks before startFromTask
     if (!reachedStartFrom) {
       if (taskName === startFromTask) {
@@ -485,9 +485,32 @@ export async function runPipelineJob(jobId: string): Promise<void> {
     await completeJob(config, finalStatus, pipelineArtifacts);
   }
   } catch (err) {
-    console.error("Unhandled error in runPipelineJob:", err);
+    const normalized = normalizeError(err);
+    console.error(normalized.message);
     if (workDir !== undefined) {
-      await cleanupPidFile(workDir);
+      try {
+        const failureIO = createTaskFileIO({
+          workDir,
+          taskName: "orchestrator",
+          trackTaskFiles: false,
+          getStage: () => "runPipelineJob",
+          statusPath: join(workDir, "tasks-status.json"),
+        });
+        const failureLogName = generateLogName(
+          "orchestrator",
+          "runPipelineJob",
+          LogEvent.FAILURE_DETAILS,
+          LogFileExtension.JSON,
+        );
+        await failureIO.writeLog(failureLogName, JSON.stringify(normalized, null, 2));
+      } catch {
+        // Do not mask the original failure if log-write fails
+      }
+      try {
+        await cleanupPidFile(workDir);
+      } catch {
+        // Do not mask the original failure if PID cleanup fails
+      }
     }
     process.exitCode = 1;
     setTimeout(() => process.exit(1), 5000).unref();

package/src/core/task-runner.ts

@@ -4,7 +4,7 @@
 import { mkdir } from "node:fs/promises";
 import { dirname, isAbsolute, join } from "node:path";
 import { pathToFileURL } from "node:url";
-import { KNOWN_STAGES } from "./progress";
+import { KNOWN_STAGES, computeDeterministicProgress } from "./progress";
 import type { StageName } from "./progress";
 import { createTaskFileIO, generateLogName, trackFile } from "./file-io";
 import type { TaskFileIO } from "./file-io";
@@ -795,17 +795,14 @@ export async function runPipeline(
 
   // Write done status (best-effort)
   try {
-    const lastStage = KNOWN_STAGES[KNOWN_STAGES.length - 1];
+    const lastStage = KNOWN_STAGES[KNOWN_STAGES.length - 1]!;
     const doneProgress = computeDeterministicProgress(
       pipelineTasks ?? [taskName],
       taskName,
       lastStage,
     );
     await writeJobStatus(jobDir, (snapshot: StatusSnapshot) => {
-      snapshot.state = TaskState.DONE;
       snapshot.progress = doneProgress;
-      snapshot.current = null;
-      snapshot.currentStage = null;
       if (!snapshot.tasks[taskName]) snapshot.tasks[taskName] = {};
       snapshot.tasks[taskName]!.state = TaskState.DONE;
       snapshot.tasks[taskName]!.currentStage = null;

package/src/providers/__tests__/base.test.ts

@@ -165,8 +165,8 @@ describe("isRetryableError", () => {
 });
 
 describe("DEFAULT_REQUEST_TIMEOUT_MS", () => {
-  it("is 120 000 ms", () => {
-    expect(DEFAULT_REQUEST_TIMEOUT_MS).toBe(120_000);
+  it("is 3 600 000 ms", () => {
+    expect(DEFAULT_REQUEST_TIMEOUT_MS).toBe(3_600_000);
   });
 });
 

package/src/providers/__tests__/openai.test.ts

@@ -352,7 +352,7 @@ describe("openaiChat", () => {
       organization: "org_test",
       baseURL: "https://example.test/v1",
       maxRetries: 0,
-      timeout: 120_000,
+      timeout: 3_600_000,
     });
   });