@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.3

package/src/core/__tests__/orchestrator.test.ts

@@ -7,6 +7,7 @@ import {
   drainPendingQueue,
   handleChildExit,
   resolveDirs,
+  startOrchestrator,
   promoteSeedFile,
   restoreSeedFile,
 } from "../orchestrator";
@@ -463,6 +464,146 @@ describe("handleChildExit", () => {
       await rm(dir, { recursive: true });
     }
   });
+
+  test("handleChildExit still attempts reconciliation when slot release fails", async () => {
+    const dir = await setupDir("exit-release-fails-");
+    try {
+      const releaseError = new Error("release failed");
+      let drained = false;
+      let reconciled = false;
+
+      await expect(handleChildExit({
+        dataDir: dir,
+        jobId: "job-release-fails",
+        triggerDrain: () => {
+          drained = true;
+        },
+        lockTimeoutMs: 1000,
+        releaseSlot: async () => {
+          throw releaseError;
+        },
+        reconcileJob: async () => {
+          reconciled = true;
+          return { reconciled: false, reason: "unreadable" };
+        },
+      })).rejects.toThrow("release failed");
+
+      expect(reconciled).toBe(true);
+      expect(drained).toBe(true);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("handleChildExit reports both cleanup errors when slot release and reconciliation fail", async () => {
+    const dir = await setupDir("exit-both-fail-");
+    try {
+      const releaseError = new Error("release failed");
+      const reconcileError = new Error("reconcile failed");
+      let drained = false;
+
+      let thrown: unknown;
+      try {
+        await handleChildExit({
+          dataDir: dir,
+          jobId: "job-both-fail",
+          triggerDrain: () => {
+            drained = true;
+          },
+          lockTimeoutMs: 1000,
+          releaseSlot: async () => {
+            throw releaseError;
+          },
+          reconcileJob: async () => {
+            throw reconcileError;
+          },
+        });
+      } catch (err) {
+        thrown = err;
+      }
+
+      expect(thrown).toBeInstanceOf(AggregateError);
+      expect((thrown as AggregateError).message).toBe("child exit cleanup failed");
+      expect((thrown as AggregateError).errors).toEqual([releaseError, reconcileError]);
+      expect(drained).toBe(true);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("AC-3: a mid-job runner exit marks the running task and job failed, releases the slot, and triggers a drain", async () => {
+    const dir = await setupDir("exit-reconcile-");
+    try {
+      const jobId = "job-running";
+      const jobDir = join(dir, "current", jobId);
+      await mkdir(jobDir, { recursive: true });
+      await writeFile(
+        join(jobDir, "tasks-status.json"),
+        JSON.stringify({
+          id: jobId,
+          state: "running",
+          current: "research",
+          currentStage: null,
+          lastUpdated: "2026-06-19T11:00:00.000Z",
+          tasks: { research: { state: "running" } },
+          files: { artifacts: [], logs: [], tmp: [] },
+        }),
+      );
+      await writeFile(join(jobDir, "runner.pid"), "999999");
+
+      let drained = false;
+      await handleChildExit({
+        dataDir: dir,
+        jobId,
+        triggerDrain: () => {
+          drained = true;
+        },
+        code: 1,
+        signal: null,
+        lockTimeoutMs: 1000,
+      });
+
+      expect(drained).toBe(true);
+      const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
+      expect(existsSync(join(runningJobsDir, `${jobId}.json`))).toBe(false);
+
+      const snapshot = JSON.parse(await readFile(join(jobDir, "tasks-status.json"), "utf-8")) as Record<string, unknown>;
+      expect(snapshot["state"]).toBe("failed");
+      const tasks = snapshot["tasks"] as Record<string, Record<string, unknown>>;
+      expect(tasks["research"]?.["state"]).toBe("failed");
+      expect(tasks["research"]?.["error"]).toBeTruthy();
+      const errorContext = tasks["research"]?.["errorContext"] as Record<string, unknown>;
+      expect(errorContext?.["kind"]).toBe("runner-interrupted");
+      expect(errorContext?.["code"]).toBe(1);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("AC-10: a job already moved to complete/ (no current/ status) is untouched, and the drain still fires", async () => {
+    const dir = await setupDir("exit-complete-");
+    try {
+      const jobId = "job-gone";
+      const statusPath = join(dir, "current", jobId, "tasks-status.json");
+
+      let drainCount = 0;
+      await handleChildExit({
+        dataDir: dir,
+        jobId,
+        triggerDrain: () => {
+          drainCount++;
+        },
+        code: 0,
+        signal: null,
+        lockTimeoutMs: 1000,
+      });
+
+      expect(drainCount).toBe(1);
+      expect(existsSync(statusPath)).toBe(false);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
 });
 
 describe("drainPendingQueue concurrency", () => {
@@ -512,3 +653,104 @@ describe("drainPendingQueue concurrency", () => {
     }
   });
 });
+
+interface FakeWatcher {
+  on(event: "add", cb: (path: string) => void): FakeWatcher;
+  on(event: "ready", cb: () => void): FakeWatcher;
+  on(event: "error", cb: (err: Error) => void): FakeWatcher;
+  close(): Promise<void>;
+}
+
+function makeFakeWatcherFactory() {
+  return (_path: string, _options: Record<string, unknown>): FakeWatcher => {
+    const watcher = {
+      on(event: "add" | "ready" | "error", cb: () => void): FakeWatcher {
+        if (event === "ready") queueMicrotask(() => cb());
+        return watcher;
+      },
+      close(): Promise<void> {
+        return Promise.resolve();
+      },
+    } as unknown as FakeWatcher;
+    return watcher;
+  };
+}
+
+async function writeRunningJob(
+  dataDir: string,
+  jobId: string,
+  lastUpdated: string,
+  runnerPid: number,
+): Promise<void> {
+  const jobDir = join(dataDir, "current", jobId);
+  await mkdir(jobDir, { recursive: true });
+  await writeFile(
+    join(jobDir, "tasks-status.json"),
+    JSON.stringify({
+      id: jobId,
+      state: "running",
+      current: "research",
+      currentStage: null,
+      lastUpdated,
+      tasks: { research: { state: "running" } },
+      files: { artifacts: [], logs: [], tmp: [] },
+    }),
+  );
+  await writeFile(join(jobDir, "runner.pid"), String(runnerPid));
+}
+
+describe("startOrchestrator boot sweep (AC-16)", () => {
+  test("a stale orphaned job in current/ is marked failed and the handle still resolves", async () => {
+    const root = await mkdtemp(join(tmpdir(), "boot-stale-"));
+    const dataDir = join(root, "pipeline-data");
+    try {
+      await writeRunningJob(dataDir, "job-stale", "2020-01-01T00:00:00.000Z", 999999);
+
+      const handle = await startOrchestrator({
+        dataDir,
+        watcherFactory: makeFakeWatcherFactory(),
+      });
+
+      expect(handle).toBeDefined();
+      expect(typeof handle.stop).toBe("function");
+
+      const snapshot = JSON.parse(
+        await readFile(join(dataDir, "current", "job-stale", "tasks-status.json"), "utf-8"),
+      ) as Record<string, unknown>;
+      expect(snapshot["state"]).toBe("failed");
+      const tasks = snapshot["tasks"] as Record<string, Record<string, unknown>>;
+      expect(tasks["research"]?.["state"]).toBe("failed");
+
+      await handle.stop();
+    } finally {
+      await rm(root, { recursive: true, force: true });
+    }
+  });
+
+  test("a live job in current/ is left running and the handle still resolves", async () => {
+    const root = await mkdtemp(join(tmpdir(), "boot-live-"));
+    const dataDir = join(root, "pipeline-data");
+    try {
+      await writeRunningJob(dataDir, "job-live", "2020-01-01T00:00:00.000Z", process.pid);
+
+      const handle = await startOrchestrator({
+        dataDir,
+        watcherFactory: makeFakeWatcherFactory(),
+      });
+
+      expect(handle).toBeDefined();
+      expect(typeof handle.stop).toBe("function");
+
+      const snapshot = JSON.parse(
+        await readFile(join(dataDir, "current", "job-live", "tasks-status.json"), "utf-8"),
+      ) as Record<string, unknown>;
+      expect(snapshot["state"]).toBe("running");
+      const tasks = snapshot["tasks"] as Record<string, Record<string, unknown>>;
+      expect(tasks["research"]?.["state"]).toBe("running");
+
+      await handle.stop();
+    } finally {
+      await rm(root, { recursive: true, force: true });
+    }
+  });
+});