@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

package/src/core/__tests__/core-boot-resolution.test.ts

@@ -0,0 +1,181 @@
+import { describe, test, expect, afterEach } from "bun:test";
+import { existsSync, mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve as resolvePath } from "node:path";
+import { mkdir, rm, writeFile } from "node:fs/promises";
+import { startOrchestrator } from "../orchestrator";
+import { resolveJobConfig, runPipelineJob } from "../pipeline-runner";
+import { resetConfig } from "../config";
+
+// PO_* keys this test mutates (capture original values in beforeEach-equivalent,
+// restore by deletion-or-assignment in afterEach per unit-testing.md §6).
+const PO_ENV_KEYS = [
+  "PO_ROOT",
+  "PO_DATA_DIR",
+  "PO_CURRENT_DIR",
+  "PO_COMPLETE_DIR",
+  "PO_PIPELINE_PATH",
+] as const;
+const savedEnv: Record<string, string | undefined> = {};
+const cleanupDirs: string[] = [];
+
+function snapshotEnv(): void {
+  for (const key of PO_ENV_KEYS) {
+    savedEnv[key] = process.env[key];
+  }
+}
+
+function restoreEnv(): void {
+  for (const key of PO_ENV_KEYS) {
+    if (savedEnv[key] === undefined) delete process.env[key];
+    else process.env[key] = savedEnv[key];
+  }
+  resetConfig();
+}
+
+interface FakeWatcher {
+  on(event: "add", cb: (path: string) => void): FakeWatcher;
+  on(event: "ready", cb: () => void): FakeWatcher;
+  on(event: "error", cb: (err: Error) => void): FakeWatcher;
+  close(): Promise<void>;
+}
+
+function fakeWatcherFactory(): (
+  path: string,
+  options: Record<string, unknown>,
+) => FakeWatcher {
+  const watcher = {
+    on(event: "add" | "ready" | "error", cb: () => void): FakeWatcher {
+      if (event === "ready") queueMicrotask(() => cb());
+      return watcher;
+    },
+    close(): Promise<void> {
+      return Promise.resolve();
+    },
+  } as unknown as FakeWatcher;
+  return (_p: string, _o: Record<string, unknown>) => watcher;
+}
+
+describe("core boot resolution (AC-7)", () => {
+  afterEach(async () => {
+    restoreEnv();
+    await Promise.all(
+      cleanupDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })),
+    );
+  });
+
+  describe("startOrchestrator", () => {
+    test("with explicit dataDir, creates pipeline-data subdirs under the resolved root", async () => {
+      snapshotEnv();
+      delete process.env["PO_ROOT"];
+
+      const root = mkdtempSync(join(tmpdir(), "boot-resolve-explicit-"));
+      cleanupDirs.push(root);
+      const dataDir = resolvePath(join(root, "pipeline-data"));
+
+      const handle = await startOrchestrator({
+        dataDir,
+        watcherFactory: fakeWatcherFactory(),
+      });
+
+      try {
+        for (const sub of ["pending", "current", "complete", "rejected", "staging"]) {
+          expect(existsSync(join(root, "pipeline-data", sub))).toBe(true);
+        }
+      } finally {
+        await handle.stop();
+      }
+    });
+
+    test("with PO_ROOT set and no explicit dataDir, resolves under PO_ROOT", async () => {
+      snapshotEnv();
+      const root = mkdtempSync(join(tmpdir(), "boot-resolve-envroot-"));
+      cleanupDirs.push(root);
+      process.env["PO_ROOT"] = root;
+
+      const handle = await startOrchestrator({
+        dataDir: undefined,
+        watcherFactory: fakeWatcherFactory(),
+      });
+
+      try {
+        for (const sub of ["pending", "current", "complete", "rejected", "staging"]) {
+          expect(existsSync(join(root, "pipeline-data", sub))).toBe(true);
+        }
+      } finally {
+        await handle.stop();
+      }
+    });
+
+    test("with neither explicit dataDir nor PO_ROOT, throws the resolver message", () => {
+      snapshotEnv();
+      delete process.env["PO_ROOT"];
+
+      expect(() =>
+        startOrchestrator({
+          dataDir: undefined,
+          watcherFactory: fakeWatcherFactory(),
+        }),
+      ).toThrow(/Workspace root is required/);
+    });
+  });
+
+  describe("resolveJobConfig", () => {
+    async function setupJobDir(tmpRoot: string, jobId: string): Promise<void> {
+      const currentDir = join(tmpRoot, "pipeline-data", "current");
+      await mkdir(currentDir, { recursive: true });
+      const jobDir = join(currentDir, jobId);
+      await mkdir(jobDir, { recursive: true });
+      await writeFile(
+        join(jobDir, "seed.json"),
+        JSON.stringify({ pipeline: "boot-test" }),
+      );
+      await writeFile(
+        join(jobDir, "tasks-status.json"),
+        JSON.stringify({
+          id: jobId,
+          state: "pending",
+          current: null,
+          currentStage: null,
+          lastUpdated: new Date().toISOString(),
+          tasks: {},
+          files: { artifacts: [], logs: [], tmp: [] },
+        }),
+      );
+    }
+
+    test("with a resolved root, poRoot equals the supplied root", async () => {
+      snapshotEnv();
+      const root = mkdtempSync(join(tmpdir(), "boot-resolvejob-po-"));
+      cleanupDirs.push(root);
+      process.env["PO_PIPELINE_PATH"] = join(root, "pipeline.json");
+      await setupJobDir(root, "boot-test-job");
+
+      const config = await resolveJobConfig("boot-test-job", resolvePath(root));
+      expect(config.poRoot).toBe(resolvePath(root));
+    });
+
+    test("with PO_ROOT deleted, still uses the supplied resolved root", async () => {
+      snapshotEnv();
+      const root = mkdtempSync(join(tmpdir(), "boot-resolvejob-explicit-"));
+      cleanupDirs.push(root);
+      delete process.env["PO_ROOT"];
+      process.env["PO_PIPELINE_PATH"] = join(root, "pipeline.json");
+      await setupJobDir(root, "boot-test-job-explicit");
+
+      const config = await resolveJobConfig("boot-test-job-explicit", resolvePath(root));
+      expect(config.poRoot).toBe(resolvePath(root));
+    });
+  });
+
+  describe("runPipelineJob", () => {
+    test("with PO_ROOT deleted, rejects with the resolver message at boot", async () => {
+      snapshotEnv();
+      delete process.env["PO_ROOT"];
+
+      await expect(runPipelineJob("any-job-id")).rejects.toThrow(
+        /Workspace root is required/,
+      );
+    });
+  });
+});

package/src/core/__tests__/job-concurrency.test.ts

@@ -7,11 +7,14 @@ import { spawnSync } from "node:child_process";
 import {
   getConcurrencyRuntimePaths,
   getJobConcurrencyStatus,
+  listActiveJobLeases,
   listQueuedSeeds,
   pruneStaleJobSlots,
+  readJobSlotLease,
   releaseJobSlot,
   tryAcquireJobSlot,
   updateJobSlotPid,
+  withRuntimeLock,
   type JobSlotLease,
 } from "../job-concurrency";
 import { readFile } from "node:fs/promises";
@@ -138,6 +141,21 @@ describe("listQueuedSeeds", () => {
       await rm(dir, { recursive: true });
     }
   });
+
+  test("matches a UUID <id>-seed.json and ignores foo.json under the shared regex", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "queued-seeds-shared-"));
+    const pendingDir = join(dir, "pending");
+    await mkdir(pendingDir, { recursive: true });
+    const uuid = "f47ac10b-58cc-4372-a567-0e02b2c3d479";
+    await writeFile(join(pendingDir, `${uuid}-seed.json`), JSON.stringify({ pipeline: "p" }));
+    await writeFile(join(pendingDir, "foo.json"), "{}");
+    try {
+      const result = await listQueuedSeeds(dir);
+      expect(result.map((s) => s.jobId)).toEqual([uuid]);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
 });
 
 async function writeLease(
@@ -333,6 +351,121 @@ describe("pruneStaleJobSlots", () => {
   });
 });
 
+describe("pruneStaleJobSlots — stale_runner backstop", () => {
+  test("reclaims an aged live-pid lease when runner.pid is absent (AC-3)", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "prune-stale-runner-"));
+    const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
+    await mkdir(runningJobsDir, { recursive: true });
+    await mkdir(join(dir, "current", "job-reuse"), { recursive: true });
+    const slotPath = await writeLease(runningJobsDir, "job-reuse", {
+      jobId: "job-reuse",
+      pid: process.pid,
+      acquiredAt: new Date(Date.now() - 60_000).toISOString(),
+      source: "orchestrator",
+      slotPath: join(runningJobsDir, "job-reuse.json"),
+    });
+    try {
+      const result = await pruneStaleJobSlots(dir, 1000, 60_000);
+      expect(result).toEqual([
+        { jobId: "job-reuse", slotPath, reason: "stale_runner" },
+      ]);
+      expect(existsSync(slotPath)).toBe(false);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("retains an aged live-pid lease when runner.pid matches (AC-8)", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "prune-stale-runner-live-"));
+    const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
+    await mkdir(runningJobsDir, { recursive: true });
+    await mkdir(join(dir, "current", "job-running"), { recursive: true });
+    await writeFile(join(dir, "current", "job-running", "runner.pid"), `${process.pid}\n`);
+    const slotPath = await writeLease(runningJobsDir, "job-running", {
+      jobId: "job-running",
+      pid: process.pid,
+      acquiredAt: new Date(Date.now() - 60_000).toISOString(),
+      source: "orchestrator",
+      slotPath: join(runningJobsDir, "job-running.json"),
+    });
+    try {
+      const result = await pruneStaleJobSlots(dir, 1000, 60_000);
+      expect(result).toEqual([]);
+      expect(existsSync(slotPath)).toBe(true);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("does not prune a null-pid lease aged past lockTimeoutMs but within staleLeaseTimeoutMs (AC-5)", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "prune-stale-split-"));
+    const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
+    await mkdir(runningJobsDir, { recursive: true });
+    await mkdir(join(dir, "current", "job-inflight"), { recursive: true });
+    const slotPath = await writeLease(runningJobsDir, "job-inflight", {
+      jobId: "job-inflight",
+      pid: null,
+      acquiredAt: new Date(Date.now() - 5_000).toISOString(),
+      source: "orchestrator",
+      slotPath: join(runningJobsDir, "job-inflight.json"),
+    });
+    try {
+      const result = await pruneStaleJobSlots(dir, 1000, 60_000);
+      expect(result).toEqual([]);
+      expect(existsSync(slotPath)).toBe(true);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("omitting staleLeaseTimeoutMs reproduces current missing_pid behavior (AC-6)", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "prune-stale-default-"));
+    const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
+    await mkdir(runningJobsDir, { recursive: true });
+    await mkdir(join(dir, "current", "job-default"), { recursive: true });
+    const slotPath = await writeLease(runningJobsDir, "job-default", {
+      jobId: "job-default",
+      pid: null,
+      acquiredAt: new Date(Date.now() - 60_000).toISOString(),
+      source: "orchestrator",
+      slotPath: join(runningJobsDir, "job-default.json"),
+    });
+    try {
+      const result = await pruneStaleJobSlots(dir, 1000);
+      expect(result).toEqual([
+        { jobId: "job-default", slotPath, reason: "missing_pid" },
+      ]);
+      expect(existsSync(slotPath)).toBe(false);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("omitting staleLeaseTimeoutMs reproduces current dead_pid behavior (AC-6)", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "prune-stale-default-dead-"));
+    const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
+    await mkdir(runningJobsDir, { recursive: true });
+    await mkdir(join(dir, "current", "job-dead-default"), { recursive: true });
+    const deadPid = getDeadPid();
+    const slotPath = await writeLease(runningJobsDir, "job-dead-default", {
+      jobId: "job-dead-default",
+      pid: deadPid,
+      acquiredAt: new Date().toISOString(),
+      source: "orchestrator",
+      slotPath: join(runningJobsDir, "job-dead-default.json"),
+    });
+    try {
+      const result = await pruneStaleJobSlots(dir, 1000);
+      expect(result).toEqual([
+        { jobId: "job-dead-default", slotPath, reason: "dead_pid" },
+      ]);
+      expect(existsSync(slotPath)).toBe(false);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+});
+
 async function setupJobsDir(prefix: string): Promise<string> {
   const dir = await mkdtemp(join(tmpdir(), prefix));
   const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
@@ -552,3 +685,130 @@ describe("getJobConcurrencyStatus", () => {
     }
   });
 });
+
+describe("readJobSlotLease", () => {
+  test("returns the lease written by tryAcquireJobSlot", async () => {
+    const dir = await setupJobsDir("read-lease-acquired-");
+    try {
+      await makeCurrent(dir, "job-a");
+      const acquired = await tryAcquireJobSlot({
+        dataDir: dir,
+        jobId: "job-a",
+        maxConcurrentJobs: 2,
+        source: "orchestrator",
+        pid: process.pid,
+      });
+      expect(acquired.ok).toBe(true);
+
+      const lease = await readJobSlotLease(dir, "job-a");
+      expect(lease).not.toBeNull();
+      expect(lease!.jobId).toBe("job-a");
+      expect(lease!.pid).toBe(process.pid);
+      expect(lease!.source).toBe("orchestrator");
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("returns null for an unknown job", async () => {
+    const dir = await setupJobsDir("read-lease-unknown-");
+    try {
+      expect(await readJobSlotLease(dir, "unknown")).toBeNull();
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+});
+
+describe("listActiveJobLeases", () => {
+  test("returns the leases written by tryAcquireJobSlot", async () => {
+    const dir = await setupJobsDir("list-active-acquired-");
+    try {
+      await makeCurrent(dir, "job-a");
+      await makeCurrent(dir, "job-b");
+      await tryAcquireJobSlot({
+        dataDir: dir,
+        jobId: "job-a",
+        maxConcurrentJobs: 2,
+        source: "orchestrator",
+        pid: process.pid,
+      });
+      await tryAcquireJobSlot({
+        dataDir: dir,
+        jobId: "job-b",
+        maxConcurrentJobs: 2,
+        source: "restart",
+        pid: process.pid,
+      });
+
+      const leases = await listActiveJobLeases(dir);
+      expect(leases.map((l) => l.jobId)).toEqual(["job-a", "job-b"]);
+      const first = leases[0]!;
+      expect(first.pid).toBe(process.pid);
+      expect(first.source).toBe("orchestrator");
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("returns [] for an absent runtime dir", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "list-active-absent-"));
+    try {
+      expect(await listActiveJobLeases(dir)).toEqual([]);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+});
+
+describe("withRuntimeLock", () => {
+  test("serializes two concurrent callers", async () => {
+    const dir = await setupJobsDir("runtime-lock-serialize-");
+    const markerPath = join(dir, "marker.txt");
+    const events: string[] = [];
+    try {
+      let markAEntered!: () => void;
+      const aEntered = new Promise<void>((resolve) => {
+        markAEntered = resolve;
+      });
+      const a = withRuntimeLock(dir, 5000, async () => {
+        markAEntered();
+        await new Promise((resolve) => setTimeout(resolve, 50));
+        await writeFile(markerPath, "from-a");
+        events.push("a-wrote");
+      });
+      await aEntered;
+      const b = withRuntimeLock(dir, 5000, async () => {
+        const observed = await readFile(markerPath, "utf-8");
+        events.push(`b-read:${observed}`);
+      });
+      await Promise.all([a, b]);
+
+      expect(events).toEqual(["a-wrote", "b-read:from-a"]);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("reclaims a stale owner file without a numeric pid using the stale age, not the acquire wait", async () => {
+    const dir = await setupJobsDir("runtime-lock-stale-owner-");
+    const { lockDir, runtimeDir } = getConcurrencyRuntimePaths(dir);
+    await mkdir(runtimeDir, { recursive: true });
+    await mkdir(lockDir, { recursive: false });
+    await writeFile(
+      join(lockDir, "owner.json"),
+      JSON.stringify({ acquiredAt: new Date(Date.now() - 60_000).toISOString() }),
+    );
+    try {
+      const result = await withRuntimeLock(
+        dir,
+        100,
+        async () => "reclaimed",
+        60_000,
+      );
+      expect(result).toBe("reclaimed");
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+});