@ryanfw/prompt-orchestration-pipeline 1.3.1 → 1.3.3

package/src/config/__tests__/statuses.test.ts

@@ -8,6 +8,7 @@ import {
   VALID_JOB_LOCATIONS,
   normalizeTaskState,
   normalizeJobStatus,
+  jobStateToStatus,
   deriveJobStatusFromTasks,
 } from "../statuses";
 
@@ -159,6 +160,10 @@ describe("normalizeJobStatus", () => {
     expect(normalizeJobStatus("completed")).toBe("complete");
   });
 
+  it("handles synonym: done → complete", () => {
+    expect(normalizeJobStatus("done")).toBe("complete");
+  });
+
   it("handles synonym: error → failed", () => {
     expect(normalizeJobStatus("error")).toBe("failed");
   });
@@ -193,6 +198,42 @@ describe("normalizeJobStatus", () => {
   });
 });
 
+describe("jobStateToStatus", () => {
+  it("returns canonical for each recognized JobState", () => {
+    expect(jobStateToStatus("pending")).toBe("pending");
+    expect(jobStateToStatus("running")).toBe("running");
+    expect(jobStateToStatus("waiting")).toBe("waiting");
+    expect(jobStateToStatus("failed")).toBe("failed");
+  });
+
+  it("maps done (persisted JobState) to complete (canonical JobStatusValue)", () => {
+    expect(jobStateToStatus("done")).toBe("complete");
+  });
+
+  it("is case-insensitive and trims whitespace", () => {
+    expect(jobStateToStatus("DONE")).toBe("complete");
+    expect(jobStateToStatus("  failed  ")).toBe("failed");
+    expect(jobStateToStatus("Waiting")).toBe("waiting");
+  });
+
+  it("returns null for non-string input", () => {
+    expect(jobStateToStatus(null)).toBeNull();
+    expect(jobStateToStatus(undefined)).toBeNull();
+    expect(jobStateToStatus(42)).toBeNull();
+    expect(jobStateToStatus({})).toBeNull();
+  });
+
+  it("returns null for unrecognized strings", () => {
+    expect(jobStateToStatus("bogus-state")).toBeNull();
+  });
+
+  it("does not accept JobStatus synonyms that are not JobState values", () => {
+    expect(jobStateToStatus("error")).toBeNull();
+    expect(jobStateToStatus("completed")).toBeNull();
+    expect(jobStateToStatus("complete")).toBeNull();
+  });
+});
+
 describe("deriveJobStatusFromTasks", () => {
   it("returns failed when any task is failed (failed priority)", () => {
     expect(deriveJobStatusFromTasks([{ state: "failed" }, { state: "done" }])).toBe("failed");

package/src/config/statuses.ts

@@ -53,8 +53,9 @@ const TASK_STATE_SYNONYMS: Readonly<Record<string, TaskStateValue>> = Object.fre
   succeeded: "done",
 });
 
-const JOB_STATUS_SYNONYMS: Readonly<Record<string, JobStatusValue>> = Object.freeze({
+export const JOB_STATUS_SYNONYMS: Readonly<Record<string, JobStatusValue>> = Object.freeze({
   completed: "complete",
+  done: "complete",
   error: "failed",
 });
 
@@ -93,6 +94,24 @@ export function normalizeJobStatus(status: unknown): JobStatusValue {
   return "pending";
 }
 
+export function jobStateToStatus(state: unknown): JobStatusValue | null {
+  if (typeof state !== "string") return null;
+  switch (state.toLowerCase().trim()) {
+    case "pending":
+      return "pending";
+    case "running":
+      return "running";
+    case "waiting":
+      return "waiting";
+    case "failed":
+      return "failed";
+    case "done":
+      return "complete";
+    default:
+      return null;
+  }
+}
+
 export function deriveJobStatusFromTasks(
   tasks: ReadonlyArray<{ state: unknown }>,
 ): JobStatusValue {

package/src/core/__tests__/agent-step.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, mock, beforeEach, afterEach } from "bun:test";
-import { mkdtempSync, rmSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { existsSync, mkdtempSync, rmSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
 import type { WriteOptions, TaskFileIO } from "../file-io.ts";
@@ -53,10 +53,11 @@ function createFakeIO(): TaskFileIO & { calls: string[] } {
 
 function createFakeMcpHandle(
   artifacts: string[] = [],
+  connection = { url: "http://127.0.0.1:9999/mcp", token: "fake-token" },
 ): McpIoServerHandle & { closeSpy: ReturnType<typeof mock> } {
   const closeSpy = mock(async () => {});
   return {
-    connection: { url: "http://127.0.0.1:9999/mcp", token: "fake-token" },
+    connection,
     artifactsWritten() {
       return [...artifacts];
     },
@@ -106,6 +107,26 @@ function makeSuccessResult(overrides?: Partial<RunResult>): RunResult {
   };
 }
 
+function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void; reject: (err: unknown) => void } {
+  let resolve!: (value: T) => void;
+  let reject!: (err: unknown) => void;
+  const promise = new Promise<T>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+  return { promise, resolve, reject };
+}
+
+async function waitFor(predicate: () => boolean, message: string): Promise<void> {
+  const deadline = Date.now() + 1_000;
+  while (!predicate()) {
+    if (Date.now() > deadline) {
+      throw new Error(message);
+    }
+    await new Promise((resolve) => setTimeout(resolve, 1));
+  }
+}
+
 function makeDeps(result: RunResult | Error) {
   const run = mock(() => makeFakeHarnessRun(result));
   const startMcpIoServer = mock(async () => createFakeMcpHandle());
@@ -178,6 +199,281 @@ describe("executeAgent", () => {
 
     expect(startMcpIoServer).not.toHaveBeenCalled();
   });
+
+  it("passes adapter-native MCP servers for claude", async () => {
+    const io = createFakeIO();
+    let capturedMcpServers: unknown;
+    const run = mock((opts: { mcpServers?: unknown }) => {
+      capturedMcpServers = opts.mcpServers;
+      return makeFakeHarnessRun(makeSuccessResult());
+    });
+    const startMcpIoServer = mock(async () => createFakeMcpHandle());
+
+    await executeAgent(
+      { io, entry: { name: "a", harness: "claude", prompt: "p" } },
+      { run, startMcpIoServer },
+    );
+
+    expect(capturedMcpServers).toEqual([{
+      name: "popio",
+      url: "http://127.0.0.1:9999/mcp",
+      headers: { Authorization: "Bearer fake-token" },
+    }]);
+  });
+
+  it("bridges MCP for codex through a temporary binary wrapper", async () => {
+    const io = createFakeIO();
+    const previousPopBin = process.env.POP_CODEX_BIN;
+    const previousAdapterBin = process.env.LLM_ADAPTER_CODEX_BIN;
+    const previousRealBin = process.env.POP_REAL_CODEX_BIN;
+    const previousMcpUrlConfig = process.env.POP_MCP_URL_CONFIG;
+    const previousMcpToken = process.env.POP_MCP_TOKEN;
+    let wrapperPath = "";
+    let capturedMcpServers: unknown;
+    try {
+      process.env.POP_CODEX_BIN = "/custom/codex";
+      delete process.env.LLM_ADAPTER_CODEX_BIN;
+      delete process.env.POP_REAL_CODEX_BIN;
+      delete process.env.POP_MCP_URL_CONFIG;
+      delete process.env.POP_MCP_TOKEN;
+      const run = mock((opts: { mcpServers?: unknown; onEvent?: (event: HarnessEvent) => void }) => {
+        capturedMcpServers = opts.mcpServers;
+        wrapperPath = process.env.LLM_ADAPTER_CODEX_BIN ?? "";
+        expect(wrapperPath).toContain("codex-mcp-");
+        expect(readFileSync(wrapperPath, "utf8")).toContain("POP_MCP_URL_CONFIG");
+        expect(process.env.POP_REAL_CODEX_BIN).toBe("/custom/codex");
+        expect(process.env.POP_MCP_URL_CONFIG).toBe(
+          'mcp_servers.popio.url="http://127.0.0.1:9999/mcp"',
+        );
+        expect(process.env.POP_MCP_TOKEN).toBe("fake-token");
+        opts.onEvent?.({
+          harness: "codex",
+          seq: 0,
+          at: Date.now(),
+          raw: { type: "result", total_cost_usd: 0.42 },
+          type: "raw",
+        });
+        return makeFakeHarnessRun(makeSuccessResult());
+      });
+      const startMcpIoServer = mock(async () => createFakeMcpHandle());
+
+      const result = await executeAgent(
+        { io, entry: { name: "a", harness: "codex", prompt: "p" } },
+        { run, startMcpIoServer },
+      );
+
+      expect(capturedMcpServers).toBeUndefined();
+      expect(result.costUsd).toBe(0.42);
+      expect(process.env.LLM_ADAPTER_CODEX_BIN).toBeUndefined();
+      expect(process.env.POP_REAL_CODEX_BIN).toBeUndefined();
+      expect(process.env.POP_MCP_URL_CONFIG).toBeUndefined();
+      expect(process.env.POP_MCP_TOKEN).toBeUndefined();
+      expect(existsSync(wrapperPath)).toBe(false);
+    } finally {
+      if (previousPopBin === undefined) {
+        delete process.env.POP_CODEX_BIN;
+      } else {
+        process.env.POP_CODEX_BIN = previousPopBin;
+      }
+      if (previousAdapterBin === undefined) {
+        delete process.env.LLM_ADAPTER_CODEX_BIN;
+      } else {
+        process.env.LLM_ADAPTER_CODEX_BIN = previousAdapterBin;
+      }
+      if (previousRealBin === undefined) {
+        delete process.env.POP_REAL_CODEX_BIN;
+      } else {
+        process.env.POP_REAL_CODEX_BIN = previousRealBin;
+      }
+      if (previousMcpUrlConfig === undefined) {
+        delete process.env.POP_MCP_URL_CONFIG;
+      } else {
+        process.env.POP_MCP_URL_CONFIG = previousMcpUrlConfig;
+      }
+      if (previousMcpToken === undefined) {
+        delete process.env.POP_MCP_TOKEN;
+      } else {
+        process.env.POP_MCP_TOKEN = previousMcpToken;
+      }
+    }
+  });
+
+  it("serializes env-backed MCP compatibility for concurrent codex runs", async () => {
+    const previousPopBin = process.env.POP_CODEX_BIN;
+    const previousAdapterBin = process.env.LLM_ADAPTER_CODEX_BIN;
+    const previousRealBin = process.env.POP_REAL_CODEX_BIN;
+    const previousMcpUrlConfig = process.env.POP_MCP_URL_CONFIG;
+    const previousMcpToken = process.env.POP_MCP_TOKEN;
+    const firstRunResult = deferred<RunResult>();
+    const secondRunResult = deferred<RunResult>();
+    const observedTokens: Array<string | undefined> = [];
+    let runCount = 0;
+    let startCount = 0;
+    try {
+      process.env.POP_CODEX_BIN = "/custom/codex";
+      delete process.env.LLM_ADAPTER_CODEX_BIN;
+      delete process.env.POP_REAL_CODEX_BIN;
+      delete process.env.POP_MCP_URL_CONFIG;
+      delete process.env.POP_MCP_TOKEN;
+      const run = mock(() => {
+        runCount += 1;
+        observedTokens.push(process.env.POP_MCP_TOKEN);
+        return {
+          result: runCount === 1 ? firstRunResult.promise : secondRunResult.promise,
+          sessionId: Promise.resolve(`sess-${runCount}`),
+          abort() {},
+        };
+      });
+      const startMcpIoServer = mock(async () => {
+        startCount += 1;
+        return createFakeMcpHandle([], {
+          url: `http://127.0.0.1:999${startCount}/mcp`,
+          token: `token-${startCount}`,
+        });
+      });
+
+      const first = executeAgent(
+        { io: createFakeIO(), entry: { name: "a", harness: "codex", prompt: "p" } },
+        { run, startMcpIoServer },
+      );
+      const second = executeAgent(
+        { io: createFakeIO(), entry: { name: "b", harness: "codex", prompt: "p" } },
+        { run, startMcpIoServer },
+      );
+
+      await waitFor(() => runCount === 1, "first codex run did not start");
+      expect(runCount).toBe(1);
+      expect(observedTokens).toHaveLength(1);
+      const firstObservedToken = observedTokens[0];
+      if (firstObservedToken === undefined) {
+        throw new Error("first codex run did not receive an MCP token");
+      }
+      expect(["token-1", "token-2"]).toContain(firstObservedToken);
+
+      firstRunResult.resolve(makeSuccessResult({ finalMessage: "run-1" }));
+      await waitFor(() => runCount === 2, "second codex run did not start");
+      expect(runCount).toBe(2);
+      expect(new Set(observedTokens)).toEqual(new Set(["token-1", "token-2"]));
+
+      secondRunResult.resolve(makeSuccessResult({ finalMessage: "run-2" }));
+      const results = await Promise.all([first, second]);
+      expect(new Set(results.map((r) => r.finalMessage))).toEqual(new Set(["run-1", "run-2"]));
+      expect(process.env.POP_MCP_TOKEN).toBeUndefined();
+      expect(process.env.POP_MCP_URL_CONFIG).toBeUndefined();
+    } finally {
+      firstRunResult.resolve(makeSuccessResult());
+      secondRunResult.resolve(makeSuccessResult());
+      if (previousPopBin === undefined) {
+        delete process.env.POP_CODEX_BIN;
+      } else {
+        process.env.POP_CODEX_BIN = previousPopBin;
+      }
+      if (previousAdapterBin === undefined) {
+        delete process.env.LLM_ADAPTER_CODEX_BIN;
+      } else {
+        process.env.LLM_ADAPTER_CODEX_BIN = previousAdapterBin;
+      }
+      if (previousRealBin === undefined) {
+        delete process.env.POP_REAL_CODEX_BIN;
+      } else {
+        process.env.POP_REAL_CODEX_BIN = previousRealBin;
+      }
+      if (previousMcpUrlConfig === undefined) {
+        delete process.env.POP_MCP_URL_CONFIG;
+      } else {
+        process.env.POP_MCP_URL_CONFIG = previousMcpUrlConfig;
+      }
+      if (previousMcpToken === undefined) {
+        delete process.env.POP_MCP_TOKEN;
+      } else {
+        process.env.POP_MCP_TOKEN = previousMcpToken;
+      }
+    }
+  });
+
+  it("bridges MCP for opencode through OPENCODE_CONFIG_DIR and extracts raw cost", async () => {
+    const io = createFakeIO();
+    const previousConfigDir = process.env.OPENCODE_CONFIG_DIR;
+    const sourceConfigDir = mkdtempSync(join(tmpdir(), "opencode-existing-"));
+    let configDir = "";
+    let capturedMcpServers: unknown;
+    try {
+      writeFileSync(
+        join(sourceConfigDir, "opencode.json"),
+        JSON.stringify({
+          provider: "kept",
+          mcp: {
+            existing: { type: "local", command: ["tool"] },
+          },
+        }),
+      );
+      process.env.OPENCODE_CONFIG_DIR = sourceConfigDir;
+      const run = mock((opts: { mcpServers?: unknown; onEvent?: (event: HarnessEvent) => void }) => {
+        capturedMcpServers = opts.mcpServers;
+        configDir = process.env.OPENCODE_CONFIG_DIR ?? "";
+        const config = JSON.parse(readFileSync(join(configDir, "opencode.json"), "utf8")) as Record<string, unknown>;
+        expect(config).toMatchObject({
+          provider: "kept",
+          mcp: {
+            existing: { type: "local", command: ["tool"] },
+            popio: {
+              type: "remote",
+              url: "http://127.0.0.1:9999/mcp",
+              headers: { Authorization: "Bearer fake-token" },
+              enabled: true,
+            },
+          },
+        });
+        opts.onEvent?.({
+          harness: "opencode",
+          seq: 0,
+          at: Date.now(),
+          raw: { type: "result", info: { costUsd: 0.13 } },
+          type: "raw",
+        });
+        return makeFakeHarnessRun(makeSuccessResult());
+      });
+      const startMcpIoServer = mock(async () => createFakeMcpHandle());
+
+      const result = await executeAgent(
+        { io, entry: { name: "a", harness: "opencode", prompt: "p" } },
+        { run, startMcpIoServer },
+      );
+
+      expect(capturedMcpServers).toBeUndefined();
+      expect(result.costUsd).toBe(0.13);
+      expect(process.env.OPENCODE_CONFIG_DIR).toBe(sourceConfigDir);
+      expect(existsSync(configDir)).toBe(false);
+    } finally {
+      rmSync(sourceConfigDir, { recursive: true, force: true });
+      if (previousConfigDir === undefined) {
+        delete process.env.OPENCODE_CONFIG_DIR;
+      } else {
+        process.env.OPENCODE_CONFIG_DIR = previousConfigDir;
+      }
+    }
+  });
+
+  it("preserves usage and cost when artifact writes fail after the harness run", async () => {
+    const io = createFakeIO();
+    io.writeArtifact = async () => {
+      throw new Error("disk full");
+    };
+    const run = mock(() => makeFakeHarnessRun(makeSuccessResult({ costUsd: 0.21 })));
+    const startMcpIoServer = mock(async () => createFakeMcpHandle(["agent-output.md"]));
+
+    const result = await executeAgent(
+      { io, entry: { name: "a", harness: "claude", prompt: "p" } },
+      { run, startMcpIoServer },
+    );
+
+    expect(result.ok).toBe(false);
+    expect(result.error).toBe("disk full");
+    expect(result.usage).toEqual({ inputTokens: 100, outputTokens: 50, totalTokens: 150 });
+    expect(result.costUsd).toBe(0.21);
+    expect(result.sessionId).toBe("sess-1");
+    expect(result.artifactsWritten).toContain("agent-output.md");
+  });
 });
 
 describe("runAgentStep", () => {
@@ -293,16 +589,39 @@ describe("runAgentStep", () => {
   it("MCP server is closed on timeout path", async () => {
     const mcpHandle = createFakeMcpHandle();
     const createTaskFileIO = mock(() => createFakeIO());
+    let capturedTimeoutMs: number | undefined;
+    let capturedIdleTimeoutMs: number | undefined;
     const deps = {
-      run: mock(() => makeFakeHarnessRun(new Error('Harness "claude" timed out after 100ms'))),
+      run: mock((opts) => {
+        capturedTimeoutMs = opts.timeoutMs;
+        capturedIdleTimeoutMs = opts.idleTimeoutMs;
+        return makeFakeHarnessRun(new Error('Harness "claude" timed out after 100ms'));
+      }),
       startMcpIoServer: mock(async () => mcpHandle),
       createTaskFileIO,
     };
 
     await runAgentStep(makeArgs({ entry: { timeoutMs: 100 } }), deps);
+    expect(capturedTimeoutMs).toBe(100);
+    expect(capturedIdleTimeoutMs).toBeUndefined();
     expect(mcpHandle.closeSpy).toHaveBeenCalled();
   });
 
+  it("passes an explicit idle timeout through to the adapter", async () => {
+    let capturedIdleTimeoutMs: number | undefined;
+    const deps = {
+      run: mock((opts) => {
+        capturedIdleTimeoutMs = opts.idleTimeoutMs;
+        return makeFakeHarnessRun(makeSuccessResult());
+      }),
+      startMcpIoServer: mock(async () => createFakeMcpHandle()),
+      createTaskFileIO: mock(() => createFakeIO()),
+    };
+
+    await runAgentStep(makeArgs({ entry: { idleTimeoutMs: 250 } }), deps);
+    expect(capturedIdleTimeoutMs).toBe(250);
+  });
+
   it("does not start MCP server when io is false", async () => {
     const createTaskFileIO = mock(() => createFakeIO());
     const deps = {

package/src/core/__tests__/config.test.ts

@@ -1,5 +1,5 @@
 import { describe, test, expect, afterEach } from "bun:test";
-import { defaultConfig, resetConfig, loadConfig, getConfig, getConfigValue, getPipelineConfig } from "../config";
+import { defaultConfig, resetConfig, loadConfig, getConfig, getConfigValue, getPipelineConfig, getOrchestratorConfig } from "../config";
 import type { AppConfig } from "../config";
 import { mkdtemp, writeFile, mkdir, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
@@ -194,6 +194,36 @@ describe("PO_MAX_RUNNING_JOBS env override", () => {
   });
 });
 
+describe("PO_STALE_RUNNING_GRACE_MS env override", () => {
+  afterEach(() => {
+    delete process.env.PO_STALE_RUNNING_GRACE_MS;
+    resetConfig();
+  });
+
+  test("getOrchestratorConfig defaults staleRunningGraceMs to 30000", () => {
+    resetConfig();
+    expect(getOrchestratorConfig().staleRunningGraceMs).toBe(30000);
+  });
+
+  test("getOrchestratorConfig reads PO_STALE_RUNNING_GRACE_MS into orchestrator.staleRunningGraceMs", () => {
+    process.env.PO_STALE_RUNNING_GRACE_MS = "60000";
+    resetConfig();
+    expect(getOrchestratorConfig().staleRunningGraceMs).toBe(60000);
+  });
+
+  test("getOrchestratorConfig rejects non-positive PO_STALE_RUNNING_GRACE_MS", () => {
+    process.env.PO_STALE_RUNNING_GRACE_MS = "-1";
+    resetConfig();
+    expect(() => getOrchestratorConfig()).toThrow("orchestrator.staleRunningGraceMs");
+  });
+
+  test("getOrchestratorConfig rejects non-integer PO_STALE_RUNNING_GRACE_MS", () => {
+    process.env.PO_STALE_RUNNING_GRACE_MS = "1.5";
+    resetConfig();
+    expect(() => getOrchestratorConfig()).toThrow("orchestrator.staleRunningGraceMs");
+  });
+});
+
 describe("getConfig", () => {
   afterEach(() => {
     resetConfig();

package/src/core/__tests__/job-concurrency.test.ts

@@ -9,9 +9,11 @@ import {
   getJobConcurrencyStatus,
   listQueuedSeeds,
   pruneStaleJobSlots,
+  readJobSlotLease,
   releaseJobSlot,
   tryAcquireJobSlot,
   updateJobSlotPid,
+  withRuntimeLock,
   type JobSlotLease,
 } from "../job-concurrency";
 import { readFile } from "node:fs/promises";
@@ -552,3 +554,61 @@ describe("getJobConcurrencyStatus", () => {
     }
   });
 });
+
+describe("readJobSlotLease", () => {
+  test("returns the lease written by tryAcquireJobSlot", async () => {
+    const dir = await setupJobsDir("read-lease-acquired-");
+    try {
+      await makeCurrent(dir, "job-a");
+      const acquired = await tryAcquireJobSlot({
+        dataDir: dir,
+        jobId: "job-a",
+        maxConcurrentJobs: 2,
+        source: "orchestrator",
+        pid: process.pid,
+      });
+      expect(acquired.ok).toBe(true);
+
+      const lease = await readJobSlotLease(dir, "job-a");
+      expect(lease).not.toBeNull();
+      expect(lease!.jobId).toBe("job-a");
+      expect(lease!.pid).toBe(process.pid);
+      expect(lease!.source).toBe("orchestrator");
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("returns null for an unknown job", async () => {
+    const dir = await setupJobsDir("read-lease-unknown-");
+    try {
+      expect(await readJobSlotLease(dir, "unknown")).toBeNull();
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+});
+
+describe("withRuntimeLock", () => {
+  test("serializes two concurrent callers", async () => {
+    const dir = await setupJobsDir("runtime-lock-serialize-");
+    const markerPath = join(dir, "marker.txt");
+    const events: string[] = [];
+    try {
+      const a = withRuntimeLock(dir, 5000, async () => {
+        await new Promise((resolve) => setTimeout(resolve, 50));
+        await writeFile(markerPath, "from-a");
+        events.push("a-wrote");
+      });
+      const b = withRuntimeLock(dir, 5000, async () => {
+        const observed = await readFile(markerPath, "utf-8");
+        events.push(`b-read:${observed}`);
+      });
+      await Promise.all([a, b]);
+
+      expect(events).toEqual(["a-wrote", "b-read:from-a"]);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+});