@ryanfw/prompt-orchestration-pipeline 1.3.1 → 1.3.2

package/docs/pop-task-guide.md

@@ -207,6 +207,7 @@ export const inference = async ({ runAgent, io, data, flags }) => {
     // model?: string         // optional, passed through to the CLI
     // io?: boolean           // default true: bridge POP read/write artifacts
     // timeoutMs?: number     // optional wall-clock cap
+    // idleTimeoutMs?: number // optional no-output cap
     // captureDiff?: boolean  // capture a git diff as 'agent.patch'
   });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ryanfw/prompt-orchestration-pipeline",
-  "version": "1.3.1",
+  "version": "1.3.2",
   "description": "A Prompt-orchestration pipeline (POP) is a framework for building, running, and experimenting with complex chains of LLM tasks.",
   "type": "module",
   "main": "src/ui/server/index.ts",
@@ -64,7 +64,6 @@
     "react": "^19.2.0",
     "react-dom": "^19.2.0",
     "react-markdown": "^10.1.0",
-    "react-mentions": "^4.4.10",
     "react-router-dom": "^7.9.4",
     "react-syntax-highlighter": "^15.6.1",
     "rehype-highlight": "^7.0.2",

package/src/core/__tests__/agent-step.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, mock, beforeEach, afterEach } from "bun:test";
-import { mkdtempSync, rmSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { existsSync, mkdtempSync, rmSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
 import type { WriteOptions, TaskFileIO } from "../file-io.ts";
@@ -53,10 +53,11 @@ function createFakeIO(): TaskFileIO & { calls: string[] } {
 
 function createFakeMcpHandle(
   artifacts: string[] = [],
+  connection = { url: "http://127.0.0.1:9999/mcp", token: "fake-token" },
 ): McpIoServerHandle & { closeSpy: ReturnType<typeof mock> } {
   const closeSpy = mock(async () => {});
   return {
-    connection: { url: "http://127.0.0.1:9999/mcp", token: "fake-token" },
+    connection,
     artifactsWritten() {
       return [...artifacts];
     },
@@ -106,6 +107,26 @@ function makeSuccessResult(overrides?: Partial<RunResult>): RunResult {
   };
 }
 
+function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void; reject: (err: unknown) => void } {
+  let resolve!: (value: T) => void;
+  let reject!: (err: unknown) => void;
+  const promise = new Promise<T>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+  return { promise, resolve, reject };
+}
+
+async function waitFor(predicate: () => boolean, message: string): Promise<void> {
+  const deadline = Date.now() + 1_000;
+  while (!predicate()) {
+    if (Date.now() > deadline) {
+      throw new Error(message);
+    }
+    await new Promise((resolve) => setTimeout(resolve, 1));
+  }
+}
+
 function makeDeps(result: RunResult | Error) {
   const run = mock(() => makeFakeHarnessRun(result));
   const startMcpIoServer = mock(async () => createFakeMcpHandle());
@@ -178,6 +199,281 @@ describe("executeAgent", () => {
 
     expect(startMcpIoServer).not.toHaveBeenCalled();
   });
+
+  it("passes adapter-native MCP servers for claude", async () => {
+    const io = createFakeIO();
+    let capturedMcpServers: unknown;
+    const run = mock((opts: { mcpServers?: unknown }) => {
+      capturedMcpServers = opts.mcpServers;
+      return makeFakeHarnessRun(makeSuccessResult());
+    });
+    const startMcpIoServer = mock(async () => createFakeMcpHandle());
+
+    await executeAgent(
+      { io, entry: { name: "a", harness: "claude", prompt: "p" } },
+      { run, startMcpIoServer },
+    );
+
+    expect(capturedMcpServers).toEqual([{
+      name: "popio",
+      url: "http://127.0.0.1:9999/mcp",
+      headers: { Authorization: "Bearer fake-token" },
+    }]);
+  });
+
+  it("bridges MCP for codex through a temporary binary wrapper", async () => {
+    const io = createFakeIO();
+    const previousPopBin = process.env.POP_CODEX_BIN;
+    const previousAdapterBin = process.env.LLM_ADAPTER_CODEX_BIN;
+    const previousRealBin = process.env.POP_REAL_CODEX_BIN;
+    const previousMcpUrlConfig = process.env.POP_MCP_URL_CONFIG;
+    const previousMcpToken = process.env.POP_MCP_TOKEN;
+    let wrapperPath = "";
+    let capturedMcpServers: unknown;
+    try {
+      process.env.POP_CODEX_BIN = "/custom/codex";
+      delete process.env.LLM_ADAPTER_CODEX_BIN;
+      delete process.env.POP_REAL_CODEX_BIN;
+      delete process.env.POP_MCP_URL_CONFIG;
+      delete process.env.POP_MCP_TOKEN;
+      const run = mock((opts: { mcpServers?: unknown; onEvent?: (event: HarnessEvent) => void }) => {
+        capturedMcpServers = opts.mcpServers;
+        wrapperPath = process.env.LLM_ADAPTER_CODEX_BIN ?? "";
+        expect(wrapperPath).toContain("codex-mcp-");
+        expect(readFileSync(wrapperPath, "utf8")).toContain("POP_MCP_URL_CONFIG");
+        expect(process.env.POP_REAL_CODEX_BIN).toBe("/custom/codex");
+        expect(process.env.POP_MCP_URL_CONFIG).toBe(
+          'mcp_servers.popio.url="http://127.0.0.1:9999/mcp"',
+        );
+        expect(process.env.POP_MCP_TOKEN).toBe("fake-token");
+        opts.onEvent?.({
+          harness: "codex",
+          seq: 0,
+          at: Date.now(),
+          raw: { type: "result", total_cost_usd: 0.42 },
+          type: "raw",
+        });
+        return makeFakeHarnessRun(makeSuccessResult());
+      });
+      const startMcpIoServer = mock(async () => createFakeMcpHandle());
+
+      const result = await executeAgent(
+        { io, entry: { name: "a", harness: "codex", prompt: "p" } },
+        { run, startMcpIoServer },
+      );
+
+      expect(capturedMcpServers).toBeUndefined();
+      expect(result.costUsd).toBe(0.42);
+      expect(process.env.LLM_ADAPTER_CODEX_BIN).toBeUndefined();
+      expect(process.env.POP_REAL_CODEX_BIN).toBeUndefined();
+      expect(process.env.POP_MCP_URL_CONFIG).toBeUndefined();
+      expect(process.env.POP_MCP_TOKEN).toBeUndefined();
+      expect(existsSync(wrapperPath)).toBe(false);
+    } finally {
+      if (previousPopBin === undefined) {
+        delete process.env.POP_CODEX_BIN;
+      } else {
+        process.env.POP_CODEX_BIN = previousPopBin;
+      }
+      if (previousAdapterBin === undefined) {
+        delete process.env.LLM_ADAPTER_CODEX_BIN;
+      } else {
+        process.env.LLM_ADAPTER_CODEX_BIN = previousAdapterBin;
+      }
+      if (previousRealBin === undefined) {
+        delete process.env.POP_REAL_CODEX_BIN;
+      } else {
+        process.env.POP_REAL_CODEX_BIN = previousRealBin;
+      }
+      if (previousMcpUrlConfig === undefined) {
+        delete process.env.POP_MCP_URL_CONFIG;
+      } else {
+        process.env.POP_MCP_URL_CONFIG = previousMcpUrlConfig;
+      }
+      if (previousMcpToken === undefined) {
+        delete process.env.POP_MCP_TOKEN;
+      } else {
+        process.env.POP_MCP_TOKEN = previousMcpToken;
+      }
+    }
+  });
+
+  it("serializes env-backed MCP compatibility for concurrent codex runs", async () => {
+    const previousPopBin = process.env.POP_CODEX_BIN;
+    const previousAdapterBin = process.env.LLM_ADAPTER_CODEX_BIN;
+    const previousRealBin = process.env.POP_REAL_CODEX_BIN;
+    const previousMcpUrlConfig = process.env.POP_MCP_URL_CONFIG;
+    const previousMcpToken = process.env.POP_MCP_TOKEN;
+    const firstRunResult = deferred<RunResult>();
+    const secondRunResult = deferred<RunResult>();
+    const observedTokens: Array<string | undefined> = [];
+    let runCount = 0;
+    let startCount = 0;
+    try {
+      process.env.POP_CODEX_BIN = "/custom/codex";
+      delete process.env.LLM_ADAPTER_CODEX_BIN;
+      delete process.env.POP_REAL_CODEX_BIN;
+      delete process.env.POP_MCP_URL_CONFIG;
+      delete process.env.POP_MCP_TOKEN;
+      const run = mock(() => {
+        runCount += 1;
+        observedTokens.push(process.env.POP_MCP_TOKEN);
+        return {
+          result: runCount === 1 ? firstRunResult.promise : secondRunResult.promise,
+          sessionId: Promise.resolve(`sess-${runCount}`),
+          abort() {},
+        };
+      });
+      const startMcpIoServer = mock(async () => {
+        startCount += 1;
+        return createFakeMcpHandle([], {
+          url: `http://127.0.0.1:999${startCount}/mcp`,
+          token: `token-${startCount}`,
+        });
+      });
+
+      const first = executeAgent(
+        { io: createFakeIO(), entry: { name: "a", harness: "codex", prompt: "p" } },
+        { run, startMcpIoServer },
+      );
+      const second = executeAgent(
+        { io: createFakeIO(), entry: { name: "b", harness: "codex", prompt: "p" } },
+        { run, startMcpIoServer },
+      );
+
+      await waitFor(() => runCount === 1, "first codex run did not start");
+      expect(runCount).toBe(1);
+      expect(observedTokens).toHaveLength(1);
+      const firstObservedToken = observedTokens[0];
+      if (firstObservedToken === undefined) {
+        throw new Error("first codex run did not receive an MCP token");
+      }
+      expect(["token-1", "token-2"]).toContain(firstObservedToken);
+
+      firstRunResult.resolve(makeSuccessResult({ finalMessage: "run-1" }));
+      await waitFor(() => runCount === 2, "second codex run did not start");
+      expect(runCount).toBe(2);
+      expect(new Set(observedTokens)).toEqual(new Set(["token-1", "token-2"]));
+
+      secondRunResult.resolve(makeSuccessResult({ finalMessage: "run-2" }));
+      const results = await Promise.all([first, second]);
+      expect(new Set(results.map((r) => r.finalMessage))).toEqual(new Set(["run-1", "run-2"]));
+      expect(process.env.POP_MCP_TOKEN).toBeUndefined();
+      expect(process.env.POP_MCP_URL_CONFIG).toBeUndefined();
+    } finally {
+      firstRunResult.resolve(makeSuccessResult());
+      secondRunResult.resolve(makeSuccessResult());
+      if (previousPopBin === undefined) {
+        delete process.env.POP_CODEX_BIN;
+      } else {
+        process.env.POP_CODEX_BIN = previousPopBin;
+      }
+      if (previousAdapterBin === undefined) {
+        delete process.env.LLM_ADAPTER_CODEX_BIN;
+      } else {
+        process.env.LLM_ADAPTER_CODEX_BIN = previousAdapterBin;
+      }
+      if (previousRealBin === undefined) {
+        delete process.env.POP_REAL_CODEX_BIN;
+      } else {
+        process.env.POP_REAL_CODEX_BIN = previousRealBin;
+      }
+      if (previousMcpUrlConfig === undefined) {
+        delete process.env.POP_MCP_URL_CONFIG;
+      } else {
+        process.env.POP_MCP_URL_CONFIG = previousMcpUrlConfig;
+      }
+      if (previousMcpToken === undefined) {
+        delete process.env.POP_MCP_TOKEN;
+      } else {
+        process.env.POP_MCP_TOKEN = previousMcpToken;
+      }
+    }
+  });
+
+  it("bridges MCP for opencode through OPENCODE_CONFIG_DIR and extracts raw cost", async () => {
+    const io = createFakeIO();
+    const previousConfigDir = process.env.OPENCODE_CONFIG_DIR;
+    const sourceConfigDir = mkdtempSync(join(tmpdir(), "opencode-existing-"));
+    let configDir = "";
+    let capturedMcpServers: unknown;
+    try {
+      writeFileSync(
+        join(sourceConfigDir, "opencode.json"),
+        JSON.stringify({
+          provider: "kept",
+          mcp: {
+            existing: { type: "local", command: ["tool"] },
+          },
+        }),
+      );
+      process.env.OPENCODE_CONFIG_DIR = sourceConfigDir;
+      const run = mock((opts: { mcpServers?: unknown; onEvent?: (event: HarnessEvent) => void }) => {
+        capturedMcpServers = opts.mcpServers;
+        configDir = process.env.OPENCODE_CONFIG_DIR ?? "";
+        const config = JSON.parse(readFileSync(join(configDir, "opencode.json"), "utf8")) as Record<string, unknown>;
+        expect(config).toMatchObject({
+          provider: "kept",
+          mcp: {
+            existing: { type: "local", command: ["tool"] },
+            popio: {
+              type: "remote",
+              url: "http://127.0.0.1:9999/mcp",
+              headers: { Authorization: "Bearer fake-token" },
+              enabled: true,
+            },
+          },
+        });
+        opts.onEvent?.({
+          harness: "opencode",
+          seq: 0,
+          at: Date.now(),
+          raw: { type: "result", info: { costUsd: 0.13 } },
+          type: "raw",
+        });
+        return makeFakeHarnessRun(makeSuccessResult());
+      });
+      const startMcpIoServer = mock(async () => createFakeMcpHandle());
+
+      const result = await executeAgent(
+        { io, entry: { name: "a", harness: "opencode", prompt: "p" } },
+        { run, startMcpIoServer },
+      );
+
+      expect(capturedMcpServers).toBeUndefined();
+      expect(result.costUsd).toBe(0.13);
+      expect(process.env.OPENCODE_CONFIG_DIR).toBe(sourceConfigDir);
+      expect(existsSync(configDir)).toBe(false);
+    } finally {
+      rmSync(sourceConfigDir, { recursive: true, force: true });
+      if (previousConfigDir === undefined) {
+        delete process.env.OPENCODE_CONFIG_DIR;
+      } else {
+        process.env.OPENCODE_CONFIG_DIR = previousConfigDir;
+      }
+    }
+  });
+
+  it("preserves usage and cost when artifact writes fail after the harness run", async () => {
+    const io = createFakeIO();
+    io.writeArtifact = async () => {
+      throw new Error("disk full");
+    };
+    const run = mock(() => makeFakeHarnessRun(makeSuccessResult({ costUsd: 0.21 })));
+    const startMcpIoServer = mock(async () => createFakeMcpHandle(["agent-output.md"]));
+
+    const result = await executeAgent(
+      { io, entry: { name: "a", harness: "claude", prompt: "p" } },
+      { run, startMcpIoServer },
+    );
+
+    expect(result.ok).toBe(false);
+    expect(result.error).toBe("disk full");
+    expect(result.usage).toEqual({ inputTokens: 100, outputTokens: 50, totalTokens: 150 });
+    expect(result.costUsd).toBe(0.21);
+    expect(result.sessionId).toBe("sess-1");
+    expect(result.artifactsWritten).toContain("agent-output.md");
+  });
 });
 
 describe("runAgentStep", () => {
@@ -293,16 +589,39 @@ describe("runAgentStep", () => {
   it("MCP server is closed on timeout path", async () => {
     const mcpHandle = createFakeMcpHandle();
     const createTaskFileIO = mock(() => createFakeIO());
+    let capturedTimeoutMs: number | undefined;
+    let capturedIdleTimeoutMs: number | undefined;
     const deps = {
-      run: mock(() => makeFakeHarnessRun(new Error('Harness "claude" timed out after 100ms'))),
+      run: mock((opts) => {
+        capturedTimeoutMs = opts.timeoutMs;
+        capturedIdleTimeoutMs = opts.idleTimeoutMs;
+        return makeFakeHarnessRun(new Error('Harness "claude" timed out after 100ms'));
+      }),
       startMcpIoServer: mock(async () => mcpHandle),
       createTaskFileIO,
     };
 
     await runAgentStep(makeArgs({ entry: { timeoutMs: 100 } }), deps);
+    expect(capturedTimeoutMs).toBe(100);
+    expect(capturedIdleTimeoutMs).toBeUndefined();
     expect(mcpHandle.closeSpy).toHaveBeenCalled();
   });
 
+  it("passes an explicit idle timeout through to the adapter", async () => {
+    let capturedIdleTimeoutMs: number | undefined;
+    const deps = {
+      run: mock((opts) => {
+        capturedIdleTimeoutMs = opts.idleTimeoutMs;
+        return makeFakeHarnessRun(makeSuccessResult());
+      }),
+      startMcpIoServer: mock(async () => createFakeMcpHandle()),
+      createTaskFileIO: mock(() => createFakeIO()),
+    };
+
+    await runAgentStep(makeArgs({ entry: { idleTimeoutMs: 250 } }), deps);
+    expect(capturedIdleTimeoutMs).toBe(250);
+  });
+
   it("does not start MCP server when io is false", async () => {
     const createTaskFileIO = mock(() => createFakeIO());
     const deps = {

package/src/core/__tests__/task-runner.test.ts

@@ -245,7 +245,13 @@ describe("runPipeline runAgent injection", () => {
       llm: {} as never,
       runAgent: async (options) => {
         calls.push(options);
-        return { ok: true, finalMessage: "did it", artifactsWritten: ["x.md"] };
+        return {
+          ok: true,
+          finalMessage: "did it",
+          artifactsWritten: ["x.md"],
+          usage: { inputTokens: 12, outputTokens: 7, totalTokens: 19 },
+          costUsd: 0.03,
+        };
       },
     });
 
@@ -257,5 +263,55 @@ describe("runPipeline runAgent injection", () => {
         finalMessage: "did it",
       });
     }
+
+    const status = JSON.parse(await readFile(path.join(workDir, "tasks-status.json"), "utf8")) as StatusSnapshot;
+    expect(status.tasks["agent-task"]?.tokenUsage).toEqual([
+      ["claude:default", 12, 7, 0.03],
+    ]);
+  });
+
+  it("records runAgent cost when normalized usage is absent", async () => {
+    const root = await makeTempRoot();
+    const workDir = path.join(root, "job-agent-cost");
+    await mkdir(workDir, { recursive: true });
+    await writeFile(
+      path.join(workDir, "tasks-status.json"),
+      JSON.stringify({ id: "job-agent-cost", tasks: {} }),
+    );
+
+    const modulePath = path.join(root, "agent-cost-task.mjs");
+    await writeFile(
+      modulePath,
+      [
+        "export const ingestion = async ({ runAgent, flags }) => {",
+        "  const r = await runAgent({ harness: 'opencode', model: 'gpt-4o', prompt: 'hello agent' });",
+        "  return { output: r, flags };",
+        "};",
+      ].join("\n"),
+    );
+
+    const result = await runPipeline(modulePath, {
+      workDir,
+      taskName: "agent-cost-task",
+      statusPath: path.join(workDir, "tasks-status.json"),
+      jobId: "job-agent-cost",
+      envLoaded: true,
+      seed: { data: {} },
+      pipelineTasks: ["agent-cost-task"],
+      llm: {} as never,
+      runAgent: async () => ({
+        ok: true,
+        finalMessage: "did it",
+        artifactsWritten: [],
+        costUsd: 0.12,
+      }),
+    });
+
+    expect(result.ok).toBe(true);
+
+    const status = JSON.parse(await readFile(path.join(workDir, "tasks-status.json"), "utf8")) as StatusSnapshot;
+    expect(status.tasks["agent-cost-task"]?.tokenUsage).toEqual([
+      ["opencode:gpt-4o", 0, 0, 0.12],
+    ]);
   });
 });