@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

package/src/core/__tests__/task-runner.test.ts

@@ -4,7 +4,11 @@ import os from "node:os";
 import path from "node:path";
 
 import { runPipeline } from "../task-runner";
+import { deriveModelKeyAndTokens, deriveAgentUsageTuple, recordAgentUsage } from "../task-runner";
+import type { AgentStepResult } from "../agent-types";
+import { getLLMEvents } from "../../llm/index";
 import type { StatusSnapshot } from "../status-writer";
+import type { TokenUsageTuple } from "../task-runner";
 
 const tempRoots: string[] = [];
 
@@ -266,11 +270,11 @@ describe("runPipeline runAgent injection", () => {
 
     const status = JSON.parse(await readFile(path.join(workDir, "tasks-status.json"), "utf8")) as StatusSnapshot;
     expect(status.tasks["agent-task"]?.tokenUsage).toEqual([
-      ["claude:default", 12, 7, 0.03],
+      ["claude:default", 12, 7, 0.03, "reported", false, false],
     ]);
   });
 
-  it("records runAgent cost when normalized usage is absent", async () => {
+  it("records runAgent cost as unavailable when normalized usage is absent", async () => {
     const root = await makeTempRoot();
     const workDir = path.join(root, "job-agent-cost");
     await mkdir(workDir, { recursive: true });
@@ -311,7 +315,594 @@ describe("runPipeline runAgent injection", () => {
 
     const status = JSON.parse(await readFile(path.join(workDir, "tasks-status.json"), "utf8")) as StatusSnapshot;
     expect(status.tasks["agent-cost-task"]?.tokenUsage).toEqual([
-      ["opencode:gpt-4o", 0, 0, 0.12],
+      ["opencode:gpt-4o", 0, 0, 0.12, "unavailable", false, true],
     ]);
   });
 });
+
+describe("TokenUsageTuple", () => {
+  it("accepts a 5-element success tuple", () => {
+    const tuple: TokenUsageTuple = ["openai:gpt-4o", 10, 20, 0.05, "reported"];
+    expect(tuple[0]).toBe("openai:gpt-4o");
+    expect(tuple[4]).toBe("reported");
+    expect(tuple[5]).toBeUndefined();
+  });
+
+  it("accepts a 6-element failed tuple", () => {
+    const tuple: TokenUsageTuple = ["openai:gpt-4o", 10, 0, 0.01, "estimated", true];
+    expect(tuple[4]).toBe("estimated");
+    expect(tuple[5]).toBe(true);
+  });
+
+  it("accepts a 7-element tuple with costEstimated after the failed marker", () => {
+    const tuple: TokenUsageTuple = ["openai:gpt-4o", 10, 20, 0, "reported", false, true];
+    expect(tuple[4]).toBe("reported");
+    expect(tuple[5]).toBe(false);
+    expect(tuple[6]).toBe(true);
+  });
+});
+
+describe("deriveModelKeyAndTokens", () => {
+  it("reads usageSource from a complete-event metric", () => {
+    const metric = {
+      provider: "openai",
+      model: "gpt-4o",
+      metadata: { alias: "gpt-4o" },
+      promptTokens: 100,
+      completionTokens: 20,
+      totalTokens: 120,
+      cost: 0.005,
+      usageSource: "reported",
+    };
+
+    expect(deriveModelKeyAndTokens(metric)).toEqual([
+      "gpt-4o",
+      100,
+      20,
+      0.005,
+      "reported",
+      false,
+      false,
+    ]);
+  });
+
+  it("preserves reported usage with estimated cost", () => {
+    const tuple = deriveModelKeyAndTokens({
+      provider: "openai",
+      model: "new-model",
+      promptTokens: 10,
+      completionTokens: 5,
+      cost: 0,
+      usageSource: "reported",
+      costEstimated: true,
+    });
+
+    expect(tuple).toEqual([
+      "openai:new-model",
+      10,
+      5,
+      0,
+      "reported",
+      false,
+      true,
+    ]);
+  });
+
+  it("preserves estimated/unavailable/zero sources", () => {
+    for (const source of ["estimated", "unavailable", "zero"] as const) {
+      const tuple = deriveModelKeyAndTokens({
+        provider: "openai",
+        model: "gpt-4o",
+        promptTokens: 1,
+        completionTokens: 2,
+        cost: 0,
+        usageSource: source,
+      });
+      expect(tuple[4]).toBe(source);
+    }
+  });
+
+  it("falls back to reported when usageSource is absent (legacy metric)", () => {
+    const tuple = deriveModelKeyAndTokens({
+      provider: "openai",
+      model: "gpt-4o",
+      promptTokens: 1,
+      completionTokens: 2,
+      cost: 0,
+    });
+    expect(tuple[4]).toBe("reported");
+  });
+
+  it("falls back to reported when usageSource is unrecognized", () => {
+    const tuple = deriveModelKeyAndTokens({
+      provider: "openai",
+      model: "gpt-4o",
+      promptTokens: 1,
+      completionTokens: 2,
+      cost: 0,
+      usageSource: "bogus",
+    });
+    expect(tuple[4]).toBe("reported");
+  });
+});
+
+describe("usage source and failed-marker persistence", () => {
+  it("writes costEstimated after the failed marker on llm:request:complete", async () => {
+    const root = await makeTempRoot();
+    const workDir = path.join(root, "job-complete");
+    await mkdir(workDir, { recursive: true });
+    await writeFile(
+      path.join(workDir, "tasks-status.json"),
+      JSON.stringify({ id: "job-complete", tasks: {} }),
+    );
+
+    const modulePath = path.join(root, "task-noop.mjs");
+    await writeFile(
+      modulePath,
+      "export const ingestion = async ({ flags }) => ({ output: { ok: true }, flags });",
+    );
+
+    const result = await runPipeline(modulePath, {
+      workDir,
+      taskName: "task-complete",
+      statusPath: path.join(workDir, "tasks-status.json"),
+      jobId: "job-complete",
+      envLoaded: true,
+      seed: { data: {} },
+      pipelineTasks: ["task-complete"],
+      llm: {} as never,
+      tasksOverride: {
+        ingestion: async ({ flags }) => {
+          getLLMEvents().emit("llm:request:complete", {
+            id: "evt-1",
+            provider: "openai",
+            model: "gpt-4o",
+            metadata: { alias: "gpt-4o" },
+            timestamp: new Date().toISOString(),
+            duration: 50,
+            promptTokens: 100,
+            completionTokens: 20,
+            totalTokens: 120,
+            cost: 0.005,
+            usageSource: "reported",
+            costEstimated: false,
+          });
+          return { output: { ok: true }, flags };
+        },
+      },
+    });
+
+    expect(result.ok).toBe(true);
+
+    const status = JSON.parse(
+      await readFile(path.join(workDir, "tasks-status.json"), "utf8"),
+    ) as StatusSnapshot;
+
+    expect(status.tasks["task-complete"]?.tokenUsage).toEqual([
+      ["gpt-4o", 100, 20, 0.005, "reported", false, false],
+    ]);
+  });
+
+  it("writes failed=true and costEstimated on llm:request:error", async () => {
+    const root = await makeTempRoot();
+    const workDir = path.join(root, "job-error");
+    await mkdir(workDir, { recursive: true });
+    await writeFile(
+      path.join(workDir, "tasks-status.json"),
+      JSON.stringify({ id: "job-error", tasks: {} }),
+    );
+
+    const modulePath = path.join(root, "task-noop.mjs");
+    await writeFile(
+      modulePath,
+      "export const ingestion = async ({ flags }) => ({ output: { ok: true }, flags });",
+    );
+
+    const result = await runPipeline(modulePath, {
+      workDir,
+      taskName: "task-error",
+      statusPath: path.join(workDir, "tasks-status.json"),
+      jobId: "job-error",
+      envLoaded: true,
+      seed: { data: {} },
+      pipelineTasks: ["task-error"],
+      llm: {} as never,
+      tasksOverride: {
+        ingestion: async ({ flags }) => {
+          getLLMEvents().emit("llm:request:error", {
+            id: "evt-err",
+            provider: "openai",
+            model: "gpt-4o",
+            metadata: { alias: "gpt-4o" },
+            timestamp: new Date().toISOString(),
+            duration: 12,
+            error: "boom",
+            promptTokens: 80,
+            completionTokens: 0,
+            totalTokens: 80,
+            cost: 0.002,
+            usageSource: "estimated",
+            costEstimated: true,
+          });
+          return { output: { ok: true }, flags };
+        },
+      },
+    });
+
+    expect(result.ok).toBe(true);
+
+    const status = JSON.parse(
+      await readFile(path.join(workDir, "tasks-status.json"), "utf8"),
+    ) as StatusSnapshot;
+
+    expect(status.tasks["task-error"]?.tokenUsage).toEqual([
+      ["gpt-4o", 80, 0, 0.002, "estimated", true, true],
+    ]);
+  });
+
+  it("preserves a legacy 4-element tokenUsage tuple across a run", async () => {
+    const root = await makeTempRoot();
+    const workDir = path.join(root, "job-legacy");
+    await mkdir(workDir, { recursive: true });
+    const statusPath = path.join(workDir, "tasks-status.json");
+    await writeFile(
+      statusPath,
+      JSON.stringify({
+        id: "job-legacy",
+        tasks: {
+          "task-legacy": {
+            tokenUsage: [["openai:gpt-3.5", 5, 5, 0.0001]],
+          },
+        },
+      }),
+    );
+
+    const modulePath = path.join(root, "task-legacy.mjs");
+    await writeFile(
+      modulePath,
+      "export const ingestion = async ({ flags }) => ({ output: { ok: true }, flags });",
+    );
+
+    const result = await runPipeline(modulePath, {
+      workDir,
+      taskName: "task-legacy",
+      statusPath,
+      jobId: "job-legacy",
+      envLoaded: true,
+      seed: { data: {} },
+      pipelineTasks: ["task-legacy"],
+      llm: {} as never,
+      tasksOverride: {
+        ingestion: async ({ flags }) => {
+          getLLMEvents().emit("llm:request:complete", {
+            id: "evt-new",
+            provider: "openai",
+            model: "gpt-4o",
+            metadata: { alias: "gpt-4o" },
+            timestamp: new Date().toISOString(),
+            duration: 1,
+            promptTokens: 10,
+            completionTokens: 4,
+            totalTokens: 14,
+            cost: 0.001,
+            usageSource: "reported",
+            costEstimated: false,
+          });
+          return { output: { ok: true }, flags };
+        },
+      },
+    });
+
+    expect(result.ok).toBe(true);
+
+    const status = JSON.parse(
+      await readFile(statusPath, "utf8"),
+    ) as StatusSnapshot;
+
+    expect(status.tasks["task-legacy"]?.tokenUsage).toEqual([
+      ["openai:gpt-3.5", 5, 5, 0.0001],
+      ["gpt-4o", 10, 4, 0.001, "reported", false, false],
+    ]);
+  });
+});
+
+describe("deriveAgentUsageTuple", () => {
+  const baseResult: AgentStepResult = {
+    ok: true,
+    finalMessage: "",
+    artifactsWritten: [],
+  };
+
+  it("classifies a present usage object as reported", () => {
+    const result: AgentStepResult = {
+      ...baseResult,
+      usage: { inputTokens: 100, outputTokens: 40, totalTokens: 140 },
+      costUsd: 0.0123,
+    };
+
+    expect(deriveAgentUsageTuple({ harness: "claude", model: "opus" }, result)).toEqual([
+      "claude:opus",
+      100,
+      40,
+      0.0123,
+      "reported",
+      false,
+      false,
+    ]);
+  });
+
+  it("classifies a zero-token usage object as reported", () => {
+    const result: AgentStepResult = {
+      ...baseResult,
+      usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },
+    };
+
+    expect(deriveAgentUsageTuple({ harness: "codex" }, result)).toEqual([
+      "codex:default",
+      0,
+      0,
+      0,
+      "reported",
+      false,
+      false,
+    ]);
+  });
+
+  it("classifies cost-only result as unavailable with tokens zeroed and cost preserved", () => {
+    const result: AgentStepResult = { ...baseResult, costUsd: 0.5 };
+
+    expect(deriveAgentUsageTuple({ harness: "opencode", model: "gpt-4o" }, result)).toEqual([
+      "opencode:gpt-4o",
+      0,
+      0,
+      0.5,
+      "unavailable",
+      false,
+      true,
+    ]);
+  });
+
+  it("returns null when neither usage nor costUsd is present", () => {
+    expect(deriveAgentUsageTuple({ harness: "claude" }, baseResult)).toBeNull();
+  });
+
+  it("returns null when costUsd is non-finite and no usage is present", () => {
+    const result: AgentStepResult = { ...baseResult, costUsd: Number.NaN };
+
+    expect(deriveAgentUsageTuple({ harness: "claude" }, result)).toBeNull();
+  });
+
+  it("sets the failed flag when opts.failed is requested", () => {
+    const usageResult: AgentStepResult = {
+      ...baseResult,
+      ok: false,
+      usage: { inputTokens: 5, outputTokens: 1, totalTokens: 6 },
+    };
+    const costResult: AgentStepResult = { ...baseResult, ok: false, costUsd: 0.2 };
+
+    expect(
+      deriveAgentUsageTuple({ harness: "claude" }, usageResult, { failed: true })?.[5],
+    ).toBe(true);
+    expect(
+      deriveAgentUsageTuple({ harness: "claude" }, costResult, { failed: true })?.[5],
+    ).toBe(true);
+  });
+});
+
+describe("recordAgentUsage", () => {
+  const baseResult: AgentStepResult = {
+    ok: true,
+    finalMessage: "",
+    artifactsWritten: [],
+  };
+
+  it("pushes the derived tuple exactly once when non-null", () => {
+    const result: AgentStepResult = {
+      ...baseResult,
+      usage: { inputTokens: 100, outputTokens: 40, totalTokens: 140 },
+      costUsd: 0.0123,
+    };
+    const pushed: TokenUsageTuple[] = [];
+
+    recordAgentUsage({ harness: "claude", model: "opus" }, result, (t) => pushed.push(t));
+
+    expect(pushed).toEqual([["claude:opus", 100, 40, 0.0123, "reported", false, false]]);
+  });
+
+  it("does not push when the derived tuple is null", () => {
+    const pushed: TokenUsageTuple[] = [];
+
+    recordAgentUsage({ harness: "claude" }, baseResult, (t) => pushed.push(t));
+
+    expect(pushed).toEqual([]);
+  });
+});
+
+describe("runPipeline console capture lifecycle", () => {
+  async function makeJob(): Promise<{ workDir: string; statusPath: string }> {
+    const root = await makeTempRoot();
+    const workDir = path.join(root, "job-console");
+    await mkdir(workDir, { recursive: true });
+    const statusPath = path.join(workDir, "tasks-status.json");
+    await writeFile(statusPath, JSON.stringify({ id: "job-console", tasks: {} }));
+    return { workDir, statusPath };
+  }
+
+  function run(modulePath: string, workDir: string, statusPath: string) {
+    return runPipeline(modulePath, {
+      workDir,
+      taskName: "research",
+      statusPath,
+      jobId: "job-console",
+      envLoaded: true,
+      seed: { data: { topic: "x" } },
+      pipelineTasks: ["research"],
+      llm: {} as never,
+    });
+  }
+
+  const succeeds =
+    "export const ingestion = async ({ flags }) => ({ output: { ok: true }, flags });";
+  const stageThrows = 'export const ingestion = async () => { throw new Error("boom"); };';
+
+  it("restores console methods and listener counts after a successful run", async () => {
+    const events = getLLMEvents();
+    const baseComplete = events.listenerCount("llm:request:complete");
+    const baseError = events.listenerCount("llm:request:error");
+
+    const origLog = console.log;
+    const origError = console.error;
+    const origWarn = console.warn;
+    const origInfo = console.info;
+    const origDebug = console.debug;
+
+    const { workDir, statusPath } = await makeJob();
+    const modulePath = path.join(workDir, "mod.mjs");
+    await writeFile(modulePath, succeeds);
+
+    const result = await run(modulePath, workDir, statusPath);
+    expect(result.ok).toBe(true);
+
+    expect(console.log).toBe(origLog);
+    expect(console.error).toBe(origError);
+    expect(console.warn).toBe(origWarn);
+    expect(console.info).toBe(origInfo);
+    expect(console.debug).toBe(origDebug);
+
+    expect(events.listenerCount("llm:request:complete")).toBe(baseComplete);
+    expect(events.listenerCount("llm:request:error")).toBe(baseError);
+  });
+
+  it("restores console methods and listener counts after a throwing stage", async () => {
+    const events = getLLMEvents();
+    const baseComplete = events.listenerCount("llm:request:complete");
+    const baseError = events.listenerCount("llm:request:error");
+
+    const origLog = console.log;
+    const origError = console.error;
+    const origWarn = console.warn;
+    const origInfo = console.info;
+    const origDebug = console.debug;
+
+    const { workDir, statusPath } = await makeJob();
+    const modulePath = path.join(workDir, "mod.mjs");
+    await writeFile(modulePath, stageThrows);
+
+    const result = await run(modulePath, workDir, statusPath);
+    expect(result.ok).toBe(false);
+
+    expect(console.log).toBe(origLog);
+    expect(console.error).toBe(origError);
+    expect(console.warn).toBe(origWarn);
+    expect(console.info).toBe(origInfo);
+    expect(console.debug).toBe(origDebug);
+
+    expect(events.listenerCount("llm:request:complete")).toBe(baseComplete);
+    expect(events.listenerCount("llm:request:error")).toBe(baseError);
+  });
+
+  it("preserves the stage failure return shape and registers the stage log", async () => {
+    const { workDir, statusPath } = await makeJob();
+    const modulePath = path.join(workDir, "mod.mjs");
+    await writeFile(modulePath, stageThrows);
+
+    const result = await run(modulePath, workDir, statusPath);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.failedStage).toBe("ingestion");
+      expect(result.error.message).toBe("boom");
+      expect(result.error.debug.stage).toBe("ingestion");
+    }
+
+    const status = JSON.parse(await readFile(statusPath, "utf8")) as {
+      tasks?: Record<string, { files?: { logs?: string[] } }>;
+    };
+    expect(status.tasks?.["research"]?.files?.logs).toContain("research-ingestion-start.log");
+  });
+});
+
+describe("runPipeline LLM listener lifecycle", () => {
+  async function makeJob(): Promise<{ workDir: string; statusPath: string }> {
+    const root = await makeTempRoot();
+    const workDir = path.join(root, "job-lifecycle");
+    await mkdir(workDir, { recursive: true });
+    await writeFile(path.join(workDir, "seed.json"), JSON.stringify({ topic: "x" }));
+    const statusPath = path.join(workDir, "tasks-status.json");
+    await writeFile(statusPath, JSON.stringify({ id: "job-lifecycle", tasks: {} }));
+    return { workDir, statusPath };
+  }
+
+  async function writeModule(workDir: string, body: string): Promise<string> {
+    const modulePath = path.join(workDir, `mod-${Math.random().toString(36).slice(2)}.mjs`);
+    await writeFile(modulePath, body);
+    return modulePath;
+  }
+
+  const succeeds = "export const ingestion = async ({ flags }) => ({ output: { ok: true }, flags });";
+  const stageThrows = 'export const ingestion = async () => { throw new Error("boom"); };';
+  // A syntax error makes loadFreshModule reject before any stage runs.
+  const loadThrows = "export const ingestion = async ({ flags }) => {{{ ";
+
+  function run(modulePath: string, workDir: string, statusPath: string) {
+    return runPipeline(modulePath, {
+      workDir,
+      taskName: "research",
+      statusPath,
+      jobId: "job-lifecycle",
+      envLoaded: true,
+      seed: { data: { topic: "x" } },
+      pipelineTasks: ["research"],
+      llm: {} as never,
+    });
+  }
+
+  it("leaves listener counts at baseline when module load throws", async () => {
+    const events = getLLMEvents();
+    const baseComplete = events.listenerCount("llm:request:complete");
+    const baseError = events.listenerCount("llm:request:error");
+
+    const { workDir, statusPath } = await makeJob();
+    const modulePath = await writeModule(workDir, loadThrows);
+
+    await expect(run(modulePath, workDir, statusPath)).rejects.toThrow();
+
+    expect(events.listenerCount("llm:request:complete")).toBe(baseComplete);
+    expect(events.listenerCount("llm:request:error")).toBe(baseError);
+  });
+
+  it("does not grow listener counts or warn across repeated runs including throwing ones", async () => {
+    const events = getLLMEvents();
+    const baseComplete = events.listenerCount("llm:request:complete");
+    const baseError = events.listenerCount("llm:request:error");
+
+    const warnings: string[] = [];
+    const onWarning = (w: Error) => warnings.push(w.name);
+    process.on("warning", onWarning);
+
+    try {
+      for (let i = 0; i < 12; i++) {
+        const { workDir, statusPath } = await makeJob();
+        if (i % 3 === 0) {
+          const modulePath = await writeModule(workDir, loadThrows);
+          await expect(run(modulePath, workDir, statusPath)).rejects.toThrow();
+        } else if (i % 3 === 1) {
+          const modulePath = await writeModule(workDir, stageThrows);
+          const result = await run(modulePath, workDir, statusPath);
+          expect(result.ok).toBe(false);
+        } else {
+          const modulePath = await writeModule(workDir, succeeds);
+          const result = await run(modulePath, workDir, statusPath);
+          expect(result.ok).toBe(true);
+        }
+
+        expect(events.listenerCount("llm:request:complete")).toBe(baseComplete);
+        expect(events.listenerCount("llm:request:error")).toBe(baseError);
+      }
+    } finally {
+      process.removeListener("warning", onWarning);
+    }
+
+    expect(warnings).not.toContain("MaxListenersExceededWarning");
+  });
+});