@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

package/src/providers/__tests__/openai.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
-import { ProviderJsonModeError, ProviderJsonParseError } from "../types.ts";
+import { ProviderJsonParseError } from "../types.ts";
 import type { OpenAIOptions } from "../types.ts";
 
 // Use var to avoid TDZ — vi.mock is hoisted above let/const declarations
@@ -27,7 +27,7 @@ import { openaiChat, _resetClient } from "../openai.ts";
  */
 function makeStreamingChatCompletion(
   textChunks: string[],
-  usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number },
+  usage?: Partial<{ prompt_tokens: number; completion_tokens: number; total_tokens: number }>,
 ): AsyncIterable<Record<string, unknown>> {
   const events: Record<string, unknown>[] = [];
 
@@ -53,11 +53,12 @@ function makeStreamingChatCompletion(
 
 /**
  * Creates an async iterable that yields streaming Responses API events.
+ * Pass `null` for `usage` to omit the usage block entirely; pass a partial
+ * object to simulate partial provider reporting.
  */
 function makeStreamingResponsesResult(
   textChunks: string[],
-  inputTokens = 10,
-  outputTokens = 20,
+  usage?: { input_tokens?: number; output_tokens?: number; total_tokens?: number } | null,
 ): AsyncIterable<Record<string, unknown>> {
   const events: Record<string, unknown>[] = [];
 
@@ -68,14 +69,19 @@ function makeStreamingResponsesResult(
     });
   }
 
+  const resolvedUsage =
+    usage === undefined
+      ? { input_tokens: 10, output_tokens: 20, total_tokens: 30 }
+      : usage;
+
+  const response: Record<string, unknown> = {};
+  if (resolvedUsage !== null) {
+    response["usage"] = resolvedUsage;
+  }
+
   events.push({
     type: "response.completed",
-    response: {
-      usage: {
-        input_tokens: inputTokens,
-        output_tokens: outputTokens,
-      },
-    },
+    response,
   });
 
   return {
@@ -224,19 +230,39 @@ describe("openaiChat", () => {
     expect(mockChatCompletionsCreate).toHaveBeenCalledTimes(1);
   });
 
-  it("throws ProviderJsonModeError when responseFormat is invalid", async () => {
-    await expect(
-      openaiChat({
-        ...baseOptions,
-        responseFormat: "text",
-      }),
-    ).rejects.toThrow(ProviderJsonModeError);
+  it("returns plain text content when responseFormat is omitted (AC-13)", async () => {
+    mockChatCompletionsCreate.mockResolvedValue(
+      makeStreamingChatCompletion(["not json, just text"]),
+    );
+
+    const result = await openaiChat({
+      messages: [{ role: "user", content: "Give me json output please." }],
+      model: "gpt-4o",
+    });
+
+    expect(typeof result.content).toBe("string");
+    expect(result.content).toBe("not json, just text");
+  });
+
+  it("does not set response_format on Chat Completions when responseFormat is omitted even if the message says 'json' (AC-13)", async () => {
+    mockChatCompletionsCreate.mockResolvedValue(
+      makeStreamingChatCompletion(["ok"]),
+    );
+
+    await openaiChat({
+      messages: [{ role: "user", content: "Return JSON please." }],
+      model: "gpt-4o",
+    });
+
+    const callArgs = mockChatCompletionsCreate.mock.calls[0]![0] as Record<string, unknown>;
+    expect(callArgs["response_format"]).toBeUndefined();
   });
 
   it("throws ProviderJsonParseError for non-JSON text in JSON mode", async () => {
     const nonJsonText = "This is plain text, not JSON at all.";
+    const usage = { prompt_tokens: 10, completion_tokens: 6, total_tokens: 16 };
     mockChatCompletionsCreate.mockResolvedValue(
-      makeStreamingChatCompletion([nonJsonText]),
+      makeStreamingChatCompletion([nonJsonText], usage),
     );
 
     try {
@@ -251,10 +277,12 @@ describe("openaiChat", () => {
       expect(parseErr.provider).toBe("openai");
       expect(parseErr.model).toBe("gpt-4o");
       expect(parseErr.sample).toBeTruthy();
+      expect(parseErr.text).toBe(nonJsonText);
+      expect(parseErr.usage).toEqual(usage);
     }
   });
 
-  it("defaults to gpt-5-chat-latest model and json_object format", async () => {
+  it("defaults to gpt-5-chat-latest model", async () => {
     const jsonPayload = { default: true };
     mockResponsesCreate.mockResolvedValue(
       makeStreamingResponsesResult([JSON.stringify(jsonPayload)]),
@@ -262,6 +290,7 @@ describe("openaiChat", () => {
 
     await openaiChat({
       messages: baseOptions.messages,
+      responseFormat: "json_object",
     });
 
     // gpt-5-chat-latest matches /^gpt-5/i, so Responses API should be used
@@ -270,28 +299,19 @@ describe("openaiChat", () => {
     expect(callArgs["model"]).toBe("gpt-5-chat-latest");
   });
 
-  it("estimates usage at ~4 chars/token when Responses API stream lacks usage", async () => {
-    const jsonPayload = { estimated: true };
-    const text = JSON.stringify(jsonPayload);
-
-    // Streaming response that completes without a response.completed event carrying usage
-    const noUsageStream: AsyncIterable<Record<string, unknown>> = {
-      async *[Symbol.asyncIterator]() {
-        yield { type: "response.output_text.delta", delta: text };
-        // No response.completed event with usage
-      },
-    };
-    mockResponsesCreate.mockResolvedValue(noUsageStream);
+  it("returns undefined usage when streaming Responses reports none", async () => {
+    const jsonPayload = { noUsage: true };
+    mockResponsesCreate.mockResolvedValue(
+      makeStreamingResponsesResult([JSON.stringify(jsonPayload)], null),
+    );
 
     const result = await openaiChat({
       ...baseOptions,
       model: "gpt-5",
     });
 
-    // Estimation: ~4 chars/token
-    expect(result.usage).toBeDefined();
-    expect(result.usage!.prompt_tokens).toBeGreaterThan(0);
-    expect(result.usage!.completion_tokens).toBeGreaterThan(0);
+    expect(result.content).toEqual(jsonPayload);
+    expect(result.usage).toBeUndefined();
   });
 
   it("retries on retryable errors and eventually succeeds", async () => {
@@ -372,48 +392,190 @@ describe("openaiChat", () => {
     );
   });
 
-  it("creates separate client instances for different timeout values", async () => {
-    mockChatCompletionsCreate.mockResolvedValue(
-      makeStreamingChatCompletion([JSON.stringify({ ok: true })]),
-    );
+    it("creates separate client instances for different timeout values", async () => {
+      mockChatCompletionsCreate.mockResolvedValue(
+        makeStreamingChatCompletion([JSON.stringify({ ok: true })]),
+      );
 
-    await openaiChat({ ...baseOptions, model: "gpt-4o", requestTimeoutMs: 10_000 });
-    await openaiChat({ ...baseOptions, model: "gpt-4o", requestTimeoutMs: 60_000 });
+      await openaiChat({ ...baseOptions, model: "gpt-4o", requestTimeoutMs: 10_000 });
+      await openaiChat({ ...baseOptions, model: "gpt-4o", requestTimeoutMs: 60_000 });
 
-    // Two distinct timeout values should produce two client constructions
-    const timeouts = MockOpenAI.mock.calls.map(
-      (call: unknown[]) => (call[0] as { timeout: number }).timeout,
-    );
-    expect(timeouts).toContain(10_000);
-    expect(timeouts).toContain(60_000);
-  });
+      // Two distinct timeout values should produce two client constructions
+      const timeouts = MockOpenAI.mock.calls.map(
+        (call: unknown[]) => (call[0] as { timeout: number }).timeout,
+      );
+      expect(timeouts).toContain(10_000);
+      expect(timeouts).toContain(60_000);
+    });
 
-  it("passes json_schema to the streaming Responses API when provided", async () => {
-    mockResponsesCreate.mockResolvedValue(
-      makeStreamingResponsesResult([JSON.stringify({ ok: true })]),
-    );
+    it("passes json_schema to the streaming Responses API when provided", async () => {
+      mockResponsesCreate.mockResolvedValue(
+        makeStreamingResponsesResult([JSON.stringify({ ok: true })]),
+      );
 
-    await openaiChat({
-      ...baseOptions,
-      model: "gpt-5",
-      responseFormat: {
-        json_schema: {
-          type: "object",
-          properties: { ok: { type: "boolean" } },
+      await openaiChat({
+        ...baseOptions,
+        model: "gpt-5",
+        responseFormat: {
+          json_schema: {
+            type: "object",
+            properties: { ok: { type: "boolean" } },
+          },
         },
-      },
+      });
+
+      const callArgs = mockResponsesCreate.mock.calls[0]![0] as Record<string, unknown>;
+      expect(callArgs["text"]).toEqual({
+        format: {
+          type: "json_schema",
+          name: "Response",
+          schema: {
+            type: "object",
+            properties: { ok: { type: "boolean" } },
+          },
+        },
+      });
     });
 
-    const callArgs = mockResponsesCreate.mock.calls[0]![0] as Record<string, unknown>;
-    expect(callArgs["text"]).toEqual({
-      format: {
+    it("passes native json_schema to Chat Completions API when a schema is supplied (AC-14)", async () => {
+      mockChatCompletionsCreate.mockResolvedValue(
+        makeStreamingChatCompletion([JSON.stringify({ ok: true })]),
+      );
+
+      const schema = {
+        type: "object",
+        properties: { ok: { type: "boolean" } },
+      };
+
+      await openaiChat({
+        messages: baseOptions.messages,
+        model: "gpt-4o",
+        responseFormat: { json_schema: schema },
+      });
+
+      const callArgs = mockChatCompletionsCreate.mock.calls[0]![0] as Record<string, unknown>;
+      expect(callArgs["response_format"]).toEqual({
         type: "json_schema",
-        name: "Response",
-        schema: {
-          type: "object",
-          properties: { ok: { type: "boolean" } },
+        json_schema: { name: "Response", schema },
+      });
+    });
+  });
+
+  describe("requestTimeoutMs (AC-12)", () => {
+    const TIMEOUT_MS = 1_000;
+
+    beforeEach(() => {
+      vi.useFakeTimers();
+      _resetClient();
+      mockResponsesCreate.mockReset();
+      mockChatCompletionsCreate.mockReset();
+      process.env["OPENAI_API_KEY"] = "test-key";
+    });
+
+    afterEach(() => {
+      vi.useRealTimers();
+      _resetClient();
+      delete process.env["OPENAI_API_KEY"];
+    });
+
+    /**
+     * Returns a stalled async iterable (OpenAI SDK stream shape) that rejects
+     * when `signal` aborts. The signal is passed inline in the SDK call's
+     * first argument (`{ ..., signal }`), not as a separate options arg.
+     */
+    function stalledAsyncIterable(
+      signal: AbortSignal | undefined,
+    ): AsyncIterable<Record<string, unknown>> {
+      return {
+        [Symbol.asyncIterator]() {
+          return {
+            next(): Promise<IteratorResult<Record<string, unknown>>> {
+              return new Promise((_resolve, reject) => {
+                if (!signal) {
+                  reject(new Error("signal missing"));
+                  return;
+                }
+                signal.addEventListener(
+                  "abort",
+                  () => {
+                    const reason = signal.reason;
+                    reject(
+                      reason instanceof Error
+                        ? reason
+                        : new DOMException("The user aborted a request.", "AbortError"),
+                    );
+                  },
+                  { once: true },
+                );
+              });
+            },
+          };
+        },
+      };
+    }
+
+    it("applies requestTimeoutMs to the Chat Completions idle abort", async () => {
+      mockChatCompletionsCreate.mockImplementation((params: { signal?: AbortSignal }) =>
+        Promise.resolve(stalledAsyncIterable(params.signal)),
+      );
+
+      const promise = openaiChat({
+        ...baseOptions,
+        model: "gpt-4o",
+        requestTimeoutMs: TIMEOUT_MS,
+        maxRetries: 0,
+      });
+
+      let state: "pending" | "resolved" | "rejected" = "pending";
+      promise.then(
+        () => {
+          state = "resolved";
+        },
+        () => {
+          state = "rejected";
+        },
+      );
+
+      vi.advanceTimersByTime(TIMEOUT_MS - 1);
+      for (let i = 0; i < 10; i++) await Promise.resolve();
+      expect(state).toBe("pending");
+
+      vi.advanceTimersByTime(1);
+      for (let i = 0; i < 50; i++) await Promise.resolve();
+
+      expect(state).toBe("rejected");
+    });
+
+    it("applies requestTimeoutMs to the Responses API idle abort", async () => {
+      mockResponsesCreate.mockImplementation((params: { signal?: AbortSignal }) =>
+        Promise.resolve(stalledAsyncIterable(params.signal)),
+      );
+
+      const promise = openaiChat({
+        ...baseOptions,
+        model: "gpt-5",
+        requestTimeoutMs: TIMEOUT_MS,
+        maxRetries: 0,
+      });
+
+      let state: "pending" | "resolved" | "rejected" = "pending";
+      promise.then(
+        () => {
+          state = "resolved";
         },
-      },
+        () => {
+          state = "rejected";
+        },
+      );
+
+      vi.advanceTimersByTime(TIMEOUT_MS - 1);
+      for (let i = 0; i < 10; i++) await Promise.resolve();
+      expect(state).toBe("pending");
+
+      vi.advanceTimersByTime(1);
+      for (let i = 0; i < 50; i++) await Promise.resolve();
+
+      expect(state).toBe("rejected");
     });
   });
 
@@ -435,7 +597,11 @@ describe("openaiChat", () => {
 
     it("accumulates text across multiple streaming chunks for Responses API", async () => {
       mockResponsesCreate.mockResolvedValue(
-        makeStreamingResponsesResult(['{"he', 'llo":"world"}'], 10, 5),
+        makeStreamingResponsesResult(['{"he', 'llo":"world"}'], {
+          input_tokens: 10,
+          output_tokens: 5,
+          total_tokens: 15,
+        }),
       );
 
       const result = await openaiChat({
@@ -451,7 +617,7 @@ describe("openaiChat", () => {
       });
     });
 
-    it("defaults usage to zeros when Chat Completions stream has no usage", async () => {
+    it("returns undefined usage when streaming Chat Completions reports none", async () => {
       mockChatCompletionsCreate.mockResolvedValue(
         makeStreamingChatCompletion([JSON.stringify({ ok: true })]),
       );
@@ -461,11 +627,33 @@ describe("openaiChat", () => {
         model: "gpt-4o",
       });
 
-      expect(result.usage).toEqual({
-        prompt_tokens: 0,
-        completion_tokens: 0,
-        total_tokens: 0,
+      expect(result.usage).toBeUndefined();
+    });
+
+    it("preserves partial Chat Completions usage, passing through only reported fields", async () => {
+      mockChatCompletionsCreate.mockResolvedValue(
+        makeStreamingChatCompletion([JSON.stringify({ ok: true })], { prompt_tokens: 7 }),
+      );
+
+      const result = await openaiChat({
+        ...baseOptions,
+        model: "gpt-4o",
       });
+
+      expect(result.usage).toBeDefined();
+      expect(result.usage!.prompt_tokens).toBe(7);
+      expect(result.usage!.completion_tokens).toBeNull();
+      expect(result.usage!.total_tokens).toBeNull();
     });
   });
+
+describe("openai adapter source hygiene (AC-1)", () => {
+  it("contains no inline usage estimate or zero-fill fallback", async () => {
+    const source = await Bun.file("src/providers/openai.ts").text();
+
+    expect(source).not.toContain("length / 4");
+    expect(source).not.toContain(
+      "{ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }",
+    );
+  });
 });

package/src/providers/__tests__/opencode.test.ts

@@ -702,6 +702,12 @@ describe("opencodeChat", () => {
         const e = err as ProviderJsonParseError;
         expect(e.provider).toBe("opencode");
         expect(e.model).toBe("anthropic/claude-sonnet-4-5");
+        expect(e.text).toBe("not json at all");
+        expect(e.usage).toEqual({
+          prompt_tokens: 10,
+          completion_tokens: 5,
+          total_tokens: 15,
+        });
       }
     });
 
@@ -1279,6 +1285,77 @@ describe("opencodeChat", () => {
       vi.restoreAllMocks();
     });
   });
+
+  describe("usage handling", () => {
+    it("returns no usage in CLI mode and leaves usageAvailable unset", async () => {
+      const mockExit = Promise.resolve(0);
+      const mockStdout = new ReadableStream({
+        start(controller) {
+          controller.enqueue(
+            new TextEncoder().encode(
+              '{"type":"text","part":{"text":"hi"}}\n',
+            ),
+          );
+          controller.close();
+        },
+      });
+      const mockStderr = new ReadableStream({
+        start(controller) {
+          controller.close();
+        },
+      });
+
+      vi.spyOn(Bun, "spawn").mockReturnValue({
+        stdout: mockStdout,
+        stderr: mockStderr,
+        exited: mockExit,
+        exitCode: 0,
+        kill: vi.fn(),
+      } as unknown as ReturnType<typeof Bun.spawn>);
+
+      const result = await opencodeChat({
+        messages: baseMessages,
+        responseFormat: "text",
+      });
+
+      expect(result.usage).toBeUndefined();
+      expect(result.usageAvailable).toBeUndefined();
+
+      vi.restoreAllMocks();
+    });
+
+    it("passes SDK-reported usage through unchanged", async () => {
+      const result = await opencodeChat({
+        messages: baseMessages,
+        opencode: { baseUrl: "http://localhost:3000" },
+      });
+
+      expect(result.usage).toEqual({
+        prompt_tokens: 10,
+        completion_tokens: 5,
+        total_tokens: 15,
+      });
+      expect(result.usageAvailable).toBeUndefined();
+    });
+
+    it("returns undefined usage when SDK reports no tokens", async () => {
+      mockPrompt.mockResolvedValueOnce({
+        data: {
+          info: { ...defaultPromptData.info, tokens: undefined },
+          parts: [{ type: "text", text: "hi" }],
+        },
+        error: undefined,
+      });
+
+      const result = await opencodeChat({
+        messages: baseMessages,
+        opencode: { baseUrl: "http://localhost:3000" },
+      });
+
+      expect(result.usage).toBeUndefined();
+      expect(result.usageAvailable).toBeUndefined();
+    });
+  });
 });
 
 describe("isOpenCodeAvailable", () => {