@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

package/src/llm/index.ts

@@ -2,8 +2,11 @@
 // Central LLM gateway: dispatcher, factories, telemetry, and utilities.
 
 import { EventEmitter } from "node:events";
-import { writeFile } from "node:fs/promises";
+import { writeFile, chmod } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { getConfig } from "../core/config.ts";
+import { redactSecrets } from "../core/redact.ts";
 import { anthropicChat } from "../providers/anthropic.ts";
 import { openaiChat } from "../providers/openai.ts";
 import { geminiChat } from "../providers/gemini.ts";
@@ -13,7 +16,7 @@ import { moonshotChat } from "../providers/moonshot.ts";
 import { zaiChat } from "../providers/zhipu.ts";
 import { claudeCodeChat, isClaudeCodeAvailable } from "../providers/claude-code.ts";
 import { opencodeChat, isOpenCodeAvailable } from "../providers/opencode.ts";
-import { ensureMessagesPresent } from "../providers/base.ts";
+import { ensureMessagesPresent, isJsonResponseFormat } from "../providers/base.ts";
 import {
   MODEL_CONFIG,
   PROVIDER_FUNCTIONS,
@@ -37,7 +40,9 @@ import type {
   LLMRequestStartEvent,
   LLMRequestCompleteEvent,
   LLMRequestErrorEvent,
+  UsageSource,
 } from "../providers/types.ts";
+import { ProviderJsonParseError } from "../providers/types.ts";
 
 // ─── Provider Name Mapping ───────────────────────────────────────────────────
 // types.ts uses "claudecode" (one word), config/models.ts uses "claude-code" (hyphenated).
@@ -60,37 +65,17 @@ function fromConfigProvider(configName: string): ProviderName {
 const llmEvents = new EventEmitter();
 let mockProvider: MockProvider | null = null;
 let requestCounter = 0;
+const warnedUnknownAliases = new Set<string>();
 
-// ─── JSON Format Inference ───────────────────────────────────────────────────
-// For OpenAI, DeepSeek, Gemini, Moonshot: if responseFormat is falsy,
-// check first two messages for "json" (case-insensitive) and infer "json_object".
-
-const JSON_INFER_PROVIDERS = new Set<ProviderName>([
-  "openai",
-  "deepseek",
-  "gemini",
-  "moonshot",
-  "alibaba",
-]);
-
-function inferJsonFormat(options: ChatOptions): ChatOptions {
-  if (options.responseFormat) return options;
-  if (!JSON_INFER_PROVIDERS.has(options.provider)) return options;
-
-  const first2 = options.messages.slice(0, 2);
-  const hasJson = first2.some((m) =>
-    m.content.toLowerCase().includes("json"),
-  );
-
-  if (hasJson) {
-    return { ...options, responseFormat: "json_object" };
-  }
-  return options;
+function warnUnknownAliasOnce(alias: string): void {
+  if (warnedUnknownAliases.has(alias)) return;
+  warnedUnknownAliases.add(alias);
+  console.warn(`[llm] unknown model alias "${alias}"; reporting cost as estimated`);
 }
 
 // ─── Adapter Dispatch ────────────────────────────────────────────────────────
 
-async function callAdapter(
+async function dispatchByProvider(
   options: ChatOptions,
 ): Promise<AdapterResponse> {
   const { provider, messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries, requestTimeoutMs } = options;
@@ -146,27 +131,198 @@ async function callAdapter(
   }
 }
 
+async function callAdapter(
+  options: ChatOptions,
+): Promise<AdapterResponse> {
+  const response = await dispatchByProvider(options);
+  response.usageAvailable = resolveUsageAvailable(options.provider, options);
+  return response;
+}
+
+// ─── JSON Repair (AC-15) ─────────────────────────────────────────────────────
+
+const REPAIR_SAMPLE_MAX_LEN = 2000;
+
+function buildJsonRepairMessages(
+  messages: ChatMessage[],
+  parseError: ProviderJsonParseError,
+): ChatMessage[] {
+  const sample = String(redactSecrets(parseError.sample)).slice(0, REPAIR_SAMPLE_MAX_LEN);
+  return [
+    ...messages,
+    {
+      role: "user",
+      content:
+        `Your previous response was not valid JSON. ` +
+        `Error sample (redacted, truncated to ${REPAIR_SAMPLE_MAX_LEN} characters):\n\n${sample}\n\n` +
+        `Please respond again with valid JSON only, no prose or code fences.`,
+    },
+  ];
+}
+
+async function performJsonRepair(
+  opts: ChatOptions,
+  repairMessages: ChatMessage[],
+): Promise<AdapterResponse> {
+  const result = await callAdapter({ ...opts, messages: repairMessages });
+  return result;
+}
+
+// ─── Usage Capability ────────────────────────────────────────────────────────
+
+/**
+ * Returns whether a provider can structurally report token usage for the given
+ * options. This is the single source of truth for `usageAvailable`; adapters
+ * never set it themselves. Forward-compatible with the #25 provider registry,
+ * where this becomes a registry column.
+ */
+export function resolveUsageAvailable(
+  provider: ProviderName,
+  options: ChatOptions,
+): boolean {
+  if (provider === "claudecode") return false;
+  if (provider === "opencode") {
+    if (options.opencode?.mode === "cli") return false;
+    if (options.opencode?.mode === "sdk") return true;
+    const baseUrl =
+      options.opencode?.baseUrl ||
+      process.env.PO_OPENCODE_BASE_URL ||
+      process.env.OPENCODE_BASE_URL;
+    return baseUrl != null;
+  }
+  return true;
+}
+
 // ─── Usage Normalization ─────────────────────────────────────────────────────
 
 function normalizeUsage(adapter: AdapterResponse, text: string): NormalizedUsage {
-  if (adapter.usage) {
-    return {
-      promptTokens: adapter.usage.prompt_tokens,
-      completionTokens: adapter.usage.completion_tokens,
-      totalTokens: adapter.usage.total_tokens,
-    };
-  }
-  // Estimate at ~4 chars/token
   const contentStr = typeof adapter.content === "string"
     ? adapter.content
     : JSON.stringify(adapter.content);
+
+  if (adapter.usage) {
+    const prompt = adapter.usage.prompt_tokens;
+    const completion = adapter.usage.completion_tokens;
+    const total = adapter.usage.total_tokens;
+
+    const promptPresent = typeof prompt === "number";
+    const completionPresent = typeof completion === "number";
+
+    if (promptPresent && completionPresent) {
+      if (prompt === 0 && completion === 0) {
+        return zeroUsage();
+      }
+      return {
+        promptTokens: prompt,
+        completionTokens: completion,
+        totalTokens: typeof total === "number" ? total : prompt + completion,
+        source: "reported",
+      };
+    }
+
+    if (typeof total === "number" && !promptPresent && !completionPresent) {
+      const promptEst = estimateTokens(text);
+      const completionEst = estimateTokens(contentStr);
+      const estSum = promptEst + completionEst;
+      const promptTokens = estSum > 0 ? Math.round((promptEst / estSum) * total) : 0;
+      return {
+        promptTokens,
+        completionTokens: total - promptTokens,
+        totalTokens: total,
+        source: "estimated",
+      };
+    }
+
+    if (promptPresent && !completionPresent) {
+      const completionEst = estimateTokens(contentStr);
+      if (typeof total === "number") {
+        return {
+          promptTokens: prompt,
+          completionTokens: Math.max(0, total - prompt),
+          totalTokens: total,
+          source: "estimated",
+        };
+      }
+      return {
+        promptTokens: prompt,
+        completionTokens: completionEst,
+        totalTokens: prompt + completionEst,
+        source: "estimated",
+      };
+    }
+
+    if (completionPresent && !promptPresent) {
+      const promptEst = estimateTokens(text);
+      if (typeof total === "number") {
+        return {
+          promptTokens: Math.max(0, total - completion),
+          completionTokens: completion,
+          totalTokens: total,
+          source: "estimated",
+        };
+      }
+      return {
+        promptTokens: promptEst,
+        completionTokens: completion,
+        totalTokens: promptEst + completion,
+        source: "estimated",
+      };
+    }
+  }
+
+  const source: UsageSource = adapter.usageAvailable === false ? "unavailable" : "estimated";
   return {
     promptTokens: estimateTokens(text),
     completionTokens: estimateTokens(contentStr),
     totalTokens: estimateTokens(text) + estimateTokens(contentStr),
+    source,
   };
 }
 
+function zeroUsage(): NormalizedUsage {
+  return {
+    promptTokens: 0,
+    completionTokens: 0,
+    totalTokens: 0,
+    source: "zero",
+  };
+}
+
+function combineUsageSource(left: UsageSource, right: UsageSource): UsageSource {
+  if (left === right) return left;
+  if (left === "unavailable" || right === "unavailable") return "unavailable";
+  if (left === "estimated" || right === "estimated") return "estimated";
+  if (left === "reported" || right === "reported") return "reported";
+  return "zero";
+}
+
+function combineNormalizedUsage(usages: NormalizedUsage[]): NormalizedUsage {
+  return usages.reduce<NormalizedUsage>(
+    (total, usage) => ({
+      promptTokens: total.promptTokens + usage.promptTokens,
+      completionTokens: total.completionTokens + usage.completionTokens,
+      totalTokens: total.totalTokens + usage.totalTokens,
+      source: combineUsageSource(total.source, usage.source),
+    }),
+    zeroUsage(),
+  );
+}
+
+function normalizeParseErrorUsage(
+  err: ProviderJsonParseError,
+  messagesText: string,
+  opts: ChatOptions,
+): NormalizedUsage {
+  return normalizeUsage(
+    {
+      content: err.text ?? err.sample,
+      usage: err.usage,
+      usageAvailable: err.usageAvailable ?? resolveUsageAvailable(opts.provider, opts),
+    },
+    messagesText,
+  );
+}
+
 // ─── Debug Log ───────────────────────────────────────────────────────────────
 
 async function writeDebugLog(options: ChatOptions, response: ChatResponse): Promise<void> {
@@ -179,7 +335,18 @@ async function writeDebugLog(options: ChatOptions, response: ChatResponse): Prom
     response: response.content,
     usage: response.usage,
   };
-  await writeFile("/tmp/messages.log", JSON.stringify(entry, null, 2) + "\n");
+  const path = join(tmpdir(), `pop-llm-debug-${process.pid}.log`);
+  try {
+    try {
+      await chmod(path, 0o600);
+    } catch {
+      // The file may not exist yet; creation below still uses 0600.
+    }
+    await writeFile(path, JSON.stringify(entry, null, 2) + "\n", { flag: "a", mode: 0o600 });
+    await chmod(path, 0o600);
+  } catch {
+    // best-effort: a debug-log write failure must never break a run
+  }
 }
 
 // ─── Public API ──────────────────────────────────────────────────────────────
@@ -187,10 +354,10 @@ async function writeDebugLog(options: ChatOptions, response: ChatResponse): Prom
 export async function chat(options: ChatOptions): Promise<ChatResponse> {
   ensureMessagesPresent(options.messages, options.provider);
   const configTimeout = getConfig().taskRunner.llmRequestTimeout;
-  const opts = inferJsonFormat({
+  const opts = {
     ...options,
     requestTimeoutMs: options.requestTimeoutMs ?? configTimeout,
-  });
+  };
   const id = `llm-${++requestCounter}-${Date.now()}`;
   const model = opts.model ?? "";
   const startTime = Date.now();
@@ -204,11 +371,52 @@ export async function chat(options: ChatOptions): Promise<ChatResponse> {
   };
   llmEvents.emit("llm:request:start", startEvent);
 
+  const failedAttemptUsage: NormalizedUsage[] = [];
+
   try {
-    const adapterResult = await callAdapter(opts);
-    const messagesText = opts.messages.map((m) => m.content).join(" ");
-    const usage = normalizeUsage(adapterResult, messagesText);
-    const cost = calculateCost(opts.provider, model, usage);
+    let adapterResult: AdapterResponse;
+    let messagesText: string;
+    try {
+      adapterResult = await callAdapter(opts);
+      messagesText = opts.messages.map((m) => m.content).join(" ");
+    } catch (err) {
+      if (
+        err instanceof ProviderJsonParseError &&
+        isJsonResponseFormat(opts.responseFormat)
+      ) {
+        const originalMessagesText = opts.messages.map((m) => m.content).join(" ");
+        failedAttemptUsage.push(normalizeParseErrorUsage(err, originalMessagesText, opts));
+        const repairMessages = buildJsonRepairMessages(opts.messages, err);
+        const repairMessagesText = repairMessages.map((m) => m.content).join(" ");
+        try {
+          adapterResult = await performJsonRepair(opts, repairMessages);
+          messagesText = repairMessagesText;
+        } catch (repairErr) {
+          if (repairErr instanceof ProviderJsonParseError) {
+            failedAttemptUsage.push(normalizeParseErrorUsage(repairErr, repairMessagesText, opts));
+          }
+          throw repairErr;
+        }
+      } else {
+        throw err;
+      }
+    }
+    const usage = combineNormalizedUsage([
+      ...failedAttemptUsage,
+      normalizeUsage(adapterResult, messagesText),
+    ]);
+    const price = findModelPrice(opts.provider, model);
+    const cost = price
+      ? Math.max(
+          0,
+          (usage.promptTokens / 1_000_000) * price.tokenCostInPerMillion +
+            (usage.completionTokens / 1_000_000) * price.tokenCostOutPerMillion,
+        )
+      : 0;
+    const costEstimated = price === null || usage.source !== "reported";
+    if (price === null) {
+      warnUnknownAliasOnce(`${toConfigProvider(opts.provider)}:${model}`);
+    }
     const duration = Date.now() - startTime;
 
     const completeEvent: LLMRequestCompleteEvent = {
@@ -218,6 +426,8 @@ export async function chat(options: ChatOptions): Promise<ChatResponse> {
       completionTokens: usage.completionTokens,
       totalTokens: usage.totalTokens,
       cost,
+      usageSource: usage.source,
+      costEstimated,
     };
     llmEvents.emit("llm:request:complete", completeEvent);
 
@@ -227,16 +437,33 @@ export async function chat(options: ChatOptions): Promise<ChatResponse> {
       raw: adapterResult.raw,
     };
 
-    // Fire-and-forget debug log
-    writeDebugLog(opts, response).catch(() => {});
+    await writeDebugLog(opts, response);
 
     return response;
   } catch (err) {
     const duration = Date.now() - startTime;
+    const messagesText = opts.messages.map((m) => m.content).join(" ");
+    const promptTokens = estimateTokens(messagesText);
+    const usage: NormalizedUsage = failedAttemptUsage.length > 0
+      ? combineNormalizedUsage(failedAttemptUsage)
+      : err instanceof ProviderJsonParseError && isJsonResponseFormat(opts.responseFormat)
+        ? normalizeParseErrorUsage(err, messagesText, opts)
+        : {
+            promptTokens,
+            completionTokens: 0,
+            totalTokens: promptTokens,
+            source: "estimated",
+          };
     const errorEvent: LLMRequestErrorEvent = {
       ...startEvent,
       duration,
       error: err instanceof Error ? err.message : String(err),
+      promptTokens: usage.promptTokens,
+      completionTokens: usage.completionTokens,
+      totalTokens: usage.totalTokens,
+      cost: calculateCost(opts.provider, model, usage),
+      usageSource: usage.source,
+      costEstimated: true,
     };
     llmEvents.emit("llm:request:error", errorEvent);
     throw err;
@@ -471,34 +698,50 @@ export function estimateTokens(text: string): number {
   return Math.ceil(text.length / 4);
 }
 
-export function calculateCost(
+export interface ModelPrice {
+  readonly tokenCostInPerMillion: number;
+  readonly tokenCostOutPerMillion: number;
+}
+
+/**
+ * Looks up the per-million-token price for a gateway `(provider, model)` pair.
+ * Returns `null` when no price is known. This is the single cost-lookup surface
+ * used by both `calculateCost` and the `costEstimated` discriminator in `chat()`.
+ */
+export function findModelPrice(
   provider: string,
   model: string,
-  usage: NormalizedUsage,
-): number {
-  // Map gateway provider name to config provider name for lookup
-  const configProvider = provider === "claudecode" ? "claude-code"
-    : provider === "zhipu" ? "zai"
-    : provider;
-
+): ModelPrice | null {
+  const configProvider = toConfigProvider(provider as ProviderName);
   const alias = `${configProvider}:${model}`;
-  const config = getModelConfig(alias);
-
-  if (!config) {
-    // Try to find by scanning MODEL_CONFIG for matching provider+model
-    for (const [, entry] of Object.entries(MODEL_CONFIG)) {
-      if (entry.provider === configProvider && entry.model === model) {
-        const inCost = (usage.promptTokens / 1_000_000) * entry.tokenCostInPerMillion;
-        const outCost = (usage.completionTokens / 1_000_000) * entry.tokenCostOutPerMillion;
-        return inCost + outCost;
-      }
+  const direct = getModelConfig(alias);
+  if (direct) {
+    return {
+      tokenCostInPerMillion: direct.tokenCostInPerMillion,
+      tokenCostOutPerMillion: direct.tokenCostOutPerMillion,
+    };
+  }
+  for (const entry of Object.values(MODEL_CONFIG)) {
+    if (entry.provider === configProvider && entry.model === model) {
+      return {
+        tokenCostInPerMillion: entry.tokenCostInPerMillion,
+        tokenCostOutPerMillion: entry.tokenCostOutPerMillion,
+      };
     }
-    return 0;
   }
+  return null;
+}
 
-  const inCost = (usage.promptTokens / 1_000_000) * config.tokenCostInPerMillion;
-  const outCost = (usage.completionTokens / 1_000_000) * config.tokenCostOutPerMillion;
-  return inCost + outCost;
+export function calculateCost(
+  provider: string,
+  model: string,
+  usage: NormalizedUsage,
+): number {
+  const price = findModelPrice(provider, model);
+  if (!price) return 0;
+  const inCost = (usage.promptTokens / 1_000_000) * price.tokenCostInPerMillion;
+  const outCost = (usage.completionTokens / 1_000_000) * price.tokenCostOutPerMillion;
+  return Math.max(0, inCost + outCost);
 }
 
 // ─── Factory Helpers ─────────────────────────────────────────────────────────

package/src/providers/__tests__/alibaba.test.ts

@@ -69,6 +69,15 @@ function mockErrorResponse(body: unknown, status: number) {
   } as unknown as Response;
 }
 
+function mockNonStreamingResponse(body: unknown, status = 200): Response {
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    json: vi.fn().mockResolvedValue(body),
+    text: vi.fn().mockResolvedValue(JSON.stringify(body)),
+  } as unknown as Response;
+}
+
 const baseOptions: AlibabaOptions = {
   messages: [
     { role: "system", content: "You are helpful." },
@@ -268,17 +277,13 @@ describe("alibabaChat", () => {
       });
     });
 
-    it("defaults usage to zeros when stream provides no usage", async () => {
+    it("returns undefined usage when stream reports none", async () => {
       const events = makeOpenAiSseEvents([JSON.stringify({ ok: true })]);
 
       fetchMock.mockResolvedValue(mockStreamingResponse(events));
 
       const result = await alibabaChat(baseOptions);
-      expect(result.usage).toEqual({
-        prompt_tokens: 0,
-        completion_tokens: 0,
-        total_tokens: 0,
-      });
+      expect(result.usage).toBeUndefined();
     });
 
     it("retries on timeout then succeeds on second attempt", async () => {
@@ -299,4 +304,60 @@ describe("alibabaChat", () => {
       expect(result.content).toEqual({ ok: true });
     });
   });
+
+  describe("non-streaming fallback", () => {
+    it("returns undefined usage when non-streaming response reports none", async () => {
+      fetchMock
+        .mockRejectedValueOnce(
+          new Error("response_format not supported in stream mode"),
+        )
+        .mockResolvedValueOnce(
+          mockNonStreamingResponse({
+            choices: [{ message: { content: '{"ok":true}' } }],
+          }),
+        );
+
+      const result = await alibabaChat(baseOptions);
+
+      expect(result.usage).toBeUndefined();
+    });
+
+    it("preserves full usage from non-streaming response", async () => {
+      fetchMock
+        .mockRejectedValueOnce(
+          new Error("response_format not supported in stream mode"),
+        )
+        .mockResolvedValueOnce(
+          mockNonStreamingResponse({
+            choices: [{ message: { content: '{"ok":true}' } }],
+            usage: { prompt_tokens: 12, completion_tokens: 8, total_tokens: 20 },
+          }),
+        );
+
+      const result = await alibabaChat(baseOptions);
+
+      expect(result.usage).toEqual({
+        prompt_tokens: 12,
+        completion_tokens: 8,
+        total_tokens: 20,
+      });
+    });
+
+    it("preserves partial usage from non-streaming response without coercion", async () => {
+      fetchMock
+        .mockRejectedValueOnce(
+          new Error("response_format not supported in stream mode"),
+        )
+        .mockResolvedValueOnce(
+          mockNonStreamingResponse({
+            choices: [{ message: { content: '{"ok":true}' } }],
+            usage: { prompt_tokens: 7 },
+          }),
+        );
+
+      const result = await alibabaChat(baseOptions);
+
+      expect(result.usage).toEqual({ prompt_tokens: 7 });
+    });
+  });
 });

package/src/providers/__tests__/anthropic.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
 import { anthropicChat } from "../anthropic.ts";
-import { ProviderJsonModeError, ProviderJsonParseError } from "../types.ts";
+import { ProviderJsonParseError } from "../types.ts";
 import type { AnthropicOptions } from "../types.ts";
 import type { Mock } from "vitest";
 
@@ -146,25 +146,30 @@ describe("anthropicChat", () => {
     expect(result.content).toEqual(jsonPayload);
   });
 
-  it("throws ProviderJsonModeError when responseFormat is invalid", async () => {
-    await expect(
-      anthropicChat({
-        ...baseOptions,
-        responseFormat: "text",
-      }),
-    ).rejects.toThrow(ProviderJsonModeError);
+  it("returns plain text content when responseFormat is omitted (AC-13)", async () => {
+    const plainText = "Hello, this is a plain text response.";
+    const events = makeAnthropicSseEvents(plainText);
+    fetchMock.mockResolvedValue(mockStreamingResponse(events));
+
+    const result = await anthropicChat({
+      messages: [{ role: "user", content: "Return JSON." }],
+    });
+
+    expect(typeof result.content).toBe("string");
+    expect(result.content).toBe(plainText);
+    expect(result.text).toBe(plainText);
   });
 
-  it("defaults to json responseFormat when responseFormat is omitted", async () => {
-    const jsonPayload = { defaultFormat: true };
-    const events = makeAnthropicSseEvents(JSON.stringify(jsonPayload));
+  it("does not parse as JSON when responseFormat is omitted even if the message says 'json' (AC-13)", async () => {
+    const events = makeAnthropicSseEvents("not json, just text");
     fetchMock.mockResolvedValue(mockStreamingResponse(events));
 
-    // responseFormat defaults to "json" — should not throw
     const result = await anthropicChat({
-      messages: baseOptions.messages,
+      messages: [{ role: "user", content: "Give me json output please." }],
     });
-    expect(result.content).toEqual(jsonPayload);
+
+    expect(result.content).toBe("not json, just text");
+    expect(fetchMock).toHaveBeenCalledTimes(1);
   });
 
   it("throws ProviderJsonParseError for non-JSON text in JSON mode", async () => {
@@ -328,6 +333,22 @@ describe("anthropicChat", () => {
       });
     });
 
+    it("returns undefined usage when accumulated stream reports none", async () => {
+      const jsonPayload = { noUsage: true };
+      const events = [
+        `event: content_block_delta\ndata: ${JSON.stringify({
+          type: "content_block_delta",
+          delta: { text: JSON.stringify(jsonPayload) },
+        })}\n\n`,
+      ];
+
+      fetchMock.mockResolvedValue(mockStreamingResponse(events));
+
+      const result = await anthropicChat(baseOptions);
+      expect(result.content).toEqual(jsonPayload);
+      expect(result.usage).toBeUndefined();
+    });
+
     it("retries on timeout then succeeds on second attempt", async () => {
       const events = makeAnthropicSseEvents(JSON.stringify({ ok: true }));