@frockbot/provider-openai-compatible 0.3.15 → 0.3.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/provider-openai-compatible",
-  "version": "0.3.15",
+  "version": "0.3.17",
   "private": false,
   "type": "module",
   "exports": {
@@ -11,8 +11,8 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/kernel-contracts": "0.3.15",
-    "@frockbot/plugin-models": "0.3.15",
+    "@frockbot/kernel-contracts": "0.3.17",
+    "@frockbot/plugin-models": "0.3.17",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {

package/src/index.test.ts

@@ -7,8 +7,10 @@ import { LlmRegistry } from "@frockbot/plugin-models";
 import { Context } from "cordis";
 import {
   OpenAICompatibleProvider,
+  planOpenAICompatibleRequestV1,
   requestToWire,
   retryAfterMillisecondsV1,
+  usageFromPayloadV1,
 } from "./index.js";
 
 const request: NormalizedModelRequest = {
@@ -41,10 +43,82 @@ const request: NormalizedModelRequest = {
 };
 
 describe("OpenAICompatibleProvider", () => {
+  test("maps schemas for OpenAI and OpenRouter-compatible endpoints", () => {
+    const structured = {
+      ...request,
+      responseFormat: {
+        type: "json_schema" as const,
+        name: "answer",
+        schema: {
+          type: "object" as const,
+          properties: { answer: { type: "string" as const } },
+          required: ["answer"],
+          additionalProperties: false,
+        },
+      },
+    };
+    const plan = planOpenAICompatibleRequestV1(structured, {
+      structuredOutput: "json_schema",
+      responseFormatDialect: "openai",
+    });
+    expect(plan.note).toBeUndefined();
+    expect(plan.body.response_format).toEqual({
+      type: "json_schema",
+      json_schema: {
+        name: "answer",
+        strict: true,
+        schema: structured.responseFormat.schema,
+      },
+    });
+    expect(plan.body.stream).toBe(true);
+  });
+
+  test("maps the direct Workers AI schema and disables streaming", () => {
+    const structured: NormalizedModelRequest = {
+      ...request,
+      responseFormat: {
+        type: "json_schema",
+        name: "answer",
+        schema: { type: "string" },
+      },
+    };
+    const plan = planOpenAICompatibleRequestV1(structured, {
+      structuredOutput: "json_schema",
+      responseFormatDialect: "workers-ai",
+    });
+    expect(plan.body.response_format).toEqual({
+      type: "json_schema",
+      json_schema: { type: "string" },
+    });
+    expect(plan.body.stream).toBe(false);
+    expect(plan.body.stream_options).toBeUndefined();
+  });
+
+  test("downgrades unsupported schemas explicitly and adds prompt guidance", () => {
+    const plan = planOpenAICompatibleRequestV1(
+      {
+        ...request,
+        responseFormat: {
+          type: "json_schema",
+          name: "answer",
+          schema: { type: "string" },
+        },
+      },
+      { structuredOutput: "none" },
+    );
+    expect(plan.note).toMatchObject({
+      code: "structured-output-downgraded",
+      effective: "prompt",
+    });
+    expect(JSON.stringify(plan.body.messages)).toContain("Return only JSON");
+    expect(plan.body.response_format).toBeUndefined();
+  });
+
   test("normalizes FrockBot messages and tools to the wire format", () => {
     expect(requestToWire(request)).toMatchObject({
       model: "test-model",
       stream: true,
+      stream_options: { include_usage: true },
       messages: [
         { role: "system", content: "Be useful." },
         { role: "user", content: "What time is it?" },
@@ -69,12 +143,49 @@ describe("OpenAICompatibleProvider", () => {
     });
   });
 
+  test("normalizes OpenAI token details", () => {
+    expect(
+      usageFromPayloadV1({
+        usage: {
+          prompt_tokens: 80,
+          completion_tokens: 20,
+          prompt_tokens_details: { cached_tokens: 32 },
+          completion_tokens_details: { reasoning_tokens: 7 },
+        },
+      }),
+    ).toEqual({
+      type: "usage",
+      usage: {
+        inputTokens: 80,
+        outputTokens: 20,
+        cachedInputTokens: 32,
+        reasoningTokens: 7,
+      },
+    });
+  });
+
+  test("normalizes Workers AI and Ollama token fields", () => {
+    expect(
+      usageFromPayloadV1({ usage: { input_tokens: 12, output_tokens: 4 } }),
+    ).toEqual({
+      type: "usage",
+      usage: { inputTokens: 12, outputTokens: 4 },
+    });
+    expect(
+      usageFromPayloadV1({ prompt_eval_count: 18, eval_count: 6 }),
+    ).toEqual({
+      type: "usage",
+      usage: { inputTokens: 18, outputTokens: 6 },
+    });
+  });
+
   test("streams text and assembles fragmented tool calls", async () => {
     const encoder = new TextEncoder();
     const payloads = [
       'data: {"choices":[{"delta":{"content":"Checking "}}]}\n\n',
       'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call-1","function":{"name":"current_","arguments":"{\\"zone\\":"}}]}}]}\n\n',
       'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"name":"time","arguments":"\\"UTC\\"}"}}]},"finish_reason":"tool_calls"}]}\n\n',
+      'data: {"choices":[],"usage":{"prompt_tokens":40,"completion_tokens":9,"prompt_tokens_details":{"cached_tokens":10}}}\n\n',
       "data: [DONE]\n\n",
     ];
     let capturedUrl = "";
@@ -120,6 +231,10 @@ describe("OpenAICompatibleProvider", () => {
     expect(capturedIdempotencyKey).toBeNull();
     expect(events).toEqual([
       { type: "text-delta", text: "Checking " },
+      {
+        type: "usage",
+        usage: { inputTokens: 40, outputTokens: 9, cachedInputTokens: 10 },
+      },
       {
         type: "tool-call",
         call: {
@@ -132,6 +247,48 @@ describe("OpenAICompatibleProvider", () => {
     ]);
   });
 
+  test("normalizes a non-streaming structured response", async () => {
+    const provider = new OpenAICompatibleProvider({
+      baseUrl: "https://models.example/v1",
+      structuredOutput: "json_schema",
+      responseFormatDialect: "workers-ai",
+      fetch: () =>
+        Promise.resolve(
+          Response.json({
+            usage: { input_tokens: 14, output_tokens: 5 },
+            choices: [
+              {
+                message: { content: '{"answer":"yes"}' },
+                finish_reason: "stop",
+              },
+            ],
+          }),
+        ),
+    });
+    const events = [];
+    for await (const event of provider.stream(
+      {
+        ...request,
+        responseFormat: {
+          type: "json_schema",
+          name: "answer",
+          schema: { type: "object", additionalProperties: true },
+        },
+      },
+      new AbortController().signal,
+    )) {
+      events.push(event);
+    }
+    expect(events).toEqual([
+      {
+        type: "usage",
+        usage: { inputTokens: 14, outputTokens: 5 },
+      },
+      { type: "text-delta", text: '{"answer":"yes"}' },
+      { type: "finish", reason: "completed" },
+    ]);
+  });
+
   test("rejects a truncated stream without a terminal marker", async () => {
     const provider = new OpenAICompatibleProvider({
       baseUrl: "https://models.example/v1",

package/src/index.ts

@@ -9,6 +9,8 @@ import {
   ModelProviderFailureError,
   type ModelProviderFailureClassV1,
   type NormalizedModelRequest,
+  type ResponseFormatNoteV1,
+  type StructuredOutputSupportV1,
 } from "@frockbot/kernel-contracts";
 import type { Plugin } from "cordis";
 
@@ -137,6 +139,10 @@ export interface OpenAICompatibleConfig {
    * model id decides through {@link modelAcceptsImagesV1}.
    */
   acceptsImages?: boolean;
+  /** The strongest structured-output mode this endpoint accepts. */
+  structuredOutput?: StructuredOutputSupportV1;
+  /** Workers AI takes the schema directly; OpenAI/OpenRouter wrap it by name. */
+  responseFormatDialect?: "openai" | "workers-ai";
   /** Overrides {@link MODEL_REQUEST_DEADLINES_V1}. */
   deadlines?: Partial<ModelRequestDeadlinesV1>;
   /**
@@ -341,8 +347,27 @@ function messageToWire(
 
 export function requestToWire(
   request: NormalizedModelRequest,
-  options: { acceptsImages?: boolean } = {},
+  options: OpenAIRequestOptionsV1 = {},
 ): Record<string, unknown> {
+  return planOpenAICompatibleRequestV1(request, options).body;
+}
+
+export interface OpenAIRequestOptionsV1 {
+  acceptsImages?: boolean;
+  structuredOutput?: StructuredOutputSupportV1;
+  responseFormatDialect?: "openai" | "workers-ai";
+}
+
+export interface OpenAIRequestPlanV1 {
+  body: Record<string, unknown>;
+  note?: ResponseFormatNoteV1;
+}
+
+/** Maps the provider-neutral format and records any fidelity downgrade. */
+export function planOpenAICompatibleRequestV1(
+  request: NormalizedModelRequest,
+  options: OpenAIRequestOptionsV1 = {},
+): OpenAIRequestPlanV1 {
   const acceptsImages =
     options.acceptsImages ?? modelAcceptsImagesV1(request.model);
   const messages: Record<string, unknown>[] = [];
@@ -351,22 +376,82 @@ export function requestToWire(
   for (const message of request.messages) {
     messages.push(...messageToWire(message, acceptsImages));
   }
-  return {
-    model: request.model,
-    stream: true,
-    messages,
-    ...(request.tools.length > 0
-      ? {
-          tools: request.tools.map((tool) => ({
-            type: "function",
-            function: {
-              name: tool.name,
-              description: tool.description,
-              parameters: tool.inputSchema,
+  const support = options.structuredOutput ?? "none";
+  const format = request.responseFormat;
+  let responseFormat: Record<string, unknown> | undefined;
+  let note: ResponseFormatNoteV1 | undefined;
+  if (format?.type === "json_schema" && support === "json_schema") {
+    responseFormat =
+      options.responseFormatDialect === "workers-ai"
+        ? { type: "json_schema", json_schema: format.schema }
+        : {
+            type: "json_schema",
+            json_schema: {
+              name: format.name,
+              strict: true,
+              schema: format.schema,
             },
-          })),
-        }
-      : {}),
+          };
+  } else if (format && support !== "none") {
+    responseFormat = { type: "json_object" };
+    if (format.type === "json_schema") {
+      note = {
+        code: "structured-output-downgraded",
+        requested: "json_schema",
+        effective: "json",
+        message: `Provider ${request.provider} supports JSON mode but not JSON Schema; the shared validator remains authoritative`,
+      };
+    }
+  } else if (format) {
+    note = {
+      code: "structured-output-downgraded",
+      requested: format.type,
+      effective: "prompt",
+      message: `Provider ${request.provider} has no native structured-output mode; the request uses prompt guidance and shared validation`,
+    };
+  }
+  if (format) {
+    const instruction =
+      format.type === "json_schema"
+        ? `Return only JSON matching this schema exactly: ${JSON.stringify(format.schema)}`
+        : "Return only one valid JSON value, with no Markdown or commentary.";
+    if (request.system) {
+      messages[0] = {
+        role: "system",
+        content: `${request.system}\n\n${instruction}`,
+      };
+    } else {
+      messages.unshift({ role: "system", content: instruction });
+    }
+  }
+  const stream = !(
+    format &&
+    support !== "none" &&
+    options.responseFormatDialect === "workers-ai"
+  );
+  return {
+    body: {
+      model: request.model,
+      // Workers AI documents JSON mode as non-streaming. The decoder accepts
+      // both response shapes while the contract remains an event stream.
+      stream,
+      ...(stream ? { stream_options: { include_usage: true } } : {}),
+      messages,
+      ...(responseFormat ? { response_format: responseFormat } : {}),
+      ...(request.tools.length > 0
+        ? {
+            tools: request.tools.map((tool) => ({
+              type: "function",
+              function: {
+                name: tool.name,
+                description: tool.description,
+                parameters: tool.inputSchema,
+              },
+            })),
+          }
+        : {}),
+    },
+    ...(note ? { note } : {}),
   };
 }
 
@@ -433,6 +518,76 @@ function asRecord(value: unknown): Record<string, unknown> | undefined {
     : undefined;
 }
 
+function usageIntegerV1(value: unknown, label: string): number | undefined {
+  if (value === undefined) return undefined;
+  if (!Number.isSafeInteger(value) || (value as number) < 0) {
+    throw new Error(`Model returned invalid ${label}`);
+  }
+  return value as number;
+}
+
+/**
+ * Normalizes token accounting returned by OpenAI-shaped, Workers AI/Gateway,
+ * and Ollama streams. Unknown payloads are ignored so the Agent loop can use
+ * its durable byte-size estimate.
+ */
+export function usageFromPayloadV1(
+  value: unknown,
+): Extract<LlmStreamEvent, { type: "usage" }> | undefined {
+  const payload = asRecord(value);
+  if (!payload) return undefined;
+  const usage = asRecord(payload.usage) ?? payload;
+  const inputTokens = usageIntegerV1(
+    usage.prompt_tokens ??
+      usage.input_tokens ??
+      usage.prompt_eval_count ??
+      payload.prompt_eval_count,
+    "input token count",
+  );
+  const outputTokens = usageIntegerV1(
+    usage.completion_tokens ??
+      usage.output_tokens ??
+      usage.eval_count ??
+      payload.eval_count,
+    "output token count",
+  );
+  if (inputTokens === undefined || outputTokens === undefined) return undefined;
+
+  const inputDetails =
+    asRecord(usage.prompt_tokens_details) ??
+    asRecord(usage.input_tokens_details);
+  const outputDetails =
+    asRecord(usage.completion_tokens_details) ??
+    asRecord(usage.output_tokens_details);
+  const cachedInputTokens = usageIntegerV1(
+    inputDetails?.cached_tokens ?? usage.cached_input_tokens,
+    "cached input token count",
+  );
+  const reasoningTokens = usageIntegerV1(
+    outputDetails?.reasoning_tokens ?? usage.reasoning_tokens,
+    "reasoning token count",
+  );
+  if (cachedInputTokens !== undefined && cachedInputTokens > inputTokens) {
+    throw new Error(
+      "Model returned cached input tokens above total input tokens",
+    );
+  }
+  if (reasoningTokens !== undefined && reasoningTokens > outputTokens) {
+    throw new Error(
+      "Model returned reasoning tokens above total output tokens",
+    );
+  }
+  return {
+    type: "usage",
+    usage: {
+      inputTokens,
+      outputTokens,
+      ...(cachedInputTokens === undefined ? {} : { cachedInputTokens }),
+      ...(reasoningTokens === undefined ? {} : { reasoningTokens }),
+    },
+  };
+}
+
 function applyToolDeltas(
   value: unknown,
   tools: Map<number, ToolAccumulator>,
@@ -475,11 +630,39 @@ export async function* streamOpenAICompatibleBody(
   body: ReadableStream<Uint8Array>,
   signal: AbortSignal,
 ): AsyncIterable<LlmStreamEvent> {
+  const [probeBody, replayBody] = body.tee();
+  const probeReader = probeBody.getReader();
+  const probeDecoder = new TextDecoder();
+  let prefix = "";
+  const cancelProbe = (): void => {
+    void probeReader.cancel(signal.reason).catch(() => undefined);
+    if (!replayBody.locked) {
+      void replayBody.cancel(signal.reason).catch(() => undefined);
+    }
+  };
+  signal.addEventListener("abort", cancelProbe, { once: true });
+  try {
+    signal.throwIfAborted();
+    while (!prefix.trimStart() && prefix.length < 4_096) {
+      const { done, value } = await probeReader.read();
+      signal.throwIfAborted();
+      if (done) break;
+      prefix += probeDecoder.decode(value, { stream: true });
+    }
+  } finally {
+    signal.removeEventListener("abort", cancelProbe);
+    void probeReader.cancel().catch(() => undefined);
+    probeReader.releaseLock();
+  }
+  if (prefix.trimStart().startsWith("{")) {
+    yield* readOpenAICompatibleJsonV1(replayBody, signal);
+    return;
+  }
   const tools = new Map<number, ToolAccumulator>();
   let finishReason: string | undefined;
   let terminal = false;
   let sawChoice = false;
-  for await (const data of readSseData(body, signal)) {
+  for await (const data of readSseData(replayBody, signal)) {
     if (data === "[DONE]") {
       terminal = true;
       break;
@@ -487,6 +670,8 @@ export async function* streamOpenAICompatibleBody(
     const payload = asRecord(
       parseJson(data, "Model returned an invalid stream event"),
     );
+    const usage = usageFromPayloadV1(payload);
+    if (usage) yield usage;
     const choices = payload?.choices;
     const choice = Array.isArray(choices) ? asRecord(choices[0]) : undefined;
     const delta = asRecord(choice?.delta);
@@ -534,6 +719,83 @@ export async function* streamOpenAICompatibleBody(
   };
 }
 
+async function* readOpenAICompatibleJsonV1(
+  body: ReadableStream<Uint8Array>,
+  signal: AbortSignal,
+): AsyncIterable<LlmStreamEvent> {
+  const reader = body.getReader();
+  const chunks: Uint8Array[] = [];
+  let bytes = 0;
+  try {
+    while (true) {
+      signal.throwIfAborted();
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value.byteLength;
+      if (bytes > MAX_SSE_RESPONSE_BYTES) await rejectOversizedSse(reader);
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  const combined = new Uint8Array(bytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    combined.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  const payload = asRecord(
+    parseJson(
+      new TextDecoder().decode(combined),
+      "Model returned an invalid response",
+    ),
+  );
+  const choices = payload?.choices;
+  const choice = Array.isArray(choices) ? asRecord(choices[0]) : undefined;
+  const message = asRecord(choice?.message);
+  if (!choice || !message) {
+    throw new Error("Model response did not include a valid choice");
+  }
+  const usage = usageFromPayloadV1(payload);
+  if (usage) yield usage;
+  if (typeof message.content === "string" && message.content) {
+    yield { type: "text-delta", text: message.content };
+  }
+  const tools = new Map<number, ToolAccumulator>();
+  if (Array.isArray(message.tool_calls)) {
+    applyToolDeltas(
+      message.tool_calls.map((candidate, index) => ({
+        ...(asRecord(candidate) ?? {}),
+        index,
+      })),
+      tools,
+    );
+  }
+  for (const tool of [...tools.values()].sort(
+    (left, right) => left.index - right.index,
+  )) {
+    if (!tool.name)
+      throw new Error("Model returned a tool call without a name");
+    yield {
+      type: "tool-call",
+      call: {
+        id: tool.id || crypto.randomUUID(),
+        name: tool.name,
+        input: parseToolInput(tool.arguments),
+      },
+    };
+  }
+  yield {
+    type: "finish",
+    reason:
+      tools.size > 0 || choice.finish_reason === "tool_calls"
+        ? "tool-calls"
+        : choice.finish_reason === "length"
+          ? "max-tokens"
+          : "completed",
+  };
+}
+
 /**
  * Wait for `opening`, but give up when the deadline clock does.
  *
@@ -621,12 +883,16 @@ export async function* streamWithModelRequestDeadlinesV1(
 
 export class OpenAICompatibleProvider implements LlmProvider {
   readonly id: string;
+  readonly supports;
   private config: OpenAICompatibleConfig;
 
   constructor(config: OpenAICompatibleConfig) {
     if (!config.baseUrl.trim())
       throw new Error("OpenAI-compatible baseUrl is required");
     this.id = config.providerId ?? "openai-compatible";
+    this.supports = {
+      structuredOutput: config.structuredOutput ?? "none",
+    } as const;
     this.config = { ...config, baseUrl: config.baseUrl.replace(/\/$/, "") };
   }
 
@@ -634,6 +900,16 @@ export class OpenAICompatibleProvider implements LlmProvider {
     request: NormalizedModelRequest,
     signal: AbortSignal,
   ): AsyncIterable<LlmStreamEvent> {
+    const plan = planOpenAICompatibleRequestV1(request, {
+      ...(this.config.acceptsImages === undefined
+        ? {}
+        : { acceptsImages: this.config.acceptsImages }),
+      structuredOutput: this.supports.structuredOutput,
+      ...(this.config.responseFormatDialect
+        ? { responseFormatDialect: this.config.responseFormatDialect }
+        : {}),
+    });
+    if (plan.note) yield { type: "response-format-note", note: plan.note };
     // Workerd rejects a detached global `fetch` ("Illegal invocation"), so the
     // default fetcher forwards through a closure rather than aliasing it.
     const fetcher =
@@ -658,13 +934,7 @@ export class OpenAICompatibleProvider implements LlmProvider {
             {
               method: "POST",
               headers,
-              body: JSON.stringify(
-                requestToWire(request, {
-                  ...(this.config.acceptsImages === undefined
-                    ? {}
-                    : { acceptsImages: this.config.acceptsImages }),
-                }),
-              ),
+              body: JSON.stringify(plan.body),
               signal: deadlineSignal,
             },
           );