@hebo-ai/gateway 0.4.1 → 0.5.0-beta.0

package/README.md

@@ -286,10 +286,9 @@ const gw = gateway({
     /**
      * Runs before any endpoint handler logic.
      * @param ctx.request Incoming request.
-     * @returns Optional RequestPatch to merge into headers / override body.
-     * Returning a Response stops execution of the endpoint.
+     * @returns Optional Response to short-circuit the request.
      */
-    onRequest: async (ctx: { request: Request }): Promise<RequestPatch | Response | void> => {
+    onRequest: async (ctx: { request: Request }): Promise<Response | void> => {
       // Example Use Cases:
       // - Verify authentication
       // - Enforce rate limits

package/dist/endpoints/chat-completions/converters.d.ts

@@ -1,11 +1,13 @@
 import type { SharedV3ProviderOptions, SharedV3ProviderMetadata } from "@ai-sdk/provider";
-import type { GenerateTextResult, StreamTextResult, FinishReason, ToolChoice, ToolSet, ModelMessage, UserContent, LanguageModelUsage, Output, TextStreamPart, ReasoningOutput, AssistantModelMessage, ToolModelMessage, UserModelMessage } from "ai";
+import type { GenerateTextResult, StreamTextResult, FinishReason, ToolChoice, ToolSet, ModelMessage, UserContent, LanguageModelUsage, TextStreamPart, ReasoningOutput, AssistantModelMessage, ToolModelMessage, UserModelMessage } from "ai";
+import { Output } from "ai";
 import type { ChatCompletionsToolCall, ChatCompletionsTool, ChatCompletionsToolChoice, ChatCompletionsContentPart, ChatCompletionsMessage, ChatCompletionsUserMessage, ChatCompletionsAssistantMessage, ChatCompletionsToolMessage, ChatCompletionsFinishReason, ChatCompletionsUsage, ChatCompletionsInputs, ChatCompletions, ChatCompletionsChunk, ChatCompletionsReasoningDetail } from "./schema";
 import { OpenAIError } from "../../errors/openai";
 export type TextCallOptions = {
     messages: ModelMessage[];
     tools?: ToolSet;
     toolChoice?: ToolChoice<ToolSet>;
+    output?: Output.Output;
     temperature?: number;
     maxOutputTokens?: number;
     frequencyPenalty?: number;

package/dist/endpoints/chat-completions/converters.js

@@ -1,16 +1,17 @@
-import { convertBase64ToUint8Array } from "@ai-sdk/provider-utils";
-import { jsonSchema, tool } from "ai";
+import { Output, jsonSchema, tool } from "ai";
+import { z } from "zod";
 import { GatewayError } from "../../errors/gateway";
 import { OpenAIError, toOpenAIError } from "../../errors/openai";
 import { toResponse } from "../../utils/response";
 // --- Request Flow ---
 export function convertToTextCallOptions(params) {
-    const { messages, tools, tool_choice, temperature, max_tokens, max_completion_tokens, reasoning_effort, reasoning, frequency_penalty, presence_penalty, seed, stop, top_p, ...rest } = params;
+    const { messages, tools, tool_choice, temperature, max_tokens, max_completion_tokens, response_format, reasoning_effort, reasoning, frequency_penalty, presence_penalty, seed, stop, top_p, ...rest } = params;
     Object.assign(rest, parseReasoningOptions(reasoning_effort, reasoning));
     return {
         messages: convertToModelMessages(messages),
         tools: convertToToolSet(tools),
         toolChoice: convertToToolChoice(tool_choice),
+        output: convertToOutput(response_format),
         temperature,
         maxOutputTokens: max_completion_tokens ?? max_tokens,
         frequencyPenalty: frequency_penalty,
@@ -23,6 +24,17 @@ export function convertToTextCallOptions(params) {
         },
     };
 }
+function convertToOutput(responseFormat) {
+    if (!responseFormat || responseFormat.type === "text") {
+        return;
+    }
+    const { name, description, schema } = responseFormat.json_schema;
+    return Output.object({
+        name,
+        description,
+        schema: jsonSchema(schema),
+    });
+}
 export function convertToModelMessages(messages) {
     const modelMessages = [];
     const toolById = indexToolMessages(messages);
@@ -63,61 +75,66 @@ export function fromChatCompletionsUserMessage(message) {
 export function fromChatCompletionsAssistantMessage(message) {
     const { tool_calls, role, content, extra_content, reasoning_details } = message;
     const parts = [];
-    if (Array.isArray(parts)) {
-        if (reasoning_details?.length) {
-            for (const detail of reasoning_details) {
-                if (detail.text && detail.type === "reasoning.text") {
-                    parts.push({
-                        type: "reasoning",
-                        text: detail.text,
-                        providerOptions: detail.signature
-                            ? {
-                                unknown: {
-                                    signature: detail.signature,
-                                },
-                            }
-                            : undefined,
-                    });
-                }
-                else if (detail.type === "reasoning.encrypted" && detail.data) {
-                    parts.push({
-                        type: "reasoning",
-                        text: "",
-                        providerOptions: {
+    if (reasoning_details?.length) {
+        for (const detail of reasoning_details) {
+            if (detail.text && detail.type === "reasoning.text") {
+                parts.push({
+                    type: "reasoning",
+                    text: detail.text,
+                    providerOptions: detail.signature
+                        ? {
                             unknown: {
-                                redactedData: detail.data,
+                                signature: detail.signature,
                             },
+                        }
+                        : undefined,
+                });
+            }
+            else if (detail.type === "reasoning.encrypted" && detail.data) {
+                parts.push({
+                    type: "reasoning",
+                    text: "",
+                    providerOptions: {
+                        unknown: {
+                            redactedData: detail.data,
                         },
-                    });
-                }
+                    },
+                });
             }
         }
-        if (tool_calls?.length) {
-            for (const tc of tool_calls) {
-                // eslint-disable-next-line no-shadow
-                const { id, function: fn, extra_content } = tc;
-                const out = {
-                    type: "tool-call",
-                    toolCallId: id,
-                    toolName: fn.name,
-                    input: parseToolOutput(fn.arguments).value,
-                };
-                if (extra_content) {
-                    out.providerOptions = extra_content;
-                }
-                parts.push(out);
+    }
+    if (content !== undefined && content !== null) {
+        const inputContent = typeof content === "string"
+            ? [{ type: "text", text: content }]
+            : content;
+        for (const part of inputContent) {
+            if (part.type === "text") {
+                parts.push({
+                    type: "text",
+                    text: part.text,
+                });
             }
         }
-        else if (content !== undefined && content !== null) {
-            parts.push({
-                type: "text",
-                text: content,
-            });
+    }
+    if (tool_calls?.length) {
+        for (const tc of tool_calls) {
+            // eslint-disable-next-line no-shadow
+            const { id, function: fn, extra_content } = tc;
+            const out = {
+                type: "tool-call",
+                toolCallId: id,
+                toolName: fn.name,
+                input: parseJsonOrText(fn.arguments).value,
+            };
+            if (extra_content) {
+                out.providerOptions = extra_content;
+            }
+            parts.push(out);
         }
     }
     const out = {
-        role: role,
-        content: Array.isArray(parts) && parts.length > 0 ? parts : (content ?? ""),
+        role,
+        content: parts.length > 0 ? parts : (content ?? ""),
     };
     if (extra_content) {
         out.providerOptions = extra_content;
@@ -137,52 +154,50 @@ export function fromChatCompletionsToolResultMessage(message, toolById) {
             type: "tool-result",
             toolCallId: tc.id,
             toolName: tc.function.name,
-            output: parseToolOutput(toolMsg.content),
+            output: parseToolResult(toolMsg.content),
         });
     }
     return toolResultParts.length > 0 ? { role: "tool", content: toolResultParts } : undefined;
 }
 export function fromChatCompletionsContent(content) {
     return content.map((part) => {
-        if (part.type === "image_url") {
-            const url = part.image_url.url;
-            if (url.startsWith("data:")) {
-                const { mimeType, base64Data } = parseDataUrl(url);
-                return mimeType.startsWith("image/")
-                    ? {
-                        type: "image",
-                        image: convertBase64ToUint8Array(base64Data),
-                        mediaType: mimeType,
-                    }
-                    : {
-                        type: "file",
-                        data: convertBase64ToUint8Array(base64Data),
-                        mediaType: mimeType,
-                    };
-            }
-            return {
-                type: "image",
-                image: new URL(url),
-            };
+        switch (part.type) {
+            case "image_url":
+                return fromImageUrlPart(part.image_url.url);
+            case "file":
+                return fromFilePart(part.file.data, part.file.media_type, part.file.filename);
+            case "input_audio":
+                return fromFilePart(part.input_audio.data, `audio/${part.input_audio.format}`);
+            default:
+                return part;
         }
-        if (part.type === "file") {
-            let { data, media_type, filename } = part.file;
-            return media_type.startsWith("image/")
-                ? {
-                    type: "image",
-                    image: convertBase64ToUint8Array(data),
-                    mediaType: media_type,
-                }
-                : {
-                    type: "file",
-                    data: convertBase64ToUint8Array(data),
-                    filename,
-                    mediaType: media_type,
-                };
-        }
-        return part;
     });
 }
+function fromImageUrlPart(url) {
+    if (url.startsWith("data:")) {
+        const { mimeType, base64Data } = parseDataUrl(url);
+        return fromFilePart(base64Data, mimeType);
+    }
+    return {
+        type: "image",
+        image: new URL(url),
+    };
+}
+function fromFilePart(base64Data, mediaType, filename) {
+    if (mediaType.startsWith("image/")) {
+        return {
+            type: "image",
+            image: z.util.base64ToUint8Array(base64Data),
+            mediaType,
+        };
+    }
+    return {
+        type: "file",
+        data: z.util.base64ToUint8Array(base64Data),
+        filename,
+        mediaType,
+    };
+}
 export const convertToToolSet = (tools) => {
     if (!tools) {
         return;
@@ -203,12 +218,28 @@ export const convertToToolChoice = (toolChoice) => {
     if (toolChoice === "none" || toolChoice === "auto" || toolChoice === "required") {
         return toolChoice;
     }
+    // FUTURE: this is right now google specific, which is not supported by AI SDK, until then, we temporarily map it to auto for now https://docs.cloud.google.com/vertex-ai/generative-ai/docs/migrate/openai/overview
+    if (toolChoice === "validated") {
+        return "auto";
+    }
     return {
         type: "tool",
         toolName: toolChoice.function.name,
     };
 };
-function parseToolOutput(content) {
+function parseToolResult(content) {
+    if (Array.isArray(content)) {
+        return {
+            type: "content",
+            value: content.map((part) => ({
+                type: "text",
+                text: part.text,
+            })),
+        };
+    }
+    return parseJsonOrText(content);
+}
+function parseJsonOrText(content) {
     try {
         return { type: "json", value: JSON.parse(content) };
     }
@@ -255,7 +286,6 @@ function parseReasoningOptions(reasoning_effort, reasoning) {
 }
 // --- Response Flow ---
 export function toChatCompletions(result, model) {
-    const finish_reason = toChatCompletionsFinishReason(result.finishReason);
     return {
         id: "chatcmpl-" + crypto.randomUUID(),
         object: "chat.completion",
@@ -265,7 +295,7 @@ export function toChatCompletions(result, model) {
             {
                 index: 0,
                 message: toChatCompletionsAssistantMessage(result),
-                finish_reason,
+                finish_reason: toChatCompletionsFinishReason(result.finishReason),
             },
         ],
         usage: result.totalUsage ? toChatCompletionsUsage(result.totalUsage) : null,
@@ -287,6 +317,7 @@ export class ChatCompletionsStream extends TransformStream {
         const creationTime = Math.floor(Date.now() / 1000);
         let toolCallIndexCounter = 0;
         const reasoningIdToIndex = new Map();
+        let finishProviderMetadata;
         const createChunk = (delta, provider_metadata, finish_reason, usage) => {
             if (provider_metadata) {
                 delta.extra_content = provider_metadata;
@@ -340,11 +371,11 @@ export class ChatCompletionsStream extends TransformStream {
                         break;
                     }
                     case "finish-step": {
-                        controller.enqueue(createChunk({}, part.providerMetadata, toChatCompletionsFinishReason(part.finishReason), toChatCompletionsUsage(part.usage)));
+                        finishProviderMetadata = part.providerMetadata;
                         break;
                     }
                     case "finish": {
-                        controller.enqueue(createChunk({}, undefined, toChatCompletionsFinishReason(part.finishReason), toChatCompletionsUsage(part.totalUsage)));
+                        controller.enqueue(createChunk({}, finishProviderMetadata, toChatCompletionsFinishReason(part.finishReason), toChatCompletionsUsage(part.totalUsage)));
                         break;
                     }
                     case "error": {

package/dist/endpoints/chat-completions/handler.js

@@ -7,7 +7,6 @@ import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
 import { recordRequestDuration, recordTimePerOutputToken, recordTokenUsage, } from "../../telemetry/gen-ai";
 import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
-import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
 import { convertToTextCallOptions, toChatCompletions, toChatCompletionsStream } from "./converters";
 import { getChatGeneralAttributes, getChatRequestAttributes, getChatResponseAttributes, } from "./otel";
@@ -22,7 +21,6 @@ export const chatCompletions = (config) => {
         if (!ctx.request || ctx.request.method !== "POST") {
             throw new GatewayError("Method Not Allowed", 405);
         }
-        const requestId = resolveRequestId(ctx.request);
         // Parse + validate input.
         try {
             ctx.body = await ctx.request.json();
@@ -68,7 +66,7 @@ export const chatCompletions = (config) => {
         // Convert inputs to AI SDK call options.
         const textOptions = convertToTextCallOptions(inputs);
         logger.trace({
-            requestId,
+            requestId: ctx.requestId,
             options: textOptions,
         }, "[chat] AI SDK options");
         addSpanEvent("hebo.options.prepared");
@@ -127,7 +125,7 @@ export const chatCompletions = (config) => {
             },
             ...textOptions,
         });
-        logger.trace({ requestId, result }, "[chat] AI SDK result");
+        logger.trace({ requestId: ctx.requestId, result }, "[chat] AI SDK result");
         addSpanEvent("hebo.ai-sdk.completed");
         // Transform result.
         ctx.result = toChatCompletions(result, ctx.resolvedModelId);

package/dist/endpoints/chat-completions/otel.js

@@ -32,6 +32,13 @@ const toMessageParts = (message) => {
                 else if (part.type === "image_url") {
                     parts.push({ type: "image", content: part.image_url.url });
                 }
+                else if (part.type === "input_audio") {
+                    parts.push({
+                        type: "audio",
+                        content: "[REDACTED_BINARY_DATA]",
+                        format: part.input_audio.format,
+                    });
+                }
                 else {
                     parts.push({
                         type: "file",