@hebo-ai/gateway 0.4.2 → 0.5.0

package/README.md

@@ -32,7 +32,7 @@ bun install @hebo-ai/gateway
 - Quickstart
   - [Setup A Gateway Instance](#setup-a-gateway-instance) | [Mount Route Handlers](#mount-route-handlers) | [Call the Gateway](#call-the-gateway)
 - Configuration Reference
-  - [Providers](#providers) | [Models](#models) | [Hooks](#hooks) | [Logger](#logger-settings) | [Telemetry](#telemetry-settings)
+  - [Providers](#providers) | [Models](#models) | [Hooks](#hooks) | [Logger](#logger-settings) | [Observability](#observability)
 - Framework Support
   - [ElysiaJS](#elysiajs) | [Hono](#hono) | [Next.js](#nextjs) | [TanStack Start](#tanstack-start)
 - Runtime Support
@@ -540,13 +540,14 @@ Normalization rules:
 
 - `enabled` -> fall-back to model default if none provided
 - `max_tokens`: fall-back to model default if model supports
-- `effort` -> budget = percentage of `max_tokens`
+- `effort` supports: `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`
+- Generic `effort` -> budget = percentage of `max_tokens`
   - `none`: 0%
   - `minimal`: 10%
   - `low`: 20%
   - `medium`: 50% (default)
   - `high`: 80%
-  - `xhigh`: 95%
+  - `xhigh` / `max`: 95%
 
 Reasoning output is surfaced as extension to the `completion` object.
 
@@ -602,9 +603,9 @@ const gw = gateway({
 > [!TIP]
 > For production workloads, we recommend `pino` for better logging performance and lower overhead.
 
-### Telemetry Settings
+### Observability
 
-Hebo Gateway can forward telemetry settings via the `telemetry` config field.
+Hebo Gateway can forward traces & metrics via the `telemetry` config field.
 
 ```ts
 import { gateway } from "@hebo-ai/gateway";
@@ -633,8 +634,9 @@ const gw = gateway({
 });
 ```
 
-Attribute names and span semantics follow OpenTelemetry GenAI semantic conventions:
+Attribute names and span & metrics semantics follow OpenTelemetry GenAI semantic conventions:
 https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/
+https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/
 
 > [!TIP]
 > To populate custom span attributes, the inbound W3C `baggage` header is supported. Keys in the `hebo.` namespace are mapped to span attributes, with the namespace stripped. For example: `baggage: hebo.user_id=u-123` becomes span attribute `user_id=u-123`.

package/dist/endpoints/chat-completions/converters.d.ts

@@ -1,11 +1,13 @@
 import type { SharedV3ProviderOptions, SharedV3ProviderMetadata } from "@ai-sdk/provider";
-import type { GenerateTextResult, StreamTextResult, FinishReason, ToolChoice, ToolSet, ModelMessage, UserContent, LanguageModelUsage, Output, TextStreamPart, ReasoningOutput, AssistantModelMessage, ToolModelMessage, UserModelMessage } from "ai";
+import type { GenerateTextResult, StreamTextResult, FinishReason, ToolChoice, ToolSet, ModelMessage, UserContent, LanguageModelUsage, TextStreamPart, ReasoningOutput, AssistantModelMessage, ToolModelMessage, UserModelMessage } from "ai";
+import { Output } from "ai";
 import type { ChatCompletionsToolCall, ChatCompletionsTool, ChatCompletionsToolChoice, ChatCompletionsContentPart, ChatCompletionsMessage, ChatCompletionsUserMessage, ChatCompletionsAssistantMessage, ChatCompletionsToolMessage, ChatCompletionsFinishReason, ChatCompletionsUsage, ChatCompletionsInputs, ChatCompletions, ChatCompletionsChunk, ChatCompletionsReasoningDetail } from "./schema";
 import { OpenAIError } from "../../errors/openai";
 export type TextCallOptions = {
     messages: ModelMessage[];
     tools?: ToolSet;
     toolChoice?: ToolChoice<ToolSet>;
+    output?: Output.Output;
     temperature?: number;
     maxOutputTokens?: number;
     frequencyPenalty?: number;

package/dist/endpoints/chat-completions/converters.js

@@ -1,16 +1,17 @@
-import { convertBase64ToUint8Array } from "@ai-sdk/provider-utils";
-import { jsonSchema, tool } from "ai";
+import { Output, jsonSchema, tool } from "ai";
+import { z } from "zod";
 import { GatewayError } from "../../errors/gateway";
 import { OpenAIError, toOpenAIError } from "../../errors/openai";
 import { toResponse } from "../../utils/response";
 // --- Request Flow ---
 export function convertToTextCallOptions(params) {
-    const { messages, tools, tool_choice, temperature, max_tokens, max_completion_tokens, reasoning_effort, reasoning, frequency_penalty, presence_penalty, seed, stop, top_p, ...rest } = params;
+    const { messages, tools, tool_choice, temperature, max_tokens, max_completion_tokens, response_format, reasoning_effort, reasoning, frequency_penalty, presence_penalty, seed, stop, top_p, ...rest } = params;
     Object.assign(rest, parseReasoningOptions(reasoning_effort, reasoning));
     return {
         messages: convertToModelMessages(messages),
         tools: convertToToolSet(tools),
         toolChoice: convertToToolChoice(tool_choice),
+        output: convertToOutput(response_format),
         temperature,
         maxOutputTokens: max_completion_tokens ?? max_tokens,
         frequencyPenalty: frequency_penalty,
@@ -23,6 +24,17 @@ export function convertToTextCallOptions(params) {
         },
     };
 }
+function convertToOutput(responseFormat) {
+    if (!responseFormat || responseFormat.type === "text") {
+        return;
+    }
+    const { name, description, schema } = responseFormat.json_schema;
+    return Output.object({
+        name,
+        description,
+        schema: jsonSchema(schema),
+    });
+}
 export function convertToModelMessages(messages) {
     const modelMessages = [];
     const toolById = indexToolMessages(messages);
@@ -63,61 +75,66 @@ export function fromChatCompletionsUserMessage(message) {
 export function fromChatCompletionsAssistantMessage(message) {
     const { tool_calls, role, content, extra_content, reasoning_details } = message;
     const parts = [];
-    if (Array.isArray(parts)) {
-        if (reasoning_details?.length) {
-            for (const detail of reasoning_details) {
-                if (detail.text && detail.type === "reasoning.text") {
-                    parts.push({
-                        type: "reasoning",
-                        text: detail.text,
-                        providerOptions: detail.signature
-                            ? {
-                                unknown: {
-                                    signature: detail.signature,
-                                },
-                            }
-                            : undefined,
-                    });
-                }
-                else if (detail.type === "reasoning.encrypted" && detail.data) {
-                    parts.push({
-                        type: "reasoning",
-                        text: "",
-                        providerOptions: {
+    if (reasoning_details?.length) {
+        for (const detail of reasoning_details) {
+            if (detail.text && detail.type === "reasoning.text") {
+                parts.push({
+                    type: "reasoning",
+                    text: detail.text,
+                    providerOptions: detail.signature
+                        ? {
                             unknown: {
-                                redactedData: detail.data,
+                                signature: detail.signature,
                             },
+                        }
+                        : undefined,
+                });
+            }
+            else if (detail.type === "reasoning.encrypted" && detail.data) {
+                parts.push({
+                    type: "reasoning",
+                    text: "",
+                    providerOptions: {
+                        unknown: {
+                            redactedData: detail.data,
                         },
-                    });
-                }
+                    },
+                });
             }
         }
-        if (tool_calls?.length) {
-            for (const tc of tool_calls) {
-                // eslint-disable-next-line no-shadow
-                const { id, function: fn, extra_content } = tc;
-                const out = {
-                    type: "tool-call",
-                    toolCallId: id,
-                    toolName: fn.name,
-                    input: parseToolOutput(fn.arguments).value,
-                };
-                if (extra_content) {
-                    out.providerOptions = extra_content;
-                }
-                parts.push(out);
+    }
+    if (content !== undefined && content !== null) {
+        const inputContent = typeof content === "string"
+            ? [{ type: "text", text: content }]
+            : content;
+        for (const part of inputContent) {
+            if (part.type === "text") {
+                parts.push({
+                    type: "text",
+                    text: part.text,
+                });
             }
         }
-        else if (content !== undefined && content !== null) {
-            parts.push({
-                type: "text",
-                text: content,
-            });
+    }
+    if (tool_calls?.length) {
+        for (const tc of tool_calls) {
+            // eslint-disable-next-line no-shadow
+            const { id, function: fn, extra_content } = tc;
+            const out = {
+                type: "tool-call",
+                toolCallId: id,
+                toolName: fn.name,
+                input: parseJsonOrText(fn.arguments).value,
+            };
+            if (extra_content) {
+                out.providerOptions = extra_content;
+            }
+            parts.push(out);
         }
     }
     const out = {
-        role: role,
-        content: Array.isArray(parts) && parts.length > 0 ? parts : (content ?? ""),
+        role,
+        content: parts.length > 0 ? parts : (content ?? ""),
     };
     if (extra_content) {
         out.providerOptions = extra_content;
@@ -137,52 +154,50 @@ export function fromChatCompletionsToolResultMessage(message, toolById) {
             type: "tool-result",
             toolCallId: tc.id,
             toolName: tc.function.name,
-            output: parseToolOutput(toolMsg.content),
+            output: parseToolResult(toolMsg.content),
         });
     }
     return toolResultParts.length > 0 ? { role: "tool", content: toolResultParts } : undefined;
 }
 export function fromChatCompletionsContent(content) {
     return content.map((part) => {
-        if (part.type === "image_url") {
-            const url = part.image_url.url;
-            if (url.startsWith("data:")) {
-                const { mimeType, base64Data } = parseDataUrl(url);
-                return mimeType.startsWith("image/")
-                    ? {
-                        type: "image",
-                        image: convertBase64ToUint8Array(base64Data),
-                        mediaType: mimeType,
-                    }
-                    : {
-                        type: "file",
-                        data: convertBase64ToUint8Array(base64Data),
-                        mediaType: mimeType,
-                    };
-            }
-            return {
-                type: "image",
-                image: new URL(url),
-            };
+        switch (part.type) {
+            case "image_url":
+                return fromImageUrlPart(part.image_url.url);
+            case "file":
+                return fromFilePart(part.file.data, part.file.media_type, part.file.filename);
+            case "input_audio":
+                return fromFilePart(part.input_audio.data, `audio/${part.input_audio.format}`);
+            default:
+                return part;
         }
-        if (part.type === "file") {
-            let { data, media_type, filename } = part.file;
-            return media_type.startsWith("image/")
-                ? {
-                    type: "image",
-                    image: convertBase64ToUint8Array(data),
-                    mediaType: media_type,
-                }
-                : {
-                    type: "file",
-                    data: convertBase64ToUint8Array(data),
-                    filename,
-                    mediaType: media_type,
-                };
-        }
-        return part;
     });
 }
+function fromImageUrlPart(url) {
+    if (url.startsWith("data:")) {
+        const { mimeType, base64Data } = parseDataUrl(url);
+        return fromFilePart(base64Data, mimeType);
+    }
+    return {
+        type: "image",
+        image: new URL(url),
+    };
+}
+function fromFilePart(base64Data, mediaType, filename) {
+    if (mediaType.startsWith("image/")) {
+        return {
+            type: "image",
+            image: z.util.base64ToUint8Array(base64Data),
+            mediaType,
+        };
+    }
+    return {
+        type: "file",
+        data: z.util.base64ToUint8Array(base64Data),
+        filename,
+        mediaType,
+    };
+}
 export const convertToToolSet = (tools) => {
     if (!tools) {
         return;
@@ -203,12 +218,28 @@ export const convertToToolChoice = (toolChoice) => {
     if (toolChoice === "none" || toolChoice === "auto" || toolChoice === "required") {
         return toolChoice;
     }
+    // FUTURE: this is right now google specific, which is not supported by AI SDK, until then, we temporarily map it to auto for now https://docs.cloud.google.com/vertex-ai/generative-ai/docs/migrate/openai/overview
+    if (toolChoice === "validated") {
+        return "auto";
+    }
     return {
         type: "tool",
         toolName: toolChoice.function.name,
     };
 };
-function parseToolOutput(content) {
+function parseToolResult(content) {
+    if (Array.isArray(content)) {
+        return {
+            type: "content",
+            value: content.map((part) => ({
+                type: "text",
+                text: part.text,
+            })),
+        };
+    }
+    return parseJsonOrText(content);
+}
+function parseJsonOrText(content) {
     try {
         return { type: "json", value: JSON.parse(content) };
     }
@@ -255,7 +286,6 @@ function parseReasoningOptions(reasoning_effort, reasoning) {
 }
 // --- Response Flow ---
 export function toChatCompletions(result, model) {
-    const finish_reason = toChatCompletionsFinishReason(result.finishReason);
     return {
         id: "chatcmpl-" + crypto.randomUUID(),
         object: "chat.completion",
@@ -265,7 +295,7 @@ export function toChatCompletions(result, model) {
             {
                 index: 0,
                 message: toChatCompletionsAssistantMessage(result),
-                finish_reason,
+                finish_reason: toChatCompletionsFinishReason(result.finishReason),
             },
         ],
         usage: result.totalUsage ? toChatCompletionsUsage(result.totalUsage) : null,
@@ -287,6 +317,7 @@ export class ChatCompletionsStream extends TransformStream {
         const creationTime = Math.floor(Date.now() / 1000);
         let toolCallIndexCounter = 0;
         const reasoningIdToIndex = new Map();
+        let finishProviderMetadata;
         const createChunk = (delta, provider_metadata, finish_reason, usage) => {
             if (provider_metadata) {
                 delta.extra_content = provider_metadata;
@@ -340,11 +371,11 @@ export class ChatCompletionsStream extends TransformStream {
                         break;
                     }
                     case "finish-step": {
-                        controller.enqueue(createChunk({}, part.providerMetadata, toChatCompletionsFinishReason(part.finishReason), toChatCompletionsUsage(part.usage)));
+                        finishProviderMetadata = part.providerMetadata;
                         break;
                     }
                     case "finish": {
-                        controller.enqueue(createChunk({}, undefined, toChatCompletionsFinishReason(part.finishReason), toChatCompletionsUsage(part.totalUsage)));
+                        controller.enqueue(createChunk({}, finishProviderMetadata, toChatCompletionsFinishReason(part.finishReason), toChatCompletionsUsage(part.totalUsage)));
                         break;
                     }
                     case "error": {

package/dist/endpoints/chat-completions/otel.js

@@ -32,6 +32,13 @@ const toMessageParts = (message) => {
                 else if (part.type === "image_url") {
                     parts.push({ type: "image", content: part.image_url.url });
                 }
+                else if (part.type === "input_audio") {
+                    parts.push({
+                        type: "audio",
+                        content: "[REDACTED_BINARY_DATA]",
+                        format: part.input_audio.format,
+                    });
+                }
                 else {
                     parts.push({
                         type: "file",