@hebo-ai/gateway 0.6.2 → 0.7.0

package/README.md

@@ -32,13 +32,13 @@ bun install @hebo-ai/gateway
 - Quickstart
   - [Setup A Gateway Instance](#setup-a-gateway-instance) | [Mount Route Handlers](#mount-route-handlers) | [Call the Gateway](#call-the-gateway)
 - Configuration Reference
-  - [Providers](#providers) | [Models](#models) | [Hooks](#hooks) | [Logger](#logger-settings) | [Observability](#observability)
+  - [Providers](#providers) | [Models](#models) | [Hooks](#hooks) | [Logger](#logger-settings) | [Observability](#observability) | [Timeouts](#timeout-settings)
 - Framework Support
   - [ElysiaJS](#elysiajs) | [Hono](#hono) | [Next.js](#nextjs) | [TanStack Start](#tanstack-start)
 - Runtime Support
   - [Vercel Edge](#vercel-edge) | [Cloudflare Workers](#cloudflare-workers) | [Deno Deploy](#deno-deploy) | [AWS Lambda](#aws-lambda)
 - OpenAI Extensions
-  - [Reasoning](#reasoning) | [Prompt Caching](#prompt-caching)
+  - [Reasoning](#reasoning) | [Service Tier](#service-tier) | [Prompt Caching](#prompt-caching)
 - Advanced Usage
   - [Passing Framework State to Hooks](#passing-framework-state-to-hooks) | [Selective Route Mounting](#selective-route-mounting) | [Low-level Schemas & Converters](#low-level-schemas--converters)
 
@@ -342,9 +342,9 @@ const gw = gateway({
      * @returns Modified result, or undefined to keep original.
      */
     after: async (ctx: {
-      result: ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings;
+      result: ChatCompletions | ChatCompletionsStream | Embeddings;
     }): Promise<
-      ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings | void
+      ChatCompletions | ChatCompletionsStream | Embeddings | void
     > => {
       // Example Use Cases:
       // - Transform result
@@ -561,6 +561,25 @@ Advanced models (like Anthropic Claude 3.7 or Gemini 3) surface structured reaso
 
 For **Gemini 3** models, returning the thought signature via `extra_content` is mandatory to resume the chain-of-thought; failing to do so may result in errors or degraded performance.
 
+### Service Tier
+
+The chat completions endpoint accepts a provider-agnostic `service_tier` extension:
+
+- `auto`, `default`, `flex`, `priority`, `scale`
+
+Provider-specific mapping:
+
+- **OpenAI**: forwards as OpenAI `serviceTier` (no middleware remap).
+- **Groq**: maps to Groq `serviceTier` (`default` -> `on_demand`, `scale`/`priority` -> `performance`).
+- **Google Vertex**: maps to request headers via middleware:
+  - `default` -> `x-vertex-ai-llm-request-type: shared`
+  - `flex` -> `x-vertex-ai-llm-request-type: shared` + `x-vertex-ai-llm-shared-request-type: flex`
+  - `priority` -> `x-vertex-ai-llm-request-type: shared` + `x-vertex-ai-llm-shared-request-type: priority`
+  - `scale` -> `x-vertex-ai-llm-request-type: dedicated`
+- **Amazon Bedrock**: maps to Bedrock `serviceTier.type` (`default`, `flex`, `priority`, `reserved`; `scale` -> `reserved`, `auto` -> omitted/default).
+
+When available, the resolved value is echoed back on response as `service_tier`.
+
 ### Prompt Caching
 
 The chat completions endpoint supports both implicit (provider-managed) and explicit prompt caching across OpenAI-compatible providers.
@@ -737,6 +756,37 @@ const gw = gateway({
 
 Langfuse credentials are read from environment variables by the Langfuse OTel SDK (`LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_BASE_URL`).
 
+### Timeout Settings
+
+You can configure request timeouts via the `timeouts` field:
+
+```ts
+import { gateway } from "@hebo-ai/gateway";
+
+const gw = gateway({
+  // ...
+  // default timeout is 300_000 (5 minutes).
+  // You can set one timeout for all tiers...
+  timeouts: 60_000,
+  // ...disable timeouts completely:
+  // timeouts: null,
+  // ...or split by service tier:
+  // - normal: all non-flex tiers (set null to disable)
+  // - flex: defaults to 3x normal when omitted (set null to disable)
+  // timeouts: { normal: 30_000, flex: null },
+});
+```
+
+> [!NOTE]
+> **Runtime/engine timeout limits**
+> Runtime-level `fetch()` clients may enforce their own timeouts. Configure those runtime/platform limits in addition to gateway `timeouts`.
+>
+> - Node.js runtimes use Undici: https://github.com/nodejs/undici/issues/1373 (Node.js, Vercel Serverless Functions, AWS Lambda)
+> - Bun context: https://github.com/oven-sh/bun/issues/16682
+>
+> **Provider/service timeout limits**
+> Serverless platforms (e.g. Cloudflare Workers, Vercel Edge/Serverless, AWS Lambda) also enforce platform time limits (roughly ~25-100s on edge paths, ~300s for streaming, and up to ~900s configurable for some).
+
 ### Passing Framework State to Hooks
 
 You can pass per-request info from your framework into the gateway via the second `state` argument on the handler, then read it in hooks through `ctx.state`.
@@ -838,7 +888,7 @@ export async function handler(req: Request): Promise<Response> {
 }
 ```
 
-Non-streaming versions are available via `createChatCompletionsResponse`. Equivalent schemas and helpers are available in the `embeddings` and `models` endpoints.
+Non-streaming versions are available via `toChatCompletionsResponse`. Equivalent schemas and helpers are available in the `embeddings` and `models` endpoints.
 
 > [!TIP]
 > Since Zod v4.3 you can generate a JSON Schema from any zod object by calling `z.toJSONSchema(...)`. This is useful for producing OpenAPI documentation from the same source of truth.

package/dist/config.js

@@ -1,7 +1,7 @@
 import { isLogger, logger, setLoggerInstance } from "./logger";
 import { createDefaultLogger } from "./logger/default";
 import { installAiSdkWarningLogger } from "./telemetry/ai-sdk";
-import { kParsed, } from "./types";
+import { DEFAULT_CHAT_TIMEOUT_MS, kParsed, } from "./types";
 export const parseConfig = (config) => {
     // If it has been parsed before, just return.
     if (kParsed in config)
@@ -66,9 +66,36 @@ export const parseConfig = (config) => {
             hebo: "off",
         };
     installAiSdkWarningLogger(telemetrySignals.gen_ai);
+    // Default timeouts
+    let normal;
+    let flex;
+    const t = config.timeouts;
+    if (t === null) {
+        normal = flex = undefined;
+    }
+    else if (typeof t === "number") {
+        normal = t;
+        flex = t * 3;
+    }
+    else {
+        if (t?.normal === null)
+            normal = undefined;
+        else if (t?.normal === undefined)
+            normal = DEFAULT_CHAT_TIMEOUT_MS;
+        else
+            normal = t.normal;
+        if (t?.flex === null)
+            flex = undefined;
+        else if (t?.flex === undefined)
+            flex = normal === undefined ? undefined : normal * 3;
+        else
+            flex = t.flex;
+    }
+    const parsedTimeouts = { normal, flex };
     // Return parsed config.
     return {
         ...config,
+        timeouts: parsedTimeouts,
         telemetry: {
             ...config.telemetry,
             enabled: telemetryEnabled,

package/dist/endpoints/chat-completions/converters.d.ts

@@ -1,8 +1,8 @@
 import type { SharedV3ProviderOptions, SharedV3ProviderMetadata } from "@ai-sdk/provider";
 import type { GenerateTextResult, StreamTextResult, FinishReason, ToolChoice, ToolSet, ModelMessage, UserContent, LanguageModelUsage, TextStreamPart, ReasoningOutput, AssistantModelMessage, ToolModelMessage, UserModelMessage } from "ai";
 import { Output } from "ai";
-import type { ChatCompletionsToolCall, ChatCompletionsTool, ChatCompletionsToolChoice, ChatCompletionsContentPart, ChatCompletionsMessage, ChatCompletionsUserMessage, ChatCompletionsAssistantMessage, ChatCompletionsToolMessage, ChatCompletionsFinishReason, ChatCompletionsUsage, ChatCompletionsInputs, ChatCompletions, ChatCompletionsChunk, ChatCompletionsReasoningDetail } from "./schema";
-import { OpenAIError } from "../../errors/openai";
+import type { ChatCompletionsToolCall, ChatCompletionsTool, ChatCompletionsToolChoice, ChatCompletionsStream, ChatCompletionsContentPart, ChatCompletionsMessage, ChatCompletionsUserMessage, ChatCompletionsAssistantMessage, ChatCompletionsToolMessage, ChatCompletionsFinishReason, ChatCompletionsUsage, ChatCompletionsInputs, ChatCompletions, ChatCompletionsChunk, ChatCompletionsReasoningDetail } from "./schema";
+import type { SseErrorFrame, SseFrame } from "../../utils/stream";
 export type TextCallOptions = {
     messages: ModelMessage[];
     tools?: ToolSet;
@@ -31,10 +31,10 @@ export declare const convertToToolChoiceOptions: (toolChoice: ChatCompletionsToo
 };
 export declare function toChatCompletions(result: GenerateTextResult<ToolSet, Output.Output>, model: string): ChatCompletions;
 export declare function toChatCompletionsResponse(result: GenerateTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
-export declare function toChatCompletionsStream<E extends boolean = false>(result: StreamTextResult<ToolSet, Output.Output>, model: string, wrapErrors?: E): ReadableStream<ChatCompletionsChunk | (E extends true ? OpenAIError : Error)>;
+export declare function toChatCompletionsStream(result: StreamTextResult<ToolSet, Output.Output>, model: string): ChatCompletionsStream;
 export declare function toChatCompletionsStreamResponse(result: StreamTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
-export declare class ChatCompletionsStream<E extends boolean = false> extends TransformStream<TextStreamPart<ToolSet>, ChatCompletionsChunk | (E extends true ? OpenAIError : Error)> {
-    constructor(model: string, wrapErrors?: E);
+export declare class ChatCompletionsTransformStream extends TransformStream<TextStreamPart<ToolSet>, SseFrame<ChatCompletionsChunk> | SseErrorFrame> {
+    constructor(model: string);
 }
 export declare const toChatCompletionsAssistantMessage: (result: GenerateTextResult<ToolSet, Output.Output>) => ChatCompletionsAssistantMessage;
 export declare function toReasoningDetail(reasoning: ReasoningOutput, id: string, index: number): ChatCompletionsReasoningDetail;

package/dist/endpoints/chat-completions/converters.js

@@ -1,7 +1,6 @@
 import { Output, jsonSchema, tool } from "ai";
 import { z } from "zod";
 import { GatewayError } from "../../errors/gateway";
-import { OpenAIError, toOpenAIError } from "../../errors/openai";
 import { toResponse } from "../../utils/response";
 import { parseDataUrl } from "../../utils/url";
 // --- Request Flow ---
@@ -379,19 +378,20 @@ export function toChatCompletions(result, model) {
         ],
         usage: result.totalUsage ? toChatCompletionsUsage(result.totalUsage) : null,
         provider_metadata: result.providerMetadata,
+        service_tier: resolveResponseServiceTier(result.providerMetadata),
     };
 }
 export function toChatCompletionsResponse(result, model, responseInit) {
     return toResponse(toChatCompletions(result, model), responseInit);
 }
-export function toChatCompletionsStream(result, model, wrapErrors) {
-    return result.fullStream.pipeThrough(new ChatCompletionsStream(model, wrapErrors));
+export function toChatCompletionsStream(result, model) {
+    return result.fullStream.pipeThrough(new ChatCompletionsTransformStream(model));
 }
 export function toChatCompletionsStreamResponse(result, model, responseInit) {
-    return toResponse(toChatCompletionsStream(result, model, true), responseInit);
+    return toResponse(toChatCompletionsStream(result, model), responseInit);
 }
-export class ChatCompletionsStream extends TransformStream {
-    constructor(model, wrapErrors) {
+export class ChatCompletionsTransformStream extends TransformStream {
+    constructor(model) {
         const streamId = `chatcmpl-${crypto.randomUUID()}`;
         const creationTime = Math.floor(Date.now() / 1000);
         let toolCallIndexCounter = 0;
@@ -402,18 +402,21 @@ export class ChatCompletionsStream extends TransformStream {
                 delta.extra_content = provider_metadata;
             }
             return {
-                id: streamId,
-                object: "chat.completion.chunk",
-                created: creationTime,
-                model,
-                choices: [
-                    {
-                        index: 0,
-                        delta,
-                        finish_reason: finish_reason ?? null,
-                    },
-                ],
-                usage: usage ?? null,
+                data: {
+                    id: streamId,
+                    object: "chat.completion.chunk",
+                    created: creationTime,
+                    model,
+                    choices: [
+                        {
+                            index: 0,
+                            delta,
+                            finish_reason: finish_reason ?? null,
+                        },
+                    ],
+                    usage: usage ?? null,
+                    service_tier: resolveResponseServiceTier(provider_metadata),
+                },
             };
         };
         super({
@@ -459,23 +462,56 @@ export class ChatCompletionsStream extends TransformStream {
                         break;
                     }
                     case "error": {
-                        let err;
-                        if (wrapErrors) {
-                            err = toOpenAIError(part.error);
-                        }
-                        else if (part.error instanceof Error) {
-                            err = part.error;
-                        }
-                        else {
-                            err = new Error(String(part.error));
-                        }
-                        controller.enqueue(err);
+                        controller.enqueue({
+                            data: part.error instanceof Error ? part.error : new Error(String(part.error)),
+                        });
                     }
                 }
             },
         });
     }
 }
+function resolveResponseServiceTier(providerMetadata) {
+    if (!providerMetadata)
+        return;
+    for (const metadata of Object.values(providerMetadata)) {
+        const tier = parseReturnedServiceTier(metadata["service_tier"] ??
+            metadata["usage_metadata"]?.["traffic_type"]);
+        if (tier)
+            return tier;
+    }
+}
+function parseReturnedServiceTier(value) {
+    if (typeof value !== "string")
+        return undefined;
+    const n = value.toLowerCase();
+    switch (n) {
+        case "traffic_type_unspecified":
+        case "auto":
+            return "auto";
+        case "default":
+        case "on_demand":
+        case "on-demand":
+        case "shared":
+            return "default";
+        case "on_demand_flex":
+        case "flex":
+            return "flex";
+        case "on_demand_priority":
+        case "priority":
+        case "performance":
+            return "priority";
+        case "provisioned_throughput":
+        case "scale":
+        case "reserved":
+        case "dedicated":
+        case "provisioned":
+        case "throughput":
+            return "scale";
+        default:
+            return undefined;
+    }
+}
 export const toChatCompletionsAssistantMessage = (result) => {
     const message = {
         role: "assistant",

package/dist/endpoints/chat-completions/handler.js

@@ -13,7 +13,7 @@ import { getChatRequestAttributes, getChatResponseAttributes } from "./otel";
 import { ChatCompletionsBodySchema } from "./schema";
 export const chatCompletions = (config) => {
     const hooks = config.hooks;
-    const handler = async (ctx) => {
+    const handler = async (ctx, cfg) => {
         const start = performance.now();
         ctx.operation = "chat";
         addSpanEvent("hebo.handler.started");
@@ -63,7 +63,7 @@ export const chatCompletions = (config) => {
         ctx.resolvedProviderId = languageModel.provider;
         logger.debug(`[chat] using ${languageModel.provider} for ${ctx.resolvedModelId}`);
         addSpanEvent("hebo.provider.resolved");
-        const genAiSignalLevel = config.telemetry?.signals?.gen_ai;
+        const genAiSignalLevel = cfg.telemetry?.signals?.gen_ai;
         const genAiGeneralAttrs = getGenAiGeneralAttributes(ctx, genAiSignalLevel);
         setSpanAttributes(genAiGeneralAttrs);
         // Convert inputs to AI SDK call options.
@@ -88,7 +88,7 @@ export const chatCompletions = (config) => {
                 headers: prepareForwardHeaders(ctx.request),
                 abortSignal: ctx.request.signal,
                 timeout: {
-                    totalMs: 5 * 60 * 1000,
+                    totalMs: ctx.body.service_tier === "flex" ? cfg.timeouts.flex : cfg.timeouts.normal,
                 },
                 onAbort: () => {
                     throw new DOMException("The operation was aborted.", "AbortError");
@@ -122,7 +122,7 @@ export const chatCompletions = (config) => {
             model: languageModelWithMiddleware,
             headers: prepareForwardHeaders(ctx.request),
             abortSignal: ctx.request.signal,
-            timeout: 5 * 60 * 1000,
+            timeout: ctx.body.service_tier === "flex" ? cfg.timeouts.flex : cfg.timeouts.normal,
             experimental_include: {
                 requestBody: false,
                 responseBody: false,

package/dist/endpoints/chat-completions/otel.d.ts

@@ -1,5 +1,5 @@
 import type { Attributes } from "@opentelemetry/api";
 import type { ChatCompletions, ChatCompletionsBody } from "./schema";
 import { type TelemetrySignalLevel } from "../../types";
-export declare const getChatRequestAttributes: (inputs: ChatCompletionsBody, signalLevel?: TelemetrySignalLevel) => Attributes;
+export declare const getChatRequestAttributes: (body: ChatCompletionsBody, signalLevel?: TelemetrySignalLevel) => Attributes;
 export declare const getChatResponseAttributes: (completions: ChatCompletions, signalLevel?: TelemetrySignalLevel) => Attributes;

package/dist/endpoints/chat-completions/otel.js

@@ -99,31 +99,32 @@ const toMessageParts = (message) => {
             throw new Error(`Unhandled content part type: ${message.role}`);
     }
 };
-export const getChatRequestAttributes = (inputs, signalLevel) => {
+export const getChatRequestAttributes = (body, signalLevel) => {
     if (!signalLevel || signalLevel === "off")
         return {};
     const attrs = {};
-    if (inputs.seed !== undefined) {
-        Object.assign(attrs, { "gen_ai.request.seed": inputs.seed });
+    if (body.seed !== undefined) {
+        Object.assign(attrs, { "gen_ai.request.seed": body.seed });
     }
     if (signalLevel !== "required") {
         Object.assign(attrs, {
             // FUTURE: add reasoning info
-            "gen_ai.request.stream": inputs.stream,
-            "gen_ai.request.frequency_penalty": inputs.frequency_penalty,
-            "gen_ai.request.max_tokens": inputs.max_completion_tokens,
-            "gen_ai.request.presence_penalty": inputs.presence_penalty,
-            "gen_ai.request.stop_sequences": inputs.stop
-                ? Array.isArray(inputs.stop)
-                    ? inputs.stop
-                    : [inputs.stop]
+            "gen_ai.request.stream": body.stream,
+            "gen_ai.request.service_tier": body.service_tier,
+            "gen_ai.request.frequency_penalty": body.frequency_penalty,
+            "gen_ai.request.max_tokens": body.max_completion_tokens,
+            "gen_ai.request.presence_penalty": body.presence_penalty,
+            "gen_ai.request.stop_sequences": body.stop
+                ? Array.isArray(body.stop)
+                    ? body.stop
+                    : [body.stop]
                 : undefined,
-            "gen_ai.request.temperature": inputs.temperature,
-            "gen_ai.request.top_p": inputs.top_p,
+            "gen_ai.request.temperature": body.temperature,
+            "gen_ai.request.top_p": body.top_p,
         });
-        if (inputs.metadata) {
-            for (const key in inputs.metadata) {
-                attrs[`gen_ai.request.metadata.${key}`] = inputs.metadata[key];
+        if (body.metadata) {
+            for (const key in body.metadata) {
+                attrs[`gen_ai.request.metadata.${key}`] = body.metadata[key];
             }
         }
     }
@@ -134,10 +135,10 @@ export const getChatRequestAttributes = (inputs, signalLevel) => {
             // "gen_ai.system_instructions": inputs.messages
             //   .filter((m) => m.role === "system")
             //   .map((m) => JSON.stringify(toTextPart(m.content))),
-            "gen_ai.input.messages": inputs.messages
+            "gen_ai.input.messages": body.messages
                 //.filter((m) => m.role !== "system")
                 .map((m) => JSON.stringify({ role: m.role, parts: toMessageParts(m) })),
-            "gen_ai.tool.definitions": inputs.tools?.map((toolDefinition) => JSON.stringify(toolDefinition)),
+            "gen_ai.tool.definitions": body.tools?.map((toolDefinition) => JSON.stringify(toolDefinition)),
         });
     }
     return attrs;
@@ -151,6 +152,7 @@ export const getChatResponseAttributes = (completions, signalLevel) => {
     if (signalLevel !== "required") {
         Object.assign(attrs, {
             "gen_ai.response.finish_reasons": completions.choices?.map((c) => c.finish_reason),
+            "gen_ai.response.service_tier": completions.service_tier,
             "gen_ai.usage.total_tokens": completions.usage?.total_tokens,
             "gen_ai.usage.input_tokens": completions.usage?.prompt_tokens,
             "gen_ai.usage.cache_read.input_tokens": completions.usage?.prompt_tokens_details?.cached_tokens,

package/dist/endpoints/chat-completions/schema.d.ts

@@ -1,4 +1,5 @@
 import * as z from "zod";
+import type { SseErrorFrame, SseFrame } from "../../utils/stream";
 export declare const ChatCompletionsCacheControlSchema: z.ZodObject<{
     type: z.ZodLiteral<"ephemeral">;
     ttl: z.ZodOptional<z.ZodString>;
@@ -469,6 +470,14 @@ export declare const ChatCompletionsResponseFormatSchema: z.ZodDiscriminatedUnio
 export type ChatCompletionsResponseFormat = z.infer<typeof ChatCompletionsResponseFormatSchema>;
 export declare const ChatCompletionsMetadataSchema: z.ZodRecord<z.ZodString, z.ZodString>;
 export type ChatCompletionsMetadata = z.infer<typeof ChatCompletionsMetadataSchema>;
+export declare const ChatCompletionsServiceTierSchema: z.ZodEnum<{
+    auto: "auto";
+    default: "default";
+    flex: "flex";
+    scale: "scale";
+    priority: "priority";
+}>;
+export type ChatCompletionsServiceTier = z.infer<typeof ChatCompletionsServiceTierSchema>;
 declare const ChatCompletionsInputsSchema: z.ZodObject<{
     messages: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
         role: z.ZodLiteral<"system">;
@@ -650,6 +659,13 @@ declare const ChatCompletionsInputsSchema: z.ZodObject<{
         medium: "medium";
         xhigh: "xhigh";
     }>>;
+    service_tier: z.ZodOptional<z.ZodEnum<{
+        auto: "auto";
+        default: "default";
+        flex: "flex";
+        scale: "scale";
+        priority: "priority";
+    }>>;
     prompt_cache_key: z.ZodOptional<z.ZodString>;
     prompt_cache_retention: z.ZodOptional<z.ZodEnum<{
         in_memory: "in_memory";
@@ -856,6 +872,13 @@ export declare const ChatCompletionsBodySchema: z.ZodObject<{
         medium: "medium";
         xhigh: "xhigh";
     }>>;
+    service_tier: z.ZodOptional<z.ZodEnum<{
+        auto: "auto";
+        default: "default";
+        flex: "flex";
+        scale: "scale";
+        priority: "priority";
+    }>>;
     prompt_cache_key: z.ZodOptional<z.ZodString>;
     prompt_cache_retention: z.ZodOptional<z.ZodEnum<{
         in_memory: "in_memory";
@@ -884,9 +907,9 @@ export declare const ChatCompletionsBodySchema: z.ZodObject<{
 }, z.core.$loose>;
 export type ChatCompletionsBody = z.infer<typeof ChatCompletionsBodySchema>;
 export declare const ChatCompletionsFinishReasonSchema: z.ZodEnum<{
-    tool_calls: "tool_calls";
     stop: "stop";
     length: "length";
+    tool_calls: "tool_calls";
     content_filter: "content_filter";
 }>;
 export type ChatCompletionsFinishReason = z.infer<typeof ChatCompletionsFinishReasonSchema>;
@@ -930,9 +953,9 @@ export declare const ChatCompletionsChoiceSchema: z.ZodObject<{
         }, z.core.$strip>>;
     }, z.core.$strip>;
     finish_reason: z.ZodEnum<{
-        tool_calls: "tool_calls";
         stop: "stop";
         length: "length";
+        tool_calls: "tool_calls";
         content_filter: "content_filter";
     }>;
     logprobs: z.ZodOptional<z.ZodUnknown>;
@@ -996,9 +1019,9 @@ export declare const ChatCompletionsSchema: z.ZodObject<{
             }, z.core.$strip>>;
         }, z.core.$strip>;
         finish_reason: z.ZodEnum<{
-            tool_calls: "tool_calls";
             stop: "stop";
             length: "length";
+            tool_calls: "tool_calls";
             content_filter: "content_filter";
         }>;
         logprobs: z.ZodOptional<z.ZodUnknown>;
@@ -1015,6 +1038,13 @@ export declare const ChatCompletionsSchema: z.ZodObject<{
             cache_write_tokens: z.ZodOptional<z.ZodInt>;
         }, z.core.$strip>>;
     }, z.core.$strip>>;
+    service_tier: z.ZodOptional<z.ZodEnum<{
+        auto: "auto";
+        default: "default";
+        flex: "flex";
+        scale: "scale";
+        priority: "priority";
+    }>>;
     provider_metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
 }, z.core.$strip>;
 export type ChatCompletions = z.infer<typeof ChatCompletionsSchema>;
@@ -1109,9 +1139,9 @@ export declare const ChatCompletionsChoiceDeltaSchema: z.ZodObject<{
         }, z.core.$strip>>>;
     }, z.core.$strip>;
     finish_reason: z.ZodNullable<z.ZodEnum<{
-        tool_calls: "tool_calls";
         stop: "stop";
         length: "length";
+        tool_calls: "tool_calls";
         content_filter: "content_filter";
     }>>;
     logprobs: z.ZodOptional<z.ZodUnknown>;
@@ -1163,9 +1193,9 @@ export declare const ChatCompletionsChunkSchema: z.ZodObject<{
             }, z.core.$strip>>>;
         }, z.core.$strip>;
         finish_reason: z.ZodNullable<z.ZodEnum<{
-            tool_calls: "tool_calls";
             stop: "stop";
             length: "length";
+            tool_calls: "tool_calls";
             content_filter: "content_filter";
         }>>;
         logprobs: z.ZodOptional<z.ZodUnknown>;
@@ -1182,7 +1212,15 @@ export declare const ChatCompletionsChunkSchema: z.ZodObject<{
             cache_write_tokens: z.ZodOptional<z.ZodInt>;
         }, z.core.$strip>>;
     }, z.core.$strip>>;
+    service_tier: z.ZodOptional<z.ZodEnum<{
+        auto: "auto";
+        default: "default";
+        flex: "flex";
+        scale: "scale";
+        priority: "priority";
+    }>>;
     provider_metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
 }, z.core.$strip>;
 export type ChatCompletionsChunk = z.infer<typeof ChatCompletionsChunkSchema>;
+export type ChatCompletionsStream = ReadableStream<SseFrame<ChatCompletionsChunk> | SseErrorFrame>;
 export {};

package/dist/endpoints/chat-completions/schema.js

@@ -192,6 +192,13 @@ export const ChatCompletionsResponseFormatSchema = z.discriminatedUnion("type",
     ChatCompletionsResponseFormatTextSchema,
 ]);
 export const ChatCompletionsMetadataSchema = z.record(z.string().min(1).max(64), z.string().max(512));
+export const ChatCompletionsServiceTierSchema = z.enum([
+    "auto",
+    "default",
+    "flex",
+    "scale",
+    "priority",
+]);
 const ChatCompletionsInputsSchema = z.object({
     messages: z.array(ChatCompletionsMessageSchema),
     tools: z.array(ChatCompletionsToolSchema).optional(),
@@ -207,6 +214,7 @@ const ChatCompletionsInputsSchema = z.object({
     metadata: ChatCompletionsMetadataSchema.optional(),
     response_format: ChatCompletionsResponseFormatSchema.optional(),
     reasoning_effort: ChatCompletionsReasoningEffortSchema.optional(),
+    service_tier: ChatCompletionsServiceTierSchema.optional(),
     prompt_cache_key: z.string().optional(),
     prompt_cache_retention: z.enum(["in_memory", "24h"]).optional(),
     // Extension origin: OpenRouter/Vercel/Anthropic
@@ -264,6 +272,7 @@ export const ChatCompletionsSchema = z.object({
     model: z.string(),
     choices: z.array(ChatCompletionsChoiceSchema),
     usage: ChatCompletionsUsageSchema.nullable(),
+    service_tier: ChatCompletionsServiceTierSchema.optional(),
     // Extension origin: Vercel AI Gateway
     provider_metadata: z
         .record(z.string(), z.record(z.string(), z.unknown()))
@@ -290,6 +299,7 @@ export const ChatCompletionsChunkSchema = z.object({
     model: z.string(),
     choices: z.array(ChatCompletionsChoiceDeltaSchema),
     usage: ChatCompletionsUsageSchema.nullable(),
+    service_tier: ChatCompletionsServiceTierSchema.optional(),
     // Extension origin: Vercel AI Gateway
     provider_metadata: z
         .record(z.string(), z.record(z.string(), z.unknown()))

package/dist/endpoints/embeddings/handler.js

@@ -13,7 +13,7 @@ import { getEmbeddingsRequestAttributes, getEmbeddingsResponseAttributes } from
 import { EmbeddingsBodySchema } from "./schema";
 export const embeddings = (config) => {
     const hooks = config.hooks;
-    const handler = async (ctx) => {
+    const handler = async (ctx, cfg) => {
         const start = performance.now();
         ctx.operation = "embeddings";
         addSpanEvent("hebo.handler.started");
@@ -62,7 +62,7 @@ export const embeddings = (config) => {
         ctx.resolvedProviderId = embeddingModel.provider;
         logger.debug(`[embeddings] using ${embeddingModel.provider} for ${ctx.resolvedModelId}`);
         addSpanEvent("hebo.provider.resolved");
-        const genAiSignalLevel = config.telemetry?.signals?.gen_ai;
+        const genAiSignalLevel = cfg.telemetry?.signals?.gen_ai;
         const genAiGeneralAttrs = getGenAiGeneralAttributes(ctx, genAiSignalLevel);
         setSpanAttributes(genAiGeneralAttrs);
         // Convert inputs to AI SDK call options.

package/dist/endpoints/embeddings/otel.d.ts

@@ -1,5 +1,5 @@
 import type { Attributes } from "@opentelemetry/api";
-import type { Embeddings, EmbeddingsInputs } from "./schema";
+import type { Embeddings, EmbeddingsBody } from "./schema";
 import { type TelemetrySignalLevel } from "../../types";
-export declare const getEmbeddingsRequestAttributes: (inputs: EmbeddingsInputs, signalLevel?: TelemetrySignalLevel) => Attributes;
+export declare const getEmbeddingsRequestAttributes: (body: EmbeddingsBody, signalLevel?: TelemetrySignalLevel) => Attributes;
 export declare const getEmbeddingsResponseAttributes: (embeddings: Embeddings, signalLevel?: TelemetrySignalLevel) => Attributes;

package/dist/endpoints/embeddings/otel.js

@@ -1,15 +1,15 @@
 import {} from "../../types";
-export const getEmbeddingsRequestAttributes = (inputs, signalLevel) => {
+export const getEmbeddingsRequestAttributes = (body, signalLevel) => {
     if (!signalLevel || signalLevel === "off")
         return {};
     const attrs = {};
     if (signalLevel !== "required") {
         Object.assign(attrs, {
-            "gen_ai.embeddings.dimension.count": inputs.dimensions,
+            "gen_ai.embeddings.dimension.count": body.dimensions,
         });
-        if (inputs.metadata) {
-            for (const key in inputs.metadata) {
-                attrs[`gen_ai.request.metadata.${key}`] = inputs.metadata[key];
+        if (body.metadata) {
+            for (const key in body.metadata) {
+                attrs[`gen_ai.request.metadata.${key}`] = body.metadata[key];
             }
         }
     }

package/dist/endpoints/models/handler.js

@@ -2,8 +2,8 @@ import { GatewayError } from "../../errors/gateway";
 import { winterCgHandler } from "../../lifecycle";
 import { toModels, toModel } from "./converters";
 export const models = (config) => {
-    // oxlint-disable-next-line require-await
-    const handler = async (ctx) => {
+    // eslint-disable-next-line require-await
+    const handler = async (ctx, _cfg) => {
         ctx.operation = "models";
         if (!ctx.request || ctx.request.method !== "GET") {
             throw new GatewayError("Method Not Allowed", 405);