@hebo-ai/gateway 0.6.2-rc1 → 0.7.0

package/README.md

@@ -32,13 +32,13 @@ bun install @hebo-ai/gateway
 - Quickstart
   - [Setup A Gateway Instance](#setup-a-gateway-instance) | [Mount Route Handlers](#mount-route-handlers) | [Call the Gateway](#call-the-gateway)
 - Configuration Reference
-  - [Providers](#providers) | [Models](#models) | [Hooks](#hooks) | [Logger](#logger-settings) | [Observability](#observability)
+  - [Providers](#providers) | [Models](#models) | [Hooks](#hooks) | [Logger](#logger-settings) | [Observability](#observability) | [Timeouts](#timeout-settings)
 - Framework Support
   - [ElysiaJS](#elysiajs) | [Hono](#hono) | [Next.js](#nextjs) | [TanStack Start](#tanstack-start)
 - Runtime Support
   - [Vercel Edge](#vercel-edge) | [Cloudflare Workers](#cloudflare-workers) | [Deno Deploy](#deno-deploy) | [AWS Lambda](#aws-lambda)
 - OpenAI Extensions
-  - [Reasoning](#reasoning) | [Prompt Caching](#prompt-caching)
+  - [Reasoning](#reasoning) | [Service Tier](#service-tier) | [Prompt Caching](#prompt-caching)
 - Advanced Usage
   - [Passing Framework State to Hooks](#passing-framework-state-to-hooks) | [Selective Route Mounting](#selective-route-mounting) | [Low-level Schemas & Converters](#low-level-schemas--converters)
 
@@ -342,9 +342,9 @@ const gw = gateway({
      * @returns Modified result, or undefined to keep original.
      */
     after: async (ctx: {
-      result: ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings;
+      result: ChatCompletions | ChatCompletionsStream | Embeddings;
     }): Promise<
-      ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings | void
+      ChatCompletions | ChatCompletionsStream | Embeddings | void
     > => {
       // Example Use Cases:
       // - Transform result
@@ -536,14 +536,14 @@ Normalization rules:
 
 - `enabled` -> fall-back to model default if none provided
 - `max_tokens`: fall-back to model default if model supports
-- `effort` supports: `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`
+- `effort` supports: `none`, `minimal`, `low`, `medium`, `high`, `xhigh`
 - Generic `effort` -> budget = percentage of `max_tokens`
   - `none`: 0%
   - `minimal`: 10%
   - `low`: 20%
   - `medium`: 50% (default)
   - `high`: 80%
-  - `xhigh` / `max`: 95%
+  - `xhigh`: 95%
 
 Reasoning output is surfaced as extension to the `completion` object.
 
@@ -561,6 +561,25 @@ Advanced models (like Anthropic Claude 3.7 or Gemini 3) surface structured reaso
 
 For **Gemini 3** models, returning the thought signature via `extra_content` is mandatory to resume the chain-of-thought; failing to do so may result in errors or degraded performance.
 
+### Service Tier
+
+The chat completions endpoint accepts a provider-agnostic `service_tier` extension:
+
+- `auto`, `default`, `flex`, `priority`, `scale`
+
+Provider-specific mapping:
+
+- **OpenAI**: forwards as OpenAI `serviceTier` (no middleware remap).
+- **Groq**: maps to Groq `serviceTier` (`default` -> `on_demand`, `scale`/`priority` -> `performance`).
+- **Google Vertex**: maps to request headers via middleware:
+  - `default` -> `x-vertex-ai-llm-request-type: shared`
+  - `flex` -> `x-vertex-ai-llm-request-type: shared` + `x-vertex-ai-llm-shared-request-type: flex`
+  - `priority` -> `x-vertex-ai-llm-request-type: shared` + `x-vertex-ai-llm-shared-request-type: priority`
+  - `scale` -> `x-vertex-ai-llm-request-type: dedicated`
+- **Amazon Bedrock**: maps to Bedrock `serviceTier.type` (`default`, `flex`, `priority`, `reserved`; `scale` -> `reserved`, `auto` -> omitted/default).
+
+When available, the resolved value is echoed back on response as `service_tier`.
+
 ### Prompt Caching
 
 The chat completions endpoint supports both implicit (provider-managed) and explicit prompt caching across OpenAI-compatible providers.
@@ -665,7 +684,7 @@ https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/
 
 > [!TIP]
 > To populate custom span attributes, the inbound W3C `baggage` header is supported. Keys in the `hebo.` namespace are mapped to span attributes, with the namespace stripped. For example: `baggage: hebo.user_id=u-123` becomes span attribute `user_id=u-123`.  
-> For `/chat/completions`, request `metadata` (`Record<string, string>`, key 1-64 chars, value up to 512 chars) is also forwarded to spans as `gen_ai.request.metadata.<key>`.
+> For `/chat/completions` and `/embeddings`, request `metadata` (`Record<string, string>`, key 1-64 chars, value up to 512 chars) is also forwarded to spans as `gen_ai.request.metadata.<key>`.
 
 For observability integration that is not otel compliant, you can disable built-in telemetry and manually instrument requests during `before` / `after` hooks.
 
@@ -737,6 +756,37 @@ const gw = gateway({
 
 Langfuse credentials are read from environment variables by the Langfuse OTel SDK (`LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_BASE_URL`).
 
+### Timeout Settings
+
+You can configure request timeouts via the `timeouts` field:
+
+```ts
+import { gateway } from "@hebo-ai/gateway";
+
+const gw = gateway({
+  // ...
+  // default timeout is 300_000 (5 minutes).
+  // You can set one timeout for all tiers...
+  timeouts: 60_000,
+  // ...disable timeouts completely:
+  // timeouts: null,
+  // ...or split by service tier:
+  // - normal: all non-flex tiers (set null to disable)
+  // - flex: defaults to 3x normal when omitted (set null to disable)
+  // timeouts: { normal: 30_000, flex: null },
+});
+```
+
+> [!NOTE]
+> **Runtime/engine timeout limits**
+> Runtime-level `fetch()` clients may enforce their own timeouts. Configure those runtime/platform limits in addition to gateway `timeouts`.
+>
+> - Node.js runtimes use Undici: https://github.com/nodejs/undici/issues/1373 (Node.js, Vercel Serverless Functions, AWS Lambda)
+> - Bun context: https://github.com/oven-sh/bun/issues/16682
+>
+> **Provider/service timeout limits**
+> Serverless platforms (e.g. Cloudflare Workers, Vercel Edge/Serverless, AWS Lambda) also enforce platform time limits (roughly ~25-100s on edge paths, ~300s for streaming, and up to ~900s configurable for some).
+
 ### Passing Framework State to Hooks
 
 You can pass per-request info from your framework into the gateway via the second `state` argument on the handler, then read it in hooks through `ctx.state`.
@@ -838,7 +888,7 @@ export async function handler(req: Request): Promise<Response> {
 }
 ```
 
-Non-streaming versions are available via `createChatCompletionsResponse`. Equivalent schemas and helpers are available in the `embeddings` and `models` endpoints.
+Non-streaming versions are available via `toChatCompletionsResponse`. Equivalent schemas and helpers are available in the `embeddings` and `models` endpoints.
 
 > [!TIP]
 > Since Zod v4.3 you can generate a JSON Schema from any zod object by calling `z.toJSONSchema(...)`. This is useful for producing OpenAPI documentation from the same source of truth.

package/dist/config.js

@@ -1,7 +1,7 @@
 import { isLogger, logger, setLoggerInstance } from "./logger";
 import { createDefaultLogger } from "./logger/default";
 import { installAiSdkWarningLogger } from "./telemetry/ai-sdk";
-import { kParsed, } from "./types";
+import { DEFAULT_CHAT_TIMEOUT_MS, kParsed, } from "./types";
 export const parseConfig = (config) => {
     // If it has been parsed before, just return.
     if (kParsed in config)
@@ -66,9 +66,36 @@ export const parseConfig = (config) => {
             hebo: "off",
         };
     installAiSdkWarningLogger(telemetrySignals.gen_ai);
+    // Default timeouts
+    let normal;
+    let flex;
+    const t = config.timeouts;
+    if (t === null) {
+        normal = flex = undefined;
+    }
+    else if (typeof t === "number") {
+        normal = t;
+        flex = t * 3;
+    }
+    else {
+        if (t?.normal === null)
+            normal = undefined;
+        else if (t?.normal === undefined)
+            normal = DEFAULT_CHAT_TIMEOUT_MS;
+        else
+            normal = t.normal;
+        if (t?.flex === null)
+            flex = undefined;
+        else if (t?.flex === undefined)
+            flex = normal === undefined ? undefined : normal * 3;
+        else
+            flex = t.flex;
+    }
+    const parsedTimeouts = { normal, flex };
     // Return parsed config.
     return {
         ...config,
+        timeouts: parsedTimeouts,
         telemetry: {
             ...config.telemetry,
             enabled: telemetryEnabled,

package/dist/endpoints/chat-completions/converters.d.ts

@@ -1,8 +1,8 @@
 import type { SharedV3ProviderOptions, SharedV3ProviderMetadata } from "@ai-sdk/provider";
 import type { GenerateTextResult, StreamTextResult, FinishReason, ToolChoice, ToolSet, ModelMessage, UserContent, LanguageModelUsage, TextStreamPart, ReasoningOutput, AssistantModelMessage, ToolModelMessage, UserModelMessage } from "ai";
 import { Output } from "ai";
-import type { ChatCompletionsToolCall, ChatCompletionsTool, ChatCompletionsToolChoice, ChatCompletionsContentPart, ChatCompletionsMessage, ChatCompletionsUserMessage, ChatCompletionsAssistantMessage, ChatCompletionsToolMessage, ChatCompletionsFinishReason, ChatCompletionsUsage, ChatCompletionsInputs, ChatCompletions, ChatCompletionsChunk, ChatCompletionsReasoningDetail } from "./schema";
-import { OpenAIError } from "../../errors/openai";
+import type { ChatCompletionsToolCall, ChatCompletionsTool, ChatCompletionsToolChoice, ChatCompletionsStream, ChatCompletionsContentPart, ChatCompletionsMessage, ChatCompletionsUserMessage, ChatCompletionsAssistantMessage, ChatCompletionsToolMessage, ChatCompletionsFinishReason, ChatCompletionsUsage, ChatCompletionsInputs, ChatCompletions, ChatCompletionsChunk, ChatCompletionsReasoningDetail } from "./schema";
+import type { SseErrorFrame, SseFrame } from "../../utils/stream";
 export type TextCallOptions = {
     messages: ModelMessage[];
     tools?: ToolSet;
@@ -31,10 +31,10 @@ export declare const convertToToolChoiceOptions: (toolChoice: ChatCompletionsToo
 };
 export declare function toChatCompletions(result: GenerateTextResult<ToolSet, Output.Output>, model: string): ChatCompletions;
 export declare function toChatCompletionsResponse(result: GenerateTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
-export declare function toChatCompletionsStream<E extends boolean = false>(result: StreamTextResult<ToolSet, Output.Output>, model: string, wrapErrors?: E): ReadableStream<ChatCompletionsChunk | (E extends true ? OpenAIError : Error)>;
+export declare function toChatCompletionsStream(result: StreamTextResult<ToolSet, Output.Output>, model: string): ChatCompletionsStream;
 export declare function toChatCompletionsStreamResponse(result: StreamTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
-export declare class ChatCompletionsStream<E extends boolean = false> extends TransformStream<TextStreamPart<ToolSet>, ChatCompletionsChunk | (E extends true ? OpenAIError : Error)> {
-    constructor(model: string, wrapErrors?: E);
+export declare class ChatCompletionsTransformStream extends TransformStream<TextStreamPart<ToolSet>, SseFrame<ChatCompletionsChunk> | SseErrorFrame> {
+    constructor(model: string);
 }
 export declare const toChatCompletionsAssistantMessage: (result: GenerateTextResult<ToolSet, Output.Output>) => ChatCompletionsAssistantMessage;
 export declare function toReasoningDetail(reasoning: ReasoningOutput, id: string, index: number): ChatCompletionsReasoningDetail;

package/dist/endpoints/chat-completions/converters.js

@@ -1,14 +1,18 @@
 import { Output, jsonSchema, tool } from "ai";
 import { z } from "zod";
 import { GatewayError } from "../../errors/gateway";
-import { OpenAIError, toOpenAIError } from "../../errors/openai";
 import { toResponse } from "../../utils/response";
 import { parseDataUrl } from "../../utils/url";
 // --- Request Flow ---
 export function convertToTextCallOptions(params) {
     const { messages, tools, tool_choice, temperature, max_tokens, max_completion_tokens, response_format, reasoning_effort, reasoning, prompt_cache_key, prompt_cache_retention, extra_body, cache_control, frequency_penalty, presence_penalty, seed, stop, top_p, ...rest } = params;
     Object.assign(rest, parseReasoningOptions(reasoning_effort, reasoning));
-    Object.assign(rest, parsePromptCachingOptions(prompt_cache_key, prompt_cache_retention, extra_body?.google?.cached_content, cache_control));
+    Object.assign(rest, parsePromptCachingOptions(prompt_cache_key, prompt_cache_retention, cache_control));
+    if (extra_body) {
+        for (const v of Object.values(extra_body)) {
+            Object.assign(rest, v);
+        }
+    }
     const { toolChoice, activeTools } = convertToToolChoiceOptions(tool_choice);
     return {
         messages: convertToModelMessages(messages),
@@ -337,29 +341,25 @@ function parseReasoningOptions(reasoning_effort, reasoning) {
     }
     return out;
 }
-function parsePromptCachingOptions(prompt_cache_key, prompt_cache_retention, cached_content, cache_control) {
+function parsePromptCachingOptions(prompt_cache_key, prompt_cache_retention, cache_control) {
     const out = {};
-    const syncedCacheKey = prompt_cache_key ?? cached_content;
-    const syncedCachedContent = cached_content ?? prompt_cache_key;
-    let syncedCacheRetention = prompt_cache_retention;
-    if (!syncedCacheRetention && cache_control?.ttl) {
-        syncedCacheRetention = cache_control.ttl === "24h" ? "24h" : "in_memory";
-    }
-    let syncedCacheControl = cache_control;
-    if (!syncedCacheControl && syncedCacheRetention) {
-        syncedCacheControl = {
+    let retention = prompt_cache_retention;
+    if (!retention && cache_control?.ttl) {
+        retention = cache_control.ttl === "24h" ? "24h" : "in_memory";
+    }
+    let control = cache_control;
+    if (!control && retention) {
+        control = {
             type: "ephemeral",
-            ttl: syncedCacheRetention === "24h" ? "24h" : "5m",
+            ttl: retention === "24h" ? "24h" : "5m",
         };
     }
-    if (syncedCacheKey)
-        out["prompt_cache_key"] = syncedCacheKey;
-    if (syncedCacheRetention)
-        out["prompt_cache_retention"] = syncedCacheRetention;
-    if (syncedCachedContent)
-        out["cached_content"] = syncedCachedContent;
-    if (syncedCacheControl)
-        out["cache_control"] = syncedCacheControl;
+    if (prompt_cache_key)
+        out["prompt_cache_key"] = prompt_cache_key;
+    if (retention)
+        out["prompt_cache_retention"] = retention;
+    if (control)
+        out["cache_control"] = control;
     return out;
 }
 // --- Response Flow ---
@@ -378,19 +378,20 @@ export function toChatCompletions(result, model) {
         ],
         usage: result.totalUsage ? toChatCompletionsUsage(result.totalUsage) : null,
         provider_metadata: result.providerMetadata,
+        service_tier: resolveResponseServiceTier(result.providerMetadata),
     };
 }
 export function toChatCompletionsResponse(result, model, responseInit) {
     return toResponse(toChatCompletions(result, model), responseInit);
 }
-export function toChatCompletionsStream(result, model, wrapErrors) {
-    return result.fullStream.pipeThrough(new ChatCompletionsStream(model, wrapErrors));
+export function toChatCompletionsStream(result, model) {
+    return result.fullStream.pipeThrough(new ChatCompletionsTransformStream(model));
 }
 export function toChatCompletionsStreamResponse(result, model, responseInit) {
-    return toResponse(toChatCompletionsStream(result, model, true), responseInit);
+    return toResponse(toChatCompletionsStream(result, model), responseInit);
 }
-export class ChatCompletionsStream extends TransformStream {
-    constructor(model, wrapErrors) {
+export class ChatCompletionsTransformStream extends TransformStream {
+    constructor(model) {
         const streamId = `chatcmpl-${crypto.randomUUID()}`;
         const creationTime = Math.floor(Date.now() / 1000);
         let toolCallIndexCounter = 0;
@@ -401,18 +402,21 @@ export class ChatCompletionsStream extends TransformStream {
                 delta.extra_content = provider_metadata;
             }
             return {
-                id: streamId,
-                object: "chat.completion.chunk",
-                created: creationTime,
-                model,
-                choices: [
-                    {
-                        index: 0,
-                        delta,
-                        finish_reason: finish_reason ?? null,
-                    },
-                ],
-                usage: usage ?? null,
+                data: {
+                    id: streamId,
+                    object: "chat.completion.chunk",
+                    created: creationTime,
+                    model,
+                    choices: [
+                        {
+                            index: 0,
+                            delta,
+                            finish_reason: finish_reason ?? null,
+                        },
+                    ],
+                    usage: usage ?? null,
+                    service_tier: resolveResponseServiceTier(provider_metadata),
+                },
             };
         };
         super({
@@ -458,23 +462,56 @@ export class ChatCompletionsStream extends TransformStream {
                         break;
                     }
                     case "error": {
-                        let err;
-                        if (wrapErrors) {
-                            err = toOpenAIError(part.error);
-                        }
-                        else if (part.error instanceof Error) {
-                            err = part.error;
-                        }
-                        else {
-                            err = new Error(String(part.error));
-                        }
-                        controller.enqueue(err);
+                        controller.enqueue({
+                            data: part.error instanceof Error ? part.error : new Error(String(part.error)),
+                        });
                     }
                 }
             },
         });
     }
 }
+function resolveResponseServiceTier(providerMetadata) {
+    if (!providerMetadata)
+        return;
+    for (const metadata of Object.values(providerMetadata)) {
+        const tier = parseReturnedServiceTier(metadata["service_tier"] ??
+            metadata["usage_metadata"]?.["traffic_type"]);
+        if (tier)
+            return tier;
+    }
+}
+function parseReturnedServiceTier(value) {
+    if (typeof value !== "string")
+        return undefined;
+    const n = value.toLowerCase();
+    switch (n) {
+        case "traffic_type_unspecified":
+        case "auto":
+            return "auto";
+        case "default":
+        case "on_demand":
+        case "on-demand":
+        case "shared":
+            return "default";
+        case "on_demand_flex":
+        case "flex":
+            return "flex";
+        case "on_demand_priority":
+        case "priority":
+        case "performance":
+            return "priority";
+        case "provisioned_throughput":
+        case "scale":
+        case "reserved":
+        case "dedicated":
+        case "provisioned":
+        case "throughput":
+            return "scale";
+        default:
+            return undefined;
+    }
+}
 export const toChatCompletionsAssistantMessage = (result) => {
     const message = {
         role: "assistant",

package/dist/endpoints/chat-completions/handler.js

@@ -13,7 +13,7 @@ import { getChatRequestAttributes, getChatResponseAttributes } from "./otel";
 import { ChatCompletionsBodySchema } from "./schema";
 export const chatCompletions = (config) => {
     const hooks = config.hooks;
-    const handler = async (ctx) => {
+    const handler = async (ctx, cfg) => {
         const start = performance.now();
         ctx.operation = "chat";
         addSpanEvent("hebo.handler.started");
@@ -63,7 +63,7 @@ export const chatCompletions = (config) => {
         ctx.resolvedProviderId = languageModel.provider;
         logger.debug(`[chat] using ${languageModel.provider} for ${ctx.resolvedModelId}`);
         addSpanEvent("hebo.provider.resolved");
-        const genAiSignalLevel = config.telemetry?.signals?.gen_ai;
+        const genAiSignalLevel = cfg.telemetry?.signals?.gen_ai;
         const genAiGeneralAttrs = getGenAiGeneralAttributes(ctx, genAiSignalLevel);
         setSpanAttributes(genAiGeneralAttrs);
         // Convert inputs to AI SDK call options.
@@ -88,7 +88,7 @@ export const chatCompletions = (config) => {
                 headers: prepareForwardHeaders(ctx.request),
                 abortSignal: ctx.request.signal,
                 timeout: {
-                    totalMs: 5 * 60 * 1000,
+                    totalMs: ctx.body.service_tier === "flex" ? cfg.timeouts.flex : cfg.timeouts.normal,
                 },
                 onAbort: () => {
                     throw new DOMException("The operation was aborted.", "AbortError");
@@ -122,7 +122,7 @@ export const chatCompletions = (config) => {
             model: languageModelWithMiddleware,
             headers: prepareForwardHeaders(ctx.request),
             abortSignal: ctx.request.signal,
-            timeout: 5 * 60 * 1000,
+            timeout: ctx.body.service_tier === "flex" ? cfg.timeouts.flex : cfg.timeouts.normal,
             experimental_include: {
                 requestBody: false,
                 responseBody: false,

package/dist/endpoints/chat-completions/otel.d.ts

@@ -1,5 +1,5 @@
 import type { Attributes } from "@opentelemetry/api";
 import type { ChatCompletions, ChatCompletionsBody } from "./schema";
 import { type TelemetrySignalLevel } from "../../types";
-export declare const getChatRequestAttributes: (inputs: ChatCompletionsBody, signalLevel?: TelemetrySignalLevel) => Attributes;
+export declare const getChatRequestAttributes: (body: ChatCompletionsBody, signalLevel?: TelemetrySignalLevel) => Attributes;
 export declare const getChatResponseAttributes: (completions: ChatCompletions, signalLevel?: TelemetrySignalLevel) => Attributes;

package/dist/endpoints/chat-completions/otel.js

@@ -99,31 +99,32 @@ const toMessageParts = (message) => {
             throw new Error(`Unhandled content part type: ${message.role}`);
     }
 };
-export const getChatRequestAttributes = (inputs, signalLevel) => {
+export const getChatRequestAttributes = (body, signalLevel) => {
     if (!signalLevel || signalLevel === "off")
         return {};
     const attrs = {};
-    if (inputs.seed !== undefined) {
-        Object.assign(attrs, { "gen_ai.request.seed": inputs.seed });
+    if (body.seed !== undefined) {
+        Object.assign(attrs, { "gen_ai.request.seed": body.seed });
     }
     if (signalLevel !== "required") {
         Object.assign(attrs, {
             // FUTURE: add reasoning info
-            "gen_ai.request.stream": inputs.stream,
-            "gen_ai.request.frequency_penalty": inputs.frequency_penalty,
-            "gen_ai.request.max_tokens": inputs.max_completion_tokens,
-            "gen_ai.request.presence_penalty": inputs.presence_penalty,
-            "gen_ai.request.stop_sequences": inputs.stop
-                ? Array.isArray(inputs.stop)
-                    ? inputs.stop
-                    : [inputs.stop]
+            "gen_ai.request.stream": body.stream,
+            "gen_ai.request.service_tier": body.service_tier,
+            "gen_ai.request.frequency_penalty": body.frequency_penalty,
+            "gen_ai.request.max_tokens": body.max_completion_tokens,
+            "gen_ai.request.presence_penalty": body.presence_penalty,
+            "gen_ai.request.stop_sequences": body.stop
+                ? Array.isArray(body.stop)
+                    ? body.stop
+                    : [body.stop]
                 : undefined,
-            "gen_ai.request.temperature": inputs.temperature,
-            "gen_ai.request.top_p": inputs.top_p,
+            "gen_ai.request.temperature": body.temperature,
+            "gen_ai.request.top_p": body.top_p,
         });
-        if (inputs.metadata) {
-            for (const key in inputs.metadata) {
-                attrs[`gen_ai.request.metadata.${key}`] = inputs.metadata[key];
+        if (body.metadata) {
+            for (const key in body.metadata) {
+                attrs[`gen_ai.request.metadata.${key}`] = body.metadata[key];
             }
         }
     }
@@ -134,10 +135,10 @@ export const getChatRequestAttributes = (inputs, signalLevel) => {
             // "gen_ai.system_instructions": inputs.messages
             //   .filter((m) => m.role === "system")
             //   .map((m) => JSON.stringify(toTextPart(m.content))),
-            "gen_ai.input.messages": inputs.messages
+            "gen_ai.input.messages": body.messages
                 //.filter((m) => m.role !== "system")
                 .map((m) => JSON.stringify({ role: m.role, parts: toMessageParts(m) })),
-            "gen_ai.tool.definitions": inputs.tools?.map((toolDefinition) => JSON.stringify(toolDefinition)),
+            "gen_ai.tool.definitions": body.tools?.map((toolDefinition) => JSON.stringify(toolDefinition)),
         });
     }
     return attrs;
@@ -151,6 +152,7 @@ export const getChatResponseAttributes = (completions, signalLevel) => {
     if (signalLevel !== "required") {
         Object.assign(attrs, {
             "gen_ai.response.finish_reasons": completions.choices?.map((c) => c.finish_reason),
+            "gen_ai.response.service_tier": completions.service_tier,
             "gen_ai.usage.total_tokens": completions.usage?.total_tokens,
             "gen_ai.usage.input_tokens": completions.usage?.prompt_tokens,
             "gen_ai.usage.cache_read.input_tokens": completions.usage?.prompt_tokens_details?.cached_tokens,