@hebo-ai/gateway 0.6.2-rc1 → 0.7.0

package/dist/models/google/middleware.js

@@ -11,14 +11,15 @@ export const geminiDimensionsMiddleware = {
         const dimensions = unknown["dimensions"];
         if (!dimensions)
             return params;
-        (params.providerOptions["google"] ??= {})["outputDimensionality"] = dimensions;
+        const target = (params.providerOptions["google"] ??= {});
+        target.outputDimensionality = dimensions;
         delete unknown["dimensions"];
         return params;
     },
 };
 // https://ai.google.dev/gemini-api/docs/thinking#thinking-levels
 export function mapGeminiReasoningEffort(effort, modelId) {
-    if (modelId.includes("gemini-3.1-pro")) {
+    if (modelId.includes("pro")) {
         switch (effort) {
             case "none":
             case "minimal":
@@ -28,26 +29,22 @@ export function mapGeminiReasoningEffort(effort, modelId) {
                 return "medium";
             case "high":
             case "xhigh":
-            case "max":
                 return "high";
         }
     }
-    if (modelId.includes("gemini-3-flash") || modelId.includes("gemini-3.1-flash")) {
-        switch (effort) {
-            case "none":
-            case "minimal":
-                return "minimal";
-            case "low":
-                return "low";
-            case "medium":
-                return "medium";
-            case "high":
-            case "xhigh":
-            case "max":
-                return "high";
-        }
+    // Flash
+    switch (effort) {
+        case "none":
+        case "minimal":
+            return "minimal";
+        case "low":
+            return "low";
+        case "medium":
+            return "medium";
+        case "high":
+        case "xhigh":
+            return "high";
     }
-    return effort;
 }
 export const GEMINI_DEFAULT_MAX_OUTPUT_TOKENS = 65536;
 export const GEMINI_2_5_PRO_MIN_THINKING_BUDGET = 128;
@@ -58,6 +55,9 @@ export const geminiReasoningMiddleware = {
         const unknown = params.providerOptions?.["unknown"];
         if (!unknown)
             return params;
+        // If thinking options exist, just pass through
+        if (unknown["thinking_config"])
+            return params;
         const reasoning = unknown["reasoning"];
         if (!reasoning)
             return params;
@@ -65,19 +65,19 @@ export const geminiReasoningMiddleware = {
         const modelId = model.modelId;
         if (modelId.includes("gemini-2")) {
             const is25Pro = modelId.includes("gemini-2.5-pro");
-            target["thinkingConfig"] = {
+            target.thinkingConfig = {
                 thinkingBudget: reasoning.max_tokens ??
                     calculateReasoningBudgetFromEffort(reasoning.effort ?? "none", params.maxOutputTokens ?? GEMINI_DEFAULT_MAX_OUTPUT_TOKENS, is25Pro ? GEMINI_2_5_PRO_MIN_THINKING_BUDGET : 0),
             };
         }
         else if (modelId.includes("gemini-3") && reasoning.effort) {
-            target["thinkingConfig"] = {
+            target.thinkingConfig = {
                 thinkingLevel: mapGeminiReasoningEffort(reasoning.effort, modelId),
             };
             // FUTURE: warn if model is gemini-3 and max_tokens (unsupported) was ignored
         }
-        (target["thinkingConfig"] ??= {})["includeThoughts"] =
-            reasoning.enabled ? !reasoning.exclude : false;
+        const thinkingConfig = (target.thinkingConfig ??= {});
+        thinkingConfig.includeThoughts = reasoning.enabled ? !reasoning.exclude : false;
         delete unknown["reasoning"];
         return params;
     },
@@ -91,9 +91,13 @@ export const geminiPromptCachingMiddleware = {
         const unknown = params.providerOptions?.["unknown"];
         if (!unknown)
             return params;
-        const cachedContent = unknown["cached_content"];
-        if (cachedContent) {
-            (params.providerOptions["google"] ??= {})["cachedContent"] = cachedContent;
+        // If cached_content options exist, just pass through
+        if (unknown["cached_content"])
+            return params;
+        const promptCacheKey = unknown["prompt_cache_key"];
+        if (promptCacheKey) {
+            (params.providerOptions["google"] ??= {}).cachedContent =
+                promptCacheKey;
         }
         delete unknown["cached_content"];
         return params;

package/dist/models/google/presets.d.ts

@@ -9,6 +9,16 @@ export declare const geminiEmbedding001: import("../../utils/preset").Preset<"go
     };
     providers: readonly ["vertex"];
 }>;
+export declare const geminiEmbedding2Preview: import("../../utils/preset").Preset<"google/gemini-embedding-2-preview", CatalogModel, {
+    name: string;
+    created: string;
+    context: number;
+    modalities: {
+        input: readonly ["text"];
+        output: readonly ["embedding"];
+    };
+    providers: readonly ["vertex"];
+}>;
 export declare const gemini3FlashPreview: import("../../utils/preset").Preset<"google/gemini-3-flash-preview", CatalogModel, {
     name: string;
     created: string;
@@ -159,6 +169,15 @@ export declare const gemini: {
             output: readonly ["embedding"];
         };
         providers: readonly ["vertex"];
+    }>, import("../../utils/preset").Preset<"google/gemini-embedding-2-preview", CatalogModel, {
+        name: string;
+        created: string;
+        context: number;
+        modalities: {
+            input: readonly ["text"];
+            output: readonly ["embedding"];
+        };
+        providers: readonly ["vertex"];
     }>];
     readonly all: (import("../../utils/preset").Preset<"google/embedding-001", CatalogModel, {
         name: string;
@@ -169,6 +188,15 @@ export declare const gemini: {
             output: readonly ["embedding"];
         };
         providers: readonly ["vertex"];
+    }> | import("../../utils/preset").Preset<"google/gemini-embedding-2-preview", CatalogModel, {
+        name: string;
+        created: string;
+        context: number;
+        modalities: {
+            input: readonly ["text"];
+            output: readonly ["embedding"];
+        };
+        providers: readonly ["vertex"];
     }> | import("../../utils/preset").Preset<"google/gemini-3-flash-preview", CatalogModel, {
         name: string;
         created: string;

package/dist/models/google/presets.js

@@ -27,6 +27,12 @@ export const geminiEmbedding001 = presetFor()("google/embedding-001", {
     created: "2025-05-20",
     context: 8192,
 });
+export const geminiEmbedding2Preview = presetFor()("google/gemini-embedding-2-preview", {
+    ...GEMINI_EMBEDDINGS_BASE,
+    name: "Gemini Embedding 2 (Preview)",
+    created: "2026-03-10",
+    context: 8192,
+});
 export const gemini3FlashPreview = presetFor()("google/gemini-3-flash-preview", {
     ...GEMINI_BASE,
     name: "Gemini 3 Flash (Preview)",
@@ -66,7 +72,7 @@ export const gemini25Pro = presetFor()("google/gemini-2.5-pro", {
 const geminiAtomic = {
     "v2.5": [gemini25FlashLite, gemini25Flash, gemini25Pro],
     "v3-preview": [gemini3FlashPreview, gemini31FlashLitePreview, gemini31ProPreview],
-    embeddings: [geminiEmbedding001],
+    embeddings: [geminiEmbedding001, geminiEmbedding2Preview],
 };
 const geminiGroups = {
     "v2.x": [...geminiAtomic["v2.5"]],

package/dist/models/openai/middleware.js

@@ -10,7 +10,8 @@ export const openAIDimensionsMiddleware = {
         const dimensions = unknown["dimensions"];
         if (!dimensions)
             return params;
-        (params.providerOptions["openai"] ??= {})["dimensions"] = dimensions;
+        const target = (params.providerOptions["openai"] ??= {});
+        target.dimensions = dimensions;
         delete unknown["dimensions"];
         return params;
     },
@@ -27,7 +28,6 @@ function mapGptOssReasoningEffort(effort) {
             return "medium";
         case "high":
         case "xhigh":
-        case "max":
             return "high";
     }
 }
@@ -45,13 +45,13 @@ export const openAIReasoningMiddleware = {
         const isGptOss = model.modelId.includes("gpt-oss");
         if (isGptOss) {
             // FUTURE: warn that unable to disable reasoning for gpt-oss models
-            target["reasoningEffort"] = mapGptOssReasoningEffort(reasoning.effort);
+            target.reasoningEffort = mapGptOssReasoningEffort(reasoning.effort);
         }
         else if (reasoning.enabled === false) {
-            target["reasoningEffort"] = "none";
+            target.reasoningEffort = "none";
         }
         else if (reasoning.effort) {
-            target["reasoningEffort"] = reasoning.effort;
+            target.reasoningEffort = reasoning.effort;
         }
         // FUTURE: warn that reasoning.max_tokens (not supported) was ignored
         delete unknown["reasoning"];
@@ -71,9 +71,9 @@ export const openAIPromptCachingMiddleware = {
         if (key || retention) {
             const target = (params.providerOptions["openai"] ??= {});
             if (key)
-                target["promptCacheKey"] = key;
+                target.promptCacheKey = key;
             if (retention)
-                target["promptCacheRetention"] = retention;
+                target.promptCacheRetention = retention;
         }
         delete unknown["prompt_cache_key"];
         delete unknown["prompt_cache_retention"];

package/dist/models/types.d.ts

@@ -1,5 +1,5 @@
 import type { ProviderId } from "../providers/types";
-export declare const CANONICAL_MODEL_IDS: readonly ["anthropic/claude-opus-4.6", "anthropic/claude-sonnet-4.6", "anthropic/claude-haiku-4.5", "anthropic/claude-sonnet-4.5", "anthropic/claude-opus-4.5", "anthropic/claude-opus-4.1", "anthropic/claude-opus-4", "anthropic/claude-sonnet-4", "anthropic/claude-sonnet-3.7", "anthropic/claude-sonnet-3.5", "anthropic/claude-haiku-3.5", "anthropic/claude-haiku-3", "openai/gpt-oss-20b", "openai/gpt-oss-120b", "openai/gpt-5", "openai/gpt-5-pro", "openai/gpt-5.2", "openai/gpt-5.2-chat", "openai/gpt-5.2-pro", "openai/gpt-5.2-codex", "openai/gpt-5.3-codex", "openai/gpt-5-mini", "openai/gpt-5-nano", "openai/gpt-5-codex", "openai/gpt-5.1-codex", "openai/gpt-5.1-codex-max", "openai/gpt-5.1-chat", "openai/gpt-5.1", "openai/text-embedding-3-small", "openai/text-embedding-3-large", "amazon/nova-micro", "amazon/nova-lite", "amazon/nova-pro", "amazon/nova-premier", "amazon/nova-2-lite", "amazon/nova-2-multimodal-embeddings", "google/gemini-2.5-flash-lite", "google/gemini-2.5-flash", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/embedding-001", "meta/llama-3.1-8b", "meta/llama-3.1-70b", "meta/llama-3.1-405b", "meta/llama-3.2-1b", "meta/llama-3.2-3b", "meta/llama-3.2-11b", "meta/llama-3.2-90b", "meta/llama-3.3-70b", "meta/llama-4-scout", "meta/llama-4-maverick", "cohere/embed-v4.0", "cohere/embed-english-v3.0", "cohere/embed-english-light-v3.0", "cohere/embed-multilingual-v3.0", "cohere/embed-multilingual-light-v3.0", "cohere/command-a", "cohere/command-r7b", "cohere/command-a-translate", "cohere/command-a-reasoning", "cohere/command-a-vision", "cohere/command-r", "cohere/command-r-plus", "voyage/voyage-2-code", "voyage/voyage-2-law", "voyage/voyage-2-finance", "voyage/voyage-3-code", "voyage/voyage-3-large", "voyage/voyage-3.5-lite", "voyage/voyage-3.5", "voyage/voyage-4-lite", "voyage/voyage-4", "voyage/voyage-4-large"];
+export declare const CANONICAL_MODEL_IDS: readonly ["anthropic/claude-opus-4.6", "anthropic/claude-sonnet-4.6", "anthropic/claude-haiku-4.5", "anthropic/claude-sonnet-4.5", "anthropic/claude-opus-4.5", "anthropic/claude-opus-4.1", "anthropic/claude-opus-4", "anthropic/claude-sonnet-4", "anthropic/claude-sonnet-3.7", "anthropic/claude-sonnet-3.5", "anthropic/claude-haiku-3.5", "anthropic/claude-haiku-3", "openai/gpt-oss-20b", "openai/gpt-oss-120b", "openai/gpt-5", "openai/gpt-5-pro", "openai/gpt-5.2", "openai/gpt-5.2-chat", "openai/gpt-5.2-pro", "openai/gpt-5.2-codex", "openai/gpt-5.3-codex", "openai/gpt-5-mini", "openai/gpt-5-nano", "openai/gpt-5-codex", "openai/gpt-5.1-codex", "openai/gpt-5.1-codex-max", "openai/gpt-5.1-chat", "openai/gpt-5.1", "openai/text-embedding-3-small", "openai/text-embedding-3-large", "amazon/nova-micro", "amazon/nova-lite", "amazon/nova-pro", "amazon/nova-premier", "amazon/nova-2-lite", "amazon/nova-2-multimodal-embeddings", "google/gemini-2.5-flash-lite", "google/gemini-2.5-flash", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-embedding-2-preview", "google/embedding-001", "meta/llama-3.1-8b", "meta/llama-3.1-70b", "meta/llama-3.1-405b", "meta/llama-3.2-1b", "meta/llama-3.2-3b", "meta/llama-3.2-11b", "meta/llama-3.2-90b", "meta/llama-3.3-70b", "meta/llama-4-scout", "meta/llama-4-maverick", "cohere/embed-v4.0", "cohere/embed-english-v3.0", "cohere/embed-english-light-v3.0", "cohere/embed-multilingual-v3.0", "cohere/embed-multilingual-light-v3.0", "cohere/command-a", "cohere/command-r7b", "cohere/command-a-translate", "cohere/command-a-reasoning", "cohere/command-a-vision", "cohere/command-r", "cohere/command-r-plus", "voyage/voyage-2-code", "voyage/voyage-2-law", "voyage/voyage-2-finance", "voyage/voyage-3-code", "voyage/voyage-3-large", "voyage/voyage-3.5-lite", "voyage/voyage-3.5", "voyage/voyage-4-lite", "voyage/voyage-4", "voyage/voyage-4-large"];
 export type CanonicalModelId = (typeof CANONICAL_MODEL_IDS)[number];
 export type ModelId = CanonicalModelId | (string & {});
 export type CatalogModel = {

package/dist/models/types.js

@@ -45,6 +45,7 @@ export const CANONICAL_MODEL_IDS = [
     "google/gemini-3-flash-preview",
     "google/gemini-3.1-flash-lite-preview",
     "google/gemini-3.1-pro-preview",
+    "google/gemini-embedding-2-preview",
     "google/embedding-001",
     // Meta
     "meta/llama-3.1-8b",

package/dist/models/voyage/middleware.js

@@ -10,7 +10,8 @@ export const voyageDimensionsMiddleware = {
         const dimensions = unknown["dimensions"];
         if (!dimensions)
             return params;
-        (params.providerOptions["voyage"] ??= {})["outputDimension"] = dimensions;
+        const target = (params.providerOptions["voyage"] ??= {});
+        target.outputDimension = dimensions;
         delete unknown["dimensions"];
         return params;
     },

package/dist/providers/bedrock/middleware.d.ts

@@ -1,4 +1,5 @@
 import type { LanguageModelMiddleware } from "ai";
+export declare const bedrockServiceTierMiddleware: LanguageModelMiddleware;
 export declare const bedrockGptReasoningMiddleware: LanguageModelMiddleware;
 export declare const bedrockClaudeReasoningMiddleware: LanguageModelMiddleware;
 export declare const bedrockPromptCachingMiddleware: LanguageModelMiddleware;

package/dist/providers/bedrock/middleware.js

@@ -1,5 +1,37 @@
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 const isClaude46 = (modelId) => modelId.includes("-4-6");
+// https://docs.aws.amazon.com/bedrock/latest/userguide/service-tiers-inference.html
+export const bedrockServiceTierMiddleware = {
+    specificationVersion: "v3",
+    // eslint-disable-next-line require-await
+    transformParams: async ({ params }) => {
+        const bedrock = params.providerOptions?.["bedrock"];
+        if (!bedrock || typeof bedrock !== "object")
+            return params;
+        // UPSTREAM: https://github.com/vercel/ai/issues/13241
+        // @ts-expect-error AI SDK missing serviceTier, need to open PR
+        const tier = bedrock["serviceTier"];
+        switch (tier) {
+            case undefined:
+                return params;
+            case "auto":
+                // Bedrock uses its default tier when omitted.
+                // @ts-expect-error AI SDK missing serviceTier, need to open PR
+                delete bedrock.serviceTier;
+                return params;
+            case "scale":
+                // @ts-expect-error AI SDK missing serviceTier, need to open PR
+                bedrock.serviceTier = { type: "reserved" };
+                return params;
+            case "default":
+            case "flex":
+            case "priority":
+                // @ts-expect-error AI SDK missing serviceTier, need to open PR
+                bedrock.serviceTier = { type: tier };
+                return params;
+        }
+    },
+};
 export const bedrockGptReasoningMiddleware = {
     specificationVersion: "v3",
     // oxlint-disable-next-line require-await
@@ -7,14 +39,15 @@ export const bedrockGptReasoningMiddleware = {
         if (!model.modelId.includes("gpt"))
             return params;
         const bedrock = params.providerOptions?.["bedrock"];
-        if (!bedrock || typeof bedrock !== "object")
+        if (!bedrock)
             return params;
-        const effort = bedrock["reasoningEffort"];
+        const effort = bedrock.reasoningEffort;
         if (effort === undefined)
             return params;
-        const target = (bedrock["reasoningConfig"] ??= {});
-        target["maxReasoningEffort"] = effort;
-        delete bedrock["reasoningEffort"];
+        const target = (bedrock.reasoningConfig ??= {});
+        // @ts-expect-error AI SDK does accept this
+        target.maxReasoningEffort = effort;
+        delete bedrock.reasoningEffort;
         return params;
     },
 };
@@ -25,28 +58,25 @@ export const bedrockClaudeReasoningMiddleware = {
         if (!model.modelId.includes("claude"))
             return params;
         const bedrock = params.providerOptions?.["bedrock"];
-        if (!bedrock || typeof bedrock !== "object")
+        if (!bedrock)
             return params;
-        const thinking = bedrock["thinking"];
-        const effort = bedrock["effort"];
+        const thinking = bedrock.thinking;
+        const effort = bedrock.effort;
         if (!thinking && effort === undefined)
             return params;
-        const target = (bedrock["reasoningConfig"] ??= {});
+        const target = (bedrock.reasoningConfig ??= {});
         if (thinking && typeof thinking === "object") {
-            const thinkingOptions = thinking;
-            if (thinkingOptions["type"] !== undefined) {
-                target["type"] = thinkingOptions["type"];
-            }
-            if (thinkingOptions["budgetTokens"] !== undefined) {
-                target["budgetTokens"] = thinkingOptions["budgetTokens"];
+            target.type = thinking.type;
+            if ("budgetTokens" in thinking && thinking.budgetTokens !== undefined) {
+                target.budgetTokens = thinking.budgetTokens;
             }
         }
         // FUTURE: bedrock currently does not support "effort" for other 4.x models
         if (effort !== undefined && isClaude46(model.modelId)) {
-            target["maxReasoningEffort"] = effort;
+            target.maxReasoningEffort = effort;
         }
-        delete bedrock["thinking"];
-        delete bedrock["effort"];
+        delete bedrock.thinking;
+        delete bedrock.effort;
         return params;
     },
 };
@@ -79,18 +109,18 @@ export const bedrockPromptCachingMiddleware = {
             delete entryBedrock["cacheControl"];
         };
         for (const message of params.prompt) {
-            processCacheControl(message["providerOptions"]);
-            if (!Array.isArray(message["content"]))
+            processCacheControl(message.providerOptions);
+            if (!Array.isArray(message.content))
                 continue;
-            for (const part of message["content"]) {
-                processCacheControl(part["providerOptions"]);
+            for (const part of message.content) {
+                processCacheControl(part.providerOptions);
             }
             lastCacheableBlock = message;
         }
         const bedrock = params.providerOptions?.["bedrock"];
         const cacheControl = bedrock?.["cacheControl"];
         if (cacheControl && !hasExplicitCacheControl && lastCacheableBlock) {
-            ((lastCacheableBlock["providerOptions"] ??= {})["bedrock"] ??= {})["cachePoint"] =
+            ((lastCacheableBlock.providerOptions ??= {})["bedrock"] ??= {})["cachePoint"] =
                 toBedrockCachePoint(model.modelId, cacheControl);
         }
         delete bedrock?.["cacheControl"];
@@ -99,6 +129,7 @@ export const bedrockPromptCachingMiddleware = {
 };
 modelMiddlewareMatcher.useForProvider("amazon-bedrock", {
     language: [
+        bedrockServiceTierMiddleware,
         bedrockGptReasoningMiddleware,
         bedrockClaudeReasoningMiddleware,
         bedrockPromptCachingMiddleware,

package/dist/providers/groq/index.d.ts

@@ -1 +1,2 @@
 export * from "./canonical";
+export * from "./middleware";

package/dist/providers/groq/index.js

@@ -1 +1,2 @@
 export * from "./canonical";
+export * from "./middleware";

package/dist/providers/groq/middleware.d.ts

@@ -0,0 +1,2 @@
+import type { LanguageModelMiddleware } from "ai";
+export declare const groqServiceTierMiddleware: LanguageModelMiddleware;

package/dist/providers/groq/middleware.js

@@ -0,0 +1,31 @@
+import { modelMiddlewareMatcher } from "../../middleware/matcher";
+// https://console.groq.com/docs/service-tiers
+export const groqServiceTierMiddleware = {
+    specificationVersion: "v3",
+    // eslint-disable-next-line require-await
+    transformParams: async ({ params }) => {
+        const groq = params.providerOptions?.["groq"];
+        if (!groq || typeof groq !== "object")
+            return params;
+        const tier = groq.serviceTier;
+        switch (tier) {
+            case undefined:
+                return params;
+            case "auto":
+            case "flex":
+                return params;
+            case "default":
+                groq.serviceTier = "on_demand";
+                return params;
+            case "scale":
+            case "priority":
+                // UPSTREAM: https://github.com/vercel/ai/issues/13235
+                // @ts-expect-error AI SDK missing "performance", need to open PR
+                groq.serviceTier = "performance";
+                return params;
+        }
+    },
+};
+modelMiddlewareMatcher.useForProvider("groq.*", {
+    language: [groqServiceTierMiddleware],
+});

package/dist/providers/vertex/index.d.ts

@@ -1 +1,2 @@
 export * from "./canonical";
+export * from "./middleware";

package/dist/providers/vertex/index.js

@@ -1 +1,2 @@
 export * from "./canonical";
+export * from "./middleware";

package/dist/providers/vertex/middleware.d.ts

@@ -0,0 +1,2 @@
+import type { LanguageModelMiddleware } from "ai";
+export declare const vertexServiceTierMiddleware: LanguageModelMiddleware;

package/dist/providers/vertex/middleware.js

@@ -0,0 +1,47 @@
+import { modelMiddlewareMatcher } from "../../middleware/matcher";
+const VERTEX_REQUEST_TYPE_HEADER = "x-vertex-ai-llm-request-type";
+const VERTEX_SHARED_REQUEST_TYPE_HEADER = "x-vertex-ai-llm-shared-request-type";
+function setHeaderIfMissing(headers, key, value) {
+    headers[key] ??= value;
+}
+// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/standard-paygo
+// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/priority-paygo
+// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/flex-paygo
+// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/provisioned-throughput/use-provisioned-throughput
+// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/reference/rest/v1/GenerateContentResponse#TrafficType
+export const vertexServiceTierMiddleware = {
+    specificationVersion: "v3",
+    // eslint-disable-next-line require-await
+    transformParams: async ({ params }) => {
+        const vertex = params.providerOptions?.["vertex"];
+        if (!vertex || typeof vertex !== "object")
+            return params;
+        const tier = vertex["serviceTier"];
+        const headers = (params.headers ??= {});
+        switch (tier) {
+            case undefined:
+                return params;
+            case "flex":
+                setHeaderIfMissing(headers, VERTEX_REQUEST_TYPE_HEADER, "shared");
+                setHeaderIfMissing(headers, VERTEX_SHARED_REQUEST_TYPE_HEADER, "flex");
+                break;
+            case "priority":
+                setHeaderIfMissing(headers, VERTEX_REQUEST_TYPE_HEADER, "shared");
+                setHeaderIfMissing(headers, VERTEX_SHARED_REQUEST_TYPE_HEADER, "priority");
+                break;
+            case "scale":
+                setHeaderIfMissing(headers, VERTEX_REQUEST_TYPE_HEADER, "dedicated");
+                break;
+            case "default":
+                setHeaderIfMissing(headers, VERTEX_REQUEST_TYPE_HEADER, "shared");
+                break;
+            case "auto":
+                break;
+        }
+        delete vertex["serviceTier"];
+        return params;
+    },
+};
+modelMiddlewareMatcher.useForProvider(["google.vertex.*"], {
+    language: [vertexServiceTierMiddleware],
+});

package/dist/types.d.ts

@@ -1,6 +1,6 @@
 import type { ProviderV3 } from "@ai-sdk/provider";
 import type { Tracer } from "@opentelemetry/api";
-import type { ChatCompletions, ChatCompletionsBody, ChatCompletionsChunk } from "./endpoints/chat-completions/schema";
+import type { ChatCompletions, ChatCompletionsBody, ChatCompletionsStream } from "./endpoints/chat-completions/schema";
 import type { Embeddings, EmbeddingsBody } from "./endpoints/embeddings/schema";
 import type { Model, ModelList } from "./endpoints/models";
 import type { Logger, LoggerConfig } from "./logger";
@@ -57,7 +57,7 @@ export type GatewayContext = {
     /**
      * Result returned by the handler (pre-response).
      */
-    result?: ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings | Model | ModelList;
+    result?: ChatCompletions | ChatCompletionsStream | Embeddings | Model | ModelList;
     /**
      * Response object returned by the handler.
      */
@@ -104,7 +104,7 @@ export type GatewayHooks = {
      * Runs after the endpoint handler.
      * @returns Result to replace, or undefined to keep original.
      */
-    after?: (ctx: AfterHookContext) => void | ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings | Promise<void | ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings>;
+    after?: (ctx: AfterHookContext) => void | ChatCompletions | ChatCompletionsStream | Embeddings | Promise<void | ChatCompletions | ChatCompletionsStream | Embeddings>;
     /**
      * Runs after the lifecycle has produced the final Response.
      * @returns Replacement Response, or undefined to keep original.
@@ -112,6 +112,18 @@ export type GatewayHooks = {
     onResponse?: (ctx: OnResponseHookContext) => void | Response | Promise<void | Response>;
 };
 export type TelemetrySignalLevel = "off" | "required" | "recommended" | "full";
+export declare const DEFAULT_CHAT_TIMEOUT_MS: number;
+export type GatewayTimeout = number | null | {
+    /**
+     * Default timeout used.
+     */
+    normal?: number | null;
+    /**
+     * Timeout used when `service_tier=flex`.
+     * Defaults to 3x `normal` when omitted.
+     */
+    flex?: number | null;
+};
 /**
  * Main configuration object for the gateway.
  */
@@ -162,9 +174,18 @@ export type GatewayConfig = {
             hebo?: TelemetrySignalLevel;
         };
     };
+    /**
+     * Optional timeout for server responses.
+     * Supports a number in milliseconds, or tiered config.
+     */
+    timeouts?: GatewayTimeout;
 };
 export declare const kParsed: unique symbol;
-export type GatewayConfigParsed = GatewayConfig & {
+export type GatewayConfigParsed = Omit<GatewayConfig, "timeouts"> & {
+    timeouts: {
+        normal?: number;
+        flex?: number;
+    };
     [kParsed]: true;
 };
 export interface Endpoint {

package/dist/types.js

@@ -1 +1,2 @@
+export const DEFAULT_CHAT_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
 export const kParsed = Symbol("hebo.gateway.parsed");

package/dist/utils/response.d.ts

@@ -1,3 +1,6 @@
+import type { SseFrame } from "./stream";
 export declare const prepareResponseInit: (requestId: string) => ResponseInit;
 export declare const mergeResponseInit: (defaultHeaders: HeadersInit, responseInit?: ResponseInit) => ResponseInit;
-export declare const toResponse: (result: ReadableStream | Uint8Array<ArrayBuffer> | object | string, responseInit?: ResponseInit) => Response;
+export declare const toResponse: (result: ReadableStream<SseFrame> | Uint8Array<ArrayBuffer> | object | string, responseInit?: ResponseInit, streamOptions?: {
+    onDone?: (status: number, reason?: unknown) => void;
+}) => Response;

package/dist/utils/response.js

@@ -1,17 +1,6 @@
 import { REQUEST_ID_HEADER } from "./headers";
+import { toSseStream } from "./stream";
 const TEXT_ENCODER = new TextEncoder();
-class JsonToSseTransformStream extends TransformStream {
-    constructor() {
-        super({
-            transform(part, controller) {
-                controller.enqueue(`data: ${JSON.stringify(part)}\n\n`);
-            },
-            flush(controller) {
-                controller.enqueue("data: [DONE]\n\n");
-            },
-        });
-    }
-}
 export const prepareResponseInit = (requestId) => ({
     headers: { [REQUEST_ID_HEADER]: requestId },
 });
@@ -31,11 +20,11 @@ export const mergeResponseInit = (defaultHeaders, responseInit) => {
         headers,
     };
 };
-export const toResponse = (result, responseInit) => {
+export const toResponse = (result, responseInit, streamOptions) => {
     let body;
     const isStream = result instanceof ReadableStream;
     if (isStream) {
-        body = result.pipeThrough(new JsonToSseTransformStream()).pipeThrough(new TextEncoderStream());
+        body = toSseStream(result, streamOptions);
     }
     else if (result instanceof Uint8Array) {
         body = result;
@@ -43,16 +32,12 @@ export const toResponse = (result, responseInit) => {
     else if (typeof result === "string") {
         body = TEXT_ENCODER.encode(result);
     }
-    else if (result instanceof Error) {
-        body = TEXT_ENCODER.encode(JSON.stringify({ message: result.message }));
-    }
     else {
         body = TEXT_ENCODER.encode(JSON.stringify(result));
     }
     if (!responseInit?.statusText) {
-        const isError = result instanceof Error;
-        const status = responseInit?.status ?? (isError ? 500 : 200);
-        const statusText = isError ? "REQUEST_FAILED" : "OK";
+        const status = responseInit?.status ?? 200;
+        const statusText = "OK";
         const headers = responseInit?.headers;
         responseInit = headers ? { status, statusText, headers } : { status, statusText };
     }

package/dist/utils/stream.d.ts

@@ -0,0 +1,9 @@
+export type SseFrame<T = unknown, E extends string | undefined = string | undefined> = {
+    data: T;
+    event?: E;
+};
+export type SseErrorFrame = SseFrame<Error, "error" | undefined>;
+export declare function toSseStream(src: ReadableStream<SseFrame>, options?: {
+    onDone?: (status: number, reason?: unknown) => void;
+    keepAliveMs?: number;
+}): ReadableStream<Uint8Array>;