@hebo-ai/gateway 0.6.2 → 0.7.0

package/dist/errors/openai.d.ts

@@ -8,12 +8,7 @@ export declare const OpenAIErrorSchema: z.ZodObject<{
     }, z.core.$strip>;
 }, z.core.$strip>;
 export declare class OpenAIError {
-    readonly error: {
-        message: string;
-        type: string;
-        code: string | undefined;
-        param: string;
-    };
+    readonly error: z.infer<typeof OpenAIErrorSchema>["error"];
     constructor(message: string, type?: string, code?: string, param?: string);
 }
 export declare function toOpenAIError(error: unknown): OpenAIError;

package/dist/lifecycle.d.ts

@@ -1,2 +1,3 @@
-import type { GatewayConfig, GatewayContext } from "./types";
-export declare const winterCgHandler: (run: (ctx: GatewayContext) => Promise<object | ReadableStream<object>>, config: GatewayConfig) => (request: Request, state?: Record<string, unknown>) => Promise<Response>;
+import type { GatewayConfig, GatewayConfigParsed, GatewayContext } from "./types";
+import type { SseFrame } from "./utils/stream";
+export declare const winterCgHandler: (run: (ctx: GatewayContext, cfg: GatewayConfigParsed) => Promise<object | ReadableStream<SseFrame>>, config: GatewayConfig) => (request: Request, state?: Record<string, unknown>) => Promise<Response>;

package/dist/lifecycle.js

@@ -8,7 +8,6 @@ import { recordRequestDuration } from "./telemetry/gen-ai";
 import { getRequestAttributes, getResponseAttributes } from "./telemetry/http";
 import { observeV8jsMemoryMetrics } from "./telemetry/memory";
 import { addSpanEvent, setSpanEventsEnabled, setSpanTracer, startSpan } from "./telemetry/span";
-import { wrapStream } from "./telemetry/stream";
 import { resolveOrCreateRequestId } from "./utils/request";
 import { prepareResponseInit, toResponse } from "./utils/response";
 export const winterCgHandler = (run, config) => {
@@ -69,11 +68,10 @@ export const winterCgHandler = (run, config) => {
                 }
             }
             if (!ctx.response) {
-                ctx.result = (await span.runWithContext(() => run(ctx)));
-                if (ctx.result instanceof ReadableStream) {
-                    ctx.result = wrapStream(ctx.result, { onDone: finalize });
-                }
-                ctx.response = toResponse(ctx.result, prepareResponseInit(ctx.requestId));
+                ctx.result = (await span.runWithContext(() => run(ctx, parsedConfig)));
+                ctx.response = toResponse(ctx.result, prepareResponseInit(ctx.requestId), {
+                    onDone: finalize,
+                });
             }
             if (parsedConfig.hooks?.onResponse) {
                 const onResponse = await parsedConfig.hooks.onResponse(ctx);

package/dist/models/google/presets.d.ts

@@ -9,6 +9,16 @@ export declare const geminiEmbedding001: import("../../utils/preset").Preset<"go
     };
     providers: readonly ["vertex"];
 }>;
+export declare const geminiEmbedding2Preview: import("../../utils/preset").Preset<"google/gemini-embedding-2-preview", CatalogModel, {
+    name: string;
+    created: string;
+    context: number;
+    modalities: {
+        input: readonly ["text"];
+        output: readonly ["embedding"];
+    };
+    providers: readonly ["vertex"];
+}>;
 export declare const gemini3FlashPreview: import("../../utils/preset").Preset<"google/gemini-3-flash-preview", CatalogModel, {
     name: string;
     created: string;
@@ -159,6 +169,15 @@ export declare const gemini: {
             output: readonly ["embedding"];
         };
         providers: readonly ["vertex"];
+    }>, import("../../utils/preset").Preset<"google/gemini-embedding-2-preview", CatalogModel, {
+        name: string;
+        created: string;
+        context: number;
+        modalities: {
+            input: readonly ["text"];
+            output: readonly ["embedding"];
+        };
+        providers: readonly ["vertex"];
     }>];
     readonly all: (import("../../utils/preset").Preset<"google/embedding-001", CatalogModel, {
         name: string;
@@ -169,6 +188,15 @@ export declare const gemini: {
             output: readonly ["embedding"];
         };
         providers: readonly ["vertex"];
+    }> | import("../../utils/preset").Preset<"google/gemini-embedding-2-preview", CatalogModel, {
+        name: string;
+        created: string;
+        context: number;
+        modalities: {
+            input: readonly ["text"];
+            output: readonly ["embedding"];
+        };
+        providers: readonly ["vertex"];
     }> | import("../../utils/preset").Preset<"google/gemini-3-flash-preview", CatalogModel, {
         name: string;
         created: string;

package/dist/models/google/presets.js

@@ -27,6 +27,12 @@ export const geminiEmbedding001 = presetFor()("google/embedding-001", {
     created: "2025-05-20",
     context: 8192,
 });
+export const geminiEmbedding2Preview = presetFor()("google/gemini-embedding-2-preview", {
+    ...GEMINI_EMBEDDINGS_BASE,
+    name: "Gemini Embedding 2 (Preview)",
+    created: "2026-03-10",
+    context: 8192,
+});
 export const gemini3FlashPreview = presetFor()("google/gemini-3-flash-preview", {
     ...GEMINI_BASE,
     name: "Gemini 3 Flash (Preview)",
@@ -66,7 +72,7 @@ export const gemini25Pro = presetFor()("google/gemini-2.5-pro", {
 const geminiAtomic = {
     "v2.5": [gemini25FlashLite, gemini25Flash, gemini25Pro],
     "v3-preview": [gemini3FlashPreview, gemini31FlashLitePreview, gemini31ProPreview],
-    embeddings: [geminiEmbedding001],
+    embeddings: [geminiEmbedding001, geminiEmbedding2Preview],
 };
 const geminiGroups = {
     "v2.x": [...geminiAtomic["v2.5"]],

package/dist/models/types.d.ts

@@ -1,5 +1,5 @@
 import type { ProviderId } from "../providers/types";
-export declare const CANONICAL_MODEL_IDS: readonly ["anthropic/claude-opus-4.6", "anthropic/claude-sonnet-4.6", "anthropic/claude-haiku-4.5", "anthropic/claude-sonnet-4.5", "anthropic/claude-opus-4.5", "anthropic/claude-opus-4.1", "anthropic/claude-opus-4", "anthropic/claude-sonnet-4", "anthropic/claude-sonnet-3.7", "anthropic/claude-sonnet-3.5", "anthropic/claude-haiku-3.5", "anthropic/claude-haiku-3", "openai/gpt-oss-20b", "openai/gpt-oss-120b", "openai/gpt-5", "openai/gpt-5-pro", "openai/gpt-5.2", "openai/gpt-5.2-chat", "openai/gpt-5.2-pro", "openai/gpt-5.2-codex", "openai/gpt-5.3-codex", "openai/gpt-5-mini", "openai/gpt-5-nano", "openai/gpt-5-codex", "openai/gpt-5.1-codex", "openai/gpt-5.1-codex-max", "openai/gpt-5.1-chat", "openai/gpt-5.1", "openai/text-embedding-3-small", "openai/text-embedding-3-large", "amazon/nova-micro", "amazon/nova-lite", "amazon/nova-pro", "amazon/nova-premier", "amazon/nova-2-lite", "amazon/nova-2-multimodal-embeddings", "google/gemini-2.5-flash-lite", "google/gemini-2.5-flash", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/embedding-001", "meta/llama-3.1-8b", "meta/llama-3.1-70b", "meta/llama-3.1-405b", "meta/llama-3.2-1b", "meta/llama-3.2-3b", "meta/llama-3.2-11b", "meta/llama-3.2-90b", "meta/llama-3.3-70b", "meta/llama-4-scout", "meta/llama-4-maverick", "cohere/embed-v4.0", "cohere/embed-english-v3.0", "cohere/embed-english-light-v3.0", "cohere/embed-multilingual-v3.0", "cohere/embed-multilingual-light-v3.0", "cohere/command-a", "cohere/command-r7b", "cohere/command-a-translate", "cohere/command-a-reasoning", "cohere/command-a-vision", "cohere/command-r", "cohere/command-r-plus", "voyage/voyage-2-code", "voyage/voyage-2-law", "voyage/voyage-2-finance", "voyage/voyage-3-code", "voyage/voyage-3-large", "voyage/voyage-3.5-lite", "voyage/voyage-3.5", "voyage/voyage-4-lite", "voyage/voyage-4", "voyage/voyage-4-large"];
+export declare const CANONICAL_MODEL_IDS: readonly ["anthropic/claude-opus-4.6", "anthropic/claude-sonnet-4.6", "anthropic/claude-haiku-4.5", "anthropic/claude-sonnet-4.5", "anthropic/claude-opus-4.5", "anthropic/claude-opus-4.1", "anthropic/claude-opus-4", "anthropic/claude-sonnet-4", "anthropic/claude-sonnet-3.7", "anthropic/claude-sonnet-3.5", "anthropic/claude-haiku-3.5", "anthropic/claude-haiku-3", "openai/gpt-oss-20b", "openai/gpt-oss-120b", "openai/gpt-5", "openai/gpt-5-pro", "openai/gpt-5.2", "openai/gpt-5.2-chat", "openai/gpt-5.2-pro", "openai/gpt-5.2-codex", "openai/gpt-5.3-codex", "openai/gpt-5-mini", "openai/gpt-5-nano", "openai/gpt-5-codex", "openai/gpt-5.1-codex", "openai/gpt-5.1-codex-max", "openai/gpt-5.1-chat", "openai/gpt-5.1", "openai/text-embedding-3-small", "openai/text-embedding-3-large", "amazon/nova-micro", "amazon/nova-lite", "amazon/nova-pro", "amazon/nova-premier", "amazon/nova-2-lite", "amazon/nova-2-multimodal-embeddings", "google/gemini-2.5-flash-lite", "google/gemini-2.5-flash", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-embedding-2-preview", "google/embedding-001", "meta/llama-3.1-8b", "meta/llama-3.1-70b", "meta/llama-3.1-405b", "meta/llama-3.2-1b", "meta/llama-3.2-3b", "meta/llama-3.2-11b", "meta/llama-3.2-90b", "meta/llama-3.3-70b", "meta/llama-4-scout", "meta/llama-4-maverick", "cohere/embed-v4.0", "cohere/embed-english-v3.0", "cohere/embed-english-light-v3.0", "cohere/embed-multilingual-v3.0", "cohere/embed-multilingual-light-v3.0", "cohere/command-a", "cohere/command-r7b", "cohere/command-a-translate", "cohere/command-a-reasoning", "cohere/command-a-vision", "cohere/command-r", "cohere/command-r-plus", "voyage/voyage-2-code", "voyage/voyage-2-law", "voyage/voyage-2-finance", "voyage/voyage-3-code", "voyage/voyage-3-large", "voyage/voyage-3.5-lite", "voyage/voyage-3.5", "voyage/voyage-4-lite", "voyage/voyage-4", "voyage/voyage-4-large"];
 export type CanonicalModelId = (typeof CANONICAL_MODEL_IDS)[number];
 export type ModelId = CanonicalModelId | (string & {});
 export type CatalogModel = {

package/dist/models/types.js

@@ -45,6 +45,7 @@ export const CANONICAL_MODEL_IDS = [
     "google/gemini-3-flash-preview",
     "google/gemini-3.1-flash-lite-preview",
     "google/gemini-3.1-pro-preview",
+    "google/gemini-embedding-2-preview",
     "google/embedding-001",
     // Meta
     "meta/llama-3.1-8b",

package/dist/providers/bedrock/middleware.d.ts

@@ -1,4 +1,5 @@
 import type { LanguageModelMiddleware } from "ai";
+export declare const bedrockServiceTierMiddleware: LanguageModelMiddleware;
 export declare const bedrockGptReasoningMiddleware: LanguageModelMiddleware;
 export declare const bedrockClaudeReasoningMiddleware: LanguageModelMiddleware;
 export declare const bedrockPromptCachingMiddleware: LanguageModelMiddleware;

package/dist/providers/bedrock/middleware.js

@@ -1,5 +1,37 @@
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 const isClaude46 = (modelId) => modelId.includes("-4-6");
+// https://docs.aws.amazon.com/bedrock/latest/userguide/service-tiers-inference.html
+export const bedrockServiceTierMiddleware = {
+    specificationVersion: "v3",
+    // eslint-disable-next-line require-await
+    transformParams: async ({ params }) => {
+        const bedrock = params.providerOptions?.["bedrock"];
+        if (!bedrock || typeof bedrock !== "object")
+            return params;
+        // UPSTREAM: https://github.com/vercel/ai/issues/13241
+        // @ts-expect-error AI SDK missing serviceTier, need to open PR
+        const tier = bedrock["serviceTier"];
+        switch (tier) {
+            case undefined:
+                return params;
+            case "auto":
+                // Bedrock uses its default tier when omitted.
+                // @ts-expect-error AI SDK missing serviceTier, need to open PR
+                delete bedrock.serviceTier;
+                return params;
+            case "scale":
+                // @ts-expect-error AI SDK missing serviceTier, need to open PR
+                bedrock.serviceTier = { type: "reserved" };
+                return params;
+            case "default":
+            case "flex":
+            case "priority":
+                // @ts-expect-error AI SDK missing serviceTier, need to open PR
+                bedrock.serviceTier = { type: tier };
+                return params;
+        }
+    },
+};
 export const bedrockGptReasoningMiddleware = {
     specificationVersion: "v3",
     // oxlint-disable-next-line require-await
@@ -97,6 +129,7 @@ export const bedrockPromptCachingMiddleware = {
 };
 modelMiddlewareMatcher.useForProvider("amazon-bedrock", {
     language: [
+        bedrockServiceTierMiddleware,
         bedrockGptReasoningMiddleware,
         bedrockClaudeReasoningMiddleware,
         bedrockPromptCachingMiddleware,

package/dist/providers/groq/index.d.ts

@@ -1 +1,2 @@
 export * from "./canonical";
+export * from "./middleware";

package/dist/providers/groq/index.js

@@ -1 +1,2 @@
 export * from "./canonical";
+export * from "./middleware";

package/dist/providers/groq/middleware.d.ts

@@ -0,0 +1,2 @@
+import type { LanguageModelMiddleware } from "ai";
+export declare const groqServiceTierMiddleware: LanguageModelMiddleware;

package/dist/providers/groq/middleware.js

@@ -0,0 +1,31 @@
+import { modelMiddlewareMatcher } from "../../middleware/matcher";
+// https://console.groq.com/docs/service-tiers
+export const groqServiceTierMiddleware = {
+    specificationVersion: "v3",
+    // eslint-disable-next-line require-await
+    transformParams: async ({ params }) => {
+        const groq = params.providerOptions?.["groq"];
+        if (!groq || typeof groq !== "object")
+            return params;
+        const tier = groq.serviceTier;
+        switch (tier) {
+            case undefined:
+                return params;
+            case "auto":
+            case "flex":
+                return params;
+            case "default":
+                groq.serviceTier = "on_demand";
+                return params;
+            case "scale":
+            case "priority":
+                // UPSTREAM: https://github.com/vercel/ai/issues/13235
+                // @ts-expect-error AI SDK missing "performance", need to open PR
+                groq.serviceTier = "performance";
+                return params;
+        }
+    },
+};
+modelMiddlewareMatcher.useForProvider("groq.*", {
+    language: [groqServiceTierMiddleware],
+});

package/dist/providers/vertex/index.d.ts

@@ -1 +1,2 @@
 export * from "./canonical";
+export * from "./middleware";

package/dist/providers/vertex/index.js

@@ -1 +1,2 @@
 export * from "./canonical";
+export * from "./middleware";

package/dist/providers/vertex/middleware.d.ts

@@ -0,0 +1,2 @@
+import type { LanguageModelMiddleware } from "ai";
+export declare const vertexServiceTierMiddleware: LanguageModelMiddleware;

package/dist/providers/vertex/middleware.js

@@ -0,0 +1,47 @@
+import { modelMiddlewareMatcher } from "../../middleware/matcher";
+const VERTEX_REQUEST_TYPE_HEADER = "x-vertex-ai-llm-request-type";
+const VERTEX_SHARED_REQUEST_TYPE_HEADER = "x-vertex-ai-llm-shared-request-type";
+function setHeaderIfMissing(headers, key, value) {
+    headers[key] ??= value;
+}
+// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/standard-paygo
+// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/priority-paygo
+// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/flex-paygo
+// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/provisioned-throughput/use-provisioned-throughput
+// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/reference/rest/v1/GenerateContentResponse#TrafficType
+export const vertexServiceTierMiddleware = {
+    specificationVersion: "v3",
+    // eslint-disable-next-line require-await
+    transformParams: async ({ params }) => {
+        const vertex = params.providerOptions?.["vertex"];
+        if (!vertex || typeof vertex !== "object")
+            return params;
+        const tier = vertex["serviceTier"];
+        const headers = (params.headers ??= {});
+        switch (tier) {
+            case undefined:
+                return params;
+            case "flex":
+                setHeaderIfMissing(headers, VERTEX_REQUEST_TYPE_HEADER, "shared");
+                setHeaderIfMissing(headers, VERTEX_SHARED_REQUEST_TYPE_HEADER, "flex");
+                break;
+            case "priority":
+                setHeaderIfMissing(headers, VERTEX_REQUEST_TYPE_HEADER, "shared");
+                setHeaderIfMissing(headers, VERTEX_SHARED_REQUEST_TYPE_HEADER, "priority");
+                break;
+            case "scale":
+                setHeaderIfMissing(headers, VERTEX_REQUEST_TYPE_HEADER, "dedicated");
+                break;
+            case "default":
+                setHeaderIfMissing(headers, VERTEX_REQUEST_TYPE_HEADER, "shared");
+                break;
+            case "auto":
+                break;
+        }
+        delete vertex["serviceTier"];
+        return params;
+    },
+};
+modelMiddlewareMatcher.useForProvider(["google.vertex.*"], {
+    language: [vertexServiceTierMiddleware],
+});

package/dist/types.d.ts

@@ -1,6 +1,6 @@
 import type { ProviderV3 } from "@ai-sdk/provider";
 import type { Tracer } from "@opentelemetry/api";
-import type { ChatCompletions, ChatCompletionsBody, ChatCompletionsChunk } from "./endpoints/chat-completions/schema";
+import type { ChatCompletions, ChatCompletionsBody, ChatCompletionsStream } from "./endpoints/chat-completions/schema";
 import type { Embeddings, EmbeddingsBody } from "./endpoints/embeddings/schema";
 import type { Model, ModelList } from "./endpoints/models";
 import type { Logger, LoggerConfig } from "./logger";
@@ -57,7 +57,7 @@ export type GatewayContext = {
     /**
      * Result returned by the handler (pre-response).
      */
-    result?: ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings | Model | ModelList;
+    result?: ChatCompletions | ChatCompletionsStream | Embeddings | Model | ModelList;
     /**
      * Response object returned by the handler.
      */
@@ -104,7 +104,7 @@ export type GatewayHooks = {
      * Runs after the endpoint handler.
      * @returns Result to replace, or undefined to keep original.
      */
-    after?: (ctx: AfterHookContext) => void | ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings | Promise<void | ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings>;
+    after?: (ctx: AfterHookContext) => void | ChatCompletions | ChatCompletionsStream | Embeddings | Promise<void | ChatCompletions | ChatCompletionsStream | Embeddings>;
     /**
      * Runs after the lifecycle has produced the final Response.
      * @returns Replacement Response, or undefined to keep original.
@@ -112,6 +112,18 @@ export type GatewayHooks = {
     onResponse?: (ctx: OnResponseHookContext) => void | Response | Promise<void | Response>;
 };
 export type TelemetrySignalLevel = "off" | "required" | "recommended" | "full";
+export declare const DEFAULT_CHAT_TIMEOUT_MS: number;
+export type GatewayTimeout = number | null | {
+    /**
+     * Default timeout used.
+     */
+    normal?: number | null;
+    /**
+     * Timeout used when `service_tier=flex`.
+     * Defaults to 3x `normal` when omitted.
+     */
+    flex?: number | null;
+};
 /**
  * Main configuration object for the gateway.
  */
@@ -162,9 +174,18 @@ export type GatewayConfig = {
             hebo?: TelemetrySignalLevel;
         };
     };
+    /**
+     * Optional timeout for server responses.
+     * Supports a number in milliseconds, or tiered config.
+     */
+    timeouts?: GatewayTimeout;
 };
 export declare const kParsed: unique symbol;
-export type GatewayConfigParsed = GatewayConfig & {
+export type GatewayConfigParsed = Omit<GatewayConfig, "timeouts"> & {
+    timeouts: {
+        normal?: number;
+        flex?: number;
+    };
     [kParsed]: true;
 };
 export interface Endpoint {

package/dist/types.js

@@ -1 +1,2 @@
+export const DEFAULT_CHAT_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
 export const kParsed = Symbol("hebo.gateway.parsed");

package/dist/utils/response.d.ts

@@ -1,3 +1,6 @@
+import type { SseFrame } from "./stream";
 export declare const prepareResponseInit: (requestId: string) => ResponseInit;
 export declare const mergeResponseInit: (defaultHeaders: HeadersInit, responseInit?: ResponseInit) => ResponseInit;
-export declare const toResponse: (result: ReadableStream | Uint8Array<ArrayBuffer> | object | string, responseInit?: ResponseInit) => Response;
+export declare const toResponse: (result: ReadableStream<SseFrame> | Uint8Array<ArrayBuffer> | object | string, responseInit?: ResponseInit, streamOptions?: {
+    onDone?: (status: number, reason?: unknown) => void;
+}) => Response;

package/dist/utils/response.js

@@ -1,17 +1,6 @@
 import { REQUEST_ID_HEADER } from "./headers";
+import { toSseStream } from "./stream";
 const TEXT_ENCODER = new TextEncoder();
-class JsonToSseTransformStream extends TransformStream {
-    constructor() {
-        super({
-            transform(part, controller) {
-                controller.enqueue(`data: ${JSON.stringify(part)}\n\n`);
-            },
-            flush(controller) {
-                controller.enqueue("data: [DONE]\n\n");
-            },
-        });
-    }
-}
 export const prepareResponseInit = (requestId) => ({
     headers: { [REQUEST_ID_HEADER]: requestId },
 });
@@ -31,11 +20,11 @@ export const mergeResponseInit = (defaultHeaders, responseInit) => {
         headers,
     };
 };
-export const toResponse = (result, responseInit) => {
+export const toResponse = (result, responseInit, streamOptions) => {
     let body;
     const isStream = result instanceof ReadableStream;
     if (isStream) {
-        body = result.pipeThrough(new JsonToSseTransformStream()).pipeThrough(new TextEncoderStream());
+        body = toSseStream(result, streamOptions);
     }
     else if (result instanceof Uint8Array) {
         body = result;
@@ -43,16 +32,12 @@ export const toResponse = (result, responseInit) => {
     else if (typeof result === "string") {
         body = TEXT_ENCODER.encode(result);
     }
-    else if (result instanceof Error) {
-        body = TEXT_ENCODER.encode(JSON.stringify({ message: result.message }));
-    }
     else {
         body = TEXT_ENCODER.encode(JSON.stringify(result));
     }
     if (!responseInit?.statusText) {
-        const isError = result instanceof Error;
-        const status = responseInit?.status ?? (isError ? 500 : 200);
-        const statusText = isError ? "REQUEST_FAILED" : "OK";
+        const status = responseInit?.status ?? 200;
+        const statusText = "OK";
         const headers = responseInit?.headers;
         responseInit = headers ? { status, statusText, headers } : { status, statusText };
     }

package/dist/utils/stream.d.ts

@@ -0,0 +1,9 @@
+export type SseFrame<T = unknown, E extends string | undefined = string | undefined> = {
+    data: T;
+    event?: E;
+};
+export type SseErrorFrame = SseFrame<Error, "error" | undefined>;
+export declare function toSseStream(src: ReadableStream<SseFrame>, options?: {
+    onDone?: (status: number, reason?: unknown) => void;
+    keepAliveMs?: number;
+}): ReadableStream<Uint8Array>;

package/dist/utils/stream.js

@@ -0,0 +1,100 @@
+import { toOpenAIError } from "../errors/openai";
+const TEXT_ENCODER = new TextEncoder();
+const SSE_DONE_CHUNK = TEXT_ENCODER.encode("data: [DONE]\n\n");
+const SSE_KEEP_ALIVE_CHUNK = TEXT_ENCODER.encode(": keep-alive\n\n");
+const SSE_DEFAULT_KEEP_ALIVE_MS = 20_000;
+export function toSseStream(src, options = {}) {
+    const keepAliveMs = options.keepAliveMs ?? SSE_DEFAULT_KEEP_ALIVE_MS;
+    let reader;
+    let timer;
+    let finished = false;
+    const done = (controller, status, reason) => {
+        if (finished)
+            return;
+        finished = true;
+        if (timer)
+            clearTimeout(timer);
+        try {
+            options.onDone?.(status, reason);
+        }
+        catch { }
+        try {
+            controller.enqueue(SSE_DONE_CHUNK);
+        }
+        catch { }
+        try {
+            controller.close();
+        }
+        catch { }
+    };
+    const heartbeat = (controller) => {
+        if (timer)
+            clearTimeout(timer);
+        if (!keepAliveMs || keepAliveMs <= 0 || finished)
+            return;
+        timer = setTimeout(() => {
+            if (finished)
+                return;
+            try {
+                controller.enqueue(SSE_KEEP_ALIVE_CHUNK);
+                heartbeat(controller);
+            }
+            catch { }
+        }, keepAliveMs);
+    };
+    return new ReadableStream({
+        start(controller) {
+            reader = src.getReader();
+            heartbeat(controller);
+        },
+        async pull(controller) {
+            if (finished)
+                return;
+            try {
+                // oxlint-disable-next-line no-await-in-loop
+                const result = await reader.read();
+                if (result.done) {
+                    done(controller, 200);
+                    return;
+                }
+                const value = result.value;
+                if (value.event === "error" || value.data instanceof Error) {
+                    const error = toOpenAIError(value.data);
+                    controller.enqueue(TEXT_ENCODER.encode(serializeSseFrame({ event: value.event, data: error })));
+                    done(controller, error.error.type === "invalid_request_error" ? 422 : 502, value.data);
+                    reader.cancel(value.data).catch(() => { });
+                    return;
+                }
+                controller.enqueue(TEXT_ENCODER.encode(serializeSseFrame(value)));
+                heartbeat(controller);
+            }
+            catch (error) {
+                try {
+                    controller.enqueue(TEXT_ENCODER.encode(serializeSseFrame({
+                        event: "error",
+                        data: toOpenAIError(error),
+                    })));
+                }
+                catch { }
+                done(controller, 502, error);
+            }
+        },
+        cancel(reason) {
+            if (finished)
+                return;
+            finished = true;
+            if (timer)
+                clearTimeout(timer);
+            options.onDone?.(499, reason);
+            return reader?.cancel(reason).catch(() => { });
+        },
+    });
+}
+function serializeSseFrame(frame) {
+    let out = "";
+    if (frame.event) {
+        out += `event: ${frame.event}\n`;
+    }
+    out += `data: ${JSON.stringify(frame.data)}\n\n`;
+    return out;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hebo-ai/gateway",
-  "version": "0.6.2",
+  "version": "0.7.0",
   "description": "AI gateway as a framework. For full control over models, routing & lifecycle. OpenAI-compatible /chat/completions, /embeddings & /models.",
   "keywords": [
     "ai",

package/dist/telemetry/stream.d.ts

@@ -1,3 +0,0 @@
-export declare const wrapStream: (src: ReadableStream, hooks: {
-    onDone?: (status: number, reason: unknown) => void;
-}) => ReadableStream;

package/dist/telemetry/stream.js

@@ -1,58 +0,0 @@
-import { toOpenAIError } from "../errors/openai";
-const isErrorChunk = (v) => v instanceof Error || (typeof v === "object" && v !== null && "error" in v);
-export const wrapStream = (src, hooks) => {
-    let finished = false;
-    let reader;
-    const done = (controller, status, reason) => {
-        if (finished)
-            return;
-        finished = true;
-        hooks.onDone?.(status, reason);
-        if (status !== 200) {
-            reader?.cancel(reason).catch(() => { });
-        }
-        try {
-            controller.close();
-        }
-        catch { }
-    };
-    return new ReadableStream({
-        async start(controller) {
-            reader = src.getReader();
-            try {
-                for (;;) {
-                    // oxlint-disable-next-line no-await-in-loop, no-unsafe-assignment
-                    const { value, done: eof } = await reader.read();
-                    if (eof)
-                        break;
-                    controller.enqueue(value);
-                    if (isErrorChunk(value)) {
-                        done(controller, toOpenAIError(value).error.type === "invalid_request_error" ? 422 : 502, value);
-                        return;
-                    }
-                }
-                done(controller, 200);
-            }
-            catch (err) {
-                try {
-                    controller.enqueue(toOpenAIError(err));
-                }
-                catch { }
-                done(controller, 502, err);
-            }
-            finally {
-                try {
-                    reader?.releaseLock();
-                }
-                catch { }
-            }
-        },
-        cancel(reason) {
-            if (finished)
-                return;
-            finished = true;
-            hooks.onDone?.(499, reason);
-            reader?.cancel(reason).catch(() => { });
-        },
-    });
-};