@hebo-ai/gateway 0.3.0-rc.0 → 0.3.0-rc.2

package/README.md

@@ -2,7 +2,7 @@
 
 Roll your own AI gateway for full control over models, providers, routing logic, guardrails, observability and more ...
 
-## Overview
+## 🐒 Overview
 
 Existing AI gateways like OpenRouter, Vercel AI Gateway, LiteLLM, and Portkey work out of the box, but they’re hard to extend once your needs go beyond configuration.
 
@@ -10,7 +10,7 @@ Hebo Gateway is an open-source, embeddable AI gateway framework built to live in
 
 Learn more in our blog post: [Yet Another AI Gateway?](https://hebo.ai/blog/260127-hebo-gateway/) (`https://hebo.ai/blog/260127-hebo-gateway/`)
 
-## Features
+## 🍌 Features
 
 - 🌐 OpenAI-compatible /chat/completions, /embeddings & /models endpoints.
 - 🔌 Integrate into your existing Hono, Elysia, Next.js & TanStack apps.
@@ -20,13 +20,28 @@ Learn more in our blog post: [Yet Another AI Gateway?](https://hebo.ai/blog/2601
 - 🪝 Hook system to customize routing, auth, rate limits, and shape responses.
 - 🧰 Low-level OpenAI-compatible schema, converters, and middleware helpers.
 
-## Installation
+## 📦 Installation
 
 ```bash
 bun install @hebo-ai/gateway
 ```
 
-## Quickstart
+## ☰ Table of Contents
+
+- Quickstart
+  - [Setup A Gateway Instance](#setup-a-gateway-instance) | [Mount Route Handlers](#mount-route-handlers) | [Call the Gateway](#call-the-gateway)
+- Configuration Reference
+  - [Providers](#providers) | [Models](#models) | [Hooks](#hooks) | [Logger](#logger-settings)
+- Framework Support
+  - [ElysiaJS](#elysiajs) | [Hono](#hono) | [Next.js](#nextjs) | [TanStack Start](#tanstack-start)
+- Runtime Support
+  - [Vercel Edge](#vercel-edge) | [Cloudflare Workers](#cloudflare-workers) | [Deno Deploy](#deno-deploy) | [AWS Lambda](#aws-lambda)
+- OpenAI Extensions
+  - [Reasoning](#reasoning)
+- Advanced Usage
+  - [Passing Framework State to Hooks](#passing-framework-state-to-hooks) | [Selective Route Mounting](#selective-route-mounting) | [Low-level Schemas & Converters](#low-level-schemas--converters)
+
+## 🚀 Quickstart
 
 ### Setup A Gateway Instance
 
@@ -115,7 +130,7 @@ const { text } = await generateText({
 console.log(text);
 ```
 
-## Configuration Reference
+## ⚙️ Configuration Reference
 
 ### Providers
 
@@ -338,7 +353,7 @@ The `ctx` object is **readonly for core fields**. Use return values to override
 > [!TIP]
 > To pass data between hooks, use `ctx.state`. It’s a per-request mutable bag in which you can stash things like auth info, routing decisions, timers, or trace IDs and read them later again in any of the other hooks.
 
-## Framework Support
+## 🧩 Framework Support
 
 Hebo Gateway exposes **WinterCG-compatible** handlers that integrate with almost any existing framework.
 
@@ -366,7 +381,9 @@ export default new Hono().mount("/v1/gateway/", gw.handler);
 console.log(`🐒 Hebo Gateway is running with Hono framework`);
 ```
 
-### Next.js (App Router)
+### Next.js
+
+#### App Router
 
 `app/api/gateway/[...all]/route.ts`
 
@@ -380,7 +397,7 @@ const gw = gateway({
 export const POST = gw.handler, GET = gw.handler;
 ```
 
-### Next.js (Pages Router)
+#### Pages Router
 
 `pages/api/gateway/[...all].ts`
 
@@ -420,7 +437,69 @@ export const Route = createFileRoute("/api/$")({
 });
 ```
 
-## OpenAI Extensions
+## 🌍 Runtime Support
+
+Hebo Gateway also works directly with runtime-level `Request -> Response` handlers.
+
+### Vercel Edge
+
+`api/gateway.ts`
+
+```ts
+export const runtime = "edge";
+
+const gw = gateway({
+  // ...
+});
+
+export default gw.handler;
+```
+
+### Cloudflare Workers
+
+`src/index.ts`
+
+```ts
+const gw = gateway({
+  // ...
+});
+
+export default {
+  fetch: gw.handler,
+};
+```
+
+### Deno Deploy
+
+`main.ts`
+
+```ts
+import { serve } from "https://deno.land/std/http/server.ts";
+
+const gw = gateway({
+  // ...
+});
+
+serve((request: Request) => gw.handler(request));
+```
+
+### AWS Lambda
+
+`src/lambda.ts`
+
+```ts
+import { awsLambdaEventHandler } from "@hattip/adapter-aws-lambda";
+
+const gw = gateway({
+  // ...
+});
+
+export const handler = awsLambdaEventHandler({
+  handler: gw.handler,
+});
+```
+
+## 🧠 OpenAI Extensions
 
 ### Reasoning
 
@@ -453,7 +532,7 @@ Reasoning output is surfaced as extension to the `completion` object.
 
 Most SDKs handle these fields out-of-the-box.
 
-## Advanced Usage
+## 🧪 Advanced Usage
 
 ### Logger Settings
 

package/dist/endpoints/chat-completions/converters.js

@@ -1,3 +1,4 @@
+import { convertBase64ToUint8Array } from "@ai-sdk/provider-utils";
 import { jsonSchema, JsonToSseTransformStream, tool } from "ai";
 import { GatewayError } from "../../errors/gateway";
 import { OpenAIError, toOpenAIError } from "../../errors/openai";
@@ -5,6 +6,7 @@ import { toResponse } from "../../utils/response";
 // --- Request Flow ---
 export function convertToTextCallOptions(params) {
     const { messages, tools, tool_choice, temperature, max_tokens, max_completion_tokens, reasoning_effort, reasoning, frequency_penalty, presence_penalty, seed, stop, top_p, ...rest } = params;
+    Object.assign(rest, parseReasoningOptions(reasoning_effort, reasoning));
     return {
         messages: convertToModelMessages(messages),
         tools: convertToToolSet(tools),
@@ -17,10 +19,7 @@ export function convertToTextCallOptions(params) {
         stopSequences: stop ? (Array.isArray(stop) ? stop : [stop]) : undefined,
         topP: top_p,
         providerOptions: {
-            unknown: {
-                ...rest,
-                ...parseReasoningOptions(reasoning_effort, reasoning),
-            },
+            unknown: rest,
         },
     };
 }
@@ -113,29 +112,16 @@ export function fromChatCompletionsContent(content) {
         if (part.type === "image_url") {
             const url = part.image_url.url;
             if (url.startsWith("data:")) {
-                const parts = url.split(",");
-                const metadata = parts[0];
-                const base64Data = parts[1];
-                if (!metadata || !base64Data) {
-                    throw new GatewayError("Invalid data URL: missing metadata or data", 400);
-                }
-                const mimeTypePart = metadata.split(":")[1];
-                if (!mimeTypePart) {
-                    throw new GatewayError("Invalid data URL: missing MIME type part", 400);
-                }
-                const mimeType = mimeTypePart.split(";")[0];
-                if (!mimeType) {
-                    throw new GatewayError("Invalid data URL: missing MIME type", 400);
-                }
+                const { mimeType, base64Data } = parseDataUrl(url);
                 return mimeType.startsWith("image/")
                     ? {
                         type: "image",
-                        image: Buffer.from(base64Data, "base64"),
+                        image: convertBase64ToUint8Array(base64Data),
                         mediaType: mimeType,
                     }
                     : {
                         type: "file",
-                        data: Buffer.from(base64Data, "base64"),
+                        data: convertBase64ToUint8Array(base64Data),
                         mediaType: mimeType,
                     };
             }
@@ -149,12 +135,12 @@ export function fromChatCompletionsContent(content) {
             return media_type.startsWith("image/")
                 ? {
                     type: "image",
-                    image: Buffer.from(data, "base64"),
+                    image: convertBase64ToUint8Array(data),
                     mediaType: media_type,
                 }
                 : {
                     type: "file",
-                    data: Buffer.from(data, "base64"),
+                    data: convertBase64ToUint8Array(data),
                     filename,
                     mediaType: media_type,
                 };
@@ -195,6 +181,20 @@ function parseToolOutput(content) {
         return { type: "text", value: content };
     }
 }
+function parseDataUrl(url) {
+    const commaIndex = url.indexOf(",");
+    if (commaIndex <= "data:".length || commaIndex === url.length - 1) {
+        throw new GatewayError("Invalid data URL: missing metadata or data", 400);
+    }
+    const metadata = url.slice("data:".length, commaIndex);
+    const base64Data = url.slice(commaIndex + 1);
+    const semicolonIndex = metadata.indexOf(";");
+    const mimeType = (semicolonIndex === -1 ? metadata : metadata.slice(0, semicolonIndex)).trim();
+    if (!mimeType) {
+        throw new GatewayError("Invalid data URL: missing MIME type", 400);
+    }
+    return { mimeType, base64Data };
+}
 function parseReasoningOptions(reasoning_effort, reasoning) {
     const effort = reasoning?.effort ?? reasoning_effort;
     const max_tokens = reasoning?.max_tokens;
@@ -285,13 +285,10 @@ export class ChatCompletionsStream extends TransformStream {
                         break;
                     }
                     case "tool-call": {
+                        const toolCall = toChatCompletionsToolCall(part.toolCallId, part.toolName, part.input, part.providerMetadata);
+                        toolCall.index = toolCallIndexCounter++;
                         controller.enqueue(createChunk({
-                            tool_calls: [
-                                {
-                                    ...toChatCompletionsToolCall(part.toolCallId, part.toolName, part.input, part.providerMetadata),
-                                    index: toolCallIndexCounter++,
-                                },
-                            ],
+                            tool_calls: [toolCall],
                         }));
                         break;
                     }
@@ -336,29 +333,23 @@ export const toChatCompletionsAssistantMessage = (result) => {
     return message;
 };
 export function toChatCompletionsUsage(usage) {
-    return {
-        ...(usage.inputTokens !== undefined && {
-            prompt_tokens: usage.inputTokens,
-        }),
-        ...(usage.outputTokens !== undefined && {
-            completion_tokens: usage.outputTokens,
-        }),
-        ...((usage.totalTokens !== undefined ||
-            usage.inputTokens !== undefined ||
-            usage.outputTokens !== undefined) && {
-            total_tokens: usage.totalTokens ?? (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0),
-        }),
-        ...(usage.outputTokenDetails?.reasoningTokens !== undefined && {
-            completion_tokens_details: {
-                reasoning_tokens: usage.outputTokenDetails.reasoningTokens,
-            },
-        }),
-        ...(usage.inputTokenDetails?.cacheReadTokens !== undefined && {
-            prompt_tokens_details: {
-                cached_tokens: usage.inputTokenDetails.cacheReadTokens,
-            },
-        }),
-    };
+    const out = {};
+    const prompt = usage.inputTokens;
+    if (prompt !== undefined)
+        out.prompt_tokens = prompt;
+    const completion = usage.outputTokens;
+    if (completion !== undefined)
+        out.completion_tokens = completion;
+    if (prompt !== undefined || completion !== undefined || usage.totalTokens !== undefined) {
+        out.total_tokens = usage.totalTokens ?? (prompt ?? 0) + (completion ?? 0);
+    }
+    const reasoning = usage.outputTokenDetails?.reasoningTokens;
+    if (reasoning !== undefined)
+        out.completion_tokens_details = { reasoning_tokens: reasoning };
+    const cached = usage.inputTokenDetails?.cacheReadTokens;
+    if (cached !== undefined)
+        out.prompt_tokens_details = { cached_tokens: cached };
+    return out;
 }
 export function toChatCompletionsToolCall(id, name, args, providerMetadata) {
     const out = {

package/dist/endpoints/chat-completions/handler.js

@@ -3,7 +3,6 @@ import * as z from "zod/mini";
 import { GatewayError } from "../../errors/gateway";
 import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
-import { forwardParamsMiddleware } from "../../middleware/common";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
 import { prepareForwardHeaders } from "../../utils/request";
@@ -55,15 +54,9 @@ export const chatCompletions = (config) => {
             options: textOptions,
         }, "[chat] AI SDK options");
         // Build middleware chain (model -> forward params -> provider).
-        const middleware = [];
-        for (const m of modelMiddlewareMatcher.forModel(ctx.resolvedModelId))
-            middleware.push(m);
-        middleware.push(forwardParamsMiddleware(languageModel.provider));
-        for (const m of modelMiddlewareMatcher.forProvider(languageModel.provider))
-            middleware.push(m);
         const languageModelWithMiddleware = wrapLanguageModel({
             model: languageModel,
-            middleware,
+            middleware: modelMiddlewareMatcher.for(ctx.resolvedModelId, languageModel.provider),
         });
         // Execute request (streaming vs. non-streaming).
         if (stream) {

package/dist/endpoints/embeddings/handler.js

@@ -3,7 +3,6 @@ import * as z from "zod/mini";
 import { GatewayError } from "../../errors/gateway";
 import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
-import { forwardParamsEmbeddingMiddleware } from "../../middleware/common";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
 import { prepareForwardHeaders } from "../../utils/request";
@@ -52,15 +51,9 @@ export const embeddings = (config) => {
         const embedOptions = convertToEmbedCallOptions(inputs);
         logger.trace({ requestId: ctx.request.headers.get("x-request-id"), options: embedOptions }, "[embeddings] AI SDK options");
         // Build middleware chain (model -> forward params -> provider).
-        const middleware = [];
-        for (const m of modelMiddlewareMatcher.forEmbeddingModel(ctx.resolvedModelId))
-            middleware.push(m);
-        middleware.push(forwardParamsEmbeddingMiddleware(embeddingModel.provider));
-        for (const m of modelMiddlewareMatcher.forEmbeddingProvider(embeddingModel.provider))
-            middleware.push(m);
         const embeddingModelWithMiddleware = wrapEmbeddingModel({
             model: embeddingModel,
-            middleware,
+            middleware: modelMiddlewareMatcher.forEmbedding(ctx.resolvedModelId, embeddingModel.provider),
         });
         // Execute request.
         const result = await embedMany({

package/dist/logger/default.d.ts

@@ -1,5 +1,5 @@
 import type { LogLevel, Logger } from "./index";
-export declare const getDefaultLogLevel: () => "debug" | "info";
+export declare const getDefaultLogLevel: () => LogLevel;
 export declare const createDefaultLogger: (config: {
     level?: LogLevel;
 }) => Logger;

package/dist/logger/default.js

@@ -1,6 +1,6 @@
 import { serializeError } from "serialize-error";
-import { isProduction } from "../utils/env";
-export const getDefaultLogLevel = () => (isProduction() ? "info" : "debug");
+import { isProduction, isTest } from "../utils/env";
+export const getDefaultLogLevel = () => isTest() ? "silent" : isProduction() ? "info" : "debug";
 const noop = () => { };
 const LEVEL = {
     trace: 5,
@@ -8,6 +8,7 @@ const LEVEL = {
     info: 20,
     warn: 30,
     error: 40,
+    silent: 50,
 };
 const LEVELS = Object.keys(LEVEL);
 const isRecord = (value) => typeof value === "object" && value !== null && !(value instanceof Error);
@@ -52,7 +53,7 @@ const buildLogObject = (level, args) => {
 };
 const makeLogFn = (level, write) => (...args) => write(JSON.stringify(buildLogObject(level, args)));
 export const createDefaultLogger = (config) => {
-    if (config.level === "silent") {
+    if (config.level === "silent" || getDefaultLogLevel() === "silent") {
         return { trace: noop, debug: noop, info: noop, warn: noop, error: noop };
     }
     const threshold = LEVEL[config.level ?? getDefaultLogLevel()];

package/dist/logger/index.js

@@ -1,3 +1,4 @@
+import { isTest } from "../utils/env";
 const KEY = Symbol.for("@hebo/logger");
 const g = globalThis;
 g[KEY] ??= {
@@ -10,6 +11,8 @@ g[KEY] ??= {
 export let logger = g[KEY];
 export const isLogger = (input) => typeof input === "object" && input !== null && "info" in input;
 export function isLoggerDisabled(input) {
+    if (isTest())
+        return true;
     if (input === null)
         return true;
     if (!input || typeof input !== "object" || "info" in input)

package/dist/middleware/common.js

@@ -16,8 +16,16 @@ function snakeToCamel(key) {
     }
     return out;
 }
+function hasUppercase(s) {
+    for (let i = 0; i < s.length; i++) {
+        const c = s[i];
+        if (c >= "A" && c <= "Z")
+            return true;
+    }
+    return false;
+}
 function camelToSnake(key) {
-    if (!/[A-Z]/.test(key))
+    if (!hasUppercase(key))
         return key;
     let out = "";
     for (let i = 0; i < key.length; i++) {
@@ -26,53 +34,31 @@ function camelToSnake(key) {
     }
     return out;
 }
-function camelizeKeysDeep(value) {
-    if (value === null || typeof value !== "object")
-        return value;
-    if (Array.isArray(value)) {
-        return value.map((v) => camelizeKeysDeep(v));
-    }
-    const out = {};
-    for (const [k, v] of Object.entries(value)) {
-        if (v === undefined || v === null)
-            continue;
-        out[snakeToCamel(k)] = camelizeKeysDeep(v);
-    }
-    return out;
-}
-function snakizeKeysDeep(value) {
+function remapDeep(value, mapKey) {
     if (value === null || typeof value !== "object")
         return value;
     if (Array.isArray(value)) {
-        return value.map((v) => snakizeKeysDeep(v));
+        return value.map((v) => remapDeep(v, mapKey));
     }
     const out = {};
-    for (const [k, v] of Object.entries(value)) {
-        if (v === undefined || v === null)
-            continue;
-        out[camelToSnake(k)] = snakizeKeysDeep(v);
+    for (const key of Object.keys(value)) {
+        out[mapKey(key)] = remapDeep(value[key], mapKey);
     }
     return out;
 }
-function processOptions(providerOptions, providerName) {
-    if (providerOptions[providerName]) {
-        providerOptions[providerName] = camelizeKeysDeep(providerOptions[providerName]);
-    }
-    const target = (providerOptions[providerName] ??= {});
-    for (const key in providerOptions) {
-        if (key === providerName)
+function processOptions(options, providerName) {
+    const target = (options[providerName] = remapDeep(options[providerName] ?? {}, snakeToCamel));
+    for (const namespace in options) {
+        if (namespace === providerName)
             continue;
-        const source = camelizeKeysDeep(providerOptions[key]);
-        for (const k in source) {
-            target[k] = source[k];
-        }
-        if (key === "unknown")
-            delete providerOptions[key];
+        Object.assign(target, remapDeep(options[namespace], snakeToCamel));
+        if (namespace === "unknown")
+            delete options[namespace];
     }
 }
-function processMetadata(providerMetadata) {
-    for (const key in providerMetadata) {
-        providerMetadata[key] = snakizeKeysDeep(providerMetadata[key]);
+function processMetadata(metadata) {
+    for (const namespace in metadata) {
+        metadata[namespace] = remapDeep(metadata[namespace], camelToSnake);
     }
 }
 /**

package/dist/middleware/matcher.d.ts

@@ -1,19 +1,27 @@
 import type { EmbeddingModelMiddleware, LanguageModelMiddleware } from "ai";
 import type { ModelId } from "../models/types";
 import type { ProviderId } from "../providers/types";
-type MiddlewareEntry = {
-    language?: LanguageModelMiddleware | LanguageModelMiddleware[];
-    embedding?: EmbeddingModelMiddleware | EmbeddingModelMiddleware[];
+type MiddlewareEntries = {
+    language?: LanguageModelMiddleware[];
+    embedding?: EmbeddingModelMiddleware[];
 };
+type ModelMiddleware = LanguageModelMiddleware | EmbeddingModelMiddleware;
 declare class ModelMiddlewareMatcher {
     private model;
     private provider;
-    useForModel(patterns: ModelId | readonly ModelId[], entry: MiddlewareEntry): void;
-    useForProvider(patterns: ProviderId | readonly ProviderId[], entry: MiddlewareEntry): void;
-    forModel(modelId: ModelId): LanguageModelMiddleware[];
-    forProvider(providerId: ProviderId): LanguageModelMiddleware[];
-    forEmbeddingModel(modelId: ModelId): EmbeddingModelMiddleware[];
-    forEmbeddingProvider(providerId: ProviderId): EmbeddingModelMiddleware[];
+    private static readonly MAX_CACHE;
+    private cache;
+    useForModel(patterns: ModelId | readonly ModelId[], entry: MiddlewareEntries): void;
+    useForProvider(patterns: ProviderId | readonly ProviderId[], entry: MiddlewareEntries): void;
+    for(modelId: ModelId, providerId: ProviderId): LanguageModelMiddleware[];
+    forEmbedding(modelId: ModelId, providerId: ProviderId): EmbeddingModelMiddleware[];
+    resolve(options: {
+        kind: "text" | "embedding";
+        modelId?: ModelId;
+        providerId?: ProviderId;
+        forward?: ModelMiddleware | (() => ModelMiddleware);
+    }): ModelMiddleware[];
+    private collect;
 }
 export declare const modelMiddlewareMatcher: ModelMiddlewareMatcher;
 export type { ModelMiddlewareMatcher };