@hebo-ai/gateway 0.1.2 → 0.2.1

package/README.md

@@ -21,7 +21,7 @@ Hebo Gateway is an open-source, embeddable AI gateway framework built to live in
 ## Installation
 
 ```bash
-bun add @hebo-ai/gateway ai @ai-sdk/groq
+bun install @hebo-ai/gateway
 ```
 
 ## Quickstart
@@ -31,33 +31,42 @@ bun add @hebo-ai/gateway ai @ai-sdk/groq
 Start by creating a gateway instance with at least one provider and a few models.
 
 ```ts
-import { gateway, createModelCatalog } from "@hebo-ai/gateway";
-import { createGroqWithCanonicalIds } from "@hebo-ai/gateway/providers/groq";
-import { gptOss20b, gptOss } from "@hebo-ai/gateway/models/gpt-oss";
+import { createGroq } from "@ai-sdk/groq";
+import { gateway, defineModelCatalog } from "@hebo-ai/gateway";
+import { withCanonicalIdsForGroq } from "@hebo-ai/gateway/providers/groq";
+import { gptOss20b, gptOss } from "@hebo-ai/gateway/models/openai";
 
 export const gw = gateway({
   // PROVIDER REGISTRY
   providers: {
-    // Any Vercel AI SDK provider +WithCanonicalIds
-    groq: createGroqWithCanonicalIds({
-      apiKey: process.env.GROQ_API_KEY,
-    },
+    // Any Vercel AI SDK provider + withCanonicalIdsForX helper
+    groq: withCanonicalIdsForGroq(
+      createGroq({
+        apiKey: process.env.GROQ_API_KEY,
+      }),
+    ),
   },
 
   // MODEL CATALOG
-  models: createModelCatalog(
-    // Choose a preset for common SOTA models
-    gptOss20b({
-      providers: ["groq"],
-    }),
-    // Or add a whole model family
-    ...gptOss["all"].map((preset) =>
-      preset({})
+  models: defineModelCatalog(
+    // Choose a pre-configured preset for common SOTA models
+    gptOss20b,
+    // Or add a whole model family with your own provider list
+    gptOss["all"].map(
+      preset => preset({
+        providers: ["groq"],
+      })
     ),
   ),
 });
 ```
 
+> [!NOTE]
+> Don't forget to install the Groq provider package too: `@ai-sdk/groq`.
+
+> [!TIP]
+> Why `withCanonicalIdsForX`? In most cases you want your gateway to route using model IDs that are consistent across providers (e.g. `openai/gpt-oss-20b` rather than `openai.gpt-oss-20b-v1:0`). We call that `Canonical IDs` - they are what enable routing, fallbacks, and policy rules. Without this wrapper, providers only understands their native IDs, which would make cross-provider routing impossible.
+
 ### Mount Route Handlers
 
 Hebo Gateway plugs into your favorite web framework. Simply mount the gateway’s `handler` under a prefix, and keep using your existing lifecycle hooks for authentication, logging, observability, and more.
@@ -104,181 +113,105 @@ const { text } = await generateText({
 console.log(text);
 ```
 
-## Framework Support
-
-Hebo Gateway exposes **WinterCG-compatible** handlers that integrate with almost any existing framework.
-
-### ElysiaJS
-
-`src/index.ts`
-
-```ts
-import { Elysia } from "elysia";
-
-const app = new Elysia().mount("/v1/gateway/", gw.handler).listen(3000);
-
-console.log(`🐒 Hebo Gateway is running with Elysia at ${app.server?.url}`);
-```
-
-### Hono
-
-`src/index.ts`
-
-```ts
-import { Hono } from "hono";
-
-export default new Hono().mount("/v1/gateway/", gw.handler);
-
-console.log(`🐒 Hebo Gateway is running with Hono framework`);
-```
-
-### Next.js (App Router)
-
-`app/api/gateway/[...all]/route.ts`
-
-```ts
-const gw = gateway({
-  // Required: add `basePath` to your gateway config
-  basePath: "/api/gateway",
-  // ...
-});
-
-export const POST = gw.handler, GET = gw.handler;
-```
-
-### Next.js (Pages Router)
-
-`pages/api/gateway/[...all].ts`
-
-```ts
-// Requires `@mjackson/node-fetch-server` npm package
-import { createRequest, sendResponse } from "@mjackson/node-fetch-server";
-
-const gw = gateway({
-  // Required: add `basePath` to your gateway config
-  basePath: "/api/gateway",
-  // ...
-});
-
-export default async function handler(req, res) {
-  await sendResponse(res, await gw.handler(createRequest(req, res)));
-}
-```
-
-### TanStack Start
-
-`routes/api/$.ts`
-
-```ts
-const gw = gateway({
-  // Required: add `basePath` to your gateway config
-  basePath: "/api/gateway",
-  // ...
-});
-
-export const Route = createFileRoute("/api/$")({
-  server: {
-    handlers: {
-      GET: ({ request }) => gw.handler(request),
-      POST: ({ request }) => gw.handler(request),
-    },
-  },
-});
-```
-
 ## Configuration Reference
 
 ### Providers
 
 Hebo Gateway’s provider registry accepts any **Vercel AI SDK Provider**. For Hebo to be able to route a model across different providers, the names need to be canonicalized to a common form, for example 'openai/gpt-4.1-mini' instead of 'gpt-4.1-mini'.
 
-Out-of-the-box canonical providers:
+We currently provide out-of-the-box canonical providers for: `Bedrock`, `Anthropic`, `Cohere`, `Vertex`, `Groq`, `OpenAI`, and `Voyage`. Import the helper from the matching package path:
 
-- Amazon Bedrock (`createAmazonBedrockWithCanonicalIds`): `@hebo-ai/gateway/providers/bedrock`
-- Anthropic (`createAnthropicWithCanonicalIds`): `@hebo-ai/gateway/providers/anthropic`
-- Cohere (`createCohereWithCanonicalIds`): `@hebo-ai/gateway/providers/cohere`
-- Google Vertex AI (`createVertexWithCanonicalIds`): `@hebo-ai/gateway/providers/vertex`
-- Groq (`createGroqWithCanonicalIds`): `@hebo-ai/gateway/providers/groq`
-- OpenAI (`createOpenAIWithCanonicalIds`): `@hebo-ai/gateway/providers/openai`
-- Voyage (`createVoyageWithCanonicalIds`): `@hebo-ai/gateway/providers/voyage`
+```ts
+// pattern: @hebo-ai/gateway/providers/<provider>
+import { withCanonicalIdsForGroq } from "@hebo-ai/gateway/providers/groq";
+```
 
 If an adapter is not yet provided, you can create your own by wrapping the provider instance with the `withCanonicalIds` helper and define your custom canonicalization mapping & rules.
 
 ```ts
-import { createOpenAI } from "@ai-sdk/openai";
+import { createAzure } from "@ai-sdk/openai";
 import {
   gateway,
-  createModelCatalog,
   withCanonicalIds,
 } from "@hebo-ai/gateway";
 
-const openai = withCanonicalIds(
-  createOpenAI({ apiKey: process.env.OPENAI_API_KEY }),
-  {
-    "openai/gpt-4.1-mini": "gpt-4.1-mini",
-    "openai/text-embedding-3-small": "text-embedding-3-small",
-  },
+const azure = withCanonicalIds(
+  createAzure({
+    resourceName: process.env["AZURE_RESOURCE_NAME"],
+    apiKey: process.env["AZURE_API_KEY"]
+  }), {
+  mapping: {
+    "openai/gpt-4.1-mini": "your-gpt-4.1-mini-deployment-name",
+    "openai/text-embedding-3-small": "your-embeddings-3-small-deployment-name",
+  }},
 );
 
 const gw = gateway({
   providers: {
-    openai,
+    azure,
   },
-  models: createModelCatalog({
+  models: {
     // ...your models pointing at canonical IDs above
-  }),
+  },
 });
 ```
 
 ### Models
 
-Registering models tells Hebo Gateway which models are available, under which canonical ID and what capabilities they have.
+Register models to tell the gateway what's available, under which canonical ID and what capabilities each one has.
 
 #### Model Presets
 
-To simplify the registration, Hebo Gateway ships a set of model presets under `@hebo-ai/gateway/models`. Use these when you want ready-to-use catalog entries with sane defaults for common SOTA models. 
+To simplify the registration, Hebo Gateway ships a set of model presets under `@hebo-ai/gateway/models`. Use these when you want ready-to-use catalog entries with sane defaults for common SOTA models.
 
 Presets come in two forms:
-- Individual presets (e.g. `gptOss20b`, `claudeSonnet45`) for a single model.
-- Family presets (e.g. `claude`, `gemini`, `llama`) which group multiple models and expose helpers like `latest`, `all`, and versioned arrays (for example `claude["v4.5"]`).
-
-Out-of-the-box model presets:
-
-- **Claude** — `@hebo-ai/gateway/models/claude`  
-  Family: `claude` (`v4.5`, `v4.x`, `latest`, `all`)
-
-- **Gemini** — `@hebo-ai/gateway/models/gemini`  
-  Family: `gemini` (`v2.5`, `v3-preview`, `v2.x`, `v3.x`, `latest`, `preview`, `all`)
 
-- **GPT-OSS** — `@hebo-ai/gateway/models/gpt-oss`  
-  Family: `gptOss` (`v1`, `v1.x`, `latest`, `all`)
-
-- **Llama** — `@hebo-ai/gateway/models/llama`  
-  Family: `llama` (`v3.1`, `v3.3`, `v4`, `v3.x`, `v4.x`, `latest`, `all`)
-
-- **Cohere** — `@hebo-ai/gateway/models/cohere`  
-  Family: `cohere` (`v4`, `v4.x`, `latest`, `all`)
-
-- **Voyage** — `@hebo-ai/gateway/models/voyage`  
-  Family: `voyage` (`v2`, `v3`, `v3.5`, `v4`, `v2.x`, `v3.x`, `v4.x`, `latest`, `all`)
+- Individual presets (e.g. `gptOss20b`, `claudeSonnet45`) for a single model.
+- Family presets (e.g. `claude`, `gemini`, `llama`) which group multiple models and expose helpers like `latest`, `all`, `vX` (e.g. `claude["v4.5"]`).
 
 ```ts
-import { createModelCatalog } from "@hebo-ai/gateway";
-import { gptOss20b } from "@hebo-ai/gateway/models/gpt-oss";
-import { claudeSonnet45, claude } from "@hebo-ai/gateway/models/claude";
+import { defineModelCatalog } from "@hebo-ai/gateway";
+import { gptOss20b } from "@hebo-ai/gateway/models/openai";
+import { claudeSonnet45, claude } from "@hebo-ai/gateway/models/anthropic";
 
 // Individual preset
-const models = createModelCatalog(
+const models = defineModelCatalog(
   gptOss20b({ providers: ["groq"] }),
   claudeSonnet45({ providers: ["bedrock"] }),
 );
 
 // Family preset (pick a group and apply the same override to each)
-const modelsFromFamily = createModelCatalog(
-  ...claude["latest"].map((preset) => preset({ providers: ["anthropic"] })),
+const modelsFromFamily = defineModelCatalog(
+  claude["latest"].map((preset) => preset({ providers: ["anthropic"] })),
 );
 ```
 
+Out-of-the-box model presets:
+
+- **Amazon** — `@hebo-ai/gateway/models/amazon`  
+  Nova: `nova` (`v1`, `v2`, `v1.x`, `v2.x`, `latest`, `embeddings`, `all`)
+
+- **Anthropic** — `@hebo-ai/gateway/models/anthropic`  
+  Claude: `claude` (`v4.5`, `v4.1`, `v4`, `v3.7`, `v3.5`, `v3`, `v4.x`, `v3.x`, `haiku`, `sonnet`, `opus`, `latest`, `all`)
+
+- **Cohere** — `@hebo-ai/gateway/models/cohere`  
+  Command: `command` (`A`, `R`, `latest`, `all`)
+  Embed: `embed` (`v4`, `v3`, `latest`, `all`)
+
+- **Google** — `@hebo-ai/gateway/models/google`  
+  Gemini: `gemini` (`v2.5`, `v3-preview`, `v2.x`, `v3.x`, `embeddings`, `latest`, `preview`, `all`)
+
+- **Meta** — `@hebo-ai/gateway/models/meta`  
+  Llama: `llama` (`v3.1`, `v3.2`, `v3.3`, `v4`, `v3.x`, `v4.x`, `latest`, `all`)
+
+- **OpenAI** — `@hebo-ai/gateway/models/openai`  
+  GPT: `gpt` (`v5`, `v5.1`, `v5.2`, `v5.x`, `chat`, `codex`, `pro`, `latest`, `all`)  
+  GPT-OSS: `gptOss` (`v1`, `v1.x`, `latest`, `all`)
+  Embeddings: `textEmbeddings` (`v3`, `v3.x`, `latest`, `all`)
+
+- **Voyage** — `@hebo-ai/gateway/models/voyage`  
+  Voyage: `voyage` (`v2`, `v3`, `v3.5`, `v4`, `v2.x`, `v3.x`, `v4.x`, `latest`, `all`)
+
 #### User-defined Models
 
 As the ecosystem is moving faster than anyone can keep-up with, you can always register your own model entries by following the `CatalogModel` type.
@@ -288,7 +221,7 @@ const gw = gateway({
   providers: {
     // ...
   },
-  models: createModelCatalog({
+  models: {
     "openai/gpt-5.2": {
       name: "GPT 5.2",
       created: "2025-12-11",
@@ -312,13 +245,16 @@ const gw = gateway({
       }
     },
     // ...
-  }),
+  },
 });
 ```
 
+> [!NOTE]
+> The only mandatory property is the `providers` array, everything else is optional metadata.
+
 ### Hooks
 
-Hooks allow you to plug-into the lifecycle of the gateway and enrich it with additional functionality. All hooks are available as async and non-async.
+Hooks allow you to plug-into the lifecycle of the gateway and enrich it with additional functionality, like your actual routing logic. All hooks are available as async and non-async.
 
 ```ts
 const gw = gateway({
@@ -345,25 +281,31 @@ const gw = gateway({
     },
     /**
      * Maps a user-provided model ID or alias to a canonical ID.
+     * @param ctx.body The parsed body object with all call parameters.
      * @param ctx.modelId Incoming model ID.
      * @returns Canonical model ID or undefined to keep original.
      */
-    resolveModelId: async (ctx: { modelId: ModelId }): Promise<ModelId | void> => {
+    resolveModelId?: (ctx: {
+      body: ChatCompletionsBody | EmbeddingsBody;
+      modelId: ModelId;
+    }) => ModelId | void | Promise<ModelId | void> {
       // Example Use Cases:
       // - Resolve modelAlias to modelId
       return undefined;
     },
     /**
      * Picks a provider instance for the request.
-     * @param ctx.providers Provider registry.
+     * @param ctx.providers ProviderRegistry from config.
      * @param ctx.models ModelCatalog from config.
+     * @param ctx.body The parsed body object with all call parameters.
      * @param ctx.modelId Resolved model ID.
      * @param ctx.operation Operation type ("text" | "embeddings").
      * @returns ProviderV3 to override, or undefined to use default.
      */
     resolveProvider: async (ctx: {
-      providers: ProviderRegistryProvider;
+      providers: ProviderRegistry;
       models: ModelCatalog;
+      body: ChatCompletionsBody | EmbeddingsBody;
       modelId: ModelId;
       operation: "text" | "embeddings";
     }): Promise<ProviderV3 | void> => {
@@ -387,8 +329,172 @@ const gw = gateway({
 });
 ```
 
+The `ctx` object is **readonly for core fields**. Use return values to override request / response and provide modelId / provider instances.
+
+> [!TIP]
+> To pass data between hooks, use `ctx.state`. It’s a per-request mutable bag in which you can stash things like auth info, routing decisions, timers, or trace IDs and read them later again in any of the other hooks.
+
+## Framework Support
+
+Hebo Gateway exposes **WinterCG-compatible** handlers that integrate with almost any existing framework.
+
+### ElysiaJS
+
+`src/index.ts`
+
+```ts
+import { Elysia } from "elysia";
+
+const app = new Elysia().mount("/v1/gateway/", gw.handler).listen(3000);
+
+console.log(`🐒 Hebo Gateway is running with Elysia at ${app.server?.url}`);
+```
+
+### Hono
+
+`src/index.ts`
+
+```ts
+import { Hono } from "hono";
+
+export default new Hono().mount("/v1/gateway/", gw.handler);
+
+console.log(`🐒 Hebo Gateway is running with Hono framework`);
+```
+
+### Next.js (App Router)
+
+`app/api/gateway/[...all]/route.ts`
+
+```ts
+const gw = gateway({
+  // Required: add `basePath` to your gateway config
+  basePath: "/api/gateway",
+  // ...
+});
+
+export const POST = gw.handler, GET = gw.handler;
+```
+
+### Next.js (Pages Router)
+
+`pages/api/gateway/[...all].ts`
+
+```ts
+// Requires `@mjackson/node-fetch-server` npm package
+import { createRequest, sendResponse } from "@mjackson/node-fetch-server";
+
+const gw = gateway({
+  // Required: add `basePath` to your gateway config
+  basePath: "/api/gateway",
+  // ...
+});
+
+export default async function handler(req, res) {
+  await sendResponse(res, await gw.handler(createRequest(req, res)));
+}
+```
+
+### TanStack Start
+
+`routes/api/$.ts`
+
+```ts
+const gw = gateway({
+  // Required: add `basePath` to your gateway config
+  basePath: "/api/gateway",
+  // ...
+});
+
+export const Route = createFileRoute("/api/$")({
+  server: {
+    handlers: {
+      GET: ({ request }) => gw.handler(request),
+      POST: ({ request }) => gw.handler(request),
+    },
+  },
+});
+```
+
+## OpenAI Extensions
+
+### Reasoning
+
+In addition to the official `reasoning_effort` parameter, the chat completions endpoint accepts a `reasoning` object for more fine-grained control of the budget. It's treated as provider-agnostic input and normalized before hitting the upstream model.
+
+```json
+{
+  "model": "anthropic/claude-4-sonnet",
+  "messages": [{ "role": "user", "content": "Explain the tradeoffs." }],
+  "reasoning": { "effort": "medium" }
+}
+```
+
+Normalization rules:
+
+- `enabled` -> fall-back to model default if none provided
+- `max_tokens`: fall-back to model default if model supports
+- `effort` -> budget = percentage of `max_tokens`
+  - `none`: 0%
+  - `minimal`: 10%
+  - `low`: 20%
+  - `medium`: 50% (default)
+  - `high`: 80%
+  - `xhigh`: 95%
+
+Reasoning output is surfaced as extension to the `completion` object.
+
+- When present, it is returned on the assistant message as `reasoning_content`. Reasoning token counts (when available) are returned on `usage.completion_tokens_details.reasoning_tokens`.
+- For stream responses, reasoning text is sent incrementally as `reasoning_content` part (separate from normal text `content` deltas). Token counts land in the final `usage` object on the terminating chunk.
+
+Most SDKs handle these fields out-of-the-box.
+
 ## Advanced Usage
 
+### Passing Framework State to Hooks
+
+You can pass per-request info from your framework into the gateway via the second `state` argument on the handler, then read it in hooks through `ctx.state`.
+
+```ts
+import { Elysia } from "elysia";
+import { gateway } from "@hebo-ai/gateway";
+
+const basePath = "/v1/gateway";
+
+const gw = gateway({
+  basePath,
+  providers: {
+    // ...
+  },
+  models: {
+    // ...
+  },
+  hooks: {
+    resolveProvider: async (ctx) => {
+      // Select provider based on userId
+      const user = ctx.state.auth.userId;
+      if (user.startsWith("vip:")) {
+        return ctx.providers["openai"];
+      } else {
+        return ctx.providers["groq"];
+      }
+    },
+  },
+});
+
+const app = new Elysia()
+  .derive(({ headers }) => ({
+    auth: {
+      userId: headers["x-user-id"],
+    },
+  }))
+  .all(`${basePath}/*`, ({ request, auth }) => gw.handler(request, { auth }), { parse: 'none' })
+  .listen(3000);
+```
+
+> [!NOTE]
+> The `parse: 'none'` hook is required to prevent Elysia from consuming the body.
+
 ### Selective Route Mounting
 
 If you want to have more flexibility, for example for custom rate limit checks per route, you can also choose to only mount individual routes from the gateway's `routes` property.
@@ -410,14 +516,15 @@ console.log(`🐒 /chat/completions mounted to ${app.server?.url}/chat`);
 We also provide full schemas, helper functions and types to convert between **OpenAI <> Vercel AI SDK** for advanced use cases like creating your own endpoint. They are available via deep-imports and completely tree-shakeable.
 
 ```ts
-import { streamText } from "ai";
+import { streamText, wrapLanguageModel } from "ai";
 import { createGroq } from "@ai-sdk/groq";
 import * as z from "zod";
 import {
-  CompletionsBodySchema,
-  transformCompletionsInputs,
-  createCompletionsStreamResponse,
+  ChatCompletionsBodySchema,
+  convertToTextCallOptions,
+  toChatCompletionsStreamResponse,
 } from "@hebo-ai/gateway/endpoints/chat-completions";
+import { forwardParamsMiddleware } from "@hebo-ai/gateway/middleware/common";
 
 const groq = createGroq({ apiKey: process.env.GROQ_API_KEY });
 
@@ -425,24 +532,28 @@ export async function handler(req: Request): Promise<Response> {
 
   const body = await req.json();
 
-  const parsed = CompletionsBodySchema.safeParse(body);
+  const parsed = ChatCompletionsBodySchema.safeParse(body);
   if (!parsed.success) {
     return new Response(z.prettifyError(parsed.error), { status: 422 });
   }
 
   const { model, ...inputs } = parsed.data;
 
-  const textOptions = transformCompletionsInputs(inputs);
+  const textOptions = convertToTextCallOptions(inputs);
 
   const result = await streamText({
-    model: groq(model),
+    model: wrapLanguageModel({
+      model: groq(model),
+      middleware: forwardParamsMiddleware("groq"),
+    }),
     ...textOptions
   });
 
-  return createCompletionsStreamResponse(result, model);
+  return toChatCompletionsStreamResponse(result, model);
 }
 ```
 
-Non-streaming versions are available via `createCompletionsResponse`. Equivalent schemas and helper are available in the `embeddings` and `models` endpoints.
+Non-streaming versions are available via `createChatCompletionsResponse`. Equivalent schemas and helpers are available in the `embeddings` and `models` endpoints.
 
-Since Zod v4.3 you can also generate a JSON Schema from any zod object by calling the `.toJSONSchema()` function. This can be useful, for example, to create OpenAPI documentation.
+> [!TIP]
+> Since Zod v4.3 you can generate a JSON Schema from any zod object by calling `z.toJSONSchema(...)`. This is useful for producing OpenAPI documentation from the same source of truth.

package/dist/config.js

@@ -1,4 +1,3 @@
-import { createProviderRegistry } from "ai";
 import { kParsed } from "./types";
 export const parseConfig = (config) => {
     // If it has been parsed before, just return
@@ -9,22 +8,13 @@ export const parseConfig = (config) => {
     if (Object.keys(providers).length === 0) {
         throw new Error("Gateway config error: no providers configured (config.providers is empty).");
     }
-    // Initialize ProviderRegistry (if nessecary)
-    let registry;
-    if ("languageModel" in providers) {
-        registry = providers;
-    }
-    else {
-        registry = createProviderRegistry(providers);
-    }
     // Strip out providers from models that are not configured
-    const providerKeys = Object.keys(registry.providers);
     const parsedModels = {};
     for (const id in models) {
         const model = models[id];
         const kept = [];
         for (const p of model.providers) {
-            if (providerKeys.includes(p))
+            if (p in providers)
                 kept.push(p);
             else
                 console.warn(`[models] ${id}: provider "${p}" removed (not configured)`);
@@ -35,5 +25,5 @@ export const parseConfig = (config) => {
     if (Object.keys(parsedModels).length === 0) {
         throw new Error("Gateway config error: no models configured (config.models is empty).");
     }
-    return { ...config, providers: registry, models: parsedModels, [kParsed]: true };
+    return { ...config, providers, models: parsedModels, [kParsed]: true };
 };