@hebo-ai/gateway 0.4.0-beta.4 → 0.4.1

package/README.md

@@ -19,6 +19,7 @@ Learn more in our blog post: [Yet Another AI Gateway?](https://hebo.ai/blog/2601
 - 🗂️ Model catalog with extensible metadata capabilities.
 - 🪝 Hook system to customize routing, auth, rate limits, and shape responses.
 - 🧰 Low-level OpenAI-compatible schema, converters, and middleware helpers.
+- 👁️ Observability via OTel GenAI semantic conventions (Langfuse-compatible).
 
 ## 📦 Installation
 
@@ -271,7 +272,7 @@ const gw = gateway({
 
 ### Hooks
 
-Hooks allow you to plug-into the lifecycle of the gateway and enrich it with additional functionality, like your actual routing logic. All hooks are available as async and non-async.
+Hooks allow you to plug into the lifecycle of the gateway and enrich it with additional functionality, like your actual routing logic. All hooks are available as async and non-async.
 
 ```ts
 const gw = gateway({
@@ -314,10 +315,10 @@ const gw = gateway({
      * @param ctx.modelId Incoming model ID.
      * @returns Canonical model ID or undefined to keep original.
      */
-    resolveModelId?: (ctx: {
+    resolveModelId: async (ctx: {
       body: ChatCompletionsBody | EmbeddingsBody;
       modelId: ModelId;
-    }) => ModelId | void | Promise<ModelId | void> {
+    }): Promise<ModelId | void> => {
       // Example Use Cases:
       // - Resolve modelAlias to modelId
       return undefined;
@@ -327,7 +328,7 @@ const gw = gateway({
      * @param ctx.providers ProviderRegistry from config.
      * @param ctx.models ModelCatalog from config.
      * @param ctx.body The parsed body object with all call parameters.
-     * @param ctx.modelId Resolved model ID.
+     * @param ctx.resolvedModelId Resolved model ID.
      * @param ctx.operation Operation type ("chat" | "embeddings").
      * @returns ProviderV3 to override, or undefined to use default.
      */
@@ -335,7 +336,7 @@ const gw = gateway({
       providers: ProviderRegistry;
       models: ModelCatalog;
       body: ChatCompletionsBody | EmbeddingsBody;
-      modelId: ModelId;
+      resolvedModelId: ModelId;
       operation: "chat" | "embeddings";
     }): Promise<ProviderV3 | void> => {
       // Example Use Cases:
@@ -349,8 +350,8 @@ const gw = gateway({
      * @returns Modified result, or undefined to keep original.
      */
     after: async (ctx: {
-      result: ChatCompletions  | ReadableStream<ChatCompletionsChunk | OpenAIError> | Embeddings
-    }): Promise<ChatCompletions  | ReadableStream<ChatCompletionsChunk | OpenAIError> | Embeddings | void> => {
+      result: ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings;
+    }): Promise<ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings | void> => {
       // Example Use Cases:
       // - Transform result
       // - Result logging
@@ -641,6 +642,32 @@ https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/
 
 For observability integration that is not otel compliant, you can disable built-in telemetry and manually instrument requests during `before` / `after` hooks.
 
+#### Langfuse
+
+Hebo telemetry spans are OpenTelemetry-compatible, so you can send them to Langfuse via `@langfuse/otel`.
+
+```ts
+import { gateway } from "@hebo-ai/gateway";
+import { LangfuseSpanProcessor } from "@langfuse/otel";
+import { context } from "@opentelemetry/api";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base";
+
+context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+
+const gw = gateway({
+  // ...
+  telemetry: {
+    enabled: true,
+    tracer = new BasicTracerProvider({
+      spanProcessors: [new LangfuseSpanProcessor()],
+    }).getTracer("hebo");,
+  },
+});
+```
+
+Langfuse credentials are read from environment variables by the Langfuse OTel SDK (`LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_BASE_URL`).
+
 ### Passing Framework State to Hooks
 
 You can pass per-request info from your framework into the gateway via the second `state` argument on the handler, then read it in hooks through `ctx.state`.

package/dist/endpoints/chat-completions/converters.d.ts

@@ -25,10 +25,10 @@ export declare const convertToToolSet: (tools: ChatCompletionsTool[] | undefined
 export declare const convertToToolChoice: (toolChoice: ChatCompletionsToolChoice | undefined) => ToolChoice<ToolSet> | undefined;
 export declare function toChatCompletions(result: GenerateTextResult<ToolSet, Output.Output>, model: string): ChatCompletions;
 export declare function toChatCompletionsResponse(result: GenerateTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
-export declare function toChatCompletionsStream(result: StreamTextResult<ToolSet, Output.Output>, model: string): ReadableStream<ChatCompletionsChunk | OpenAIError>;
+export declare function toChatCompletionsStream<E extends boolean = false>(result: StreamTextResult<ToolSet, Output.Output>, model: string, wrapErrors?: E): ReadableStream<ChatCompletionsChunk | (E extends true ? OpenAIError : Error)>;
 export declare function toChatCompletionsStreamResponse(result: StreamTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
-export declare class ChatCompletionsStream extends TransformStream<TextStreamPart<ToolSet>, ChatCompletionsChunk | OpenAIError> {
-    constructor(model: string);
+export declare class ChatCompletionsStream<E extends boolean = false> extends TransformStream<TextStreamPart<ToolSet>, ChatCompletionsChunk | (E extends true ? OpenAIError : Error)> {
+    constructor(model: string, wrapErrors?: E);
 }
 export declare const toChatCompletionsAssistantMessage: (result: GenerateTextResult<ToolSet, Output.Output>) => ChatCompletionsAssistantMessage;
 export declare function toReasoningDetail(reasoning: ReasoningOutput, id: string, index: number): ChatCompletionsReasoningDetail;

package/dist/endpoints/chat-completions/converters.js

@@ -275,14 +275,14 @@ export function toChatCompletions(result, model) {
 export function toChatCompletionsResponse(result, model, responseInit) {
     return toResponse(toChatCompletions(result, model), responseInit);
 }
-export function toChatCompletionsStream(result, model) {
-    return result.fullStream.pipeThrough(new ChatCompletionsStream(model));
+export function toChatCompletionsStream(result, model, wrapErrors) {
+    return result.fullStream.pipeThrough(new ChatCompletionsStream(model, wrapErrors));
 }
 export function toChatCompletionsStreamResponse(result, model, responseInit) {
-    return toResponse(toChatCompletionsStream(result, model), responseInit);
+    return toResponse(toChatCompletionsStream(result, model, true), responseInit);
 }
 export class ChatCompletionsStream extends TransformStream {
-    constructor(model) {
+    constructor(model, wrapErrors) {
         const streamId = `chatcmpl-${crypto.randomUUID()}`;
         const creationTime = Math.floor(Date.now() / 1000);
         let toolCallIndexCounter = 0;
@@ -348,9 +348,17 @@ export class ChatCompletionsStream extends TransformStream {
                         break;
                     }
                     case "error": {
-                        const error = part.error;
-                        controller.enqueue(toOpenAIError(error));
-                        controller.terminate();
+                        let err;
+                        if (wrapErrors) {
+                            err = toOpenAIError(part.error);
+                        }
+                        else if (part.error instanceof Error) {
+                            err = part.error;
+                        }
+                        else {
+                            err = new Error(String(part.error));
+                        }
+                        controller.enqueue(err);
                     }
                 }
             },

package/dist/endpoints/chat-completions/handler.js

@@ -5,7 +5,7 @@ import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
-import { recordRequestDuration, recordTokenUsage } from "../../telemetry/gen-ai";
+import { recordRequestDuration, recordTimePerOutputToken, recordTokenUsage, } from "../../telemetry/gen-ai";
 import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
 import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
@@ -33,8 +33,8 @@ export const chatCompletions = (config) => {
         addSpanEvent("hebo.request.deserialized");
         const parsed = ChatCompletionsBodySchema.safeParse(ctx.body);
         if (!parsed.success) {
-            // FUTURE: add body shape to error message
-            throw new GatewayError(z.prettifyError(parsed.error), 400);
+            // FUTURE: consider adding body shape to metadata
+            throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
         }
         ctx.body = parsed.data;
         addSpanEvent("hebo.request.parsed");
@@ -84,13 +84,12 @@ export const chatCompletions = (config) => {
             const result = streamText({
                 model: languageModelWithMiddleware,
                 headers: prepareForwardHeaders(ctx.request),
-                // No abort signal here, otherwise we can't detect upstream from client cancellations
-                // abortSignal: ctx.request.signal,
+                abortSignal: ctx.request.signal,
                 timeout: {
                     totalMs: 5 * 60 * 1000,
                 },
                 onAbort: () => {
-                    throw new DOMException("Upstream failed", "AbortError");
+                    throw new DOMException("The operation was aborted.", "AbortError");
                 },
                 onError: () => { },
                 onFinish: (res) => {
@@ -100,7 +99,8 @@ export const chatCompletions = (config) => {
                     const genAiResponseAttrs = getChatResponseAttributes(streamResult, genAiSignalLevel);
                     setSpanAttributes(genAiResponseAttrs);
                     recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-                    recordRequestDuration(performance.now() - start, genAiGeneralAttrs, genAiSignalLevel);
+                    recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+                    recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
                 },
                 experimental_include: {
                     requestBody: false,
@@ -119,7 +119,6 @@ export const chatCompletions = (config) => {
         const result = await generateText({
             model: languageModelWithMiddleware,
             headers: prepareForwardHeaders(ctx.request),
-            // FUTURE: currently can't tell whether upstream or downstream abort
             abortSignal: ctx.request.signal,
             timeout: 5 * 60 * 1000,
             experimental_include: {
@@ -140,7 +139,8 @@ export const chatCompletions = (config) => {
             ctx.result = (await hooks.after(ctx)) ?? ctx.result;
             addSpanEvent("hebo.hooks.after.completed");
         }
-        recordRequestDuration(performance.now() - start, genAiGeneralAttrs, genAiSignalLevel);
+        recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+        recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
         return ctx.result;
     };
     return { handler: winterCgHandler(handler, config) };

package/dist/endpoints/chat-completions/otel.js

@@ -44,6 +44,10 @@ const toMessageParts = (message) => {
         }
         return parts;
     }
+    // FUTURE: remove once Langfuse supports gen_ai.system_instructions
+    if (message.role === "system") {
+        return [toTextPart(message.content)];
+    }
     return [];
 };
 export const getChatGeneralAttributes = (ctx, signalLevel) => {
@@ -81,11 +85,13 @@ export const getChatRequestAttributes = (inputs, signalLevel) => {
     }
     if (signalLevel === "full") {
         Object.assign(attrs, {
-            "gen_ai.system_instructions": inputs.messages
-                .filter((m) => m.role === "system")
-                .map((m) => JSON.stringify({ parts: [toTextPart(m.content)] })),
+            // FUTURE: move system instructions from messages to here
+            // blocker: https://github.com/langfuse/langfuse/issues/11607
+            // "gen_ai.system_instructions": inputs.messages
+            //   .filter((m) => m.role === "system")
+            //   .map((m) => JSON.stringify(toTextPart(m.content))),
             "gen_ai.input.messages": inputs.messages
-                .filter((m) => m.role !== "system")
+                //.filter((m) => m.role !== "system")
                 .map((m) => JSON.stringify({ role: m.role, parts: toMessageParts(m) })),
             "gen_ai.tool.definitions": JSON.stringify(inputs.tools),
         });

package/dist/endpoints/embeddings/handler.js

@@ -5,7 +5,7 @@ import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
-import { recordRequestDuration, recordTokenUsage } from "../../telemetry/gen-ai";
+import { recordRequestDuration, recordTimePerOutputToken, recordTokenUsage, } from "../../telemetry/gen-ai";
 import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
 import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
@@ -33,8 +33,8 @@ export const embeddings = (config) => {
         addSpanEvent("hebo.request.deserialized");
         const parsed = EmbeddingsBodySchema.safeParse(ctx.body);
         if (!parsed.success) {
-            // FUTURE: add body shape to error message
-            throw new GatewayError(z.prettifyError(parsed.error), 400);
+            // FUTURE: consider adding body shape to metadata
+            throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
         }
         ctx.body = parsed.data;
         addSpanEvent("hebo.request.parsed");
@@ -95,7 +95,8 @@ export const embeddings = (config) => {
             ctx.result = (await hooks.after(ctx)) ?? ctx.result;
             addSpanEvent("hebo.hooks.after.completed");
         }
-        recordRequestDuration(performance.now() - start, genAiGeneralAttrs, genAiSignalLevel);
+        recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+        recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
         return ctx.result;
     };
     return { handler: winterCgHandler(handler, config) };

package/dist/errors/gateway.d.ts

@@ -1,5 +1,5 @@
 export declare class GatewayError extends Error {
     readonly status: number;
     readonly code: string;
-    constructor(error: string | Error, status: number, code?: string, cause?: unknown);
+    constructor(error: unknown, status: number, code?: string, cause?: unknown);
 }

package/dist/errors/gateway.js

@@ -3,10 +3,10 @@ export class GatewayError extends Error {
     status;
     code;
     constructor(error, status, code, cause) {
-        const msg = typeof error === "string" ? error : error.message;
-        super(msg);
+        const isError = error instanceof Error;
+        super(isError ? error.message : String(error));
+        this.cause = cause ?? (isError ? error : undefined);
         this.status = status;
         this.code = code ?? STATUS_CODE(status);
-        this.cause = cause ?? (typeof error === "string" ? undefined : error);
     }
 }

package/dist/errors/openai.js

@@ -19,7 +19,8 @@ export class OpenAIError {
 }
 const mapType = (status) => (status < 500 ? "invalid_request_error" : "server_error");
 const maybeMaskMessage = (meta, requestId) => {
-    if (!(isProduction() && (meta.status >= 500 || meta.code.includes("UPSTREAM")))) {
+    // FUTURE: consider masking all upstream errors, also 4xx
+    if (!(isProduction() && meta.status >= 500)) {
         return meta.message;
     }
     // FUTURE: always attach requestId to errors (masked and unmasked)

package/dist/errors/utils.d.ts

@@ -8,12 +8,13 @@ export declare const STATUS_CODES: {
     readonly 409: "CONFLICT";
     readonly 422: "UNPROCESSABLE_ENTITY";
     readonly 429: "TOO_MANY_REQUESTS";
+    readonly 499: "CLIENT_CLOSED_REQUEST";
     readonly 500: "INTERNAL_SERVER_ERROR";
     readonly 502: "BAD_GATEWAY";
     readonly 503: "SERVICE_UNAVAILABLE";
     readonly 504: "GATEWAY_TIMEOUT";
 };
-export declare const STATUS_CODE: (status: number) => "BAD_REQUEST" | "UNAUTHORIZED" | "PAYMENT_REQUIRED" | "FORBIDDEN" | "NOT_FOUND" | "METHOD_NOT_ALLOWED" | "CONFLICT" | "UNPROCESSABLE_ENTITY" | "TOO_MANY_REQUESTS" | "INTERNAL_SERVER_ERROR" | "BAD_GATEWAY" | "SERVICE_UNAVAILABLE" | "GATEWAY_TIMEOUT";
+export declare const STATUS_CODE: (status: number) => "BAD_REQUEST" | "UNAUTHORIZED" | "PAYMENT_REQUIRED" | "FORBIDDEN" | "NOT_FOUND" | "METHOD_NOT_ALLOWED" | "CONFLICT" | "UNPROCESSABLE_ENTITY" | "TOO_MANY_REQUESTS" | "CLIENT_CLOSED_REQUEST" | "INTERNAL_SERVER_ERROR" | "BAD_GATEWAY" | "SERVICE_UNAVAILABLE" | "GATEWAY_TIMEOUT";
 export declare function getErrorMeta(error: unknown): {
     status: number;
     code: string;

package/dist/errors/utils.js

@@ -10,6 +10,7 @@ export const STATUS_CODES = {
     409: "CONFLICT",
     422: "UNPROCESSABLE_ENTITY",
     429: "TOO_MANY_REQUESTS",
+    499: "CLIENT_CLOSED_REQUEST",
     500: "INTERNAL_SERVER_ERROR",
     502: "BAD_GATEWAY",
     503: "SERVICE_UNAVAILABLE",

package/dist/lifecycle.js

@@ -1,9 +1,11 @@
 import { parseConfig } from "./config";
+import { GatewayError } from "./errors/gateway";
 import { toOpenAIErrorResponse } from "./errors/openai";
 import { logger } from "./logger";
 import { getBaggageAttributes } from "./telemetry/baggage";
 import { initFetch } from "./telemetry/fetch";
 import { getRequestAttributes, getResponseAttributes } from "./telemetry/http";
+import { recordV8jsMemory } from "./telemetry/memory";
 import { addSpanEvent, setSpanEventsEnabled, setSpanTracer, startSpan } from "./telemetry/span";
 import { wrapStream } from "./telemetry/stream";
 import { resolveRequestId } from "./utils/headers";
@@ -11,7 +13,7 @@ import { maybeApplyRequestPatch, prepareRequestHeaders } from "./utils/request";
 import { prepareResponseInit, toResponse } from "./utils/response";
 export const winterCgHandler = (run, config) => {
     const parsedConfig = parseConfig(config);
-    if (parsedConfig.telemetry.enabled) {
+    if (parsedConfig.telemetry?.enabled) {
         setSpanTracer(parsedConfig.telemetry?.tracer);
         setSpanEventsEnabled(parsedConfig.telemetry?.signals?.hebo);
         initFetch(parsedConfig.telemetry?.signals?.hebo);
@@ -39,17 +41,21 @@ export const winterCgHandler = (run, config) => {
                 // FUTURE add http.server.request.duration
                 span.setAttributes(getResponseAttributes(ctx.response, parsedConfig.telemetry?.signals?.http));
             }
-            const realStatus = status === 200 ? (ctx.response?.status ?? status) : status;
+            let realStatus = status;
+            if (ctx.request.signal.aborted)
+                realStatus = 499;
+            else if (status === 200 && ctx.response?.status)
+                realStatus = ctx.response.status;
             if (realStatus !== 200) {
-                // FUTURE: in-stream errors are redacted in prod
                 (realStatus >= 500 ? logger.error : logger.warn)({
                     requestId: resolveRequestId(ctx.request),
-                    err: reason,
+                    err: reason ?? ctx.request.signal.reason,
                 });
                 if (realStatus >= 500)
                     span.recordError(reason);
             }
             span.setAttributes({ "http.response.status_code_effective": realStatus });
+            recordV8jsMemory(parsedConfig.telemetry?.signals?.hebo);
             span.finish();
         };
         try {
@@ -66,7 +72,7 @@ export const winterCgHandler = (run, config) => {
             if (!ctx.response) {
                 ctx.result = (await span.runWithContext(() => run(ctx)));
                 if (ctx.result instanceof ReadableStream) {
-                    ctx.result = wrapStream(ctx.result, { onDone: finalize }, ctx.request.signal);
+                    ctx.result = wrapStream(ctx.result, { onDone: finalize });
                 }
                 ctx.response = toResponse(ctx.result, prepareResponseInit(ctx.request));
             }
@@ -83,7 +89,9 @@ export const winterCgHandler = (run, config) => {
             }
         }
         catch (error) {
-            ctx.response = toOpenAIErrorResponse(error, prepareResponseInit(ctx.request));
+            ctx.response = toOpenAIErrorResponse(ctx.request.signal.aborted
+                ? new GatewayError(error ?? ctx.request.signal.reason, 499)
+                : error, prepareResponseInit(ctx.request));
             finalize(ctx.response.status, error);
         }
         return ctx.response ?? new Response("Internal Server Error", { status: 500 });