@hebo-ai/gateway 0.4.0-beta.4 → 0.4.0

package/README.md

@@ -19,6 +19,7 @@ Learn more in our blog post: [Yet Another AI Gateway?](https://hebo.ai/blog/2601
 - 🗂️ Model catalog with extensible metadata capabilities.
 - 🪝 Hook system to customize routing, auth, rate limits, and shape responses.
 - 🧰 Low-level OpenAI-compatible schema, converters, and middleware helpers.
+- 👁️ OpenTelemetry support for GenAI semantic conventions (Langfuse-compatible).
 
 ## 📦 Installation
 
@@ -641,6 +642,32 @@ https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/
 
 For observability integration that is not otel compliant, you can disable built-in telemetry and manually instrument requests during `before` / `after` hooks.
 
+#### Langfuse
+
+Hebo telemetry spans are OpenTelemetry-compatible, so you can send them to Langfuse via `@langfuse/otel`.
+
+```ts
+import { gateway } from "@hebo-ai/gateway";
+import { LangfuseSpanProcessor } from "@langfuse/otel";
+import { context } from "@opentelemetry/api";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base";
+
+context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+
+const gw = gateway({
+  // ...
+  telemetry: {
+    enabled: true,
+    tracer = new BasicTracerProvider({
+      spanProcessors: [new LangfuseSpanProcessor()],
+    }).getTracer("hebo");,
+  },
+});
+```
+
+Langfuse credentials are read from environment variables by the Langfuse OTel SDK (`LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_BASE_URL`).
+
 ### Passing Framework State to Hooks
 
 You can pass per-request info from your framework into the gateway via the second `state` argument on the handler, then read it in hooks through `ctx.state`.

package/dist/endpoints/chat-completions/converters.d.ts

@@ -25,10 +25,10 @@ export declare const convertToToolSet: (tools: ChatCompletionsTool[] | undefined
 export declare const convertToToolChoice: (toolChoice: ChatCompletionsToolChoice | undefined) => ToolChoice<ToolSet> | undefined;
 export declare function toChatCompletions(result: GenerateTextResult<ToolSet, Output.Output>, model: string): ChatCompletions;
 export declare function toChatCompletionsResponse(result: GenerateTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
-export declare function toChatCompletionsStream(result: StreamTextResult<ToolSet, Output.Output>, model: string): ReadableStream<ChatCompletionsChunk | OpenAIError>;
+export declare function toChatCompletionsStream<E extends boolean = false>(result: StreamTextResult<ToolSet, Output.Output>, model: string, wrapErrors?: E): ReadableStream<ChatCompletionsChunk | (E extends true ? OpenAIError : Error)>;
 export declare function toChatCompletionsStreamResponse(result: StreamTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
-export declare class ChatCompletionsStream extends TransformStream<TextStreamPart<ToolSet>, ChatCompletionsChunk | OpenAIError> {
-    constructor(model: string);
+export declare class ChatCompletionsStream<E extends boolean = false> extends TransformStream<TextStreamPart<ToolSet>, ChatCompletionsChunk | (E extends true ? OpenAIError : Error)> {
+    constructor(model: string, wrapErrors?: E);
 }
 export declare const toChatCompletionsAssistantMessage: (result: GenerateTextResult<ToolSet, Output.Output>) => ChatCompletionsAssistantMessage;
 export declare function toReasoningDetail(reasoning: ReasoningOutput, id: string, index: number): ChatCompletionsReasoningDetail;

package/dist/endpoints/chat-completions/converters.js

@@ -275,14 +275,14 @@ export function toChatCompletions(result, model) {
 export function toChatCompletionsResponse(result, model, responseInit) {
     return toResponse(toChatCompletions(result, model), responseInit);
 }
-export function toChatCompletionsStream(result, model) {
-    return result.fullStream.pipeThrough(new ChatCompletionsStream(model));
+export function toChatCompletionsStream(result, model, wrapErrors) {
+    return result.fullStream.pipeThrough(new ChatCompletionsStream(model, wrapErrors));
 }
 export function toChatCompletionsStreamResponse(result, model, responseInit) {
-    return toResponse(toChatCompletionsStream(result, model), responseInit);
+    return toResponse(toChatCompletionsStream(result, model, true), responseInit);
 }
 export class ChatCompletionsStream extends TransformStream {
-    constructor(model) {
+    constructor(model, wrapErrors) {
         const streamId = `chatcmpl-${crypto.randomUUID()}`;
         const creationTime = Math.floor(Date.now() / 1000);
         let toolCallIndexCounter = 0;
@@ -348,9 +348,17 @@ export class ChatCompletionsStream extends TransformStream {
                         break;
                     }
                     case "error": {
-                        const error = part.error;
-                        controller.enqueue(toOpenAIError(error));
-                        controller.terminate();
+                        let err;
+                        if (wrapErrors) {
+                            err = toOpenAIError(part.error);
+                        }
+                        else if (part.error instanceof Error) {
+                            err = part.error;
+                        }
+                        else {
+                            err = new Error(String(part.error));
+                        }
+                        controller.enqueue(err);
                     }
                 }
             },

package/dist/endpoints/chat-completions/handler.js

@@ -5,7 +5,7 @@ import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
-import { recordRequestDuration, recordTokenUsage } from "../../telemetry/gen-ai";
+import { recordRequestDuration, recordTimePerOutputToken, recordTokenUsage, } from "../../telemetry/gen-ai";
 import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
 import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
@@ -33,8 +33,8 @@ export const chatCompletions = (config) => {
         addSpanEvent("hebo.request.deserialized");
         const parsed = ChatCompletionsBodySchema.safeParse(ctx.body);
         if (!parsed.success) {
-            // FUTURE: add body shape to error message
-            throw new GatewayError(z.prettifyError(parsed.error), 400);
+            // FUTURE: consider adding body shape to metadata
+            throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
         }
         ctx.body = parsed.data;
         addSpanEvent("hebo.request.parsed");
@@ -84,13 +84,12 @@ export const chatCompletions = (config) => {
             const result = streamText({
                 model: languageModelWithMiddleware,
                 headers: prepareForwardHeaders(ctx.request),
-                // No abort signal here, otherwise we can't detect upstream from client cancellations
-                // abortSignal: ctx.request.signal,
+                abortSignal: ctx.request.signal,
                 timeout: {
                     totalMs: 5 * 60 * 1000,
                 },
                 onAbort: () => {
-                    throw new DOMException("Upstream failed", "AbortError");
+                    throw new DOMException("The operation was aborted.", "AbortError");
                 },
                 onError: () => { },
                 onFinish: (res) => {
@@ -100,7 +99,8 @@ export const chatCompletions = (config) => {
                     const genAiResponseAttrs = getChatResponseAttributes(streamResult, genAiSignalLevel);
                     setSpanAttributes(genAiResponseAttrs);
                     recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-                    recordRequestDuration(performance.now() - start, genAiGeneralAttrs, genAiSignalLevel);
+                    recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+                    recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
                 },
                 experimental_include: {
                     requestBody: false,
@@ -119,7 +119,6 @@ export const chatCompletions = (config) => {
         const result = await generateText({
             model: languageModelWithMiddleware,
             headers: prepareForwardHeaders(ctx.request),
-            // FUTURE: currently can't tell whether upstream or downstream abort
             abortSignal: ctx.request.signal,
             timeout: 5 * 60 * 1000,
             experimental_include: {
@@ -140,7 +139,8 @@ export const chatCompletions = (config) => {
             ctx.result = (await hooks.after(ctx)) ?? ctx.result;
             addSpanEvent("hebo.hooks.after.completed");
         }
-        recordRequestDuration(performance.now() - start, genAiGeneralAttrs, genAiSignalLevel);
+        recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+        recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
         return ctx.result;
     };
     return { handler: winterCgHandler(handler, config) };

package/dist/endpoints/chat-completions/otel.js

@@ -44,6 +44,10 @@ const toMessageParts = (message) => {
         }
         return parts;
     }
+    // FUTURE: remove once Langfuse supports gen_ai.system_instructions
+    if (message.role === "system") {
+        return [toTextPart(message.content)];
+    }
     return [];
 };
 export const getChatGeneralAttributes = (ctx, signalLevel) => {
@@ -81,11 +85,13 @@ export const getChatRequestAttributes = (inputs, signalLevel) => {
     }
     if (signalLevel === "full") {
         Object.assign(attrs, {
-            "gen_ai.system_instructions": inputs.messages
-                .filter((m) => m.role === "system")
-                .map((m) => JSON.stringify({ parts: [toTextPart(m.content)] })),
+            // FUTURE: move system instructions from messages to here
+            // blocker: https://github.com/langfuse/langfuse/issues/11607
+            // "gen_ai.system_instructions": inputs.messages
+            //   .filter((m) => m.role === "system")
+            //   .map((m) => JSON.stringify(toTextPart(m.content))),
             "gen_ai.input.messages": inputs.messages
-                .filter((m) => m.role !== "system")
+                //.filter((m) => m.role !== "system")
                 .map((m) => JSON.stringify({ role: m.role, parts: toMessageParts(m) })),
             "gen_ai.tool.definitions": JSON.stringify(inputs.tools),
         });

package/dist/endpoints/embeddings/handler.js

@@ -5,7 +5,7 @@ import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
-import { recordRequestDuration, recordTokenUsage } from "../../telemetry/gen-ai";
+import { recordRequestDuration, recordTimePerOutputToken, recordTokenUsage, } from "../../telemetry/gen-ai";
 import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
 import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
@@ -33,8 +33,8 @@ export const embeddings = (config) => {
         addSpanEvent("hebo.request.deserialized");
         const parsed = EmbeddingsBodySchema.safeParse(ctx.body);
         if (!parsed.success) {
-            // FUTURE: add body shape to error message
-            throw new GatewayError(z.prettifyError(parsed.error), 400);
+            // FUTURE: consider adding body shape to metadata
+            throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
         }
         ctx.body = parsed.data;
         addSpanEvent("hebo.request.parsed");
@@ -95,7 +95,8 @@ export const embeddings = (config) => {
             ctx.result = (await hooks.after(ctx)) ?? ctx.result;
             addSpanEvent("hebo.hooks.after.completed");
         }
-        recordRequestDuration(performance.now() - start, genAiGeneralAttrs, genAiSignalLevel);
+        recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+        recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
         return ctx.result;
     };
     return { handler: winterCgHandler(handler, config) };

package/dist/errors/gateway.d.ts

@@ -1,5 +1,5 @@
 export declare class GatewayError extends Error {
     readonly status: number;
     readonly code: string;
-    constructor(error: string | Error, status: number, code?: string, cause?: unknown);
+    constructor(error: unknown, status: number, code?: string, cause?: unknown);
 }

package/dist/errors/gateway.js

@@ -3,10 +3,10 @@ export class GatewayError extends Error {
     status;
     code;
     constructor(error, status, code, cause) {
-        const msg = typeof error === "string" ? error : error.message;
-        super(msg);
+        const isError = error instanceof Error;
+        super(isError ? error.message : String(error));
+        this.cause = cause ?? (isError ? error : undefined);
         this.status = status;
         this.code = code ?? STATUS_CODE(status);
-        this.cause = cause ?? (typeof error === "string" ? undefined : error);
     }
 }

package/dist/errors/openai.js

@@ -19,7 +19,8 @@ export class OpenAIError {
 }
 const mapType = (status) => (status < 500 ? "invalid_request_error" : "server_error");
 const maybeMaskMessage = (meta, requestId) => {
-    if (!(isProduction() && (meta.status >= 500 || meta.code.includes("UPSTREAM")))) {
+    // FUTURE: consider masking all upstream errors, also 4xx
+    if (!(isProduction() && meta.status >= 500)) {
         return meta.message;
     }
     // FUTURE: always attach requestId to errors (masked and unmasked)

package/dist/errors/utils.d.ts

@@ -8,12 +8,13 @@ export declare const STATUS_CODES: {
     readonly 409: "CONFLICT";
     readonly 422: "UNPROCESSABLE_ENTITY";
     readonly 429: "TOO_MANY_REQUESTS";
+    readonly 499: "CLIENT_CLOSED_REQUEST";
     readonly 500: "INTERNAL_SERVER_ERROR";
     readonly 502: "BAD_GATEWAY";
     readonly 503: "SERVICE_UNAVAILABLE";
     readonly 504: "GATEWAY_TIMEOUT";
 };
-export declare const STATUS_CODE: (status: number) => "BAD_REQUEST" | "UNAUTHORIZED" | "PAYMENT_REQUIRED" | "FORBIDDEN" | "NOT_FOUND" | "METHOD_NOT_ALLOWED" | "CONFLICT" | "UNPROCESSABLE_ENTITY" | "TOO_MANY_REQUESTS" | "INTERNAL_SERVER_ERROR" | "BAD_GATEWAY" | "SERVICE_UNAVAILABLE" | "GATEWAY_TIMEOUT";
+export declare const STATUS_CODE: (status: number) => "BAD_REQUEST" | "UNAUTHORIZED" | "PAYMENT_REQUIRED" | "FORBIDDEN" | "NOT_FOUND" | "METHOD_NOT_ALLOWED" | "CONFLICT" | "UNPROCESSABLE_ENTITY" | "TOO_MANY_REQUESTS" | "CLIENT_CLOSED_REQUEST" | "INTERNAL_SERVER_ERROR" | "BAD_GATEWAY" | "SERVICE_UNAVAILABLE" | "GATEWAY_TIMEOUT";
 export declare function getErrorMeta(error: unknown): {
     status: number;
     code: string;

package/dist/errors/utils.js

@@ -10,6 +10,7 @@ export const STATUS_CODES = {
     409: "CONFLICT",
     422: "UNPROCESSABLE_ENTITY",
     429: "TOO_MANY_REQUESTS",
+    499: "CLIENT_CLOSED_REQUEST",
     500: "INTERNAL_SERVER_ERROR",
     502: "BAD_GATEWAY",
     503: "SERVICE_UNAVAILABLE",

package/dist/lifecycle.js

@@ -1,9 +1,11 @@
 import { parseConfig } from "./config";
+import { GatewayError } from "./errors/gateway";
 import { toOpenAIErrorResponse } from "./errors/openai";
 import { logger } from "./logger";
 import { getBaggageAttributes } from "./telemetry/baggage";
 import { initFetch } from "./telemetry/fetch";
 import { getRequestAttributes, getResponseAttributes } from "./telemetry/http";
+import { recordV8jsMemory } from "./telemetry/memory";
 import { addSpanEvent, setSpanEventsEnabled, setSpanTracer, startSpan } from "./telemetry/span";
 import { wrapStream } from "./telemetry/stream";
 import { resolveRequestId } from "./utils/headers";
@@ -11,7 +13,7 @@ import { maybeApplyRequestPatch, prepareRequestHeaders } from "./utils/request";
 import { prepareResponseInit, toResponse } from "./utils/response";
 export const winterCgHandler = (run, config) => {
     const parsedConfig = parseConfig(config);
-    if (parsedConfig.telemetry.enabled) {
+    if (parsedConfig.telemetry?.enabled) {
         setSpanTracer(parsedConfig.telemetry?.tracer);
         setSpanEventsEnabled(parsedConfig.telemetry?.signals?.hebo);
         initFetch(parsedConfig.telemetry?.signals?.hebo);
@@ -39,17 +41,21 @@ export const winterCgHandler = (run, config) => {
                 // FUTURE add http.server.request.duration
                 span.setAttributes(getResponseAttributes(ctx.response, parsedConfig.telemetry?.signals?.http));
             }
-            const realStatus = status === 200 ? (ctx.response?.status ?? status) : status;
+            let realStatus = status;
+            if (ctx.request.signal.aborted)
+                realStatus = 499;
+            else if (status === 200 && ctx.response?.status)
+                realStatus = ctx.response.status;
             if (realStatus !== 200) {
-                // FUTURE: in-stream errors are redacted in prod
                 (realStatus >= 500 ? logger.error : logger.warn)({
                     requestId: resolveRequestId(ctx.request),
-                    err: reason,
+                    err: reason ?? ctx.request.signal.reason,
                 });
                 if (realStatus >= 500)
                     span.recordError(reason);
             }
             span.setAttributes({ "http.response.status_code_effective": realStatus });
+            recordV8jsMemory(parsedConfig.telemetry?.signals?.hebo);
             span.finish();
         };
         try {
@@ -66,7 +72,7 @@ export const winterCgHandler = (run, config) => {
             if (!ctx.response) {
                 ctx.result = (await span.runWithContext(() => run(ctx)));
                 if (ctx.result instanceof ReadableStream) {
-                    ctx.result = wrapStream(ctx.result, { onDone: finalize }, ctx.request.signal);
+                    ctx.result = wrapStream(ctx.result, { onDone: finalize });
                 }
                 ctx.response = toResponse(ctx.result, prepareResponseInit(ctx.request));
             }
@@ -83,7 +89,9 @@ export const winterCgHandler = (run, config) => {
             }
         }
         catch (error) {
-            ctx.response = toOpenAIErrorResponse(error, prepareResponseInit(ctx.request));
+            ctx.response = toOpenAIErrorResponse(ctx.request.signal.aborted
+                ? new GatewayError(error ?? ctx.request.signal.reason, 499)
+                : error, prepareResponseInit(ctx.request));
             finalize(ctx.response.status, error);
         }
         return ctx.response ?? new Response("Internal Server Error", { status: 500 });

package/dist/telemetry/gen-ai.d.ts

@@ -1,4 +1,5 @@
 import { type Attributes } from "@opentelemetry/api";
 import type { TelemetrySignalLevel } from "../types";
-export declare const recordRequestDuration: (duration: number, attrs: Attributes, signalLevel?: TelemetrySignalLevel) => void;
+export declare const recordRequestDuration: (start: number, attrs: Attributes, signalLevel?: TelemetrySignalLevel) => void;
+export declare const recordTimePerOutputToken: (start: number, tokenAttrs: Attributes, metricAttrs: Attributes, signalLevel?: TelemetrySignalLevel) => void;
 export declare const recordTokenUsage: (tokenAttrs: Attributes, metricAttrs: Attributes, signalLevel?: TelemetrySignalLevel) => void;

package/dist/telemetry/gen-ai.js

@@ -1,5 +1,5 @@
 import { metrics } from "@opentelemetry/api";
-const meter = metrics.getMeter("@hebo-ai/gateway");
+const meter = metrics.getMeter("@hebo/gateway");
 const requestDurationHistogram = meter.createHistogram("gen_ai.server.request.duration", {
     description: "End-to-end gateway request duration",
     unit: "s",
@@ -9,6 +9,15 @@ const requestDurationHistogram = meter.createHistogram("gen_ai.server.request.du
         ],
     },
 });
+const timePerOutputTokenHistogram = meter.createHistogram("gen_ai.server.time_per_output_token", {
+    description: "End-to-end gateway request duration per output token",
+    unit: "s",
+    advice: {
+        explicitBucketBoundaries: [
+            0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 2.5,
+        ],
+    },
+});
 const tokenUsageHistogram = meter.createHistogram("gen_ai.client.token.usage", {
     description: "Token usage reported by upstream model responses",
     unit: "{token}",
@@ -20,10 +29,19 @@ const tokenUsageHistogram = meter.createHistogram("gen_ai.client.token.usage", {
     },
 });
 // FUTURE: record unsuccessful calls
-export const recordRequestDuration = (duration, attrs, signalLevel) => {
+export const recordRequestDuration = (start, attrs, signalLevel) => {
     if (!signalLevel || signalLevel === "off")
         return;
-    requestDurationHistogram.record(duration / 1000, attrs);
+    requestDurationHistogram.record((performance.now() - start) / 1000, attrs);
+};
+// FUTURE: record unsuccessful calls
+export const recordTimePerOutputToken = (start, tokenAttrs, metricAttrs, signalLevel) => {
+    if (!signalLevel || (signalLevel !== "recommended" && signalLevel !== "full"))
+        return;
+    const outputTokens = tokenAttrs["gen_ai.usage.output_tokens"];
+    if (typeof outputTokens !== "number" || outputTokens <= 0)
+        return;
+    timePerOutputTokenHistogram.record((performance.now() - start) / 1000 / outputTokens, metricAttrs);
 };
 // FUTURE: record unsuccessful calls
 export const recordTokenUsage = (tokenAttrs, metricAttrs, signalLevel) => {

package/dist/telemetry/memory.d.ts

@@ -0,0 +1,2 @@
+import type { TelemetrySignalLevel } from "../types";
+export declare const recordV8jsMemory: (level?: TelemetrySignalLevel) => void;

package/dist/telemetry/memory.js

@@ -0,0 +1,27 @@
+import { metrics } from "@opentelemetry/api";
+const meter = metrics.getMeter("@hebo/gateway");
+const defaultHeapSpaceAttrs = { "v8js.heap.space.name": "total" };
+const heapUsedCounter = meter.createUpDownCounter("v8js.memory.heap.used", {
+    description: "Used bytes in the V8 heap",
+    unit: "By",
+});
+const heapSpacePhysicalSizeCounter = meter.createUpDownCounter("v8js.memory.heap.space.physical_size", {
+    description: "Physical bytes allocated for the V8 heap space",
+    unit: "By",
+});
+const isEnabled = (level) => level === "recommended" || level === "full";
+export const recordV8jsMemory = (level) => {
+    if (!isEnabled(level))
+        return;
+    let usage;
+    try {
+        usage = globalThis.process?.memoryUsage?.();
+    }
+    catch {
+        return;
+    }
+    if (!usage)
+        return;
+    heapUsedCounter.add(usage.heapUsed, defaultHeapSpaceAttrs);
+    heapSpacePhysicalSizeCounter.add(usage.rss, defaultHeapSpaceAttrs);
+};

package/dist/telemetry/span.js

@@ -1,5 +1,5 @@
 import { INVALID_SPAN_CONTEXT, SpanKind, SpanStatusCode, context, trace } from "@opentelemetry/api";
-const DEFAULT_TRACER_NAME = "@hebo-ai/gateway";
+const DEFAULT_TRACER_NAME = "@hebo/gateway";
 let spanTracer;
 let spanEventsEnabled = false;
 const NOOP_SPAN = {

package/dist/telemetry/stream.d.ts

@@ -1,3 +1,3 @@
 export declare const wrapStream: (src: ReadableStream, hooks: {
     onDone?: (status: number, reason: unknown) => void;
-}, signal?: AbortSignal) => ReadableStream;
+}) => ReadableStream;

package/dist/telemetry/stream.js

@@ -1,43 +1,37 @@
-const isErrorChunk = (v) => !!v?.error;
-export const wrapStream = (src, hooks, signal) => {
-    let finishOnce = false;
-    const finish = (status, reason) => {
-        if (finishOnce)
-            return;
-        finishOnce = true;
-        hooks.onDone?.(status, reason ?? signal?.reason);
+import { toOpenAIError } from "#/errors/openai";
+const isErrorChunk = (v) => v instanceof Error || !!v?.error;
+export const wrapStream = (src, hooks) => {
+    let finished = false;
+    const done = (reader, controller, status, reason) => {
+        if (!finished) {
+            finished = true;
+            hooks.onDone?.(status, reason);
+        }
+        reader.cancel(reason).catch(() => { });
+        controller.close();
     };
     return new ReadableStream({
         async start(controller) {
             const reader = src.getReader();
-            const close = (status, reason) => {
-                finish(status, reason);
-                reader.cancel(reason).catch(() => { });
-                controller.close();
-            };
             try {
                 for (;;) {
-                    if (signal?.aborted) {
-                        close(499, signal.reason);
-                        return;
-                    }
                     // eslint-disable-next-line no-await-in-loop
-                    const { value, done } = await reader.read();
-                    if (done)
+                    const { value, done: eof } = await reader.read();
+                    if (eof)
                         break;
-                    controller.enqueue(value);
-                    if (isErrorChunk(value)) {
-                        const status = value.error.type === "invalid_request_error" ? 422 : 502;
-                        close(status, value.error.message);
+                    const out = isErrorChunk(value) ? toOpenAIError(value) : value;
+                    controller.enqueue(out);
+                    if (out !== value) {
+                        const status = out.error?.type === "invalid_request_error" ? 422 : 502;
+                        done(reader, controller, status, value);
                         return;
                     }
                 }
-                finish(200);
-                controller.close();
+                done(reader, controller, 200);
             }
             catch (err) {
-                const status = signal?.aborted ? 499 : err?.name === "AbortError" ? 503 : 502;
-                close(status, err);
+                controller.enqueue(toOpenAIError(err));
+                done(reader, controller, 502, err);
             }
             finally {
                 try {
@@ -47,7 +41,10 @@ export const wrapStream = (src, hooks, signal) => {
             }
         },
         cancel(reason) {
-            finish(499, reason);
+            if (!finished) {
+                finished = true;
+                hooks.onDone?.(499, reason);
+            }
             src.cancel(reason).catch(() => { });
         },
     });

package/dist/types.d.ts

@@ -3,7 +3,6 @@ import type { Tracer } from "@opentelemetry/api";
 import type { ChatCompletions, ChatCompletionsBody, ChatCompletionsChunk } from "./endpoints/chat-completions/schema";
 import type { Embeddings, EmbeddingsBody } from "./endpoints/embeddings/schema";
 import type { Model, ModelList } from "./endpoints/models";
-import type { OpenAIError } from "./errors/openai";
 import type { Logger, LoggerConfig } from "./logger";
 import type { ModelCatalog, ModelId } from "./models/types";
 import type { ProviderId, ProviderRegistry } from "./providers/types";
@@ -67,7 +66,7 @@ export type GatewayContext = {
     /**
      * Result returned by the handler (pre-response).
      */
-    result?: ChatCompletions | ReadableStream<ChatCompletionsChunk | OpenAIError> | Embeddings | Model | ModelList;
+    result?: ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings | Model | ModelList;
     /**
      * Response object returned by the handler.
      */
@@ -115,7 +114,7 @@ export type GatewayHooks = {
      * Runs after the endpoint handler.
      * @returns Result to replace, or undefined to keep original.
      */
-    after?: (ctx: AfterHookContext) => void | ChatCompletions | ReadableStream<ChatCompletionsChunk | OpenAIError> | Embeddings | Promise<void | ChatCompletions | ReadableStream<ChatCompletionsChunk | OpenAIError> | Embeddings>;
+    after?: (ctx: AfterHookContext) => void | ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings | Promise<void | ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings>;
     /**
      * Runs after the lifecycle has produced the final Response.
      * @returns Replacement Response, or undefined to keep original.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hebo-ai/gateway",
-  "version": "0.4.0-beta.4",
+  "version": "0.4.0",
   "description": "AI gateway as a framework. For full control over models, routing & lifecycle. OpenAI-compatible /chat/completions, /embeddings & /models.",
   "keywords": [
     "ai",
@@ -168,6 +168,7 @@
     "@ai-sdk/groq": "^3.0.19",
     "@ai-sdk/openai": "^3.0.23",
     "@aws-sdk/credential-providers": "^3.981.0",
+    "@langfuse/otel": "^4.6.1",
     "@mjackson/node-fetch-server": "^0.7.0",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/context-async-hooks": "^2.5.1",

package/src/endpoints/chat-completions/converters.ts

@@ -405,11 +405,12 @@ export function toChatCompletionsResponse(
   return toResponse(toChatCompletions(result, model), responseInit);
 }
 
-export function toChatCompletionsStream(
+export function toChatCompletionsStream<E extends boolean = false>(
   result: StreamTextResult<ToolSet, Output.Output>,
   model: string,
-): ReadableStream<ChatCompletionsChunk | OpenAIError> {
-  return result.fullStream.pipeThrough(new ChatCompletionsStream(model));
+  wrapErrors?: E,
+): ReadableStream<ChatCompletionsChunk | (E extends true ? OpenAIError : Error)> {
+  return result.fullStream.pipeThrough(new ChatCompletionsStream(model, wrapErrors));
 }
 
 export function toChatCompletionsStreamResponse(
@@ -417,14 +418,14 @@ export function toChatCompletionsStreamResponse(
   model: string,
   responseInit?: ResponseInit,
 ): Response {
-  return toResponse(toChatCompletionsStream(result, model), responseInit);
+  return toResponse(toChatCompletionsStream(result, model, true), responseInit);
 }
 
-export class ChatCompletionsStream extends TransformStream<
+export class ChatCompletionsStream<E extends boolean = false> extends TransformStream<
   TextStreamPart<ToolSet>,
-  ChatCompletionsChunk | OpenAIError
+  ChatCompletionsChunk | (E extends true ? OpenAIError : Error)
 > {
-  constructor(model: string) {
+  constructor(model: string, wrapErrors?: E) {
     const streamId = `chatcmpl-${crypto.randomUUID()}`;
     const creationTime = Math.floor(Date.now() / 1000);
     let toolCallIndexCounter = 0;
@@ -535,9 +536,15 @@ export class ChatCompletionsStream extends TransformStream<
           }
 
           case "error": {
-            const error = part.error;
-            controller.enqueue(toOpenAIError(error));
-            controller.terminate();
+            let err: Error | OpenAIError;
+            if (wrapErrors) {
+              err = toOpenAIError(part.error);
+            } else if (part.error instanceof Error) {
+              err = part.error;
+            } else {
+              err = new Error(String(part.error));
+            }
+            controller.enqueue(err as E extends true ? OpenAIError : Error);
           }
         }
       },

package/src/endpoints/chat-completions/handler.ts

@@ -23,7 +23,11 @@ import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
-import { recordRequestDuration, recordTokenUsage } from "../../telemetry/gen-ai";
+import {
+  recordRequestDuration,
+  recordTimePerOutputToken,
+  recordTokenUsage,
+} from "../../telemetry/gen-ai";
 import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
 import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
@@ -60,8 +64,8 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
 
     const parsed = ChatCompletionsBodySchema.safeParse(ctx.body);
     if (!parsed.success) {
-      // FUTURE: add body shape to error message
-      throw new GatewayError(z.prettifyError(parsed.error), 400);
+      // FUTURE: consider adding body shape to metadata
+      throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
     }
     ctx.body = parsed.data;
     addSpanEvent("hebo.request.parsed");
@@ -123,13 +127,12 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
       const result = streamText({
         model: languageModelWithMiddleware,
         headers: prepareForwardHeaders(ctx.request),
-        // No abort signal here, otherwise we can't detect upstream from client cancellations
-        // abortSignal: ctx.request.signal,
+        abortSignal: ctx.request.signal,
         timeout: {
           totalMs: 5 * 60 * 1000,
         },
         onAbort: () => {
-          throw new DOMException("Upstream failed", "AbortError");
+          throw new DOMException("The operation was aborted.", "AbortError");
         },
         onError: () => {},
         onFinish: (res) => {
@@ -143,7 +146,8 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
           const genAiResponseAttrs = getChatResponseAttributes(streamResult, genAiSignalLevel);
           setSpanAttributes(genAiResponseAttrs);
           recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-          recordRequestDuration(performance.now() - start, genAiGeneralAttrs, genAiSignalLevel);
+          recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+          recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
         },
         experimental_include: {
           requestBody: false,
@@ -166,7 +170,6 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
     const result = await generateText({
       model: languageModelWithMiddleware,
       headers: prepareForwardHeaders(ctx.request),
-      // FUTURE: currently can't tell whether upstream or downstream abort
       abortSignal: ctx.request.signal,
       timeout: 5 * 60 * 1000,
       experimental_include: {
@@ -191,7 +194,8 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
       addSpanEvent("hebo.hooks.after.completed");
     }
 
-    recordRequestDuration(performance.now() - start, genAiGeneralAttrs, genAiSignalLevel);
+    recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+    recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
     return ctx.result;
   };
 

package/src/endpoints/chat-completions/otel.ts

@@ -54,6 +54,11 @@ const toMessageParts = (message: ChatCompletionsMessage): Record<string, unknown
     return parts;
   }
 
+  // FUTURE: remove once Langfuse supports gen_ai.system_instructions
+  if (message.role === "system") {
+    return [toTextPart(message.content)];
+  }
+
   return [];
 };
 
@@ -103,11 +108,13 @@ export const getChatRequestAttributes = (
 
   if (signalLevel === "full") {
     Object.assign(attrs, {
-      "gen_ai.system_instructions": inputs.messages
-        .filter((m) => m.role === "system")
-        .map((m) => JSON.stringify({ parts: [toTextPart(m.content)] })),
+      // FUTURE: move system instructions from messages to here
+      // blocker: https://github.com/langfuse/langfuse/issues/11607
+      // "gen_ai.system_instructions": inputs.messages
+      //   .filter((m) => m.role === "system")
+      //   .map((m) => JSON.stringify(toTextPart(m.content))),
       "gen_ai.input.messages": inputs.messages
-        .filter((m) => m.role !== "system")
+        //.filter((m) => m.role !== "system")
         .map((m) => JSON.stringify({ role: m.role, parts: toMessageParts(m) })),
       "gen_ai.tool.definitions": JSON.stringify(inputs.tools),
     });

package/src/endpoints/embeddings/handler.ts

@@ -16,7 +16,11 @@ import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
-import { recordRequestDuration, recordTokenUsage } from "../../telemetry/gen-ai";
+import {
+  recordRequestDuration,
+  recordTimePerOutputToken,
+  recordTokenUsage,
+} from "../../telemetry/gen-ai";
 import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
 import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
@@ -53,8 +57,8 @@ export const embeddings = (config: GatewayConfig): Endpoint => {
 
     const parsed = EmbeddingsBodySchema.safeParse(ctx.body);
     if (!parsed.success) {
-      // FUTURE: add body shape to error message
-      throw new GatewayError(z.prettifyError(parsed.error), 400);
+      // FUTURE: consider adding body shape to metadata
+      throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
     }
     ctx.body = parsed.data;
     addSpanEvent("hebo.request.parsed");
@@ -127,7 +131,8 @@ export const embeddings = (config: GatewayConfig): Endpoint => {
       addSpanEvent("hebo.hooks.after.completed");
     }
 
-    recordRequestDuration(performance.now() - start, genAiGeneralAttrs, genAiSignalLevel);
+    recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+    recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
     return ctx.result;
   };
 

package/src/errors/gateway.ts

@@ -4,11 +4,12 @@ export class GatewayError extends Error {
   readonly status: number;
   readonly code: string;
 
-  constructor(error: string | Error, status: number, code?: string, cause?: unknown) {
-    const msg = typeof error === "string" ? error : error.message;
-    super(msg);
+  constructor(error: unknown, status: number, code?: string, cause?: unknown) {
+    const isError = error instanceof Error;
+    super(isError ? error.message : String(error));
+    this.cause = cause ?? (isError ? error : undefined);
+
     this.status = status;
     this.code = code ?? STATUS_CODE(status);
-    this.cause = cause ?? (typeof error === "string" ? undefined : error);
   }
 }

package/src/errors/openai.ts

@@ -25,7 +25,8 @@ export class OpenAIError {
 const mapType = (status: number) => (status < 500 ? "invalid_request_error" : "server_error");
 
 const maybeMaskMessage = (meta: ReturnType<typeof getErrorMeta>, requestId?: string) => {
-  if (!(isProduction() && (meta.status >= 500 || meta.code.includes("UPSTREAM")))) {
+  // FUTURE: consider masking all upstream errors, also 4xx
+  if (!(isProduction() && meta.status >= 500)) {
     return meta.message;
   }
   // FUTURE: always attach requestId to errors (masked and unmasked)

package/src/errors/utils.ts

@@ -11,6 +11,7 @@ export const STATUS_CODES = {
   409: "CONFLICT",
   422: "UNPROCESSABLE_ENTITY",
   429: "TOO_MANY_REQUESTS",
+  499: "CLIENT_CLOSED_REQUEST",
   500: "INTERNAL_SERVER_ERROR",
   502: "BAD_GATEWAY",
   503: "SERVICE_UNAVAILABLE",

package/src/lifecycle.ts

@@ -6,11 +6,13 @@ import type {
 } from "./types";
 
 import { parseConfig } from "./config";
+import { GatewayError } from "./errors/gateway";
 import { toOpenAIErrorResponse } from "./errors/openai";
 import { logger } from "./logger";
 import { getBaggageAttributes } from "./telemetry/baggage";
 import { initFetch } from "./telemetry/fetch";
 import { getRequestAttributes, getResponseAttributes } from "./telemetry/http";
+import { recordV8jsMemory } from "./telemetry/memory";
 import { addSpanEvent, setSpanEventsEnabled, setSpanTracer, startSpan } from "./telemetry/span";
 import { wrapStream } from "./telemetry/stream";
 import { resolveRequestId } from "./utils/headers";
@@ -23,7 +25,7 @@ export const winterCgHandler = (
 ) => {
   const parsedConfig = parseConfig(config);
 
-  if (parsedConfig.telemetry!.enabled) {
+  if (parsedConfig.telemetry?.enabled) {
     setSpanTracer(parsedConfig.telemetry?.tracer);
     setSpanEventsEnabled(parsedConfig.telemetry?.signals?.hebo);
     initFetch(parsedConfig.telemetry?.signals?.hebo);
@@ -58,18 +60,22 @@ export const winterCgHandler = (
         );
       }
 
-      const realStatus = status === 200 ? (ctx.response?.status ?? status) : status;
+      let realStatus = status;
+      if (ctx.request.signal.aborted) realStatus = 499;
+      else if (status === 200 && ctx.response?.status) realStatus = ctx.response.status;
+
       if (realStatus !== 200) {
-        // FUTURE: in-stream errors are redacted in prod
         (realStatus >= 500 ? logger.error : logger.warn)({
           requestId: resolveRequestId(ctx.request),
-          err: reason,
+          err: reason ?? ctx.request.signal.reason,
         });
 
         if (realStatus >= 500) span.recordError(reason);
       }
       span.setAttributes({ "http.response.status_code_effective": realStatus });
 
+      recordV8jsMemory(parsedConfig.telemetry?.signals?.hebo);
+
       span.finish();
     };
 
@@ -89,7 +95,7 @@ export const winterCgHandler = (
         ctx.result = (await span.runWithContext(() => run(ctx))) as typeof ctx.result;
 
         if (ctx.result instanceof ReadableStream) {
-          ctx.result = wrapStream(ctx.result, { onDone: finalize }, ctx.request.signal);
+          ctx.result = wrapStream(ctx.result, { onDone: finalize });
         }
 
         ctx.response = toResponse(ctx.result!, prepareResponseInit(ctx.request));
@@ -108,7 +114,12 @@ export const winterCgHandler = (
         finalize(ctx.response.status);
       }
     } catch (error) {
-      ctx.response = toOpenAIErrorResponse(error, prepareResponseInit(ctx.request));
+      ctx.response = toOpenAIErrorResponse(
+        ctx.request.signal.aborted
+          ? new GatewayError(error ?? ctx.request.signal.reason, 499)
+          : error,
+        prepareResponseInit(ctx.request),
+      );
       finalize(ctx.response.status, error);
     }
 

package/src/telemetry/gen-ai.ts

@@ -2,7 +2,7 @@ import { metrics, type Attributes } from "@opentelemetry/api";
 
 import type { TelemetrySignalLevel } from "../types";
 
-const meter = metrics.getMeter("@hebo-ai/gateway");
+const meter = metrics.getMeter("@hebo/gateway");
 
 const requestDurationHistogram = meter.createHistogram("gen_ai.server.request.duration", {
   description: "End-to-end gateway request duration",
@@ -14,6 +14,16 @@ const requestDurationHistogram = meter.createHistogram("gen_ai.server.request.du
   },
 });
 
+const timePerOutputTokenHistogram = meter.createHistogram("gen_ai.server.time_per_output_token", {
+  description: "End-to-end gateway request duration per output token",
+  unit: "s",
+  advice: {
+    explicitBucketBoundaries: [
+      0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 2.5,
+    ],
+  },
+});
+
 const tokenUsageHistogram = meter.createHistogram("gen_ai.client.token.usage", {
   description: "Token usage reported by upstream model responses",
   unit: "{token}",
@@ -27,13 +37,31 @@ const tokenUsageHistogram = meter.createHistogram("gen_ai.client.token.usage", {
 
 // FUTURE: record unsuccessful calls
 export const recordRequestDuration = (
-  duration: number,
+  start: number,
   attrs: Attributes,
   signalLevel?: TelemetrySignalLevel,
 ) => {
   if (!signalLevel || signalLevel === "off") return;
 
-  requestDurationHistogram.record(duration / 1000, attrs);
+  requestDurationHistogram.record((performance.now() - start) / 1000, attrs);
+};
+
+// FUTURE: record unsuccessful calls
+export const recordTimePerOutputToken = (
+  start: number,
+  tokenAttrs: Attributes,
+  metricAttrs: Attributes,
+  signalLevel?: TelemetrySignalLevel,
+) => {
+  if (!signalLevel || (signalLevel !== "recommended" && signalLevel !== "full")) return;
+
+  const outputTokens = tokenAttrs["gen_ai.usage.output_tokens"];
+  if (typeof outputTokens !== "number" || outputTokens <= 0) return;
+
+  timePerOutputTokenHistogram.record(
+    (performance.now() - start) / 1000 / outputTokens,
+    metricAttrs,
+  );
 };
 
 // FUTURE: record unsuccessful calls

package/src/telemetry/memory.ts

@@ -0,0 +1,36 @@
+import { metrics } from "@opentelemetry/api";
+
+import type { TelemetrySignalLevel } from "../types";
+
+const meter = metrics.getMeter("@hebo/gateway");
+const defaultHeapSpaceAttrs = { "v8js.heap.space.name": "total" } as const;
+
+const heapUsedCounter = meter.createUpDownCounter("v8js.memory.heap.used", {
+  description: "Used bytes in the V8 heap",
+  unit: "By",
+});
+
+const heapSpacePhysicalSizeCounter = meter.createUpDownCounter(
+  "v8js.memory.heap.space.physical_size",
+  {
+    description: "Physical bytes allocated for the V8 heap space",
+    unit: "By",
+  },
+);
+
+const isEnabled = (level?: TelemetrySignalLevel) => level === "recommended" || level === "full";
+
+export const recordV8jsMemory = (level?: TelemetrySignalLevel) => {
+  if (!isEnabled(level)) return;
+
+  let usage;
+  try {
+    usage = globalThis.process?.memoryUsage?.();
+  } catch {
+    return;
+  }
+  if (!usage) return;
+
+  heapUsedCounter.add(usage.heapUsed, defaultHeapSpaceAttrs);
+  heapSpacePhysicalSizeCounter.add(usage.rss, defaultHeapSpaceAttrs);
+};

package/src/telemetry/span.ts

@@ -4,7 +4,7 @@ import { INVALID_SPAN_CONTEXT, SpanKind, SpanStatusCode, context, trace } from "
 
 import type { TelemetrySignalLevel } from "../types";
 
-const DEFAULT_TRACER_NAME = "@hebo-ai/gateway";
+const DEFAULT_TRACER_NAME = "@hebo/gateway";
 
 let spanTracer: Tracer | undefined;
 let spanEventsEnabled = false;

package/src/telemetry/stream.ts

@@ -1,54 +1,51 @@
-const isErrorChunk = (v: unknown) => !!(v as any)?.error;
+import { toOpenAIError } from "#/errors/openai";
+
+const isErrorChunk = (v: unknown) => v instanceof Error || !!(v as any)?.error;
 
 export const wrapStream = (
   src: ReadableStream,
   hooks: { onDone?: (status: number, reason: unknown) => void },
-  signal?: AbortSignal,
 ): ReadableStream => {
-  let finishOnce = false;
-
-  const finish = (status: number, reason?: unknown) => {
-    if (finishOnce) return;
-    finishOnce = true;
+  let finished = false;
 
-    hooks.onDone?.(status, reason ?? signal?.reason);
+  const done = (
+    reader: ReadableStreamDefaultReader,
+    controller: ReadableStreamDefaultController,
+    status: number,
+    reason?: unknown,
+  ) => {
+    if (!finished) {
+      finished = true;
+      hooks.onDone?.(status, reason);
+    }
+    reader.cancel(reason).catch(() => {});
+    controller.close();
   };
 
   return new ReadableStream({
     async start(controller) {
       const reader = src.getReader();
 
-      const close = (status: number, reason?: unknown) => {
-        finish(status, reason);
-        reader.cancel(reason).catch(() => {});
-        controller.close();
-      };
-
       try {
         for (;;) {
-          if (signal?.aborted) {
-            close(499, signal.reason);
-            return;
-          }
-
           // eslint-disable-next-line no-await-in-loop
-          const { value, done } = await reader.read();
-          if (done) break;
+          const { value, done: eof } = await reader.read();
+          if (eof) break;
 
-          controller.enqueue(value);
+          const out = isErrorChunk(value) ? toOpenAIError(value) : value;
+          controller.enqueue(out);
 
-          if (isErrorChunk(value)) {
-            const status = value.error.type === "invalid_request_error" ? 422 : 502;
-            close(status, value.error.message);
+          if (out !== value) {
+            const status = out.error?.type === "invalid_request_error" ? 422 : 502;
+            done(reader, controller, status, value);
             return;
           }
         }
 
-        finish(200);
-        controller.close();
+        done(reader, controller, 200);
       } catch (err) {
-        const status = signal?.aborted ? 499 : (err as any)?.name === "AbortError" ? 503 : 502;
-        close(status, err);
+        controller.enqueue(toOpenAIError(err));
+        done(reader, controller, 502, err);
       } finally {
         try {
           reader.releaseLock();
@@ -56,8 +53,11 @@ export const wrapStream = (
       }
     },
 
-    cancel(reason?: unknown) {
-      finish(499, reason);
+    cancel(reason) {
+      if (!finished) {
+        finished = true;
+        hooks.onDone?.(499, reason);
+      }
       src.cancel(reason).catch(() => {});
     },
   });

package/src/types.ts

@@ -8,7 +8,6 @@ import type {
 } from "./endpoints/chat-completions/schema";
 import type { Embeddings, EmbeddingsBody } from "./endpoints/embeddings/schema";
 import type { Model, ModelList } from "./endpoints/models";
-import type { OpenAIError } from "./errors/openai";
 import type { Logger, LoggerConfig } from "./logger";
 import type { ModelCatalog, ModelId } from "./models/types";
 import type { ProviderId, ProviderRegistry } from "./providers/types";
@@ -76,7 +75,7 @@ export type GatewayContext = {
    */
   result?:
     | ChatCompletions
-    | ReadableStream<ChatCompletionsChunk | OpenAIError>
+    | ReadableStream<ChatCompletionsChunk | Error>
     | Embeddings
     | Model
     | ModelList;
@@ -150,11 +149,9 @@ export type GatewayHooks = {
   ) =>
     | void
     | ChatCompletions
-    | ReadableStream<ChatCompletionsChunk | OpenAIError>
+    | ReadableStream<ChatCompletionsChunk | Error>
     | Embeddings
-    | Promise<
-        void | ChatCompletions | ReadableStream<ChatCompletionsChunk | OpenAIError> | Embeddings
-      >;
+    | Promise<void | ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings>;
   /**
    * Runs after the lifecycle has produced the final Response.
    * @returns Replacement Response, or undefined to keep original.