@hebo-ai/gateway 0.3.0-rc.2 → 0.3.0

package/dist/endpoints/chat-completions/handler.js

@@ -5,6 +5,8 @@ import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
+import { markPerf } from "../../telemetry/perf";
+import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
 import { convertToTextCallOptions, toChatCompletions, toChatCompletionsStream } from "./converters";
 import { ChatCompletionsBodySchema } from "./schema";
@@ -50,7 +52,7 @@ export const chatCompletions = (config) => {
         // Convert inputs to AI SDK call options.
         const textOptions = convertToTextCallOptions(inputs);
         logger.trace({
-            requestId: ctx.request.headers.get("x-request-id"),
+            requestId: resolveRequestId(ctx.request),
             options: textOptions,
         }, "[chat] AI SDK options");
         // Build middleware chain (model -> forward params -> provider).
@@ -59,6 +61,7 @@ export const chatCompletions = (config) => {
             middleware: modelMiddlewareMatcher.for(ctx.resolvedModelId, languageModel.provider),
         });
         // Execute request (streaming vs. non-streaming).
+        markPerf(ctx.request, "aiSdkStart");
         if (stream) {
             const result = streamText({
                 model: languageModelWithMiddleware,
@@ -66,33 +69,41 @@ export const chatCompletions = (config) => {
                 // No abort signal here, otherwise we can't detect upstream from client cancellations
                 // abortSignal: ctx.request.signal,
                 onError: ({ error }) => {
-                    logger.error(error instanceof Error ? error : new Error(String(error)), {
-                        requestId: ctx.request.headers.get("x-request-id"),
+                    logger.error({
+                        requestId: resolveRequestId(ctx.request),
+                        err: error instanceof Error ? error : new Error(String(error)),
                     });
                     throw error;
                 },
                 onAbort: () => {
                     throw new DOMException("Upstream failed", "AbortError");
                 },
+                timeout: {
+                    chunkMs: 5 * 60 * 1000,
+                },
                 experimental_include: {
                     requestBody: false,
                 },
                 includeRawChunks: false,
                 ...textOptions,
             });
+            markPerf(ctx.request, "aiSdkEnd");
             return toChatCompletionsStream(result, ctx.modelId);
         }
         const result = await generateText({
             model: languageModelWithMiddleware,
             headers: prepareForwardHeaders(ctx.request),
+            // FUTURE: currently can't tell whether upstream or downstream abort
             abortSignal: ctx.request.signal,
             experimental_include: {
                 requestBody: false,
                 responseBody: false,
             },
+            timeout: 5 * 60 * 1000,
             ...textOptions,
         });
-        logger.trace({ requestId: ctx.request.headers.get("x-request-id"), result }, "[chat] AI SDK result");
+        markPerf(ctx.request, "aiSdkEnd");
+        logger.trace({ requestId: resolveRequestId(ctx.request), result }, "[chat] AI SDK result");
         return toChatCompletions(result, ctx.modelId);
     };
     return { handler: winterCgHandler(handler, config) };

package/dist/endpoints/embeddings/handler.js

@@ -5,6 +5,8 @@ import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
+import { markPerf } from "../../telemetry/perf";
+import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
 import { convertToEmbedCallOptions, toEmbeddings } from "./converters";
 import { EmbeddingsBodySchema } from "./schema";
@@ -49,20 +51,22 @@ export const embeddings = (config) => {
         logger.debug(`[embeddings] using ${embeddingModel.provider} for ${ctx.resolvedModelId}`);
         // Convert inputs to AI SDK call options.
         const embedOptions = convertToEmbedCallOptions(inputs);
-        logger.trace({ requestId: ctx.request.headers.get("x-request-id"), options: embedOptions }, "[embeddings] AI SDK options");
+        logger.trace({ requestId: resolveRequestId(ctx.request), options: embedOptions }, "[embeddings] AI SDK options");
         // Build middleware chain (model -> forward params -> provider).
         const embeddingModelWithMiddleware = wrapEmbeddingModel({
             model: embeddingModel,
             middleware: modelMiddlewareMatcher.forEmbedding(ctx.resolvedModelId, embeddingModel.provider),
         });
         // Execute request.
+        markPerf(ctx.request, "aiSdkStart");
         const result = await embedMany({
             model: embeddingModelWithMiddleware,
             headers: prepareForwardHeaders(ctx.request),
             abortSignal: ctx.request.signal,
             ...embedOptions,
         });
-        logger.trace({ requestId: ctx.request.headers.get("x-request-id"), result }, "[embeddings] AI SDK result");
+        markPerf(ctx.request, "aiSdkEnd");
+        logger.trace({ requestId: resolveRequestId(ctx.request), result }, "[embeddings] AI SDK result");
         return toEmbeddings(result, ctx.modelId);
     };
     return { handler: winterCgHandler(handler, config) };

package/dist/errors/openai.js

@@ -1,6 +1,8 @@
 import * as z from "zod";
+import { isProduction } from "../utils/env";
+import { resolveRequestId } from "../utils/headers";
 import { toResponse } from "../utils/response";
-import { getErrorMeta } from "./utils";
+import { getErrorMeta, STATUS_CODE } from "./utils";
 export const OpenAIErrorSchema = z.object({
     error: z.object({
         message: z.string(),
@@ -21,7 +23,16 @@ export function toOpenAIError(error) {
 }
 export function toOpenAIErrorResponse(error, responseInit) {
     const meta = getErrorMeta(error);
-    return toResponse(new OpenAIError(meta.message, meta.type, meta.code), {
+    const shouldMask = isProduction() && (meta.status >= 500 || meta.code.includes("UPSTREAM"));
+    let message;
+    if (shouldMask) {
+        const requestId = resolveRequestId(responseInit);
+        message = `${STATUS_CODE(meta.status)} (${requestId})`;
+    }
+    else {
+        message = meta.message;
+    }
+    return toResponse(new OpenAIError(message, meta.type, meta.code), {
         ...responseInit,
         status: meta.status,
         statusText: meta.code,

package/dist/errors/utils.d.ts

@@ -20,5 +20,4 @@ export declare function getErrorMeta(error: unknown): {
     param: string;
     type: string;
     message: string;
-    rawMessage: string;
 };

package/dist/errors/utils.js

@@ -1,4 +1,3 @@
-import { isProduction } from "../utils/env";
 import { normalizeAiSdkError } from "./ai-sdk";
 import { GatewayError } from "./gateway";
 export const STATUS_CODES = {
@@ -23,7 +22,7 @@ export const STATUS_CODE = (status) => {
     return status >= 400 && status < 500 ? STATUS_CODES[400] : STATUS_CODES[500];
 };
 export function getErrorMeta(error) {
-    const rawMessage = error instanceof Error ? error.message : String(error);
+    const message = error instanceof Error ? error.message : String(error);
     let code;
     let status;
     let param = "";
@@ -41,7 +40,5 @@ export function getErrorMeta(error) {
         }
     }
     const type = status < 500 ? "invalid_request_error" : "server_error";
-    const shouldMask = !code.includes("UPSTREAM") && status >= 500 && isProduction();
-    const message = shouldMask ? STATUS_CODE(status) : rawMessage;
-    return { code, status, param, type, message, rawMessage };
+    return { code, status, param, type, message };
 }

package/dist/lifecycle.js

@@ -2,15 +2,13 @@ import { parseConfig } from "./config";
 import { toOpenAIErrorResponse } from "./errors/openai";
 import { isLoggerDisabled, logger } from "./logger";
 import { withAccessLog } from "./telemetry/access-log";
+import { resolveRequestId } from "./utils/headers";
 import { maybeApplyRequestPatch, prepareRequestHeaders } from "./utils/request";
-import { toResponse } from "./utils/response";
+import { prepareResponseInit, toResponse } from "./utils/response";
 export const winterCgHandler = (run, config) => {
     const parsedConfig = parseConfig(config);
     const core = async (ctx) => {
         try {
-            const headers = prepareRequestHeaders(ctx.request);
-            if (headers)
-                ctx.request = new Request(ctx.request, { headers });
             const before = await parsedConfig.hooks?.before?.(ctx);
             if (before) {
                 if (before instanceof Response) {
@@ -23,13 +21,18 @@ export const winterCgHandler = (run, config) => {
             const after = await parsedConfig.hooks?.after?.(ctx);
             if (after)
                 ctx.result = after;
-            ctx.response = ctx.result instanceof Response ? ctx.result : toResponse(ctx.result);
+            if (ctx.result instanceof Response) {
+                ctx.response = ctx.result;
+                return;
+            }
+            ctx.response = toResponse(ctx.result, prepareResponseInit(ctx.request));
         }
         catch (error) {
-            logger.error(error instanceof Error ? error : new Error(String(error)), {
-                requestId: ctx.request.headers.get("x-request-id"),
+            logger.error({
+                requestId: resolveRequestId(ctx.request),
+                err: error instanceof Error ? error : new Error(String(error)),
             });
-            ctx.response = toOpenAIErrorResponse(error);
+            ctx.response = toOpenAIErrorResponse(error, prepareResponseInit(ctx.request));
         }
     };
     const handler = isLoggerDisabled(parsedConfig.logger) ? core : withAccessLog(core);
@@ -40,6 +43,9 @@ export const winterCgHandler = (run, config) => {
             providers: parsedConfig.providers,
             models: parsedConfig.models,
         };
+        const headers = prepareRequestHeaders(ctx.request);
+        if (headers)
+            ctx.request = new Request(ctx.request, { headers });
         await handler(ctx);
         return ctx.response ?? new Response("Internal Server Error", { status: 500 });
     };

package/dist/logger/default.js

@@ -15,39 +15,34 @@ const isRecord = (value) => typeof value === "object" && value !== null && !(val
 const buildLogObject = (level, args) => {
     if (args.length === 0)
         return {};
-    const [first, second, third] = args;
+    const [first, second] = args;
     let obj;
     let err;
     let msg;
     if (first instanceof Error) {
         err = first;
-        if (isRecord(second)) {
-            obj = second;
-            if (third !== undefined) {
-                msg = String(third);
-            }
-        }
-        else if (second !== undefined) {
-            msg = String(second);
-        }
     }
     else if (isRecord(first)) {
-        obj = first;
-        if (second !== undefined) {
-            msg = String(second);
+        if (first["err"] !== undefined) {
+            err = first["err"];
+            delete first["err"];
         }
+        obj = first;
     }
     else {
         msg = String(first);
     }
+    if (second !== undefined) {
+        msg = String(second);
+    }
     if (err && msg === undefined) {
-        msg = err.message;
+        msg = err instanceof Error ? err.message : String(err);
     }
     return {
         level,
         time: Date.now(),
         ...(msg ? { msg } : {}),
-        ...(err ? { err: serializeError(err) } : {}),
+        ...(err ? { err: err instanceof Error ? serializeError(err) : err } : {}),
         ...obj,
     };
 };

package/dist/logger/index.d.ts

@@ -2,7 +2,6 @@ export type LogFn = {
     (msg: string): void;
     (obj: Record<string, unknown>, msg?: string): void;
     (err: Error, msg?: string): void;
-    (err: Error, obj?: Record<string, unknown>, msg?: string): void;
 };
 export type Logger = Record<"trace" | "debug" | "info" | "warn" | "error", LogFn>;
 export type LogLevel = "trace" | "debug" | "info" | "warn" | "error" | "silent";

package/dist/telemetry/access-log.js

@@ -1,26 +1,27 @@
 import { logger } from "../logger";
+import { resolveRequestId } from "../utils/headers";
+import { clearPerf, getMemoryMeta, getPerfMeta, initPerf, markPerf } from "./perf";
 import { instrumentStream } from "./stream";
 import { getAIMeta, getRequestMeta, getResponseMeta } from "./utils";
 export const withAccessLog = (run) => async (ctx) => {
-    const start = performance.now();
+    initPerf(ctx.request);
     const requestBytes = (() => {
         const n = Number(ctx.request.headers.get("content-length"));
         return Number.isFinite(n) ? n : undefined;
     })();
     const logAccess = (status, stats) => {
-        const totalDuration = +((stats?.streamEnd ?? performance.now()) - start).toFixed(2);
-        const responseTime = stats?.streamStart && +(stats.streamStart - start).toFixed(2);
+        if (!stats)
+            markPerf(ctx.request, "responseTime");
+        markPerf(ctx.request, "totalDuration");
         const requestMeta = getRequestMeta(ctx.request);
         const responseMeta = getResponseMeta(ctx.response);
         const meta = {
-            requestId: ctx.request.headers.get("x-request-id"),
+            requestId: resolveRequestId(ctx.request),
             ai: getAIMeta(ctx),
             request: requestMeta,
             response: responseMeta,
-            timings: {
-                totalDuration,
-                responseTime: responseTime ?? totalDuration,
-            },
+            timings: getPerfMeta(ctx.request),
+            memory: getMemoryMeta(ctx.request),
             bytes: {
                 in: requestBytes,
                 out: stats?.bytes ?? responseMeta["contentLength"],
@@ -29,6 +30,7 @@ export const withAccessLog = (run) => async (ctx) => {
         const realStatus = status === 200 ? (ctx.response?.status ?? status) : status;
         const msg = `${ctx.request.method} ${requestMeta["path"]} ${realStatus}`;
         logger.info(meta, msg);
+        clearPerf(ctx.request);
     };
     await run(ctx);
     if (ctx.response.body instanceof ReadableStream) {
@@ -40,6 +42,7 @@ export const withAccessLog = (run) => async (ctx) => {
             statusText: ctx.response.statusText,
             headers: ctx.response.headers,
         });
+        markPerf(ctx.request, "responseTime");
         return;
     }
     logAccess(ctx.response.status);

package/dist/telemetry/fetch.d.ts

@@ -0,0 +1 @@
+export declare const initFetch: () => void;

package/dist/telemetry/fetch.js

@@ -0,0 +1,16 @@
+import { markPerf, markPerfOnce } from "./perf";
+const ORIGINAL_FETCH_KEY = Symbol.for("@hebo/fetch/original-fetch");
+const g = globalThis;
+const perfFetch = async (input, init) => {
+    const original = g[ORIGINAL_FETCH_KEY];
+    markPerfOnce(init ?? input, "fetchStart");
+    const response = await original(input, init);
+    markPerf(init ?? input, "fetchEnd");
+    return response;
+};
+export const initFetch = () => {
+    if (g[ORIGINAL_FETCH_KEY])
+        return;
+    g[ORIGINAL_FETCH_KEY] = globalThis.fetch.bind(globalThis);
+    globalThis.fetch = perfFetch;
+};

package/dist/telemetry/perf.d.ts

@@ -0,0 +1,11 @@
+type RequestIdSource = string | URL | Request | RequestInit;
+export declare const initPerf: (source: RequestIdSource) => void;
+export declare const markPerf: (source: RequestIdSource, name: string) => number | undefined;
+export declare const markPerfOnce: (source: RequestIdSource, name: string) => number | undefined;
+export declare const clearPerf: (source: RequestIdSource) => void;
+export declare const getPerfMeta: (source: RequestIdSource) => Record<string, number>;
+export declare const getMemoryMeta: (source: RequestIdSource) => {
+    total: number | undefined;
+    request: number;
+} | undefined;
+export {};

package/dist/telemetry/perf.js

@@ -0,0 +1,60 @@
+import { resolveRequestId } from "../utils/headers";
+import { initFetch } from "./fetch";
+const REQ_PERF_KEY = Symbol.for("@hebo/perf/by-request");
+const g = globalThis;
+const perfByRequestId = (g[REQ_PERF_KEY] ??= new Map());
+const toMb = (bytes) => +(bytes / (1024 * 1024)).toFixed(2);
+const mem = () => process?.memoryUsage?.();
+const samplePeakMemory = (perf) => {
+    const heapUsed = mem()?.heapUsed;
+    if (perf.memory && heapUsed && heapUsed > perf.memory.peakHeapUsed)
+        perf.memory.peakHeapUsed = heapUsed;
+};
+const getPerfStore = (source) => {
+    const id = resolveRequestId(source);
+    return id ? perfByRequestId.get(id) : undefined;
+};
+export const initPerf = (source) => {
+    initFetch();
+    const id = resolveRequestId(source);
+    if (!id || perfByRequestId.has(id))
+        return;
+    const heapUsed = mem()?.heapUsed;
+    perfByRequestId.set(id, {
+        timers: {},
+        origin: performance.now(),
+        // eslint-disable-next-line eqeqeq
+        memory: heapUsed == null ? undefined : { steadyHeapUsed: heapUsed, peakHeapUsed: heapUsed },
+    });
+};
+const mark = (source, name, once) => {
+    const perf = getPerfStore(source);
+    if (!perf)
+        return;
+    const existing = perf.timers[name];
+    if (once && existing !== undefined)
+        return existing;
+    const value = +(performance.now() - perf.origin).toFixed(2);
+    perf.timers[name] = value;
+    samplePeakMemory(perf);
+    return value;
+};
+export const markPerf = (source, name) => mark(source, name, false);
+export const markPerfOnce = (source, name) => mark(source, name, true);
+export const clearPerf = (source) => {
+    const id = resolveRequestId(source);
+    if (id)
+        perfByRequestId.delete(id);
+};
+export const getPerfMeta = (source) => getPerfStore(source)?.timers ?? {};
+export const getMemoryMeta = (source) => {
+    const perf = getPerfStore(source);
+    if (!perf?.memory)
+        return;
+    samplePeakMemory(perf);
+    const memory = mem();
+    return {
+        total: memory ? toMb(memory.rss) : undefined,
+        request: toMb(perf.memory.peakHeapUsed - perf.memory.steadyHeapUsed),
+    };
+};

package/dist/utils/headers.d.ts

@@ -0,0 +1,4 @@
+export declare const REQUEST_ID_HEADER = "x-request-id";
+type HeaderSource = string | URL | Headers | Request | Response | RequestInit | ResponseInit | HeadersInit | undefined;
+export declare const resolveRequestId: (source: HeaderSource) => string | undefined;
+export {};

package/dist/utils/headers.js

@@ -0,0 +1,24 @@
+export const REQUEST_ID_HEADER = "x-request-id";
+export const resolveRequestId = (source) => {
+    if (!source || typeof source === "string" || source instanceof URL)
+        return undefined;
+    if (source instanceof Request || source instanceof Response) {
+        return source.headers.get(REQUEST_ID_HEADER) ?? undefined;
+    }
+    const headers = "headers" in source ? source.headers : source;
+    if (!headers || typeof headers === "string")
+        return undefined;
+    if (Object.getPrototypeOf(headers) === Object.prototype) {
+        return headers[REQUEST_ID_HEADER] ?? undefined;
+    }
+    if (headers instanceof Headers)
+        return headers.get(REQUEST_ID_HEADER) ?? undefined;
+    if (Array.isArray(headers)) {
+        for (const [key, value] of headers) {
+            if (key.toLowerCase() === REQUEST_ID_HEADER)
+                return value;
+        }
+        return undefined;
+    }
+    return undefined;
+};

package/dist/utils/request.js

@@ -1,14 +1,15 @@
 import pkg from "../../package.json" with { type: "json" };
+import { REQUEST_ID_HEADER } from "./headers";
 const GATEWAY_VERSION = pkg.version;
 export const prepareRequestHeaders = (request) => {
-    const existingRequestId = request.headers.get("x-request-id");
+    const existingRequestId = request.headers.get(REQUEST_ID_HEADER);
     if (existingRequestId)
         return;
     const requestId = request.headers.get("x-correlation-id") ??
         request.headers.get("x-trace-id") ??
         crypto.randomUUID();
     const headers = new Headers(request.headers);
-    headers.set("x-request-id", requestId);
+    headers.set(REQUEST_ID_HEADER, requestId);
     return headers;
 };
 export const prepareRequestBody = async (request) => {
@@ -26,7 +27,7 @@ export const prepareForwardHeaders = (request) => {
         ? `${userAgent} @hebo-ai/gateway/${GATEWAY_VERSION}`
         : `@hebo-ai/gateway/${GATEWAY_VERSION}`;
     return {
-        "x-request-id": request.headers.get("x-request-id"),
+        [REQUEST_ID_HEADER]: request.headers.get(REQUEST_ID_HEADER),
         "user-agent": appendedUserAgent,
     };
 };

package/dist/utils/response.d.ts

@@ -1,2 +1,3 @@
+export declare const prepareResponseInit: (request: Request) => ResponseInit;
 export declare const mergeResponseInit: (defaultHeaders: HeadersInit, responseInit?: ResponseInit) => ResponseInit;
 export declare const toResponse: (result: ReadableStream<Uint8Array> | Uint8Array<ArrayBuffer> | object | string, responseInit?: ResponseInit) => Response;

package/dist/utils/response.js

@@ -1,11 +1,21 @@
+import { REQUEST_ID_HEADER, resolveRequestId } from "./headers";
 const TEXT_ENCODER = new TextEncoder();
+export const prepareResponseInit = (request) => ({
+    headers: { [REQUEST_ID_HEADER]: resolveRequestId(request.headers) },
+});
 export const mergeResponseInit = (defaultHeaders, responseInit) => {
     const headers = new Headers(defaultHeaders);
     const override = responseInit?.headers;
     if (override) {
         new Headers(override).forEach((value, key) => headers.set(key, value));
     }
-    return responseInit ? { ...responseInit, headers } : { headers };
+    if (!responseInit)
+        return { headers };
+    return {
+        status: responseInit.status,
+        statusText: responseInit.statusText,
+        headers,
+    };
 };
 export const toResponse = (result, responseInit) => {
     let body;
@@ -23,11 +33,13 @@ export const toResponse = (result, responseInit) => {
         body = TEXT_ENCODER.encode(JSON.stringify(result));
     }
     const contentLength = body instanceof Uint8Array ? String(body.byteLength) : "";
-    if (!responseInit)
-        responseInit =
-            result instanceof Error
-                ? { status: 500, statusText: "REQUEST_FAILED" }
-                : { status: 200, statusText: "OK" };
+    const isError = result instanceof Error;
+    if (!responseInit?.statusText) {
+        const status = responseInit?.status ?? (isError ? 500 : 200);
+        const statusText = isError ? "REQUEST_FAILED" : "OK";
+        const headers = responseInit?.headers;
+        responseInit = headers ? { status, statusText, headers } : { status, statusText };
+    }
     const init = mergeResponseInit(isStream
         ? {
             "content-type": "text/event-stream",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hebo-ai/gateway",
-  "version": "0.3.0-rc.2",
+  "version": "0.3.0",
   "description": "AI gateway as a framework. For full control over models, routing & lifecycle. OpenAI-compatible /chat/completions, /embeddings & /models.",
   "keywords": [
     "ai",

package/src/endpoints/chat-completions/handler.ts

@@ -14,6 +14,8 @@ import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
+import { markPerf } from "../../telemetry/perf";
+import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
 import { convertToTextCallOptions, toChatCompletions, toChatCompletionsStream } from "./converters";
 import { ChatCompletionsBodySchema } from "./schema";
@@ -68,7 +70,7 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
     const textOptions = convertToTextCallOptions(inputs);
     logger.trace(
       {
-        requestId: ctx.request.headers.get("x-request-id"),
+        requestId: resolveRequestId(ctx.request),
         options: textOptions,
       },
       "[chat] AI SDK options",
@@ -81,6 +83,7 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
     });
 
     // Execute request (streaming vs. non-streaming).
+    markPerf(ctx.request, "aiSdkStart");
     if (stream) {
       const result = streamText({
         model: languageModelWithMiddleware,
@@ -88,20 +91,25 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
         // No abort signal here, otherwise we can't detect upstream from client cancellations
         // abortSignal: ctx.request.signal,
         onError: ({ error }) => {
-          logger.error(error instanceof Error ? error : new Error(String(error)), {
-            requestId: ctx.request.headers.get("x-request-id"),
+          logger.error({
+            requestId: resolveRequestId(ctx.request),
+            err: error instanceof Error ? error : new Error(String(error)),
           });
           throw error;
         },
         onAbort: () => {
           throw new DOMException("Upstream failed", "AbortError");
         },
+        timeout: {
+          chunkMs: 5 * 60 * 1000,
+        },
         experimental_include: {
           requestBody: false,
         },
         includeRawChunks: false,
         ...textOptions,
       });
+      markPerf(ctx.request, "aiSdkEnd");
 
       return toChatCompletionsStream(result, ctx.modelId);
     }
@@ -109,18 +117,18 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
     const result = await generateText({
       model: languageModelWithMiddleware,
       headers: prepareForwardHeaders(ctx.request),
+      // FUTURE: currently can't tell whether upstream or downstream abort
       abortSignal: ctx.request.signal,
       experimental_include: {
         requestBody: false,
         responseBody: false,
       },
+      timeout: 5 * 60 * 1000,
       ...textOptions,
     });
+    markPerf(ctx.request, "aiSdkEnd");
 
-    logger.trace(
-      { requestId: ctx.request.headers.get("x-request-id"), result },
-      "[chat] AI SDK result",
-    );
+    logger.trace({ requestId: resolveRequestId(ctx.request), result }, "[chat] AI SDK result");
 
     return toChatCompletions(result, ctx.modelId);
   };

package/src/endpoints/embeddings/handler.ts

@@ -14,6 +14,8 @@ import { winterCgHandler } from "../../lifecycle";
 import { logger } from "../../logger";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { resolveProvider } from "../../providers/registry";
+import { markPerf } from "../../telemetry/perf";
+import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
 import { convertToEmbedCallOptions, toEmbeddings } from "./converters";
 import { EmbeddingsBodySchema } from "./schema";
@@ -67,7 +69,7 @@ export const embeddings = (config: GatewayConfig): Endpoint => {
     // Convert inputs to AI SDK call options.
     const embedOptions = convertToEmbedCallOptions(inputs);
     logger.trace(
-      { requestId: ctx.request.headers.get("x-request-id"), options: embedOptions },
+      { requestId: resolveRequestId(ctx.request), options: embedOptions },
       "[embeddings] AI SDK options",
     );
 
@@ -78,15 +80,17 @@ export const embeddings = (config: GatewayConfig): Endpoint => {
     });
 
     // Execute request.
+    markPerf(ctx.request, "aiSdkStart");
     const result = await embedMany({
       model: embeddingModelWithMiddleware,
       headers: prepareForwardHeaders(ctx.request),
       abortSignal: ctx.request.signal,
       ...embedOptions,
     });
+    markPerf(ctx.request, "aiSdkEnd");
 
     logger.trace(
-      { requestId: ctx.request.headers.get("x-request-id"), result },
+      { requestId: resolveRequestId(ctx.request), result },
       "[embeddings] AI SDK result",
     );
 

package/src/errors/openai.ts

@@ -1,7 +1,9 @@
 import * as z from "zod";
 
+import { isProduction } from "../utils/env";
+import { resolveRequestId } from "../utils/headers";
 import { toResponse } from "../utils/response";
-import { getErrorMeta } from "./utils";
+import { getErrorMeta, STATUS_CODE } from "./utils";
 
 export const OpenAIErrorSchema = z.object({
   error: z.object({
@@ -27,7 +29,18 @@ export function toOpenAIError(error: unknown): OpenAIError {
 
 export function toOpenAIErrorResponse(error: unknown, responseInit?: ResponseInit) {
   const meta = getErrorMeta(error);
-  return toResponse(new OpenAIError(meta.message, meta.type, meta.code), {
+
+  const shouldMask = isProduction() && (meta.status >= 500 || meta.code.includes("UPSTREAM"));
+
+  let message;
+  if (shouldMask) {
+    const requestId = resolveRequestId(responseInit);
+    message = `${STATUS_CODE(meta.status)} (${requestId})`;
+  } else {
+    message = meta.message;
+  }
+
+  return toResponse(new OpenAIError(message, meta.type, meta.code), {
     ...responseInit,
     status: meta.status,
     statusText: meta.code,

package/src/errors/utils.ts

@@ -1,4 +1,3 @@
-import { isProduction } from "../utils/env";
 import { normalizeAiSdkError } from "./ai-sdk";
 import { GatewayError } from "./gateway";
 
@@ -25,7 +24,7 @@ export const STATUS_CODE = (status: number) => {
 };
 
 export function getErrorMeta(error: unknown) {
-  const rawMessage = error instanceof Error ? error.message : String(error);
+  const message = error instanceof Error ? error.message : String(error);
 
   let code: string;
   let status: number;
@@ -44,8 +43,6 @@ export function getErrorMeta(error: unknown) {
   }
 
   const type = status < 500 ? "invalid_request_error" : "server_error";
-  const shouldMask = !code.includes("UPSTREAM") && status >= 500 && isProduction();
-  const message = shouldMask ? STATUS_CODE(status) : rawMessage;
 
-  return { code, status, param, type, message, rawMessage };
+  return { code, status, param, type, message };
 }

package/src/lifecycle.ts

@@ -4,8 +4,9 @@ import { parseConfig } from "./config";
 import { toOpenAIErrorResponse } from "./errors/openai";
 import { isLoggerDisabled, logger } from "./logger";
 import { withAccessLog } from "./telemetry/access-log";
+import { resolveRequestId } from "./utils/headers";
 import { maybeApplyRequestPatch, prepareRequestHeaders } from "./utils/request";
-import { toResponse } from "./utils/response";
+import { prepareResponseInit, toResponse } from "./utils/response";
 
 export const winterCgHandler = (
   run: (ctx: GatewayContext) => Promise<object | ReadableStream<Uint8Array>>,
@@ -15,9 +16,6 @@ export const winterCgHandler = (
 
   const core = async (ctx: GatewayContext): Promise<void> => {
     try {
-      const headers = prepareRequestHeaders(ctx.request);
-      if (headers) ctx.request = new Request(ctx.request, { headers });
-
       const before = await parsedConfig.hooks?.before?.(ctx as BeforeHookContext);
       if (before) {
         if (before instanceof Response) {
@@ -32,12 +30,17 @@ export const winterCgHandler = (
       const after = await parsedConfig.hooks?.after?.(ctx as AfterHookContext);
       if (after) ctx.result = after;
 
-      ctx.response = ctx.result instanceof Response ? ctx.result : toResponse(ctx.result);
+      if (ctx.result instanceof Response) {
+        ctx.response = ctx.result;
+        return;
+      }
+      ctx.response = toResponse(ctx.result, prepareResponseInit(ctx.request));
     } catch (error) {
-      logger.error(error instanceof Error ? error : new Error(String(error)), {
-        requestId: ctx.request.headers.get("x-request-id"),
+      logger.error({
+        requestId: resolveRequestId(ctx.request)!,
+        err: error instanceof Error ? error : new Error(String(error)),
       });
-      ctx.response = toOpenAIErrorResponse(error);
+      ctx.response = toOpenAIErrorResponse(error, prepareResponseInit(ctx.request));
     }
   };
 
@@ -51,6 +54,9 @@ export const winterCgHandler = (
       models: parsedConfig.models,
     };
 
+    const headers = prepareRequestHeaders(ctx.request);
+    if (headers) ctx.request = new Request(ctx.request, { headers });
+
     await handler(ctx);
 
     return ctx.response ?? new Response("Internal Server Error", { status: 500 });

package/src/logger/default.ts

@@ -25,39 +25,37 @@ const isRecord = (value: unknown): value is Record<string, unknown> =>
 const buildLogObject = (level: LogLevel, args: unknown[]): Record<string, unknown> => {
   if (args.length === 0) return {};
 
-  const [first, second, third] = args;
+  const [first, second] = args;
+
   let obj: Record<string, unknown> | undefined;
-  let err: Error | undefined;
+  let err: unknown;
   let msg: string | undefined;
 
   if (first instanceof Error) {
     err = first;
-    if (isRecord(second)) {
-      obj = second;
-      if (third !== undefined) {
-        msg = String(third);
-      }
-    } else if (second !== undefined) {
-      msg = String(second);
-    }
   } else if (isRecord(first)) {
-    obj = first;
-    if (second !== undefined) {
-      msg = String(second);
+    if (first["err"] !== undefined) {
+      err = first["err"];
+      delete first["err"];
     }
+    obj = first;
   } else {
     msg = String(first);
   }
 
+  if (second !== undefined) {
+    msg = String(second);
+  }
+
   if (err && msg === undefined) {
-    msg = err.message;
+    msg = err instanceof Error ? err.message : String(err);
   }
 
   return {
     level,
     time: Date.now(),
     ...(msg ? { msg } : {}),
-    ...(err ? { err: serializeError(err) } : {}),
+    ...(err ? { err: err instanceof Error ? serializeError(err) : err } : {}),
     ...obj,
   };
 };

package/src/logger/index.ts

@@ -4,7 +4,6 @@ export type LogFn = {
   (msg: string): void;
   (obj: Record<string, unknown>, msg?: string): void;
   (err: Error, msg?: string): void;
-  (err: Error, obj?: Record<string, unknown>, msg?: string): void;
 };
 
 export type Logger = Record<"trace" | "debug" | "info" | "warn" | "error", LogFn>;

package/src/telemetry/access-log.ts

@@ -1,36 +1,34 @@
 import type { GatewayContext } from "../types";
 
 import { logger } from "../logger";
+import { resolveRequestId } from "../utils/headers";
+import { clearPerf, getMemoryMeta, getPerfMeta, initPerf, markPerf } from "./perf";
 import { instrumentStream } from "./stream";
 import { getAIMeta, getRequestMeta, getResponseMeta } from "./utils";
 
 export const withAccessLog =
   (run: (ctx: GatewayContext) => Promise<void>) => async (ctx: GatewayContext) => {
-    const start = performance.now();
+    initPerf(ctx.request);
 
     const requestBytes = (() => {
       const n = Number(ctx.request.headers.get("content-length"));
       return Number.isFinite(n) ? n : undefined;
     })();
 
-    const logAccess = (
-      status: number,
-      stats?: { bytes?: number; streamStart?: number; streamEnd?: number },
-    ) => {
-      const totalDuration = +((stats?.streamEnd ?? performance.now()) - start).toFixed(2);
-      const responseTime = stats?.streamStart && +(stats.streamStart - start).toFixed(2);
+    const logAccess = (status: number, stats?: { bytes?: number }) => {
+      if (!stats) markPerf(ctx.request, "responseTime");
+      markPerf(ctx.request, "totalDuration");
+
       const requestMeta = getRequestMeta(ctx.request);
       const responseMeta = getResponseMeta(ctx.response);
 
       const meta: Record<string, unknown> = {
-        requestId: ctx.request.headers.get("x-request-id"),
+        requestId: resolveRequestId(ctx.request),
         ai: getAIMeta(ctx),
         request: requestMeta,
         response: responseMeta,
-        timings: {
-          totalDuration,
-          responseTime: responseTime ?? totalDuration,
-        },
+        timings: getPerfMeta(ctx.request),
+        memory: getMemoryMeta(ctx.request),
         bytes: {
           in: requestBytes,
           out: stats?.bytes ?? responseMeta["contentLength"],
@@ -42,6 +40,8 @@ export const withAccessLog =
       const msg = `${ctx.request.method} ${requestMeta["path"]} ${realStatus}`;
 
       logger.info(meta, msg);
+
+      clearPerf(ctx.request);
     };
 
     await run(ctx);
@@ -61,6 +61,8 @@ export const withAccessLog =
         headers: ctx.response!.headers,
       });
 
+      markPerf(ctx.request, "responseTime");
+
       return;
     }
 

package/src/telemetry/fetch.ts

@@ -0,0 +1,24 @@
+import { markPerf, markPerfOnce } from "./perf";
+
+const ORIGINAL_FETCH_KEY = Symbol.for("@hebo/fetch/original-fetch");
+
+type GlobalFetchState = typeof globalThis & {
+  [ORIGINAL_FETCH_KEY]?: typeof fetch;
+};
+
+const g = globalThis as GlobalFetchState;
+
+const perfFetch = async (input: RequestInfo | URL, init?: RequestInit) => {
+  const original = g[ORIGINAL_FETCH_KEY]!;
+  markPerfOnce(init ?? input, "fetchStart");
+  const response = await original(input, init);
+  markPerf(init ?? input, "fetchEnd");
+  return response;
+};
+
+export const initFetch = () => {
+  if (g[ORIGINAL_FETCH_KEY]) return;
+
+  g[ORIGINAL_FETCH_KEY] = globalThis.fetch.bind(globalThis);
+  globalThis.fetch = perfFetch as typeof fetch;
+};

package/src/telemetry/perf.ts

@@ -0,0 +1,89 @@
+import { resolveRequestId } from "../utils/headers";
+import { initFetch } from "./fetch";
+
+type PerfStore = {
+  timers: Record<string, number>;
+  origin: number;
+  memory?: {
+    steadyHeapUsed: number;
+    peakHeapUsed: number;
+  };
+};
+type RequestIdSource = string | URL | Request | RequestInit;
+
+const REQ_PERF_KEY = Symbol.for("@hebo/perf/by-request");
+
+type GlobalPerfState = typeof globalThis & {
+  [REQ_PERF_KEY]?: Map<string, PerfStore>;
+};
+const g = globalThis as GlobalPerfState;
+const perfByRequestId = (g[REQ_PERF_KEY] ??= new Map<string, PerfStore>());
+
+const toMb = (bytes: number) => +(bytes / (1024 * 1024)).toFixed(2);
+const mem = () => process?.memoryUsage?.();
+
+const samplePeakMemory = (perf: PerfStore) => {
+  const heapUsed = mem()?.heapUsed;
+  if (perf.memory && heapUsed && heapUsed > perf.memory.peakHeapUsed)
+    perf.memory.peakHeapUsed = heapUsed;
+};
+
+const getPerfStore = (source: RequestIdSource) => {
+  const id = resolveRequestId(source);
+  return id ? perfByRequestId.get(id) : undefined;
+};
+
+export const initPerf = (source: RequestIdSource) => {
+  initFetch();
+
+  const id = resolveRequestId(source);
+  if (!id || perfByRequestId.has(id)) return;
+
+  const heapUsed = mem()?.heapUsed;
+
+  perfByRequestId.set(id, {
+    timers: {},
+    origin: performance.now(),
+    // eslint-disable-next-line eqeqeq
+    memory: heapUsed == null ? undefined : { steadyHeapUsed: heapUsed, peakHeapUsed: heapUsed },
+  });
+};
+
+const mark = (source: RequestIdSource, name: string, once: boolean) => {
+  const perf = getPerfStore(source);
+  if (!perf) return;
+
+  const existing = perf.timers[name];
+  if (once && existing !== undefined) return existing;
+
+  const value = +(performance.now() - perf.origin).toFixed(2);
+  perf.timers[name] = value;
+
+  samplePeakMemory(perf);
+
+  return value;
+};
+
+export const markPerf = (source: RequestIdSource, name: string) => mark(source, name, false);
+
+export const markPerfOnce = (source: RequestIdSource, name: string) => mark(source, name, true);
+
+export const clearPerf = (source: RequestIdSource) => {
+  const id = resolveRequestId(source);
+  if (id) perfByRequestId.delete(id);
+};
+
+export const getPerfMeta = (source: RequestIdSource) => getPerfStore(source)?.timers ?? {};
+
+export const getMemoryMeta = (source: RequestIdSource) => {
+  const perf = getPerfStore(source);
+  if (!perf?.memory) return;
+
+  samplePeakMemory(perf);
+  const memory = mem();
+
+  return {
+    total: memory ? toMb(memory.rss) : undefined,
+    request: toMb(perf.memory.peakHeapUsed - perf.memory.steadyHeapUsed),
+  };
+};

package/src/utils/headers.ts

@@ -0,0 +1,38 @@
+export const REQUEST_ID_HEADER = "x-request-id";
+
+type HeaderSource =
+  | string
+  | URL
+  | Headers
+  | Request
+  | Response
+  | RequestInit
+  | ResponseInit
+  | HeadersInit
+  | undefined;
+
+export const resolveRequestId = (source: HeaderSource): string | undefined => {
+  if (!source || typeof source === "string" || source instanceof URL) return undefined;
+
+  if (source instanceof Request || source instanceof Response) {
+    return source.headers.get(REQUEST_ID_HEADER) ?? undefined;
+  }
+
+  const headers = "headers" in source ? source.headers : source;
+  if (!headers || typeof headers === "string") return undefined;
+
+  if (Object.getPrototypeOf(headers) === Object.prototype) {
+    return (headers as Record<string, string>)[REQUEST_ID_HEADER] ?? undefined;
+  }
+
+  if (headers instanceof Headers) return headers.get(REQUEST_ID_HEADER) ?? undefined;
+
+  if (Array.isArray(headers)) {
+    for (const [key, value] of headers) {
+      if (key.toLowerCase() === REQUEST_ID_HEADER) return value;
+    }
+    return undefined;
+  }
+
+  return undefined;
+};

package/src/utils/request.ts

@@ -1,11 +1,12 @@
 import type { RequestPatch } from "../types";
 
 import pkg from "../../package.json" with { type: "json" };
+import { REQUEST_ID_HEADER } from "./headers";
 
 const GATEWAY_VERSION = pkg.version;
 
 export const prepareRequestHeaders = (request: Request) => {
-  const existingRequestId = request.headers.get("x-request-id");
+  const existingRequestId = request.headers.get(REQUEST_ID_HEADER);
   if (existingRequestId) return;
 
   const requestId =
@@ -14,7 +15,7 @@ export const prepareRequestHeaders = (request: Request) => {
     crypto.randomUUID();
 
   const headers = new Headers(request.headers);
-  headers.set("x-request-id", requestId);
+  headers.set(REQUEST_ID_HEADER, requestId);
 
   return headers;
 };
@@ -37,7 +38,7 @@ export const prepareForwardHeaders = (request: Request): Record<string, string>
     : `@hebo-ai/gateway/${GATEWAY_VERSION}`;
 
   return {
-    "x-request-id": request.headers.get("x-request-id")!,
+    [REQUEST_ID_HEADER]: request.headers.get(REQUEST_ID_HEADER)!,
     "user-agent": appendedUserAgent,
   };
 };

package/src/utils/response.ts

@@ -1,5 +1,11 @@
+import { REQUEST_ID_HEADER, resolveRequestId } from "./headers";
+
 const TEXT_ENCODER = new TextEncoder();
 
+export const prepareResponseInit = (request: Request): ResponseInit => ({
+  headers: { [REQUEST_ID_HEADER]: resolveRequestId(request.headers)! },
+});
+
 export const mergeResponseInit = (
   defaultHeaders: HeadersInit,
   responseInit?: ResponseInit,
@@ -9,7 +15,13 @@ export const mergeResponseInit = (
   if (override) {
     new Headers(override).forEach((value, key) => headers.set(key, value));
   }
-  return responseInit ? { ...responseInit, headers } : { headers };
+  if (!responseInit) return { headers };
+
+  return {
+    status: responseInit.status,
+    statusText: responseInit.statusText,
+    headers,
+  };
 };
 
 export const toResponse = (
@@ -30,12 +42,15 @@ export const toResponse = (
   }
 
   const contentLength = body instanceof Uint8Array ? String(body.byteLength) : "";
+  const isError = result instanceof Error;
 
-  if (!responseInit)
-    responseInit =
-      result instanceof Error
-        ? { status: 500, statusText: "REQUEST_FAILED" }
-        : { status: 200, statusText: "OK" };
+  if (!responseInit?.statusText) {
+    const status = responseInit?.status ?? (isError ? 500 : 200);
+    const statusText = isError ? "REQUEST_FAILED" : "OK";
+    const headers = responseInit?.headers;
+
+    responseInit = headers ? { status, statusText, headers } : { status, statusText };
+  }
 
   const init = mergeResponseInit(
     isStream