@hebo-ai/gateway 0.3.0-rc.3 → 0.3.0

package/dist/endpoints/chat-completions/handler.js

@@ -69,14 +69,18 @@ export const chatCompletions = (config) => {
                 // No abort signal here, otherwise we can't detect upstream from client cancellations
                 // abortSignal: ctx.request.signal,
                 onError: ({ error }) => {
-                    logger.error(error instanceof Error ? error : new Error(String(error)), {
+                    logger.error({
                         requestId: resolveRequestId(ctx.request),
+                        err: error instanceof Error ? error : new Error(String(error)),
                     });
                     throw error;
                 },
                 onAbort: () => {
                     throw new DOMException("Upstream failed", "AbortError");
                 },
+                timeout: {
+                    chunkMs: 5 * 60 * 1000,
+                },
                 experimental_include: {
                     requestBody: false,
                 },
@@ -89,11 +93,13 @@ export const chatCompletions = (config) => {
         const result = await generateText({
             model: languageModelWithMiddleware,
             headers: prepareForwardHeaders(ctx.request),
+            // FUTURE: currently can't tell whether upstream or downstream abort
             abortSignal: ctx.request.signal,
             experimental_include: {
                 requestBody: false,
                 responseBody: false,
             },
+            timeout: 5 * 60 * 1000,
             ...textOptions,
         });
         markPerf(ctx.request, "aiSdkEnd");

package/dist/lifecycle.js

@@ -28,8 +28,9 @@ export const winterCgHandler = (run, config) => {
             ctx.response = toResponse(ctx.result, prepareResponseInit(ctx.request));
         }
         catch (error) {
-            logger.error(error instanceof Error ? error : new Error(String(error)), {
+            logger.error({
                 requestId: resolveRequestId(ctx.request),
+                err: error instanceof Error ? error : new Error(String(error)),
             });
             ctx.response = toOpenAIErrorResponse(error, prepareResponseInit(ctx.request));
         }

package/dist/logger/default.js

@@ -15,39 +15,34 @@ const isRecord = (value) => typeof value === "object" && value !== null && !(val
 const buildLogObject = (level, args) => {
     if (args.length === 0)
         return {};
-    const [first, second, third] = args;
+    const [first, second] = args;
     let obj;
     let err;
     let msg;
     if (first instanceof Error) {
         err = first;
-        if (isRecord(second)) {
-            obj = second;
-            if (third !== undefined) {
-                msg = String(third);
-            }
-        }
-        else if (second !== undefined) {
-            msg = String(second);
-        }
     }
     else if (isRecord(first)) {
-        obj = first;
-        if (second !== undefined) {
-            msg = String(second);
+        if (first["err"] !== undefined) {
+            err = first["err"];
+            delete first["err"];
         }
+        obj = first;
     }
     else {
         msg = String(first);
     }
+    if (second !== undefined) {
+        msg = String(second);
+    }
     if (err && msg === undefined) {
-        msg = err.message;
+        msg = err instanceof Error ? err.message : String(err);
     }
     return {
         level,
         time: Date.now(),
         ...(msg ? { msg } : {}),
-        ...(err ? { err: serializeError(err) } : {}),
+        ...(err ? { err: err instanceof Error ? serializeError(err) : err } : {}),
         ...obj,
     };
 };

package/dist/logger/index.d.ts

@@ -2,7 +2,6 @@ export type LogFn = {
     (msg: string): void;
     (obj: Record<string, unknown>, msg?: string): void;
     (err: Error, msg?: string): void;
-    (err: Error, obj?: Record<string, unknown>, msg?: string): void;
 };
 export type Logger = Record<"trace" | "debug" | "info" | "warn" | "error", LogFn>;
 export type LogLevel = "trace" | "debug" | "info" | "warn" | "error" | "silent";

package/dist/telemetry/access-log.js

@@ -1,6 +1,6 @@
 import { logger } from "../logger";
 import { resolveRequestId } from "../utils/headers";
-import { clearPerf, getPerfMeta, initPerf, markPerf } from "./perf";
+import { clearPerf, getMemoryMeta, getPerfMeta, initPerf, markPerf } from "./perf";
 import { instrumentStream } from "./stream";
 import { getAIMeta, getRequestMeta, getResponseMeta } from "./utils";
 export const withAccessLog = (run) => async (ctx) => {
@@ -21,6 +21,7 @@ export const withAccessLog = (run) => async (ctx) => {
             request: requestMeta,
             response: responseMeta,
             timings: getPerfMeta(ctx.request),
+            memory: getMemoryMeta(ctx.request),
             bytes: {
                 in: requestBytes,
                 out: stats?.bytes ?? responseMeta["contentLength"],

package/dist/telemetry/perf.d.ts

@@ -4,4 +4,8 @@ export declare const markPerf: (source: RequestIdSource, name: string) => number
 export declare const markPerfOnce: (source: RequestIdSource, name: string) => number | undefined;
 export declare const clearPerf: (source: RequestIdSource) => void;
 export declare const getPerfMeta: (source: RequestIdSource) => Record<string, number>;
+export declare const getMemoryMeta: (source: RequestIdSource) => {
+    total: number | undefined;
+    request: number;
+} | undefined;
 export {};

package/dist/telemetry/perf.js

@@ -3,22 +3,28 @@ import { initFetch } from "./fetch";
 const REQ_PERF_KEY = Symbol.for("@hebo/perf/by-request");
 const g = globalThis;
 const perfByRequestId = (g[REQ_PERF_KEY] ??= new Map());
+const toMb = (bytes) => +(bytes / (1024 * 1024)).toFixed(2);
+const mem = () => process?.memoryUsage?.();
+const samplePeakMemory = (perf) => {
+    const heapUsed = mem()?.heapUsed;
+    if (perf.memory && heapUsed && heapUsed > perf.memory.peakHeapUsed)
+        perf.memory.peakHeapUsed = heapUsed;
+};
 const getPerfStore = (source) => {
-    const requestId = resolveRequestId(source);
-    if (!requestId)
-        return undefined;
-    return perfByRequestId.get(requestId);
+    const id = resolveRequestId(source);
+    return id ? perfByRequestId.get(id) : undefined;
 };
 export const initPerf = (source) => {
     initFetch();
-    const requestId = resolveRequestId(source);
-    if (!requestId)
-        return;
-    if (perfByRequestId.has(requestId))
+    const id = resolveRequestId(source);
+    if (!id || perfByRequestId.has(id))
         return;
-    perfByRequestId.set(requestId, {
+    const heapUsed = mem()?.heapUsed;
+    perfByRequestId.set(id, {
         timers: {},
         origin: performance.now(),
+        // eslint-disable-next-line eqeqeq
+        memory: heapUsed == null ? undefined : { steadyHeapUsed: heapUsed, peakHeapUsed: heapUsed },
     });
 };
 const mark = (source, name, once) => {
@@ -30,21 +36,25 @@ const mark = (source, name, once) => {
         return existing;
     const value = +(performance.now() - perf.origin).toFixed(2);
     perf.timers[name] = value;
+    samplePeakMemory(perf);
     return value;
 };
-export const markPerf = (source, name) => {
-    return mark(source, name, false);
-};
-export const markPerfOnce = (source, name) => {
-    return mark(source, name, true);
-};
+export const markPerf = (source, name) => mark(source, name, false);
+export const markPerfOnce = (source, name) => mark(source, name, true);
 export const clearPerf = (source) => {
-    const requestId = resolveRequestId(source);
-    if (!requestId)
-        return;
-    perfByRequestId.delete(requestId);
+    const id = resolveRequestId(source);
+    if (id)
+        perfByRequestId.delete(id);
 };
-export const getPerfMeta = (source) => {
+export const getPerfMeta = (source) => getPerfStore(source)?.timers ?? {};
+export const getMemoryMeta = (source) => {
     const perf = getPerfStore(source);
-    return perf?.timers ?? {};
+    if (!perf?.memory)
+        return;
+    samplePeakMemory(perf);
+    const memory = mem();
+    return {
+        total: memory ? toMb(memory.rss) : undefined,
+        request: toMb(perf.memory.peakHeapUsed - perf.memory.steadyHeapUsed),
+    };
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hebo-ai/gateway",
-  "version": "0.3.0-rc.3",
+  "version": "0.3.0",
   "description": "AI gateway as a framework. For full control over models, routing & lifecycle. OpenAI-compatible /chat/completions, /embeddings & /models.",
   "keywords": [
     "ai",

package/src/endpoints/chat-completions/handler.ts

@@ -91,14 +91,18 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
         // No abort signal here, otherwise we can't detect upstream from client cancellations
         // abortSignal: ctx.request.signal,
         onError: ({ error }) => {
-          logger.error(error instanceof Error ? error : new Error(String(error)), {
+          logger.error({
             requestId: resolveRequestId(ctx.request),
+            err: error instanceof Error ? error : new Error(String(error)),
           });
           throw error;
         },
         onAbort: () => {
           throw new DOMException("Upstream failed", "AbortError");
         },
+        timeout: {
+          chunkMs: 5 * 60 * 1000,
+        },
         experimental_include: {
           requestBody: false,
         },
@@ -113,11 +117,13 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
     const result = await generateText({
       model: languageModelWithMiddleware,
       headers: prepareForwardHeaders(ctx.request),
+      // FUTURE: currently can't tell whether upstream or downstream abort
       abortSignal: ctx.request.signal,
       experimental_include: {
         requestBody: false,
         responseBody: false,
       },
+      timeout: 5 * 60 * 1000,
       ...textOptions,
     });
     markPerf(ctx.request, "aiSdkEnd");

package/src/lifecycle.ts

@@ -36,8 +36,9 @@ export const winterCgHandler = (
       }
       ctx.response = toResponse(ctx.result, prepareResponseInit(ctx.request));
     } catch (error) {
-      logger.error(error instanceof Error ? error : new Error(String(error)), {
+      logger.error({
         requestId: resolveRequestId(ctx.request)!,
+        err: error instanceof Error ? error : new Error(String(error)),
       });
       ctx.response = toOpenAIErrorResponse(error, prepareResponseInit(ctx.request));
     }

package/src/logger/default.ts

@@ -25,39 +25,37 @@ const isRecord = (value: unknown): value is Record<string, unknown> =>
 const buildLogObject = (level: LogLevel, args: unknown[]): Record<string, unknown> => {
   if (args.length === 0) return {};
 
-  const [first, second, third] = args;
+  const [first, second] = args;
+
   let obj: Record<string, unknown> | undefined;
-  let err: Error | undefined;
+  let err: unknown;
   let msg: string | undefined;
 
   if (first instanceof Error) {
     err = first;
-    if (isRecord(second)) {
-      obj = second;
-      if (third !== undefined) {
-        msg = String(third);
-      }
-    } else if (second !== undefined) {
-      msg = String(second);
-    }
   } else if (isRecord(first)) {
-    obj = first;
-    if (second !== undefined) {
-      msg = String(second);
+    if (first["err"] !== undefined) {
+      err = first["err"];
+      delete first["err"];
     }
+    obj = first;
   } else {
     msg = String(first);
   }
 
+  if (second !== undefined) {
+    msg = String(second);
+  }
+
   if (err && msg === undefined) {
-    msg = err.message;
+    msg = err instanceof Error ? err.message : String(err);
   }
 
   return {
     level,
     time: Date.now(),
     ...(msg ? { msg } : {}),
-    ...(err ? { err: serializeError(err) } : {}),
+    ...(err ? { err: err instanceof Error ? serializeError(err) : err } : {}),
     ...obj,
   };
 };

package/src/logger/index.ts

@@ -4,7 +4,6 @@ export type LogFn = {
   (msg: string): void;
   (obj: Record<string, unknown>, msg?: string): void;
   (err: Error, msg?: string): void;
-  (err: Error, obj?: Record<string, unknown>, msg?: string): void;
 };
 
 export type Logger = Record<"trace" | "debug" | "info" | "warn" | "error", LogFn>;

package/src/telemetry/access-log.ts

@@ -2,7 +2,7 @@ import type { GatewayContext } from "../types";
 
 import { logger } from "../logger";
 import { resolveRequestId } from "../utils/headers";
-import { clearPerf, getPerfMeta, initPerf, markPerf } from "./perf";
+import { clearPerf, getMemoryMeta, getPerfMeta, initPerf, markPerf } from "./perf";
 import { instrumentStream } from "./stream";
 import { getAIMeta, getRequestMeta, getResponseMeta } from "./utils";
 
@@ -28,6 +28,7 @@ export const withAccessLog =
         request: requestMeta,
         response: responseMeta,
         timings: getPerfMeta(ctx.request),
+        memory: getMemoryMeta(ctx.request),
         bytes: {
           in: requestBytes,
           out: stats?.bytes ?? responseMeta["contentLength"],

package/src/telemetry/perf.ts

@@ -4,6 +4,10 @@ import { initFetch } from "./fetch";
 type PerfStore = {
   timers: Record<string, number>;
   origin: number;
+  memory?: {
+    steadyHeapUsed: number;
+    peakHeapUsed: number;
+  };
 };
 type RequestIdSource = string | URL | Request | RequestInit;
 
@@ -15,49 +19,71 @@ type GlobalPerfState = typeof globalThis & {
 const g = globalThis as GlobalPerfState;
 const perfByRequestId = (g[REQ_PERF_KEY] ??= new Map<string, PerfStore>());
 
-const getPerfStore = (source: RequestIdSource): PerfStore | undefined => {
-  const requestId = resolveRequestId(source);
-  if (!requestId) return undefined;
-  return perfByRequestId.get(requestId);
+const toMb = (bytes: number) => +(bytes / (1024 * 1024)).toFixed(2);
+const mem = () => process?.memoryUsage?.();
+
+const samplePeakMemory = (perf: PerfStore) => {
+  const heapUsed = mem()?.heapUsed;
+  if (perf.memory && heapUsed && heapUsed > perf.memory.peakHeapUsed)
+    perf.memory.peakHeapUsed = heapUsed;
+};
+
+const getPerfStore = (source: RequestIdSource) => {
+  const id = resolveRequestId(source);
+  return id ? perfByRequestId.get(id) : undefined;
 };
 
 export const initPerf = (source: RequestIdSource) => {
   initFetch();
-  const requestId = resolveRequestId(source);
-  if (!requestId) return;
-  if (perfByRequestId.has(requestId)) return;
-  perfByRequestId.set(requestId, {
+
+  const id = resolveRequestId(source);
+  if (!id || perfByRequestId.has(id)) return;
+
+  const heapUsed = mem()?.heapUsed;
+
+  perfByRequestId.set(id, {
     timers: {},
     origin: performance.now(),
+    // eslint-disable-next-line eqeqeq
+    memory: heapUsed == null ? undefined : { steadyHeapUsed: heapUsed, peakHeapUsed: heapUsed },
   });
 };
 
 const mark = (source: RequestIdSource, name: string, once: boolean) => {
   const perf = getPerfStore(source);
   if (!perf) return;
+
   const existing = perf.timers[name];
   if (once && existing !== undefined) return existing;
 
   const value = +(performance.now() - perf.origin).toFixed(2);
   perf.timers[name] = value;
+
+  samplePeakMemory(perf);
+
   return value;
 };
 
-export const markPerf = (source: RequestIdSource, name: string) => {
-  return mark(source, name, false);
-};
+export const markPerf = (source: RequestIdSource, name: string) => mark(source, name, false);
 
-export const markPerfOnce = (source: RequestIdSource, name: string) => {
-  return mark(source, name, true);
-};
+export const markPerfOnce = (source: RequestIdSource, name: string) => mark(source, name, true);
 
 export const clearPerf = (source: RequestIdSource) => {
-  const requestId = resolveRequestId(source);
-  if (!requestId) return;
-  perfByRequestId.delete(requestId);
+  const id = resolveRequestId(source);
+  if (id) perfByRequestId.delete(id);
 };
 
-export const getPerfMeta = (source: RequestIdSource) => {
+export const getPerfMeta = (source: RequestIdSource) => getPerfStore(source)?.timers ?? {};
+
+export const getMemoryMeta = (source: RequestIdSource) => {
   const perf = getPerfStore(source);
-  return perf?.timers ?? {};
+  if (!perf?.memory) return;
+
+  samplePeakMemory(perf);
+  const memory = mem();
+
+  return {
+    total: memory ? toMb(memory.rss) : undefined,
+    request: toMb(perf.memory.peakHeapUsed - perf.memory.steadyHeapUsed),
+  };
 };