@hebo-ai/gateway 0.4.1 → 0.5.0-beta.0

package/src/endpoints/chat-completions/schema.ts

@@ -4,12 +4,13 @@ export const ChatCompletionsContentPartTextSchema = z.object({
   type: z.literal("text"),
   text: z.string(),
 });
+export type ChatCompletionsContentPartText = z.infer<typeof ChatCompletionsContentPartTextSchema>;
 
 export const ChatCompletionsContentPartImageSchema = z.object({
   type: z.literal("image_url"),
   image_url: z.object({
     url: z.string(),
-    detail: z.union([z.literal("low"), z.literal("high"), z.literal("auto")]).optional(),
+    detail: z.enum(["low", "high", "auto"]).optional(),
   }),
 });
 
@@ -22,11 +23,35 @@ export const ChatCompletionsContentPartFileSchema = z.object({
   }),
 });
 
-// FUTURE: missing ContentPartAudio
-export type ChatCompletionsContentPart =
-  | z.infer<typeof ChatCompletionsContentPartTextSchema>
-  | z.infer<typeof ChatCompletionsContentPartImageSchema>
-  | z.infer<typeof ChatCompletionsContentPartFileSchema>;
+export const ChatCompletionsContentPartAudioSchema = z.object({
+  type: z.literal("input_audio"),
+  input_audio: z.object({
+    data: z.string(),
+    // only wav and mp3 are official by OpenAI, rest is taken from Gemini support:
+    // https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/audio-understanding
+    format: z.enum([
+      "x-aac",
+      "flac",
+      "mp3",
+      "m4a",
+      "mpeg",
+      "mpga",
+      "mp4",
+      "ogg",
+      "pcm",
+      "wav",
+      "webm",
+    ]),
+  }),
+});
+
+export const ChatCompletionsContentPartSchema = z.discriminatedUnion("type", [
+  ChatCompletionsContentPartTextSchema,
+  ChatCompletionsContentPartImageSchema,
+  ChatCompletionsContentPartFileSchema,
+  ChatCompletionsContentPartAudioSchema,
+]);
+export type ChatCompletionsContentPart = z.infer<typeof ChatCompletionsContentPartSchema>;
 
 export const ChatCompletionsToolCallSchema = z.object({
   type: z.literal("function"),
@@ -35,7 +60,10 @@ export const ChatCompletionsToolCallSchema = z.object({
     arguments: z.string(),
     name: z.string(),
   }),
-  extra_content: z.record(z.string(), z.any()).optional().meta({ extension: true }),
+  extra_content: z
+    .record(z.string(), z.record(z.string(), z.unknown()))
+    .optional()
+    .meta({ extension: true }),
 });
 export type ChatCompletionsToolCall = z.infer<typeof ChatCompletionsToolCallSchema>;
 
@@ -48,16 +76,7 @@ export type ChatCompletionsSystemMessage = z.infer<typeof ChatCompletionsSystemM
 
 export const ChatCompletionsUserMessageSchema = z.object({
   role: z.literal("user"),
-  content: z.union([
-    z.string(),
-    z.array(
-      z.union([
-        ChatCompletionsContentPartTextSchema,
-        ChatCompletionsContentPartImageSchema,
-        ChatCompletionsContentPartFileSchema,
-      ]),
-    ),
-  ]),
+  content: z.union([z.string(), z.array(ChatCompletionsContentPartSchema)]),
   name: z.string().optional(),
 });
 export type ChatCompletionsUserMessage = z.infer<typeof ChatCompletionsUserMessageSchema>;
@@ -76,8 +95,9 @@ export type ChatCompletionsReasoningDetail = z.infer<typeof ChatCompletionsReaso
 
 export const ChatCompletionsAssistantMessageSchema = z.object({
   role: z.literal("assistant"),
-  // FUTURE: this should support arrays of TextContentPart and RefusalContentPart
-  content: z.union([z.string(), z.null()]).optional(),
+  content: z
+    .union([z.string(), z.null(), z.array(ChatCompletionsContentPartTextSchema)])
+    .optional(),
   name: z.string().optional(),
   // FUTURE: This should also support Custom Tool Calls
   tool_calls: z.array(ChatCompletionsToolCallSchema).optional(),
@@ -87,19 +107,21 @@ export const ChatCompletionsAssistantMessageSchema = z.object({
     .array(ChatCompletionsReasoningDetailSchema)
     .optional()
     .meta({ extension: true }),
-  extra_content: z.record(z.string(), z.any()).optional().meta({ extension: true }),
+  extra_content: z
+    .record(z.string(), z.record(z.string(), z.unknown()))
+    .optional()
+    .meta({ extension: true }),
 });
 export type ChatCompletionsAssistantMessage = z.infer<typeof ChatCompletionsAssistantMessageSchema>;
 
 export const ChatCompletionsToolMessageSchema = z.object({
   role: z.literal("tool"),
-  // FUTURE: this should also support arrays of TextContentParts
-  content: z.string(),
+  content: z.union([z.string(), z.array(ChatCompletionsContentPartTextSchema)]),
   tool_call_id: z.string(),
 });
 export type ChatCompletionsToolMessage = z.infer<typeof ChatCompletionsToolMessageSchema>;
 
-export const ChatCompletionsMessageSchema = z.union([
+export const ChatCompletionsMessageSchema = z.discriminatedUnion("role", [
   ChatCompletionsSystemMessageSchema,
   ChatCompletionsUserMessageSchema,
   ChatCompletionsAssistantMessageSchema,
@@ -112,16 +134,14 @@ export const ChatCompletionsToolSchema = z.object({
   function: z.object({
     name: z.string(),
     description: z.string().optional(),
-    parameters: z.record(z.string(), z.any()),
+    parameters: z.record(z.string(), z.unknown()),
     // Missing strict parameter
   }),
 });
 export type ChatCompletionsTool = z.infer<typeof ChatCompletionsToolSchema>;
 
 export const ChatCompletionsToolChoiceSchema = z.union([
-  z.literal("none"),
-  z.literal("auto"),
-  z.literal("required"),
+  z.enum(["none", "auto", "required", "validated"]),
   // FUTURE: missing AllowedTools and CustomToolChoice
   z.object({
     type: z.literal("function"),
@@ -132,13 +152,14 @@ export const ChatCompletionsToolChoiceSchema = z.union([
 ]);
 export type ChatCompletionsToolChoice = z.infer<typeof ChatCompletionsToolChoiceSchema>;
 
-export const ChatCompletionsReasoningEffortSchema = z.union([
-  z.literal("none"),
-  z.literal("minimal"),
-  z.literal("low"),
-  z.literal("medium"),
-  z.literal("high"),
-  z.literal("xhigh"),
+export const ChatCompletionsReasoningEffortSchema = z.enum([
+  "none",
+  "minimal",
+  "low",
+  "medium",
+  "high",
+  "xhigh",
+  "max",
 ]);
 export type ChatCompletionsReasoningEffort = z.infer<typeof ChatCompletionsReasoningEffortSchema>;
 
@@ -150,6 +171,26 @@ export const ChatCompletionsReasoningConfigSchema = z.object({
 });
 export type ChatCompletionsReasoningConfig = z.infer<typeof ChatCompletionsReasoningConfigSchema>;
 
+export const ChatCompletionsResponseFormatJsonSchema = z.object({
+  // FUTURE: consider support for legacy json_object (if demand)
+  type: z.literal("json_schema"),
+  json_schema: z.object({
+    name: z.string(),
+    description: z.string().optional(),
+    schema: z.record(z.string(), z.unknown()),
+    // FUTURE: consider support for non-strict mode (for providers that support it)
+    strict: z.boolean().optional(),
+  }),
+});
+export const ChatCompletionsResponseFormatTextSchema = z.object({
+  type: z.literal("text"),
+});
+export const ChatCompletionsResponseFormatSchema = z.discriminatedUnion("type", [
+  ChatCompletionsResponseFormatJsonSchema,
+  ChatCompletionsResponseFormatTextSchema,
+]);
+export type ChatCompletionsResponseFormat = z.infer<typeof ChatCompletionsResponseFormatSchema>;
+
 const ChatCompletionsInputsSchema = z.object({
   messages: z.array(ChatCompletionsMessageSchema),
   tools: z
@@ -167,6 +208,7 @@ const ChatCompletionsInputsSchema = z.object({
   seed: z.int().optional(),
   stop: z.union([z.string(), z.array(z.string())]).optional(),
   top_p: z.number().min(0).max(1.0).optional(),
+  response_format: ChatCompletionsResponseFormatSchema.optional(),
   reasoning_effort: ChatCompletionsReasoningEffortSchema.optional(),
   // Extensions
   reasoning: ChatCompletionsReasoningConfigSchema.optional().meta({ extension: true }),
@@ -180,11 +222,11 @@ export const ChatCompletionsBodySchema = z.looseObject({
 });
 export type ChatCompletionsBody = z.infer<typeof ChatCompletionsBodySchema>;
 
-export const ChatCompletionsFinishReasonSchema = z.union([
-  z.literal("stop"),
-  z.literal("length"),
-  z.literal("content_filter"),
-  z.literal("tool_calls"),
+export const ChatCompletionsFinishReasonSchema = z.enum([
+  "stop",
+  "length",
+  "content_filter",
+  "tool_calls",
 ]);
 export type ChatCompletionsFinishReason = z.infer<typeof ChatCompletionsFinishReasonSchema>;
 
@@ -193,7 +235,7 @@ export const ChatCompletionsChoiceSchema = z.object({
   message: ChatCompletionsAssistantMessageSchema,
   finish_reason: ChatCompletionsFinishReasonSchema,
   // FUTURE: model this out
-  logprobs: z.any().optional(),
+  logprobs: z.unknown().optional(),
 });
 export type ChatCompletionsChoice = z.infer<typeof ChatCompletionsChoiceSchema>;
 
@@ -224,7 +266,7 @@ export const ChatCompletionsSchema = z.object({
   choices: z.array(ChatCompletionsChoiceSchema),
   usage: ChatCompletionsUsageSchema.nullable(),
   // Extensions
-  provider_metadata: z.any().optional().meta({ extension: true }),
+  provider_metadata: z.unknown().optional().meta({ extension: true }),
 });
 export type ChatCompletions = z.infer<typeof ChatCompletionsSchema>;
 
@@ -246,7 +288,7 @@ export const ChatCompletionsChoiceDeltaSchema = z.object({
   delta: ChatCompletionsAssistantMessageDeltaSchema,
   finish_reason: ChatCompletionsFinishReasonSchema.nullable(),
   // FUTURE: model this out
-  logprobs: z.any().optional(),
+  logprobs: z.unknown().optional(),
 });
 export type ChatCompletionsChoiceDelta = z.infer<typeof ChatCompletionsChoiceDeltaSchema>;
 
@@ -258,6 +300,6 @@ export const ChatCompletionsChunkSchema = z.object({
   choices: z.array(ChatCompletionsChoiceDeltaSchema),
   usage: ChatCompletionsUsageSchema.nullable(),
   // Extensions
-  provider_metadata: z.any().optional().meta({ extension: true }),
+  provider_metadata: z.unknown().optional().meta({ extension: true }),
 });
 export type ChatCompletionsChunk = z.infer<typeof ChatCompletionsChunkSchema>;

package/src/endpoints/embeddings/handler.ts

@@ -22,7 +22,6 @@ import {
   recordTokenUsage,
 } from "../../telemetry/gen-ai";
 import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
-import { resolveRequestId } from "../../utils/headers";
 import { prepareForwardHeaders } from "../../utils/request";
 import { convertToEmbedCallOptions, toEmbeddings } from "./converters";
 import {
@@ -45,8 +44,6 @@ export const embeddings = (config: GatewayConfig): Endpoint => {
       throw new GatewayError("Method Not Allowed", 405);
     }
 
-    const requestId = resolveRequestId(ctx.request);
-
     // Parse + validate input.
     try {
       ctx.body = await ctx.request.json();
@@ -98,7 +95,10 @@ export const embeddings = (config: GatewayConfig): Endpoint => {
 
     // Convert inputs to AI SDK call options.
     const embedOptions = convertToEmbedCallOptions(inputs);
-    logger.trace({ requestId, options: embedOptions }, "[embeddings] AI SDK options");
+    logger.trace(
+      { requestId: ctx.requestId, options: embedOptions },
+      "[embeddings] AI SDK options",
+    );
     addSpanEvent("hebo.options.prepared");
     setSpanAttributes(getEmbeddingsRequestAttributes(inputs, genAiSignalLevel));
 
@@ -116,7 +116,7 @@ export const embeddings = (config: GatewayConfig): Endpoint => {
       abortSignal: ctx.request.signal,
       ...embedOptions,
     });
-    logger.trace({ requestId, result }, "[embeddings] AI SDK result");
+    logger.trace({ requestId: ctx.requestId, result }, "[embeddings] AI SDK result");
     addSpanEvent("hebo.ai-sdk.completed");
 
     // Transform result.

package/src/endpoints/embeddings/schema.ts

@@ -31,6 +31,6 @@ export const EmbeddingsSchema = z.object({
   model: z.string(),
   usage: EmbeddingsUsageSchema.nullable(),
   // Extensions
-  provider_metadata: z.optional(z.any()).meta({ extension: true }),
+  provider_metadata: z.unknown().optional().meta({ extension: true }),
 });
 export type Embeddings = z.infer<typeof EmbeddingsSchema>;

package/src/errors/gateway.ts

@@ -7,6 +7,8 @@ export class GatewayError extends Error {
   constructor(error: unknown, status: number, code?: string, cause?: unknown) {
     const isError = error instanceof Error;
     super(isError ? error.message : String(error));
+
+    this.name = "GatewayError";
     this.cause = cause ?? (isError ? error : undefined);
 
     this.status = status;

package/src/lifecycle.ts

@@ -15,8 +15,7 @@ import { getRequestAttributes, getResponseAttributes } from "./telemetry/http";
 import { recordV8jsMemory } from "./telemetry/memory";
 import { addSpanEvent, setSpanEventsEnabled, setSpanTracer, startSpan } from "./telemetry/span";
 import { wrapStream } from "./telemetry/stream";
-import { resolveRequestId } from "./utils/headers";
-import { maybeApplyRequestPatch, prepareRequestHeaders } from "./utils/request";
+import { resolveOrCreateRequestId } from "./utils/request";
 import { prepareResponseInit, toResponse } from "./utils/response";
 
 export const winterCgHandler = (
@@ -37,15 +36,14 @@ export const winterCgHandler = (
       state: state ?? {},
       providers: parsedConfig.providers,
       models: parsedConfig.models,
+      requestId: resolveOrCreateRequestId(request),
     };
 
-    const headers = prepareRequestHeaders(ctx.request);
-    if (headers) ctx.request = new Request(ctx.request, { headers });
-
     const span = startSpan(ctx.request.url);
     span.setAttributes(getBaggageAttributes(ctx.request));
     if (!span.isExisting) {
       span.setAttributes(getRequestAttributes(ctx.request, parsedConfig.telemetry?.signals?.http));
+      span.setAttributes({ "http.request.id": ctx.requestId });
     }
 
     const finalize = (status: number, reason?: unknown) => {
@@ -65,8 +63,8 @@ export const winterCgHandler = (
       else if (status === 200 && ctx.response?.status) realStatus = ctx.response.status;
 
       if (realStatus !== 200) {
-        (realStatus >= 500 ? logger.error : logger.warn)({
-          requestId: resolveRequestId(ctx.request),
+        logger[realStatus >= 500 ? "error" : "warn"]({
+          requestId: ctx.requestId,
           err: reason ?? ctx.request.signal.reason,
         });
 
@@ -86,8 +84,6 @@ export const winterCgHandler = (
 
         if (onRequest instanceof Response) {
           ctx.response = onRequest;
-        } else if (onRequest) {
-          ctx.request = maybeApplyRequestPatch(ctx.request, onRequest);
         }
       }
 
@@ -98,7 +94,7 @@ export const winterCgHandler = (
           ctx.result = wrapStream(ctx.result, { onDone: finalize });
         }
 
-        ctx.response = toResponse(ctx.result!, prepareResponseInit(ctx.request));
+        ctx.response = toResponse(ctx.result!, prepareResponseInit(ctx.requestId));
       }
 
       if (parsedConfig.hooks?.onResponse) {
@@ -118,7 +114,7 @@ export const winterCgHandler = (
         ctx.request.signal.aborted
           ? new GatewayError(error ?? ctx.request.signal.reason, 499)
           : error,
-        prepareResponseInit(ctx.request),
+        prepareResponseInit(ctx.requestId),
       );
       finalize(ctx.response.status, error);
     }

package/src/logger/default.ts

@@ -1,10 +1,8 @@
-import { serializeError } from "serialize-error";
-
 import type { LogFn, LogLevel, Logger } from "./index";
 
 import { isProduction, isTest } from "../utils/env";
 
-export const getDefaultLogLevel = (): LogLevel =>
+const getDefaultLogLevel = (): LogLevel =>
   isTest() ? "silent" : isProduction() ? "info" : "debug";
 
 const noop: LogFn = () => {};
@@ -22,20 +20,48 @@ const LEVELS = Object.keys(LEVEL) as (keyof typeof LEVEL)[];
 const isRecord = (value: unknown): value is Record<string, unknown> =>
   typeof value === "object" && value !== null && !(value instanceof Error);
 
+function serializeError(err: unknown, _seen?: WeakSet<object>): Record<string, unknown> {
+  if (!(err instanceof Error)) return { message: String(err) };
+
+  const seen = _seen ?? new WeakSet();
+  if (seen.has(err)) return { name: err.name, message: err.message, circular: true };
+  seen.add(err);
+
+  const out: Record<string, unknown> = {};
+
+  for (const k of Object.getOwnPropertyNames(err)) {
+    if (k.startsWith("_")) continue;
+
+    let val: unknown;
+    try {
+      val = (err as any)[k];
+    } catch {
+      val = "[Unreadable]";
+    }
+
+    if (typeof val === "bigint") val = `${val}n`;
+
+    // FUTURE: check for circular references within val
+    out[String(k)] = val instanceof Error ? serializeError(val, seen) : val;
+  }
+
+  return out;
+}
+
 const buildLogObject = (level: LogLevel, args: unknown[]): Record<string, unknown> => {
   if (args.length === 0) return {};
 
   const [first, second] = args;
 
   let obj: Record<string, unknown> | undefined;
-  let err: unknown;
+  let err: Record<string, unknown> | undefined;
   let msg: string | undefined;
 
   if (first instanceof Error) {
-    err = first;
+    err = serializeError(first);
   } else if (isRecord(first)) {
     if (first["err"] !== undefined) {
-      err = first["err"];
+      err = serializeError(first["err"]);
       delete first["err"];
     }
     obj = first;
@@ -48,14 +74,14 @@ const buildLogObject = (level: LogLevel, args: unknown[]): Record<string, unknow
   }
 
   if (err && msg === undefined) {
-    msg = err instanceof Error ? err.message : String(err);
+    msg = err["message"] as string;
   }
 
   return {
     level,
     time: Date.now(),
     ...(msg ? { msg } : {}),
-    ...(err ? { err: err instanceof Error ? serializeError(err) : err } : {}),
+    ...(err ? { err } : {}),
     ...obj,
   };
 };

package/src/middleware/utils.ts

@@ -23,6 +23,7 @@ export function calculateReasoningBudgetFromEffort(
       percentage = 0.8;
       break;
     case "xhigh":
+    case "max":
       percentage = 0.95;
       break;
     default:

package/src/models/amazon/middleware.ts

@@ -34,6 +34,7 @@ function mapNovaEffort(effort: ChatCompletionsReasoningEffort) {
       return "medium";
     case "high":
     case "xhigh":
+    case "max":
       return "high";
   }
 }