@hebo-ai/gateway 0.3.0-rc.1 → 0.3.0-rc.2

package/dist/config.d.ts

@@ -0,0 +1,2 @@
+import { type GatewayConfig, type GatewayConfigParsed } from "./types";
+export declare const parseConfig: (config: GatewayConfig) => GatewayConfigParsed;

package/dist/config.js

@@ -0,0 +1,61 @@
+import { isLogger, logger, setLoggerInstance } from "./logger";
+import { createDefaultLogger } from "./logger/default";
+import { kParsed } from "./types";
+export const parseConfig = (config) => {
+    // If it has been parsed before, just return
+    if (kParsed in config)
+        return config;
+    const providers = config.providers ?? {};
+    const parsedProviders = {};
+    const models = config.models ?? {};
+    // Set the global logger instance
+    if (config.logger === undefined) {
+        setLoggerInstance(createDefaultLogger({}));
+    }
+    else if (config.logger !== null) {
+        setLoggerInstance(isLogger(config.logger) ? config.logger : createDefaultLogger(config.logger));
+        logger.info(isLogger(config.logger)
+            ? `[logger] custom logger configured`
+            : `[logger] logger configured: level=${config.logger.level}`);
+    }
+    // Strip providers that are not configured
+    for (const id in providers) {
+        const provider = providers[id];
+        if (provider === undefined) {
+            logger.warn(`[config] ${id} provider removed (undefined)`);
+            continue;
+        }
+        parsedProviders[id] = provider;
+    }
+    if (Object.keys(parsedProviders).length === 0) {
+        throw new Error("No providers configured (config.providers is empty)");
+    }
+    // Strip providers that are not configured from models
+    const parsedModels = {};
+    const warnings = new Set();
+    for (const id in models) {
+        const model = models[id];
+        const kept = [];
+        for (const p of model.providers) {
+            if (p in parsedProviders)
+                kept.push(p);
+            else
+                warnings.add(p);
+        }
+        if (kept.length > 0)
+            parsedModels[id] = { ...model, providers: kept };
+    }
+    for (const warning of warnings) {
+        logger.warn(`[config] ${warning} provider removed (not configured)`);
+    }
+    if (Object.keys(parsedModels).length === 0) {
+        throw new Error("No models configured (config.models is empty)");
+    }
+    return {
+        ...config,
+        logger: config.logger,
+        providers: parsedProviders,
+        models: parsedModels,
+        [kParsed]: true,
+    };
+};

package/dist/endpoints/chat-completions/converters.d.ts

@@ -0,0 +1,36 @@
+import type { SharedV3ProviderOptions, SharedV3ProviderMetadata } from "@ai-sdk/provider";
+import type { GenerateTextResult, StreamTextResult, FinishReason, ToolChoice, ToolSet, ModelMessage, UserContent, LanguageModelUsage, Output, TextStreamPart, AssistantModelMessage, ToolModelMessage, UserModelMessage } from "ai";
+import type { ChatCompletionsToolCall, ChatCompletionsTool, ChatCompletionsToolChoice, ChatCompletionsContentPart, ChatCompletionsMessage, ChatCompletionsUserMessage, ChatCompletionsAssistantMessage, ChatCompletionsToolMessage, ChatCompletionsFinishReason, ChatCompletionsUsage, ChatCompletionsInputs, ChatCompletions, ChatCompletionsChunk } from "./schema";
+import { OpenAIError } from "../../errors/openai";
+export type TextCallOptions = {
+    messages: ModelMessage[];
+    tools?: ToolSet;
+    toolChoice?: ToolChoice<ToolSet>;
+    temperature?: number;
+    maxOutputTokens?: number;
+    frequencyPenalty?: number;
+    presencePenalty?: number;
+    seed?: number;
+    stopSequences?: string[];
+    topP?: number;
+    providerOptions: SharedV3ProviderOptions;
+};
+export declare function convertToTextCallOptions(params: ChatCompletionsInputs): TextCallOptions;
+export declare function convertToModelMessages(messages: ChatCompletionsMessage[]): ModelMessage[];
+export declare function fromChatCompletionsUserMessage(message: ChatCompletionsUserMessage): UserModelMessage;
+export declare function fromChatCompletionsAssistantMessage(message: ChatCompletionsAssistantMessage): AssistantModelMessage;
+export declare function fromChatCompletionsToolResultMessage(message: ChatCompletionsAssistantMessage, toolById: Map<string, ChatCompletionsToolMessage>): ToolModelMessage | undefined;
+export declare function fromChatCompletionsContent(content: ChatCompletionsContentPart[]): UserContent;
+export declare const convertToToolSet: (tools: ChatCompletionsTool[] | undefined) => ToolSet | undefined;
+export declare const convertToToolChoice: (toolChoice: ChatCompletionsToolChoice | undefined) => ToolChoice<ToolSet> | undefined;
+export declare function toChatCompletions(result: GenerateTextResult<ToolSet, Output.Output>, model: string): ChatCompletions;
+export declare function toChatCompletionsResponse(result: GenerateTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
+export declare function toChatCompletionsStream(result: StreamTextResult<ToolSet, Output.Output>, model: string): ReadableStream<Uint8Array>;
+export declare function toChatCompletionsStreamResponse(result: StreamTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
+export declare class ChatCompletionsStream extends TransformStream<TextStreamPart<ToolSet>, ChatCompletionsChunk | OpenAIError> {
+    constructor(model: string);
+}
+export declare const toChatCompletionsAssistantMessage: (result: GenerateTextResult<ToolSet, Output.Output>) => ChatCompletionsAssistantMessage;
+export declare function toChatCompletionsUsage(usage: LanguageModelUsage): ChatCompletionsUsage;
+export declare function toChatCompletionsToolCall(id: string, name: string, args: unknown, providerMetadata?: SharedV3ProviderMetadata): ChatCompletionsToolCall;
+export declare const toChatCompletionsFinishReason: (finishReason: FinishReason) => ChatCompletionsFinishReason;

package/dist/endpoints/chat-completions/converters.js

@@ -0,0 +1,373 @@
+import { convertBase64ToUint8Array } from "@ai-sdk/provider-utils";
+import { jsonSchema, JsonToSseTransformStream, tool } from "ai";
+import { GatewayError } from "../../errors/gateway";
+import { OpenAIError, toOpenAIError } from "../../errors/openai";
+import { toResponse } from "../../utils/response";
+// --- Request Flow ---
+export function convertToTextCallOptions(params) {
+    const { messages, tools, tool_choice, temperature, max_tokens, max_completion_tokens, reasoning_effort, reasoning, frequency_penalty, presence_penalty, seed, stop, top_p, ...rest } = params;
+    Object.assign(rest, parseReasoningOptions(reasoning_effort, reasoning));
+    return {
+        messages: convertToModelMessages(messages),
+        tools: convertToToolSet(tools),
+        toolChoice: convertToToolChoice(tool_choice),
+        temperature,
+        maxOutputTokens: max_completion_tokens ?? max_tokens,
+        frequencyPenalty: frequency_penalty,
+        presencePenalty: presence_penalty,
+        seed,
+        stopSequences: stop ? (Array.isArray(stop) ? stop : [stop]) : undefined,
+        topP: top_p,
+        providerOptions: {
+            unknown: rest,
+        },
+    };
+}
+export function convertToModelMessages(messages) {
+    const modelMessages = [];
+    const toolById = indexToolMessages(messages);
+    for (const message of messages) {
+        if (message.role === "tool")
+            continue;
+        if (message.role === "system") {
+            modelMessages.push(message);
+            continue;
+        }
+        if (message.role === "user") {
+            modelMessages.push(fromChatCompletionsUserMessage(message));
+            continue;
+        }
+        modelMessages.push(fromChatCompletionsAssistantMessage(message));
+        const toolResult = fromChatCompletionsToolResultMessage(message, toolById);
+        if (toolResult)
+            modelMessages.push(toolResult);
+    }
+    return modelMessages;
+}
+function indexToolMessages(messages) {
+    const map = new Map();
+    for (const m of messages) {
+        if (m.role === "tool")
+            map.set(m.tool_call_id, m);
+    }
+    return map;
+}
+export function fromChatCompletionsUserMessage(message) {
+    return {
+        role: "user",
+        content: Array.isArray(message.content)
+            ? fromChatCompletionsContent(message.content)
+            : message.content,
+    };
+}
+export function fromChatCompletionsAssistantMessage(message) {
+    const { tool_calls, role, content, extra_content } = message;
+    if (!tool_calls?.length) {
+        const out = {
+            role: role,
+            content: content ?? "",
+        };
+        if (extra_content) {
+            out.providerOptions = extra_content;
+        }
+        return out;
+    }
+    return {
+        role: role,
+        content: tool_calls.map((tc) => {
+            const { id, function: fn, extra_content } = tc;
+            const out = {
+                type: "tool-call",
+                toolCallId: id,
+                toolName: fn.name,
+                input: parseToolOutput(fn.arguments).value,
+            };
+            if (extra_content) {
+                out.providerOptions = extra_content;
+            }
+            return out;
+        }),
+    };
+}
+export function fromChatCompletionsToolResultMessage(message, toolById) {
+    const toolCalls = message.tool_calls ?? [];
+    if (toolCalls.length === 0)
+        return undefined;
+    const toolResultParts = [];
+    for (const tc of toolCalls) {
+        const toolMsg = toolById.get(tc.id);
+        if (!toolMsg)
+            continue;
+        toolResultParts.push({
+            type: "tool-result",
+            toolCallId: tc.id,
+            toolName: tc.function.name,
+            output: parseToolOutput(toolMsg.content),
+        });
+    }
+    return toolResultParts.length > 0 ? { role: "tool", content: toolResultParts } : undefined;
+}
+export function fromChatCompletionsContent(content) {
+    return content.map((part) => {
+        if (part.type === "image_url") {
+            const url = part.image_url.url;
+            if (url.startsWith("data:")) {
+                const { mimeType, base64Data } = parseDataUrl(url);
+                return mimeType.startsWith("image/")
+                    ? {
+                        type: "image",
+                        image: convertBase64ToUint8Array(base64Data),
+                        mediaType: mimeType,
+                    }
+                    : {
+                        type: "file",
+                        data: convertBase64ToUint8Array(base64Data),
+                        mediaType: mimeType,
+                    };
+            }
+            return {
+                type: "image",
+                image: new URL(url),
+            };
+        }
+        if (part.type === "file") {
+            let { data, media_type, filename } = part.file;
+            return media_type.startsWith("image/")
+                ? {
+                    type: "image",
+                    image: convertBase64ToUint8Array(data),
+                    mediaType: media_type,
+                }
+                : {
+                    type: "file",
+                    data: convertBase64ToUint8Array(data),
+                    filename,
+                    mediaType: media_type,
+                };
+        }
+        return part;
+    });
+}
+export const convertToToolSet = (tools) => {
+    if (!tools) {
+        return;
+    }
+    const toolSet = {};
+    for (const t of tools) {
+        toolSet[t.function.name] = tool({
+            description: t.function.description,
+            inputSchema: jsonSchema(t.function.parameters),
+        });
+    }
+    return toolSet;
+};
+export const convertToToolChoice = (toolChoice) => {
+    if (!toolChoice) {
+        return undefined;
+    }
+    if (toolChoice === "none" || toolChoice === "auto" || toolChoice === "required") {
+        return toolChoice;
+    }
+    return {
+        type: "tool",
+        toolName: toolChoice.function.name,
+    };
+};
+function parseToolOutput(content) {
+    try {
+        return { type: "json", value: JSON.parse(content) };
+    }
+    catch {
+        return { type: "text", value: content };
+    }
+}
+function parseDataUrl(url) {
+    const commaIndex = url.indexOf(",");
+    if (commaIndex <= "data:".length || commaIndex === url.length - 1) {
+        throw new GatewayError("Invalid data URL: missing metadata or data", 400);
+    }
+    const metadata = url.slice("data:".length, commaIndex);
+    const base64Data = url.slice(commaIndex + 1);
+    const semicolonIndex = metadata.indexOf(";");
+    const mimeType = (semicolonIndex === -1 ? metadata : metadata.slice(0, semicolonIndex)).trim();
+    if (!mimeType) {
+        throw new GatewayError("Invalid data URL: missing MIME type", 400);
+    }
+    return { mimeType, base64Data };
+}
+function parseReasoningOptions(reasoning_effort, reasoning) {
+    const effort = reasoning?.effort ?? reasoning_effort;
+    const max_tokens = reasoning?.max_tokens;
+    if (reasoning?.enabled === false || effort === "none") {
+        return { reasoning: { enabled: false }, reasoning_effort: "none" };
+    }
+    if (!reasoning && effort === undefined)
+        return {};
+    const out = { reasoning: {} };
+    if (effort) {
+        out.reasoning.enabled = true;
+        out.reasoning.effort = effort;
+        out.reasoning_effort = effort;
+    }
+    if (max_tokens) {
+        out.reasoning.enabled = true;
+        out.reasoning.max_tokens = max_tokens;
+    }
+    if (out.reasoning.enabled) {
+        out.reasoning.exclude = reasoning?.exclude;
+    }
+    return out;
+}
+// --- Response Flow ---
+export function toChatCompletions(result, model) {
+    const finish_reason = toChatCompletionsFinishReason(result.finishReason);
+    return {
+        id: "chatcmpl-" + crypto.randomUUID(),
+        object: "chat.completion",
+        created: Math.floor(Date.now() / 1000),
+        model,
+        choices: [
+            {
+                index: 0,
+                message: toChatCompletionsAssistantMessage(result),
+                finish_reason,
+            },
+        ],
+        usage: result.totalUsage ? toChatCompletionsUsage(result.totalUsage) : null,
+        provider_metadata: result.providerMetadata,
+    };
+}
+export function toChatCompletionsResponse(result, model, responseInit) {
+    return toResponse(toChatCompletions(result, model), responseInit);
+}
+export function toChatCompletionsStream(result, model) {
+    return result.fullStream
+        .pipeThrough(new ChatCompletionsStream(model))
+        .pipeThrough(new JsonToSseTransformStream())
+        .pipeThrough(new TextEncoderStream());
+}
+export function toChatCompletionsStreamResponse(result, model, responseInit) {
+    return toResponse(toChatCompletionsStream(result, model), responseInit);
+}
+export class ChatCompletionsStream extends TransformStream {
+    constructor(model) {
+        const streamId = `chatcmpl-${crypto.randomUUID()}`;
+        const creationTime = Math.floor(Date.now() / 1000);
+        let toolCallIndexCounter = 0;
+        const createChunk = (delta, provider_metadata, finish_reason, usage) => {
+            if (provider_metadata) {
+                delta.extra_content = provider_metadata;
+            }
+            return {
+                id: streamId,
+                object: "chat.completion.chunk",
+                created: creationTime,
+                model,
+                choices: [
+                    {
+                        index: 0,
+                        delta,
+                        finish_reason: finish_reason ?? null,
+                    },
+                ],
+                usage: usage ?? null,
+            };
+        };
+        super({
+            transform(part, controller) {
+                switch (part.type) {
+                    case "text-delta": {
+                        controller.enqueue(createChunk({ role: "assistant", content: part.text }, part.providerMetadata));
+                        break;
+                    }
+                    case "reasoning-delta": {
+                        controller.enqueue(createChunk({ reasoning_content: part.text }, part.providerMetadata));
+                        break;
+                    }
+                    case "tool-call": {
+                        const toolCall = toChatCompletionsToolCall(part.toolCallId, part.toolName, part.input, part.providerMetadata);
+                        toolCall.index = toolCallIndexCounter++;
+                        controller.enqueue(createChunk({
+                            tool_calls: [toolCall],
+                        }));
+                        break;
+                    }
+                    case "finish-step": {
+                        controller.enqueue(createChunk({}, part.providerMetadata, toChatCompletionsFinishReason(part.finishReason), toChatCompletionsUsage(part.usage)));
+                        break;
+                    }
+                    case "finish": {
+                        controller.enqueue(createChunk({}, undefined, toChatCompletionsFinishReason(part.finishReason), toChatCompletionsUsage(part.totalUsage)));
+                        break;
+                    }
+                    case "error": {
+                        const error = part.error;
+                        controller.enqueue(toOpenAIError(error));
+                        break;
+                    }
+                }
+            },
+        });
+    }
+}
+export const toChatCompletionsAssistantMessage = (result) => {
+    const message = {
+        role: "assistant",
+        content: null,
+    };
+    if (result.toolCalls && result.toolCalls.length > 0) {
+        message.tool_calls = result.toolCalls.map((toolCall) => toChatCompletionsToolCall(toolCall.toolCallId, toolCall.toolName, toolCall.input, toolCall.providerMetadata));
+    }
+    for (const part of result.content) {
+        if (part.type === "text") {
+            message.content = part.text;
+            if (part.providerMetadata) {
+                message.extra_content = part.providerMetadata;
+            }
+            break;
+        }
+    }
+    if (result.reasoningText) {
+        message.reasoning_content = result.reasoningText;
+    }
+    return message;
+};
+export function toChatCompletionsUsage(usage) {
+    const out = {};
+    const prompt = usage.inputTokens;
+    if (prompt !== undefined)
+        out.prompt_tokens = prompt;
+    const completion = usage.outputTokens;
+    if (completion !== undefined)
+        out.completion_tokens = completion;
+    if (prompt !== undefined || completion !== undefined || usage.totalTokens !== undefined) {
+        out.total_tokens = usage.totalTokens ?? (prompt ?? 0) + (completion ?? 0);
+    }
+    const reasoning = usage.outputTokenDetails?.reasoningTokens;
+    if (reasoning !== undefined)
+        out.completion_tokens_details = { reasoning_tokens: reasoning };
+    const cached = usage.inputTokenDetails?.cacheReadTokens;
+    if (cached !== undefined)
+        out.prompt_tokens_details = { cached_tokens: cached };
+    return out;
+}
+export function toChatCompletionsToolCall(id, name, args, providerMetadata) {
+    const out = {
+        id,
+        type: "function",
+        function: {
+            name,
+            arguments: typeof args === "string" ? args : JSON.stringify(args),
+        },
+    };
+    if (providerMetadata) {
+        out.extra_content = providerMetadata;
+    }
+    return out;
+}
+export const toChatCompletionsFinishReason = (finishReason) => {
+    if (finishReason === "error" || finishReason === "other") {
+        return "stop";
+    }
+    return finishReason.replaceAll("-", "_");
+};

package/dist/endpoints/chat-completions/handler.d.ts

@@ -0,0 +1,2 @@
+import type { GatewayConfig, Endpoint } from "../../types";
+export declare const chatCompletions: (config: GatewayConfig) => Endpoint;

package/dist/endpoints/chat-completions/handler.js

@@ -0,0 +1,99 @@
+import { generateText, streamText, wrapLanguageModel } from "ai";
+import * as z from "zod/mini";
+import { GatewayError } from "../../errors/gateway";
+import { winterCgHandler } from "../../lifecycle";
+import { logger } from "../../logger";
+import { modelMiddlewareMatcher } from "../../middleware/matcher";
+import { resolveProvider } from "../../providers/registry";
+import { prepareForwardHeaders } from "../../utils/request";
+import { convertToTextCallOptions, toChatCompletions, toChatCompletionsStream } from "./converters";
+import { ChatCompletionsBodySchema } from "./schema";
+export const chatCompletions = (config) => {
+    const hooks = config.hooks;
+    const handler = async (ctx) => {
+        // Guard: enforce HTTP method early.
+        if (!ctx.request || ctx.request.method !== "POST") {
+            throw new GatewayError("Method Not Allowed", 405);
+        }
+        // Parse + validate input.
+        let body;
+        try {
+            body = await ctx.request.json();
+        }
+        catch {
+            throw new GatewayError("Invalid JSON", 400);
+        }
+        const parsed = ChatCompletionsBodySchema.safeParse(body);
+        if (!parsed.success) {
+            throw new GatewayError(z.prettifyError(parsed.error), 400);
+        }
+        ctx.body = parsed.data;
+        // Resolve model + provider (hooks may override defaults).
+        let inputs, stream;
+        ({ model: ctx.modelId, stream, ...inputs } = parsed.data);
+        ctx.resolvedModelId =
+            (await hooks?.resolveModelId?.(ctx)) ?? ctx.modelId;
+        logger.debug(`[chat] resolved ${ctx.modelId} to ${ctx.resolvedModelId}`);
+        ctx.operation = "text";
+        const override = await hooks?.resolveProvider?.(ctx);
+        ctx.provider =
+            override ??
+                resolveProvider({
+                    providers: ctx.providers,
+                    models: ctx.models,
+                    modelId: ctx.resolvedModelId,
+                    operation: ctx.operation,
+                });
+        const languageModel = ctx.provider.languageModel(ctx.resolvedModelId);
+        ctx.resolvedProviderId = languageModel.provider;
+        logger.debug(`[chat] using ${languageModel.provider} for ${ctx.resolvedModelId}`);
+        // Convert inputs to AI SDK call options.
+        const textOptions = convertToTextCallOptions(inputs);
+        logger.trace({
+            requestId: ctx.request.headers.get("x-request-id"),
+            options: textOptions,
+        }, "[chat] AI SDK options");
+        // Build middleware chain (model -> forward params -> provider).
+        const languageModelWithMiddleware = wrapLanguageModel({
+            model: languageModel,
+            middleware: modelMiddlewareMatcher.for(ctx.resolvedModelId, languageModel.provider),
+        });
+        // Execute request (streaming vs. non-streaming).
+        if (stream) {
+            const result = streamText({
+                model: languageModelWithMiddleware,
+                headers: prepareForwardHeaders(ctx.request),
+                // No abort signal here, otherwise we can't detect upstream from client cancellations
+                // abortSignal: ctx.request.signal,
+                onError: ({ error }) => {
+                    logger.error(error instanceof Error ? error : new Error(String(error)), {
+                        requestId: ctx.request.headers.get("x-request-id"),
+                    });
+                    throw error;
+                },
+                onAbort: () => {
+                    throw new DOMException("Upstream failed", "AbortError");
+                },
+                experimental_include: {
+                    requestBody: false,
+                },
+                includeRawChunks: false,
+                ...textOptions,
+            });
+            return toChatCompletionsStream(result, ctx.modelId);
+        }
+        const result = await generateText({
+            model: languageModelWithMiddleware,
+            headers: prepareForwardHeaders(ctx.request),
+            abortSignal: ctx.request.signal,
+            experimental_include: {
+                requestBody: false,
+                responseBody: false,
+            },
+            ...textOptions,
+        });
+        logger.trace({ requestId: ctx.request.headers.get("x-request-id"), result }, "[chat] AI SDK result");
+        return toChatCompletions(result, ctx.modelId);
+    };
+    return { handler: winterCgHandler(handler, config) };
+};

package/dist/endpoints/chat-completions/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./converters";
+export * from "./handler";
+export * from "./schema";

package/dist/endpoints/chat-completions/index.js

@@ -0,0 +1,3 @@
+export * from "./converters";
+export * from "./handler";
+export * from "./schema";