@hebo-ai/gateway 0.6.2 → 0.8.0-rc0

package/dist/endpoints/chat-completions/handler.js

@@ -1,149 +0,0 @@
-import { generateText, Output, streamText, wrapLanguageModel, } from "ai";
-import * as z from "zod/mini";
-import { GatewayError } from "../../errors/gateway";
-import { winterCgHandler } from "../../lifecycle";
-import { logger } from "../../logger";
-import { modelMiddlewareMatcher } from "../../middleware/matcher";
-import { resolveProvider } from "../../providers/registry";
-import { getGenAiGeneralAttributes, recordTimePerOutputToken, recordTokenUsage, } from "../../telemetry/gen-ai";
-import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
-import { prepareForwardHeaders } from "../../utils/request";
-import { convertToTextCallOptions, toChatCompletions, toChatCompletionsStream } from "./converters";
-import { getChatRequestAttributes, getChatResponseAttributes } from "./otel";
-import { ChatCompletionsBodySchema } from "./schema";
-export const chatCompletions = (config) => {
-    const hooks = config.hooks;
-    const handler = async (ctx) => {
-        const start = performance.now();
-        ctx.operation = "chat";
-        addSpanEvent("hebo.handler.started");
-        // Guard: enforce HTTP method early.
-        if (!ctx.request || ctx.request.method !== "POST") {
-            throw new GatewayError("Method Not Allowed", 405);
-        }
-        // Parse + validate input.
-        try {
-            // oxlint-disable-next-line no-unsafe-assignment
-            ctx.body = await ctx.request.json();
-        }
-        catch {
-            throw new GatewayError("Invalid JSON", 400);
-        }
-        logger.trace({ requestId: ctx.requestId, body: ctx.body }, "[chat] ChatCompletionsBody");
-        addSpanEvent("hebo.request.deserialized");
-        const parsed = ChatCompletionsBodySchema.safeParse(ctx.body);
-        if (!parsed.success) {
-            // FUTURE: consider adding body shape to metadata
-            throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
-        }
-        ctx.body = parsed.data;
-        addSpanEvent("hebo.request.parsed");
-        if (hooks?.before) {
-            ctx.body =
-                (await hooks.before(ctx)) ?? ctx.body;
-            addSpanEvent("hebo.hooks.before.completed");
-        }
-        // Resolve model + provider (hooks may override defaults).
-        let inputs, stream;
-        ({ model: ctx.modelId, stream, ...inputs } = ctx.body);
-        ctx.resolvedModelId =
-            (await hooks?.resolveModelId?.(ctx)) ?? ctx.modelId;
-        logger.debug(`[chat] resolved ${ctx.modelId} to ${ctx.resolvedModelId}`);
-        addSpanEvent("hebo.model.resolved");
-        const override = await hooks?.resolveProvider?.(ctx);
-        ctx.provider =
-            override ??
-                resolveProvider({
-                    providers: ctx.providers,
-                    models: ctx.models,
-                    modelId: ctx.resolvedModelId,
-                    operation: ctx.operation,
-                });
-        const languageModel = ctx.provider.languageModel(ctx.resolvedModelId);
-        ctx.resolvedProviderId = languageModel.provider;
-        logger.debug(`[chat] using ${languageModel.provider} for ${ctx.resolvedModelId}`);
-        addSpanEvent("hebo.provider.resolved");
-        const genAiSignalLevel = config.telemetry?.signals?.gen_ai;
-        const genAiGeneralAttrs = getGenAiGeneralAttributes(ctx, genAiSignalLevel);
-        setSpanAttributes(genAiGeneralAttrs);
-        // Convert inputs to AI SDK call options.
-        // oxlint-disable-next-line no-unsafe-argument
-        const textOptions = convertToTextCallOptions(inputs);
-        logger.trace({
-            requestId: ctx.requestId,
-            options: textOptions,
-        }, "[chat] AI SDK options");
-        addSpanEvent("hebo.options.prepared");
-        setSpanAttributes(getChatRequestAttributes(ctx.body, genAiSignalLevel));
-        // Build middleware chain (model -> forward params -> provider).
-        const languageModelWithMiddleware = wrapLanguageModel({
-            model: languageModel,
-            middleware: modelMiddlewareMatcher.for(ctx.resolvedModelId, languageModel.provider),
-        });
-        // Execute request (streaming vs. non-streaming).
-        if (stream) {
-            addSpanEvent("hebo.ai-sdk.started");
-            const result = streamText({
-                model: languageModelWithMiddleware,
-                headers: prepareForwardHeaders(ctx.request),
-                abortSignal: ctx.request.signal,
-                timeout: {
-                    totalMs: 5 * 60 * 1000,
-                },
-                onAbort: () => {
-                    throw new DOMException("The operation was aborted.", "AbortError");
-                },
-                onError: () => { },
-                onFinish: (res) => {
-                    addSpanEvent("hebo.ai-sdk.completed");
-                    const streamResult = toChatCompletions(res, ctx.resolvedModelId);
-                    logger.trace({ requestId: ctx.requestId, result: streamResult }, "[chat] ChatCompletions");
-                    addSpanEvent("hebo.result.transformed");
-                    const genAiResponseAttrs = getChatResponseAttributes(streamResult, genAiSignalLevel);
-                    setSpanAttributes(genAiResponseAttrs);
-                    recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-                    recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-                },
-                experimental_include: {
-                    requestBody: false,
-                },
-                includeRawChunks: false,
-                ...textOptions,
-            });
-            ctx.result = toChatCompletionsStream(result, ctx.resolvedModelId);
-            if (hooks?.after) {
-                ctx.result = (await hooks.after(ctx)) ?? ctx.result;
-                addSpanEvent("hebo.hooks.after.completed");
-            }
-            return ctx.result;
-        }
-        addSpanEvent("hebo.ai-sdk.started");
-        const result = await generateText({
-            model: languageModelWithMiddleware,
-            headers: prepareForwardHeaders(ctx.request),
-            abortSignal: ctx.request.signal,
-            timeout: 5 * 60 * 1000,
-            experimental_include: {
-                requestBody: false,
-                responseBody: false,
-            },
-            ...textOptions,
-        });
-        logger.trace({ requestId: ctx.requestId, result }, "[chat] AI SDK result");
-        addSpanEvent("hebo.ai-sdk.completed");
-        // Transform result.
-        ctx.result = toChatCompletions(result, ctx.resolvedModelId);
-        logger.trace({ requestId: ctx.requestId, result: ctx.result }, "[chat] ChatCompletions");
-        addSpanEvent("hebo.result.transformed");
-        const genAiResponseAttrs = getChatResponseAttributes(ctx.result, genAiSignalLevel);
-        setSpanAttributes(genAiResponseAttrs);
-        recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-        if (hooks?.after) {
-            ctx.result = (await hooks.after(ctx)) ?? ctx.result;
-            addSpanEvent("hebo.hooks.after.completed");
-        }
-        recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-        return ctx.result;
-    };
-    return { handler: winterCgHandler(handler, config) };
-};

package/dist/endpoints/chat-completions/index.d.ts

@@ -1,4 +0,0 @@
-export * from "./converters";
-export * from "./handler";
-export * from "./schema";
-export * from "./otel";

package/dist/endpoints/chat-completions/index.js

@@ -1,4 +0,0 @@
-export * from "./converters";
-export * from "./handler";
-export * from "./schema";
-export * from "./otel";

package/dist/endpoints/chat-completions/otel.d.ts

@@ -1,5 +0,0 @@
-import type { Attributes } from "@opentelemetry/api";
-import type { ChatCompletions, ChatCompletionsBody } from "./schema";
-import { type TelemetrySignalLevel } from "../../types";
-export declare const getChatRequestAttributes: (inputs: ChatCompletionsBody, signalLevel?: TelemetrySignalLevel) => Attributes;
-export declare const getChatResponseAttributes: (completions: ChatCompletions, signalLevel?: TelemetrySignalLevel) => Attributes;

package/dist/endpoints/chat-completions/otel.js

@@ -1,171 +0,0 @@
-import {} from "../../types";
-import { parseDataUrl } from "../../utils/url";
-const toTextParts = (content) => {
-    if (typeof content === "string") {
-        return [{ type: "text", content }];
-    }
-    const result = [];
-    if (Array.isArray(content)) {
-        for (const part of content) {
-            if (part.type === "text") {
-                result.push({ type: "text", content: part.text });
-            }
-        }
-    }
-    return result;
-};
-const toBlobPart = (modality, mimeType) => {
-    const part = {
-        type: "blob",
-        modality,
-        content: "[REDACTED_BINARY_DATA]",
-    };
-    if (mimeType)
-        part["mime_type"] = mimeType;
-    return part;
-};
-const toToolResponsePart = (id, content) => ({
-    type: "tool_call_response",
-    id,
-    response: typeof content === "string" ? content : content.map((p) => p.text).join(""),
-});
-const toAssistantParts = (message) => {
-    const parts = [];
-    if (typeof message.reasoning === "string") {
-        parts.push({ type: "reasoning", content: message.reasoning });
-    }
-    for (const part of toTextParts(message.content)) {
-        parts.push(part);
-    }
-    if (Array.isArray(message.tool_calls)) {
-        for (const call of message.tool_calls) {
-            parts.push({
-                type: "tool_call",
-                id: call.id,
-                name: call.function.name,
-                arguments: call.function.arguments,
-            });
-        }
-    }
-    return parts;
-};
-const toUserParts = (content) => {
-    if (typeof content === "string")
-        return [{ type: "text", content }];
-    const parts = [];
-    for (const part of content) {
-        switch (part.type) {
-            case "text":
-                parts.push({ type: "text", content: part.text });
-                break;
-            case "image_url": {
-                const url = part.image_url.url;
-                if (url.slice(0, 5).toLowerCase() === "data:") {
-                    const { mimeType } = parseDataUrl(url);
-                    parts.push(toBlobPart("image", mimeType || undefined));
-                }
-                else {
-                    parts.push({ type: "uri", modality: "image", uri: url });
-                }
-                break;
-            }
-            case "input_audio":
-                parts.push(toBlobPart("audio", `audio/${part.input_audio.format}`));
-                break;
-            case "file": {
-                const filePart = toBlobPart("file", part.file.media_type);
-                if (part.file.filename)
-                    filePart["file_name"] = part.file.filename;
-                parts.push(filePart);
-                break;
-            }
-        }
-    }
-    return parts;
-};
-const toMessageParts = (message) => {
-    switch (message.role) {
-        case "assistant":
-            return toAssistantParts(message);
-        case "tool":
-            return [toToolResponsePart(message.tool_call_id, message.content)];
-        case "user":
-            return toUserParts(message.content);
-        // FUTURE: remove once Langfuse supports gen_ai.system_instructions
-        // https://github.com/langfuse/langfuse/issues/11607
-        case "system":
-            return toTextParts(message.content);
-        default:
-            throw new Error(`Unhandled content part type: ${message.role}`);
-    }
-};
-export const getChatRequestAttributes = (inputs, signalLevel) => {
-    if (!signalLevel || signalLevel === "off")
-        return {};
-    const attrs = {};
-    if (inputs.seed !== undefined) {
-        Object.assign(attrs, { "gen_ai.request.seed": inputs.seed });
-    }
-    if (signalLevel !== "required") {
-        Object.assign(attrs, {
-            // FUTURE: add reasoning info
-            "gen_ai.request.stream": inputs.stream,
-            "gen_ai.request.frequency_penalty": inputs.frequency_penalty,
-            "gen_ai.request.max_tokens": inputs.max_completion_tokens,
-            "gen_ai.request.presence_penalty": inputs.presence_penalty,
-            "gen_ai.request.stop_sequences": inputs.stop
-                ? Array.isArray(inputs.stop)
-                    ? inputs.stop
-                    : [inputs.stop]
-                : undefined,
-            "gen_ai.request.temperature": inputs.temperature,
-            "gen_ai.request.top_p": inputs.top_p,
-        });
-        if (inputs.metadata) {
-            for (const key in inputs.metadata) {
-                attrs[`gen_ai.request.metadata.${key}`] = inputs.metadata[key];
-            }
-        }
-    }
-    if (signalLevel === "full") {
-        Object.assign(attrs, {
-            // FUTURE: move system instructions from messages to here
-            // blocker: https://github.com/langfuse/langfuse/issues/11607
-            // "gen_ai.system_instructions": inputs.messages
-            //   .filter((m) => m.role === "system")
-            //   .map((m) => JSON.stringify(toTextPart(m.content))),
-            "gen_ai.input.messages": inputs.messages
-                //.filter((m) => m.role !== "system")
-                .map((m) => JSON.stringify({ role: m.role, parts: toMessageParts(m) })),
-            "gen_ai.tool.definitions": inputs.tools?.map((toolDefinition) => JSON.stringify(toolDefinition)),
-        });
-    }
-    return attrs;
-};
-export const getChatResponseAttributes = (completions, signalLevel) => {
-    if (!signalLevel || signalLevel === "off")
-        return {};
-    const attrs = {
-        "gen_ai.response.id": completions.id,
-    };
-    if (signalLevel !== "required") {
-        Object.assign(attrs, {
-            "gen_ai.response.finish_reasons": completions.choices?.map((c) => c.finish_reason),
-            "gen_ai.usage.total_tokens": completions.usage?.total_tokens,
-            "gen_ai.usage.input_tokens": completions.usage?.prompt_tokens,
-            "gen_ai.usage.cache_read.input_tokens": completions.usage?.prompt_tokens_details?.cached_tokens,
-            "gen_ai.usage.output_tokens": completions.usage?.completion_tokens,
-            "gen_ai.usage.reasoning.output_tokens": completions.usage?.completion_tokens_details?.reasoning_tokens,
-        });
-    }
-    if (signalLevel === "full") {
-        Object.assign(attrs, {
-            "gen_ai.output.messages": completions.choices?.map((c) => JSON.stringify({
-                role: c.message.role,
-                parts: toMessageParts(c.message),
-                finish_reason: c.finish_reason,
-            })),
-        });
-    }
-    return attrs;
-};