@hebo-ai/gateway 0.9.2 → 0.9.4

package/dist/endpoints/chat-completions/handler.js

@@ -0,0 +1,152 @@
+import { generateText, Output, streamText, wrapLanguageModel, } from "ai";
+import * as z from "zod/mini";
+import { GatewayError } from "../../errors/gateway";
+import { winterCgHandler } from "../../lifecycle";
+import { logger } from "../../logger";
+import { modelMiddlewareMatcher } from "../../middleware/matcher";
+import { resolveProvider } from "../../providers/registry";
+import { getGenAiGeneralAttributes, recordTimePerOutputToken, recordTimeToFirstToken, recordTokenUsage, } from "../../telemetry/gen-ai";
+import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
+import { parseRequestBody } from "../../utils/body";
+import { prepareForwardHeaders } from "../../utils/request";
+import { convertToTextCallOptions, toChatCompletions, toChatCompletionsStream } from "./converters";
+import { getChatRequestAttributes, getChatResponseAttributes } from "./otel";
+import { ChatCompletionsBodySchema, } from "./schema";
+export const chatCompletions = (config) => {
+    const hooks = config.hooks;
+    const handler = async (ctx, cfg) => {
+        const start = performance.now();
+        ctx.operation = "chat";
+        setSpanAttributes({ "gen_ai.operation.name": ctx.operation });
+        addSpanEvent("hebo.handler.started");
+        // Guard: enforce HTTP method early.
+        if (!ctx.request || ctx.request.method !== "POST") {
+            throw new GatewayError("Method Not Allowed", 405);
+        }
+        // Parse + validate input (handles Content-Encoding decompression + body size limits).
+        ctx.body = (await parseRequestBody(ctx.request, cfg.maxBodySize));
+        logger.trace({ requestId: ctx.requestId, body: ctx.body }, "[chat] ChatCompletionsBody");
+        addSpanEvent("hebo.request.deserialized");
+        const parsed = ChatCompletionsBodySchema.safeParse(ctx.body);
+        if (!parsed.success) {
+            // FUTURE: consider adding body shape to metadata
+            throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
+        }
+        ctx.body = parsed.data;
+        addSpanEvent("hebo.request.parsed");
+        if (hooks?.before) {
+            ctx.body =
+                (await hooks.before(ctx)) ?? ctx.body;
+            addSpanEvent("hebo.hooks.before.completed");
+        }
+        // Resolve model + provider (hooks may override defaults).
+        ctx.modelId = ctx.body.model;
+        ctx.resolvedModelId =
+            (await hooks?.resolveModelId?.(ctx)) ?? ctx.modelId;
+        logger.debug(`[chat] resolved ${ctx.modelId} to ${ctx.resolvedModelId}`);
+        addSpanEvent("hebo.model.resolved");
+        const override = await hooks?.resolveProvider?.(ctx);
+        ctx.provider =
+            override ??
+                resolveProvider({
+                    providers: ctx.providers,
+                    models: ctx.models,
+                    modelId: ctx.resolvedModelId,
+                    operation: ctx.operation,
+                });
+        const languageModel = ctx.provider.languageModel(ctx.resolvedModelId);
+        ctx.resolvedProviderId = languageModel.provider;
+        logger.debug(`[chat] using ${languageModel.provider} for ${ctx.resolvedModelId}`);
+        addSpanEvent("hebo.provider.resolved");
+        const genAiSignalLevel = cfg.telemetry?.signals?.gen_ai;
+        const genAiGeneralAttrs = getGenAiGeneralAttributes(ctx, genAiSignalLevel);
+        setSpanAttributes(genAiGeneralAttrs);
+        // Convert inputs to AI SDK call options.
+        const { model: _model, stream, ...inputs } = ctx.body;
+        const textOptions = convertToTextCallOptions(inputs);
+        logger.trace({
+            requestId: ctx.requestId,
+            options: textOptions,
+        }, "[chat] AI SDK options");
+        addSpanEvent("hebo.options.prepared");
+        setSpanAttributes(getChatRequestAttributes(ctx.body, genAiSignalLevel));
+        // Build middleware chain (model -> forward params -> provider).
+        const languageModelWithMiddleware = wrapLanguageModel({
+            model: languageModel,
+            middleware: modelMiddlewareMatcher.for(ctx.resolvedModelId, languageModel.provider),
+        });
+        // Execute request (streaming vs. non-streaming).
+        if (stream) {
+            addSpanEvent("hebo.ai-sdk.started");
+            let ttft = 0;
+            const result = streamText({
+                model: languageModelWithMiddleware,
+                headers: prepareForwardHeaders(ctx.request),
+                abortSignal: ctx.request.signal,
+                timeout: {
+                    totalMs: ctx.body.service_tier === "flex" ? cfg.timeouts.flex : cfg.timeouts.normal,
+                },
+                onAbort: () => {
+                    throw new DOMException("The operation was aborted.", "AbortError");
+                },
+                onError: () => { },
+                onChunk: () => {
+                    if (!ttft) {
+                        ttft = performance.now() - start;
+                        recordTimeToFirstToken(ttft, genAiGeneralAttrs, genAiSignalLevel);
+                    }
+                },
+                onFinish: (res) => {
+                    addSpanEvent("hebo.ai-sdk.completed");
+                    const streamResult = toChatCompletions(res, ctx.resolvedModelId);
+                    logger.trace({ requestId: ctx.requestId, result: streamResult }, "[chat] ChatCompletions");
+                    addSpanEvent("hebo.result.transformed");
+                    const genAiResponseAttrs = getChatResponseAttributes(streamResult, genAiSignalLevel);
+                    setSpanAttributes(genAiResponseAttrs);
+                    recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+                    recordTimePerOutputToken(start, ttft, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+                },
+                experimental_include: {
+                    requestBody: false,
+                },
+                includeRawChunks: false,
+                ...textOptions,
+            });
+            ctx.result = toChatCompletionsStream(result, ctx.resolvedModelId);
+            if (hooks?.after) {
+                ctx.result = (await hooks.after(ctx)) ?? ctx.result;
+                addSpanEvent("hebo.hooks.after.completed");
+            }
+            return ctx.result;
+        }
+        addSpanEvent("hebo.ai-sdk.started");
+        const result = await generateText({
+            model: languageModelWithMiddleware,
+            headers: prepareForwardHeaders(ctx.request),
+            abortSignal: ctx.request.signal,
+            timeout: ctx.body.service_tier === "flex" ? cfg.timeouts.flex : cfg.timeouts.normal,
+            experimental_include: {
+                requestBody: false,
+                responseBody: false,
+            },
+            ...textOptions,
+        });
+        logger.trace({ requestId: ctx.requestId, result }, "[chat] AI SDK result");
+        addSpanEvent("hebo.ai-sdk.completed");
+        recordTimeToFirstToken(performance.now() - start, genAiGeneralAttrs, genAiSignalLevel);
+        // Transform result.
+        ctx.result = toChatCompletions(result, ctx.resolvedModelId);
+        logger.trace({ requestId: ctx.requestId, result: ctx.result }, "[chat] ChatCompletions");
+        addSpanEvent("hebo.result.transformed");
+        const genAiResponseAttrs = getChatResponseAttributes(ctx.result, genAiSignalLevel);
+        setSpanAttributes(genAiResponseAttrs);
+        recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+        if (hooks?.after) {
+            ctx.result = (await hooks.after(ctx)) ?? ctx.result;
+            addSpanEvent("hebo.hooks.after.completed");
+        }
+        recordTimePerOutputToken(start, 0, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+        return ctx.result;
+    };
+    return { handler: winterCgHandler(handler, config) };
+};

package/dist/endpoints/chat-completions/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./converters";
+export * from "./handler";
+export * from "./schema";
+export * from "./otel";

package/dist/endpoints/chat-completions/index.js

@@ -0,0 +1,4 @@
+export * from "./converters";
+export * from "./handler";
+export * from "./schema";
+export * from "./otel";

package/dist/endpoints/chat-completions/otel.d.ts

@@ -0,0 +1,5 @@
+import type { Attributes } from "@opentelemetry/api";
+import { type TelemetrySignalLevel } from "../../types";
+import type { ChatCompletions, ChatCompletionsBody } from "./schema";
+export declare const getChatRequestAttributes: (body: ChatCompletionsBody, signalLevel?: TelemetrySignalLevel) => Attributes;
+export declare const getChatResponseAttributes: (completions: ChatCompletions, signalLevel?: TelemetrySignalLevel) => Attributes;

package/dist/endpoints/chat-completions/otel.js

@@ -0,0 +1,177 @@
+import {} from "../../types";
+import { parseDataUrl } from "../../utils/url";
+const toTextParts = (content) => {
+    if (typeof content === "string") {
+        return [{ type: "text", content }];
+    }
+    const result = [];
+    if (Array.isArray(content)) {
+        for (const part of content) {
+            if (part.type === "text") {
+                result.push({ type: "text", content: part.text });
+            }
+        }
+    }
+    return result;
+};
+const toBlobPart = (modality, mimeType) => {
+    const part = {
+        type: "blob",
+        modality,
+        content: "[REDACTED_BINARY_DATA]",
+    };
+    if (mimeType)
+        part["mime_type"] = mimeType;
+    return part;
+};
+const toToolResponsePart = (id, content) => ({
+    type: "tool_call_response",
+    id,
+    response: typeof content === "string" ? content : content.map((p) => p.text).join(""),
+});
+const toAssistantParts = (message) => {
+    const parts = [];
+    if (typeof message.reasoning === "string") {
+        parts.push({ type: "reasoning", content: message.reasoning });
+    }
+    for (const part of toTextParts(message.content)) {
+        parts.push(part);
+    }
+    if (Array.isArray(message.tool_calls)) {
+        for (const call of message.tool_calls) {
+            parts.push({
+                type: "tool_call",
+                id: call.id,
+                name: call.function.name,
+                arguments: call.function.arguments,
+            });
+        }
+    }
+    return parts;
+};
+const toUserParts = (content) => {
+    if (typeof content === "string")
+        return [{ type: "text", content }];
+    const parts = [];
+    for (const part of content) {
+        switch (part.type) {
+            case "text":
+                parts.push({ type: "text", content: part.text });
+                break;
+            case "image_url": {
+                const url = part.image_url.url;
+                if (url.slice(0, 5).toLowerCase() === "data:") {
+                    const { mimeType } = parseDataUrl(url);
+                    parts.push(toBlobPart("image", mimeType || undefined));
+                }
+                else {
+                    parts.push({ type: "uri", modality: "image", uri: url });
+                }
+                break;
+            }
+            case "input_audio":
+                parts.push(toBlobPart("audio", `audio/${part.input_audio.format}`));
+                break;
+            case "file": {
+                const filePart = toBlobPart("file", part.file.media_type);
+                if (part.file.filename)
+                    filePart["file_name"] = part.file.filename;
+                parts.push(filePart);
+                break;
+            }
+            default:
+                parts.push({ type: part.type, content: "[UNHANDLED_CONTENT_PART]" });
+                break;
+        }
+    }
+    return parts;
+};
+const toMessageParts = (message) => {
+    switch (message.role) {
+        case "assistant":
+            return toAssistantParts(message);
+        case "tool":
+            return [toToolResponsePart(message.tool_call_id, message.content)];
+        case "user":
+            return toUserParts(message.content);
+        // FUTURE: remove once Langfuse supports gen_ai.system_instructions
+        // https://github.com/langfuse/langfuse/issues/11607
+        case "system":
+            return toTextParts(message.content);
+        default:
+            return [{ type: message.role, content: "[UNHANDLED_ROLE]" }];
+    }
+};
+export const getChatRequestAttributes = (body, signalLevel) => {
+    if (!signalLevel || signalLevel === "off")
+        return {};
+    const attrs = {};
+    if (body.seed !== undefined) {
+        Object.assign(attrs, { "gen_ai.request.seed": body.seed });
+    }
+    if (signalLevel !== "required") {
+        Object.assign(attrs, {
+            "gen_ai.request.reasoning.enabled": body.reasoning?.enabled,
+            "gen_ai.request.reasoning.effort": body.reasoning?.effort,
+            "gen_ai.request.reasoning.max_tokens": body.reasoning?.max_tokens,
+            "gen_ai.request.stream": body.stream,
+            "gen_ai.request.frequency_penalty": body.frequency_penalty,
+            "gen_ai.request.max_tokens": body.max_completion_tokens,
+            "gen_ai.request.presence_penalty": body.presence_penalty,
+            "gen_ai.request.stop_sequences": body.stop
+                ? Array.isArray(body.stop)
+                    ? body.stop
+                    : [body.stop]
+                : undefined,
+            "gen_ai.request.temperature": body.temperature,
+            "gen_ai.request.top_p": body.top_p,
+        });
+        if (body.metadata) {
+            for (const key in body.metadata) {
+                attrs[`gen_ai.request.metadata.${key}`] = body.metadata[key];
+            }
+        }
+    }
+    if (signalLevel === "full") {
+        Object.assign(attrs, {
+            // FUTURE: move system instructions from messages to here
+            // blocker: https://github.com/langfuse/langfuse/issues/11607
+            // "gen_ai.system_instructions": inputs.messages
+            //   .filter((m) => m.role === "system")
+            //   .map((m) => JSON.stringify(toTextPart(m.content))),
+            "gen_ai.input.messages": body.messages
+                //.filter((m) => m.role !== "system")
+                .map((m) => JSON.stringify({ role: m.role, parts: toMessageParts(m) })),
+            "gen_ai.tool.definitions": body.tools?.map((toolDefinition) => JSON.stringify(toolDefinition)),
+        });
+    }
+    return attrs;
+};
+export const getChatResponseAttributes = (completions, signalLevel) => {
+    if (!signalLevel || signalLevel === "off")
+        return {};
+    const attrs = {
+        "gen_ai.response.id": completions.id,
+    };
+    if (signalLevel !== "required") {
+        Object.assign(attrs, {
+            "gen_ai.response.finish_reasons": completions.choices?.map((c) => c.finish_reason),
+            "gen_ai.response.service_tier": completions.service_tier,
+            "gen_ai.usage.total_tokens": completions.usage?.total_tokens,
+            "gen_ai.usage.input_tokens": completions.usage?.prompt_tokens,
+            "gen_ai.usage.cache_read.input_tokens": completions.usage?.prompt_tokens_details?.cached_tokens,
+            "gen_ai.usage.output_tokens": completions.usage?.completion_tokens,
+            "gen_ai.usage.reasoning.output_tokens": completions.usage?.completion_tokens_details?.reasoning_tokens,
+        });
+    }
+    if (signalLevel === "full") {
+        Object.assign(attrs, {
+            "gen_ai.output.messages": completions.choices?.map((c) => JSON.stringify({
+                role: c.message.role,
+                parts: toMessageParts(c.message),
+                finish_reason: c.finish_reason,
+            })),
+        });
+    }
+    return attrs;
+};