@hebo-ai/gateway 0.5.2 → 0.6.0-rc0

package/dist/endpoints/chat-completions/handler.js

@@ -1,145 +0,0 @@
-import { generateText, Output, streamText, wrapLanguageModel, } from "ai";
-import * as z from "zod/mini";
-import { GatewayError } from "../../errors/gateway";
-import { winterCgHandler } from "../../lifecycle";
-import { logger } from "../../logger";
-import { modelMiddlewareMatcher } from "../../middleware/matcher";
-import { resolveProvider } from "../../providers/registry";
-import { recordRequestDuration, recordTimePerOutputToken, recordTokenUsage, } from "../../telemetry/gen-ai";
-import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
-import { prepareForwardHeaders } from "../../utils/request";
-import { convertToTextCallOptions, toChatCompletions, toChatCompletionsStream } from "./converters";
-import { getChatGeneralAttributes, getChatRequestAttributes, getChatResponseAttributes, } from "./otel";
-import { ChatCompletionsBodySchema } from "./schema";
-export const chatCompletions = (config) => {
-    const hooks = config.hooks;
-    const handler = async (ctx) => {
-        const start = performance.now();
-        ctx.operation = "chat";
-        addSpanEvent("hebo.handler.started");
-        // Guard: enforce HTTP method early.
-        if (!ctx.request || ctx.request.method !== "POST") {
-            throw new GatewayError("Method Not Allowed", 405);
-        }
-        // Parse + validate input.
-        try {
-            ctx.body = await ctx.request.json();
-        }
-        catch {
-            throw new GatewayError("Invalid JSON", 400);
-        }
-        addSpanEvent("hebo.request.deserialized");
-        const parsed = ChatCompletionsBodySchema.safeParse(ctx.body);
-        if (!parsed.success) {
-            // FUTURE: consider adding body shape to metadata
-            throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
-        }
-        ctx.body = parsed.data;
-        addSpanEvent("hebo.request.parsed");
-        if (hooks?.before) {
-            ctx.body = (await hooks.before(ctx)) ?? ctx.body;
-            addSpanEvent("hebo.hooks.before.completed");
-        }
-        // Resolve model + provider (hooks may override defaults).
-        let inputs, stream;
-        ({ model: ctx.modelId, stream, ...inputs } = ctx.body);
-        ctx.resolvedModelId =
-            (await hooks?.resolveModelId?.(ctx)) ?? ctx.modelId;
-        logger.debug(`[chat] resolved ${ctx.modelId} to ${ctx.resolvedModelId}`);
-        addSpanEvent("hebo.model.resolved");
-        const override = await hooks?.resolveProvider?.(ctx);
-        ctx.provider =
-            override ??
-                resolveProvider({
-                    providers: ctx.providers,
-                    models: ctx.models,
-                    modelId: ctx.resolvedModelId,
-                    operation: ctx.operation,
-                });
-        const languageModel = ctx.provider.languageModel(ctx.resolvedModelId);
-        ctx.resolvedProviderId = languageModel.provider;
-        logger.debug(`[chat] using ${languageModel.provider} for ${ctx.resolvedModelId}`);
-        addSpanEvent("hebo.provider.resolved");
-        const genAiSignalLevel = config.telemetry?.signals?.gen_ai;
-        const genAiGeneralAttrs = getChatGeneralAttributes(ctx, genAiSignalLevel);
-        setSpanAttributes(genAiGeneralAttrs);
-        // Convert inputs to AI SDK call options.
-        const textOptions = convertToTextCallOptions(inputs);
-        logger.trace({
-            requestId: ctx.requestId,
-            options: textOptions,
-        }, "[chat] AI SDK options");
-        addSpanEvent("hebo.options.prepared");
-        setSpanAttributes(getChatRequestAttributes(inputs, genAiSignalLevel));
-        // Build middleware chain (model -> forward params -> provider).
-        const languageModelWithMiddleware = wrapLanguageModel({
-            model: languageModel,
-            middleware: modelMiddlewareMatcher.for(ctx.resolvedModelId, languageModel.provider),
-        });
-        // Execute request (streaming vs. non-streaming).
-        if (stream) {
-            addSpanEvent("hebo.ai-sdk.started");
-            const result = streamText({
-                model: languageModelWithMiddleware,
-                headers: prepareForwardHeaders(ctx.request),
-                abortSignal: ctx.request.signal,
-                timeout: {
-                    totalMs: 5 * 60 * 1000,
-                },
-                onAbort: () => {
-                    throw new DOMException("The operation was aborted.", "AbortError");
-                },
-                onError: () => { },
-                onFinish: (res) => {
-                    addSpanEvent("hebo.ai-sdk.completed");
-                    const streamResult = toChatCompletions(res, ctx.resolvedModelId);
-                    addSpanEvent("hebo.result.transformed");
-                    const genAiResponseAttrs = getChatResponseAttributes(streamResult, genAiSignalLevel);
-                    setSpanAttributes(genAiResponseAttrs);
-                    recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-                    recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-                    recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
-                },
-                experimental_include: {
-                    requestBody: false,
-                },
-                includeRawChunks: false,
-                ...textOptions,
-            });
-            ctx.result = toChatCompletionsStream(result, ctx.resolvedModelId);
-            if (hooks?.after) {
-                ctx.result = (await hooks.after(ctx)) ?? ctx.result;
-                addSpanEvent("hebo.hooks.after.completed");
-            }
-            return ctx.result;
-        }
-        addSpanEvent("hebo.ai-sdk.started");
-        const result = await generateText({
-            model: languageModelWithMiddleware,
-            headers: prepareForwardHeaders(ctx.request),
-            abortSignal: ctx.request.signal,
-            timeout: 5 * 60 * 1000,
-            experimental_include: {
-                requestBody: false,
-                responseBody: false,
-            },
-            ...textOptions,
-        });
-        logger.trace({ requestId: ctx.requestId, result }, "[chat] AI SDK result");
-        addSpanEvent("hebo.ai-sdk.completed");
-        // Transform result.
-        ctx.result = toChatCompletions(result, ctx.resolvedModelId);
-        addSpanEvent("hebo.result.transformed");
-        const genAiResponseAttrs = getChatResponseAttributes(ctx.result, genAiSignalLevel);
-        setSpanAttributes(genAiResponseAttrs);
-        recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-        if (hooks?.after) {
-            ctx.result = (await hooks.after(ctx)) ?? ctx.result;
-            addSpanEvent("hebo.hooks.after.completed");
-        }
-        recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-        recordRequestDuration(start, genAiGeneralAttrs, genAiSignalLevel);
-        return ctx.result;
-    };
-    return { handler: winterCgHandler(handler, config) };
-};

package/dist/endpoints/chat-completions/index.d.ts

@@ -1,3 +0,0 @@
-export * from "./converters";
-export * from "./handler";
-export * from "./schema";

package/dist/endpoints/chat-completions/index.js

@@ -1,3 +0,0 @@
-export * from "./converters";
-export * from "./handler";
-export * from "./schema";

package/dist/endpoints/chat-completions/otel.d.ts

@@ -1,6 +0,0 @@
-import type { Attributes } from "@opentelemetry/api";
-import type { ChatCompletions, ChatCompletionsBody } from "./schema";
-import { type GatewayContext, type TelemetrySignalLevel } from "../../types";
-export declare const getChatGeneralAttributes: (ctx: GatewayContext, signalLevel?: TelemetrySignalLevel) => Attributes;
-export declare const getChatRequestAttributes: (inputs: ChatCompletionsBody, signalLevel?: TelemetrySignalLevel) => Attributes;
-export declare const getChatResponseAttributes: (completions: ChatCompletions, signalLevel?: TelemetrySignalLevel) => Attributes;

package/dist/endpoints/chat-completions/otel.js

@@ -1,134 +0,0 @@
-import {} from "../../types";
-const toTextPart = (content) => ({ type: "text", content });
-const toMessageParts = (message) => {
-    if (message.role === "assistant") {
-        const parts = [];
-        if (typeof message.content === "string")
-            parts.push(toTextPart(message.content));
-        if (Array.isArray(message.tool_calls)) {
-            for (const call of message.tool_calls) {
-                parts.push({
-                    type: "tool_call",
-                    id: call.id,
-                    name: call.function.name,
-                    arguments: call.function.arguments,
-                });
-            }
-        }
-        return parts;
-    }
-    if (message.role === "tool") {
-        return [{ type: "tool_call_response", id: message.tool_call_id, content: message.content }];
-    }
-    if (message.role === "user") {
-        const parts = [];
-        if (typeof message.content === "string")
-            parts.push(toTextPart(message.content));
-        if (Array.isArray(message.content)) {
-            for (const part of message.content) {
-                if (part.type === "text") {
-                    parts.push(toTextPart(part.text));
-                }
-                else if (part.type === "image_url") {
-                    parts.push({ type: "image", content: part.image_url.url });
-                }
-                else if (part.type === "input_audio") {
-                    parts.push({
-                        type: "audio",
-                        content: "[REDACTED_BINARY_DATA]",
-                        format: part.input_audio.format,
-                    });
-                }
-                else {
-                    parts.push({
-                        type: "file",
-                        // FUTURE: optionally expose safe metadata without raw binary payloads.
-                        content: part.file.filename ?? "[REDACTED_BINARY_DATA]",
-                        media_type: part.file.media_type,
-                    });
-                }
-            }
-        }
-        return parts;
-    }
-    // FUTURE: remove once Langfuse supports gen_ai.system_instructions
-    if (message.role === "system") {
-        return [toTextPart(message.content)];
-    }
-    return [];
-};
-export const getChatGeneralAttributes = (ctx, signalLevel) => {
-    if (!signalLevel || signalLevel === "off")
-        return {};
-    const requestModel = typeof ctx.body?.model === "string" ? ctx.body.model : ctx.modelId;
-    return {
-        "gen_ai.operation.name": ctx.operation,
-        "gen_ai.request.model": requestModel,
-        "gen_ai.response.model": ctx.resolvedModelId,
-        "gen_ai.provider.name": ctx.resolvedProviderId,
-    };
-};
-export const getChatRequestAttributes = (inputs, signalLevel) => {
-    if (!signalLevel || signalLevel === "off")
-        return {};
-    const attrs = {};
-    if (inputs.seed !== undefined) {
-        Object.assign(attrs, { "gen_ai.request.seed": inputs.seed });
-    }
-    if (signalLevel !== "required") {
-        Object.assign(attrs, {
-            "gen_ai.request.stream": inputs.stream,
-            "gen_ai.request.frequency_penalty": inputs.frequency_penalty,
-            "gen_ai.request.max_tokens": inputs.max_completion_tokens,
-            "gen_ai.request.presence_penalty": inputs.presence_penalty,
-            "gen_ai.request.stop_sequences": inputs.stop
-                ? Array.isArray(inputs.stop)
-                    ? inputs.stop
-                    : [inputs.stop]
-                : undefined,
-            "gen_ai.request.temperature": inputs.temperature,
-            "gen_ai.request.top_p": inputs.top_p,
-        });
-    }
-    if (signalLevel === "full") {
-        Object.assign(attrs, {
-            // FUTURE: move system instructions from messages to here
-            // blocker: https://github.com/langfuse/langfuse/issues/11607
-            // "gen_ai.system_instructions": inputs.messages
-            //   .filter((m) => m.role === "system")
-            //   .map((m) => JSON.stringify(toTextPart(m.content))),
-            "gen_ai.input.messages": inputs.messages
-                //.filter((m) => m.role !== "system")
-                .map((m) => JSON.stringify({ role: m.role, parts: toMessageParts(m) })),
-            "gen_ai.tool.definitions": JSON.stringify(inputs.tools),
-        });
-    }
-    return attrs;
-};
-export const getChatResponseAttributes = (completions, signalLevel) => {
-    if (!signalLevel || signalLevel === "off")
-        return {};
-    const attrs = {
-        "gen_ai.response.id": completions.id,
-    };
-    if (signalLevel !== "required") {
-        Object.assign(attrs, {
-            "gen_ai.response.finish_reasons": completions.choices?.map((c) => c.finish_reason),
-            "gen_ai.usage.total_tokens": completions.usage?.total_tokens,
-            "gen_ai.usage.input_tokens": completions.usage?.prompt_tokens,
-            "gen_ai.usage.cached_tokens": completions.usage?.prompt_tokens_details?.cached_tokens,
-            "gen_ai.usage.output_tokens": completions.usage?.completion_tokens,
-            "gen_ai.usage.reasoning_tokens": completions.usage?.completion_tokens_details?.reasoning_tokens,
-        });
-    }
-    if (signalLevel === "full") {
-        Object.assign(attrs, {
-            "gen_ai.output.messages": completions.choices?.map((c) => JSON.stringify({
-                role: c.message.role,
-                parts: toMessageParts(c.message),
-                finish_reason: c.finish_reason,
-            })),
-        });
-    }
-    return attrs;
-};