@hebo-ai/gateway 0.8.2 → 0.9.0

package/dist/endpoints/responses/handler.d.ts

@@ -0,0 +1,2 @@
+import type { GatewayConfig, Endpoint } from "../../types";
+export declare const responses: (config: GatewayConfig) => Endpoint;

package/dist/endpoints/responses/handler.js

@@ -0,0 +1,137 @@
+import { generateText, Output, streamText, wrapLanguageModel, } from "ai";
+import * as z from "zod";
+import { GatewayError } from "../../errors/gateway";
+import { winterCgHandler } from "../../lifecycle";
+import { logger } from "../../logger";
+import { modelMiddlewareMatcher } from "../../middleware/matcher";
+import { resolveProvider } from "../../providers/registry";
+import { getGenAiGeneralAttributes, recordTimePerOutputToken, recordTokenUsage, } from "../../telemetry/gen-ai";
+import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
+import { prepareForwardHeaders } from "../../utils/request";
+import { convertToTextCallOptions, toResponses, toResponsesStream } from "./converters";
+import { getResponsesRequestAttributes, getResponsesResponseAttributes } from "./otel";
+import { ResponsesBodySchema } from "./schema";
+export const responses = (config) => {
+    const hooks = config.hooks;
+    const handler = async (ctx, cfg) => {
+        const start = performance.now();
+        ctx.operation = "responses";
+        setSpanAttributes({ "gen_ai.operation.name": ctx.operation });
+        addSpanEvent("hebo.handler.started");
+        if (!ctx.request || ctx.request.method !== "POST") {
+            throw new GatewayError("Method Not Allowed", 405);
+        }
+        try {
+            // oxlint-disable-next-line no-unsafe-assignment
+            ctx.body = await ctx.request.json();
+        }
+        catch {
+            throw new GatewayError("Invalid JSON", 400);
+        }
+        logger.trace({ requestId: ctx.requestId, body: ctx.body }, "[responses] ResponsesBody");
+        addSpanEvent("hebo.request.deserialized");
+        const parsed = ResponsesBodySchema.safeParse(ctx.body);
+        if (!parsed.success) {
+            throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
+        }
+        ctx.body = parsed.data;
+        addSpanEvent("hebo.request.parsed");
+        if (hooks?.before) {
+            ctx.body = (await hooks.before(ctx)) ?? ctx.body;
+            addSpanEvent("hebo.hooks.before.completed");
+        }
+        ctx.modelId = ctx.body.model;
+        ctx.resolvedModelId =
+            (await hooks?.resolveModelId?.(ctx)) ?? ctx.modelId;
+        logger.debug(`[responses] resolved ${ctx.modelId} to ${ctx.resolvedModelId}`);
+        addSpanEvent("hebo.model.resolved");
+        const override = await hooks?.resolveProvider?.(ctx);
+        ctx.provider =
+            override ??
+                resolveProvider({
+                    providers: ctx.providers,
+                    models: ctx.models,
+                    modelId: ctx.resolvedModelId,
+                    operation: ctx.operation,
+                });
+        const languageModel = ctx.provider.languageModel(ctx.resolvedModelId);
+        ctx.resolvedProviderId = languageModel.provider;
+        logger.debug(`[responses] using ${languageModel.provider} for ${ctx.resolvedModelId}`);
+        addSpanEvent("hebo.provider.resolved");
+        const genAiSignalLevel = cfg.telemetry?.signals?.gen_ai;
+        const genAiGeneralAttrs = getGenAiGeneralAttributes(ctx, genAiSignalLevel);
+        setSpanAttributes(genAiGeneralAttrs);
+        const { model: _model, stream, ...inputs } = ctx.body;
+        const textOptions = convertToTextCallOptions(inputs);
+        logger.trace({ requestId: ctx.requestId, options: textOptions }, "[responses] AI SDK options");
+        addSpanEvent("hebo.options.prepared");
+        setSpanAttributes(getResponsesRequestAttributes(ctx.body, genAiSignalLevel));
+        const languageModelWithMiddleware = wrapLanguageModel({
+            model: languageModel,
+            middleware: modelMiddlewareMatcher.for(ctx.resolvedModelId, languageModel.provider),
+        });
+        if (stream) {
+            addSpanEvent("hebo.ai-sdk.started");
+            const result = streamText({
+                model: languageModelWithMiddleware,
+                headers: prepareForwardHeaders(ctx.request),
+                abortSignal: ctx.request.signal,
+                timeout: {
+                    totalMs: ctx.body.service_tier === "flex" ? cfg.timeouts.flex : cfg.timeouts.normal,
+                },
+                onAbort: () => {
+                    throw new DOMException("The operation was aborted.", "AbortError");
+                },
+                onError: () => { },
+                onFinish: (res) => {
+                    addSpanEvent("hebo.ai-sdk.completed");
+                    const streamResult = toResponses(res, ctx.resolvedModelId, ctx.body.metadata);
+                    logger.trace({ requestId: ctx.requestId, result: streamResult }, "[responses] Responses");
+                    addSpanEvent("hebo.result.transformed");
+                    const genAiResponseAttrs = getResponsesResponseAttributes(streamResult, genAiSignalLevel, res.finishReason);
+                    setSpanAttributes(genAiResponseAttrs);
+                    recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+                    recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+                },
+                experimental_include: {
+                    requestBody: false,
+                },
+                includeRawChunks: false,
+                ...textOptions,
+            });
+            ctx.result = toResponsesStream(result, ctx.resolvedModelId, ctx.body.metadata);
+            if (hooks?.after) {
+                ctx.result = (await hooks.after(ctx)) ?? ctx.result;
+                addSpanEvent("hebo.hooks.after.completed");
+            }
+            return ctx.result;
+        }
+        addSpanEvent("hebo.ai-sdk.started");
+        const result = await generateText({
+            model: languageModelWithMiddleware,
+            headers: prepareForwardHeaders(ctx.request),
+            abortSignal: ctx.request.signal,
+            timeout: ctx.body.service_tier === "flex" ? cfg.timeouts.flex : cfg.timeouts.normal,
+            experimental_include: {
+                requestBody: false,
+                responseBody: false,
+            },
+            ...textOptions,
+        });
+        logger.trace({ requestId: ctx.requestId, result }, "[responses] AI SDK result");
+        addSpanEvent("hebo.ai-sdk.completed");
+        ctx.result = toResponses(result, ctx.resolvedModelId, ctx.body.metadata);
+        logger.trace({ requestId: ctx.requestId, result: ctx.result }, "[responses] Responses");
+        addSpanEvent("hebo.result.transformed");
+        const genAiResponseAttrs = getResponsesResponseAttributes(ctx.result, genAiSignalLevel, result.finishReason);
+        setSpanAttributes(genAiResponseAttrs);
+        recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+        if (hooks?.after) {
+            ctx.result = (await hooks.after(ctx)) ?? ctx.result;
+            addSpanEvent("hebo.hooks.after.completed");
+        }
+        recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
+        return ctx.result;
+    };
+    return { handler: winterCgHandler(handler, config) };
+};

package/dist/endpoints/responses/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./converters";
+export * from "./handler";
+export * from "./schema";
+export * from "./otel";

package/dist/endpoints/responses/index.js

@@ -0,0 +1,4 @@
+export * from "./converters";
+export * from "./handler";
+export * from "./schema";
+export * from "./otel";

package/dist/endpoints/responses/otel.d.ts

@@ -0,0 +1,6 @@
+import type { Attributes } from "@opentelemetry/api";
+import type { FinishReason } from "ai";
+import type { Responses, ResponsesBody } from "./schema";
+import { type TelemetrySignalLevel } from "../../types";
+export declare const getResponsesRequestAttributes: (body: ResponsesBody, signalLevel?: TelemetrySignalLevel) => Attributes;
+export declare const getResponsesResponseAttributes: (responses: Responses, signalLevel?: TelemetrySignalLevel, finishReason?: FinishReason) => Attributes;

package/dist/endpoints/responses/otel.js

@@ -0,0 +1,221 @@
+import {} from "../../types";
+import { parseDataUrl } from "../../utils/url";
+const toBlobPart = (modality, mimeType) => {
+    const part = {
+        type: "blob",
+        modality,
+        content: "[REDACTED_BINARY_DATA]",
+    };
+    if (mimeType)
+        part["mime_type"] = mimeType;
+    return part;
+};
+const toInputParts = (content) => {
+    if (typeof content === "string")
+        return [{ type: "text", content }];
+    const parts = [];
+    for (const part of content) {
+        switch (part.type) {
+            case "input_text":
+                parts.push({ type: "text", content: part.text });
+                break;
+            case "input_image": {
+                const url = part.image_url;
+                if (url && url.slice(0, 5).toLowerCase() === "data:") {
+                    const { mimeType } = parseDataUrl(url);
+                    parts.push(toBlobPart("image", mimeType || undefined));
+                }
+                else if (url) {
+                    parts.push({ type: "uri", modality: "image", uri: url });
+                }
+                else if (part.file_id) {
+                    parts.push({ type: "blob", modality: "image", content: `file_id:${part.file_id}` });
+                }
+                break;
+            }
+            case "input_audio":
+                parts.push(toBlobPart("audio", `audio/${part.input_audio.format}`));
+                break;
+            case "input_file": {
+                if (part.file_data) {
+                    parts.push(toBlobPart("file"));
+                }
+                else if (part.file_url) {
+                    parts.push({ type: "uri", modality: "file", uri: part.file_url });
+                }
+                else if (part.file_id) {
+                    parts.push({ type: "blob", modality: "file", content: `file_id:${part.file_id}` });
+                }
+                break;
+            }
+        }
+    }
+    return parts;
+};
+const toOutputTextParts = (content) => {
+    if (typeof content === "string") {
+        return [{ type: "text", content }];
+    }
+    return content.map((part) => ({ type: "text", content: part.text }));
+};
+const toItemParts = (item) => {
+    switch (item.type) {
+        case "message":
+            return toMessageParts(item);
+        case "function_call":
+            return [
+                {
+                    type: "tool_call",
+                    id: item.call_id,
+                    name: item.name,
+                    arguments: item.arguments,
+                },
+            ];
+        case "function_call_output":
+            return [
+                {
+                    type: "tool_call_response",
+                    id: item.call_id,
+                    response: typeof item.output === "string"
+                        ? item.output
+                        : // FUTURE: Use toInputParts() to preserve multimodal content once telemetry
+                            // backends (like Langfuse) have a standard representation for multimodal
+                            // tool responses. Currently collapsed to text-only for compatibility.
+                            item.output.map((p) => (p.type === "input_text" ? p.text : "")).join(""),
+                },
+            ];
+        case "reasoning": {
+            const parts = [];
+            for (const s of item.summary) {
+                parts.push({ type: "reasoning", content: s.text });
+            }
+            if (item.content) {
+                for (const c of item.content) {
+                    parts.push({ type: "reasoning", content: c.text });
+                }
+            }
+            if (item.encrypted_content) {
+                parts.push({ type: "reasoning", content: "[ENCRYPTED_REASONING]" });
+            }
+            return parts;
+        }
+    }
+};
+const toMessageParts = (item) => {
+    switch (item.role) {
+        case "assistant":
+            return toOutputTextParts(item.content);
+        case "user":
+        case "developer":
+        case "system":
+            // FUTURE: remove once Langfuse supports gen_ai.system_instructions
+            // https://github.com/langfuse/langfuse/issues/11607
+            return toInputParts(item.content);
+        default:
+            return [];
+    }
+};
+export const getResponsesRequestAttributes = (body, signalLevel) => {
+    if (!signalLevel || signalLevel === "off")
+        return {};
+    const attrs = {};
+    if (signalLevel !== "required") {
+        Object.assign(attrs, {
+            "gen_ai.request.stream": body.stream,
+            "gen_ai.request.service_tier": body.service_tier,
+            "gen_ai.request.frequency_penalty": body.frequency_penalty,
+            "gen_ai.request.max_tokens": body.max_output_tokens,
+            "gen_ai.request.presence_penalty": body.presence_penalty,
+            "gen_ai.request.temperature": body.temperature,
+            "gen_ai.request.top_p": body.top_p,
+            // FUTURE: Support text.verbosity configuration
+        });
+        if (body.metadata) {
+            for (const key in body.metadata) {
+                attrs[`gen_ai.request.metadata.${key}`] = body.metadata[key];
+            }
+        }
+    }
+    if (signalLevel === "full") {
+        const inputMessages = [];
+        if (body.instructions) {
+            // FUTURE: move system instructions from messages to here
+            // blocker: https://github.com/langfuse/langfuse/issues/11607
+            inputMessages.push(JSON.stringify({
+                role: "system",
+                parts: [{ type: "text", content: body.instructions }],
+            }));
+        }
+        if (typeof body.input === "string") {
+            inputMessages.push(JSON.stringify({
+                role: "user",
+                parts: [{ type: "text", content: body.input }],
+            }));
+        }
+        else if (Array.isArray(body.input)) {
+            for (const item of body.input) {
+                if (item.type === "message") {
+                    inputMessages.push(JSON.stringify({
+                        role: item.role,
+                        parts: toItemParts(item),
+                    }));
+                }
+                else {
+                    inputMessages.push(JSON.stringify({
+                        type: item.type,
+                        parts: toItemParts(item),
+                    }));
+                }
+            }
+        }
+        Object.assign(attrs, {
+            "gen_ai.input.messages": inputMessages,
+            "gen_ai.tool.definitions": body.tools?.map((toolDef) => JSON.stringify(toolDef)),
+        });
+    }
+    return attrs;
+};
+export const getResponsesResponseAttributes = (responses, signalLevel, finishReason) => {
+    if (!signalLevel || signalLevel === "off")
+        return {};
+    const attrs = {
+        "gen_ai.response.id": responses.id,
+    };
+    if (signalLevel !== "required") {
+        Object.assign(attrs, {
+            "gen_ai.response.finish_reasons": finishReason ? [finishReason] : [responses.status],
+            "gen_ai.response.service_tier": responses.service_tier,
+            "gen_ai.usage.total_tokens": responses.usage?.total_tokens,
+            "gen_ai.usage.input_tokens": responses.usage?.input_tokens,
+            "gen_ai.usage.cache_read.input_tokens": responses.usage?.input_tokens_details?.cached_tokens,
+            "gen_ai.usage.output_tokens": responses.usage?.output_tokens,
+            "gen_ai.usage.reasoning.output_tokens": responses.usage?.output_tokens_details?.reasoning_tokens,
+        });
+    }
+    if (signalLevel === "full") {
+        Object.assign(attrs, {
+            "gen_ai.output.messages": responses.output?.map((item) => {
+                const base = {
+                    type: item.type,
+                    status: item.status,
+                    parts: [],
+                };
+                if (item.type === "message") {
+                    base.role = item.role;
+                    base.parts = item.content.map((c) => ({ type: "text", content: c.text }));
+                }
+                else if (item.type === "function_call") {
+                    base.name = item.name;
+                    base.arguments = item.arguments;
+                }
+                else if (item.type === "reasoning") {
+                    // Casting needed because ResponsesOutputItem and ResponsesInputItem share
+                    // the ResponsesReasoningItem definition.
+                    base.parts = toItemParts(item);
+                }
+                return JSON.stringify(base);
+            }),
+        });
+    }
+    return attrs;
+};