@hebo-ai/gateway 0.6.2-rc0 → 0.6.2

package/src/endpoints/chat-completions/handler.test.ts

@@ -1,391 +0,0 @@
-import { simulateReadableStream } from "ai";
-import { MockLanguageModelV3, MockProviderV3 } from "ai/test";
-import { describe, expect, test } from "bun:test";
-
-import { parseResponse, postJson } from "../../../test/helpers/http";
-import { defineModelCatalog } from "../../models/catalog";
-import { chatCompletions } from "./handler";
-
-const baseUrl = "http://localhost/chat/completions";
-
-describe("Chat Completions Handler", () => {
-  const mockLanguageModel = new MockLanguageModelV3({
-    // oxlint-disable-next-line require-await
-    doGenerate: async (options) => {
-      const isStructuredOutput = options.responseFormat?.type === "json";
-      const isToolCall = options.tools && options.tools.length > 0;
-
-      if (isToolCall) {
-        return {
-          finishReason: { unified: "tool-calls", raw: "tool-calls" },
-          usage: {
-            inputTokens: { total: 15, noCache: 15, cacheRead: 20, cacheWrite: 0 },
-            outputTokens: { total: 25, text: 0, reasoning: 10 },
-          },
-          content: [
-            {
-              type: "tool-call",
-              toolCallId: "call_123",
-              toolName: "get_current_weather",
-              input: '{"location":"San Francisco, CA"}',
-            },
-          ],
-          providerMetadata: { provider: { key: "value" } },
-          warnings: [],
-        };
-      }
-
-      if (isStructuredOutput) {
-        return {
-          finishReason: { unified: "stop", raw: "stop" },
-          usage: {
-            inputTokens: { total: 10, noCache: 10, cacheRead: 20, cacheWrite: 0 },
-            outputTokens: { total: 20, text: 20, reasoning: 10 },
-          },
-          content: [
-            {
-              type: "text",
-              text: '{"city":"San Francisco","temp_c":18}',
-            },
-          ],
-          providerMetadata: { provider: { key: "value" } },
-          warnings: [],
-        };
-      }
-
-      return {
-        finishReason: { unified: "stop", raw: "stop" },
-        usage: {
-          inputTokens: { total: 10, noCache: 10, cacheRead: 20, cacheWrite: 0 },
-          outputTokens: { total: 20, text: 20, reasoning: 10 },
-        },
-        content: [
-          {
-            type: "text",
-            text: "Hello from AI",
-          },
-        ],
-        providerMetadata: { provider: { key: "value" } },
-        warnings: [],
-      };
-    },
-    // oxlint-disable-next-line require-await
-    doStream: async () => ({
-      stream: simulateReadableStream({
-        chunks: [
-          { type: "text-start", id: "1" },
-          { type: "text-delta", delta: "Hello", id: "1" },
-          { type: "text-delta", delta: " world", id: "1" },
-          { type: "text-end", id: "1" },
-          {
-            type: "finish",
-            finishReason: { unified: "stop", raw: "stop" },
-            usage: {
-              inputTokens: { total: 5, noCache: 5, cacheRead: 20, cacheWrite: 0 },
-              outputTokens: { total: 5, text: 5, reasoning: 10 },
-            },
-          },
-        ],
-      }),
-    }),
-  });
-
-  const endpoint = chatCompletions({
-    providers: {
-      groq: new MockProviderV3({
-        languageModels: {
-          "openai/gpt-oss-20b": mockLanguageModel,
-        },
-      }),
-    },
-    models: defineModelCatalog({
-      "openai/gpt-oss-20b": {
-        name: "GPT-OSS 20B",
-        modalities: { input: ["text", "file"], output: ["text"] },
-        providers: ["groq"],
-      },
-    }),
-  });
-
-  test("should return 405 for non-POST requests", async () => {
-    const request = new Request(baseUrl, { method: "GET" });
-    const res = await endpoint.handler(request);
-    const data = await parseResponse(res);
-    expect(data).toMatchObject({
-      error: {
-        code: "method_not_allowed",
-        message: "Method Not Allowed",
-        type: "invalid_request_error",
-      },
-    });
-  });
-
-  test("should return 400 for invalid JSON", async () => {
-    const request = new Request(baseUrl, {
-      method: "POST",
-      body: "invalid-json",
-    });
-    const res = await endpoint.handler(request);
-    const data = await parseResponse(res);
-    expect(data).toMatchObject({
-      error: {
-        code: "bad_request",
-        message: "Invalid JSON",
-        type: "invalid_request_error",
-      },
-    });
-  });
-
-  test("should return 400 for validation errors (missing messages)", async () => {
-    const request = postJson(baseUrl, { model: "openai/gpt-oss-20b" });
-    const res = await endpoint.handler(request);
-    const data = await parseResponse(res);
-    expect(data).toMatchObject({
-      error: {
-        code: "bad_request",
-        message: "✖ Invalid input: expected array, received undefined\n  → at messages",
-        type: "invalid_request_error",
-        param: "",
-      },
-    });
-  });
-
-  test("should return 422 for non-existent model", async () => {
-    const request = postJson(baseUrl, {
-      model: "non-existent",
-      messages: [{ role: "user", content: "hi" }],
-    });
-    const res = await endpoint.handler(request);
-    const data = await parseResponse(res);
-    expect(data).toMatchObject({
-      error: {
-        code: "model_not_found",
-        message: "Model 'non-existent' not found in catalog",
-        type: "invalid_request_error",
-      },
-    });
-  });
-
-  test("should generate non-streaming completion successfully", async () => {
-    const request = postJson(baseUrl, {
-      model: "openai/gpt-oss-20b",
-      messages: [{ role: "user", content: "hi" }],
-    });
-    const res = await endpoint.handler(request);
-    const data = await parseResponse(res);
-    expect(data).toEqual({
-      id: expect.stringMatching(/^chatcmpl-/),
-      object: "chat.completion",
-      created: expect.any(Number),
-      model: "openai/gpt-oss-20b",
-      choices: [
-        {
-          index: 0,
-          message: {
-            role: "assistant",
-            content: "Hello from AI",
-          },
-          finish_reason: "stop",
-        },
-      ],
-      usage: {
-        prompt_tokens: 10,
-        completion_tokens: 20,
-        total_tokens: 30,
-        completion_tokens_details: {
-          reasoning_tokens: 10,
-        },
-        prompt_tokens_details: {
-          cached_tokens: 20,
-          cache_write_tokens: 0,
-        },
-      },
-      provider_metadata: { provider: { key: "value" } },
-    });
-  });
-
-  test("should accept input_audio content parts", async () => {
-    const request = postJson(baseUrl, {
-      model: "openai/gpt-oss-20b",
-      messages: [
-        {
-          role: "user",
-          content: [
-            {
-              type: "input_audio",
-              input_audio: {
-                data: "aGVsbG8=",
-                format: "wav",
-              },
-            },
-          ],
-        },
-      ],
-    });
-
-    const res = await endpoint.handler(request);
-    expect(res.status).toBe(200);
-    const data = await parseResponse(res);
-    expect(data.model).toBe("openai/gpt-oss-20b");
-  });
-
-  test("should generate completion with tool calls successfully", async () => {
-    const request = postJson(baseUrl, {
-      model: "openai/gpt-oss-20b",
-      messages: [{ role: "user", content: "What is the weather in SF?" }],
-      tools: [
-        {
-          type: "function",
-          function: {
-            name: "get_current_weather",
-            description: "Get the current weather",
-            parameters: {
-              type: "object",
-              properties: {
-                location: { type: "string" },
-              },
-            },
-          },
-        },
-      ],
-    });
-    const res = await endpoint.handler(request);
-    const data = await parseResponse(res);
-    expect(data).toEqual({
-      id: expect.stringMatching(/^chatcmpl-/),
-      object: "chat.completion",
-      created: expect.any(Number),
-      model: "openai/gpt-oss-20b",
-      choices: [
-        {
-          index: 0,
-          message: {
-            role: "assistant",
-            content: null,
-            tool_calls: [
-              {
-                id: "call_123",
-                type: "function",
-                function: {
-                  name: "get_current_weather",
-                  arguments: '{"location":"San Francisco, CA"}',
-                },
-              },
-            ],
-          },
-          finish_reason: "tool_calls",
-        },
-      ],
-      usage: {
-        prompt_tokens: 15,
-        completion_tokens: 25,
-        total_tokens: 40,
-        completion_tokens_details: {
-          reasoning_tokens: 10,
-        },
-        prompt_tokens_details: {
-          cached_tokens: 20,
-          cache_write_tokens: 0,
-        },
-      },
-      provider_metadata: { provider: { key: "value" } },
-    });
-  });
-
-  test("should generate streaming completion successfully", async () => {
-    const request = postJson(baseUrl, {
-      model: "openai/gpt-oss-20b",
-      messages: [{ role: "user", content: "hi" }],
-      stream: true,
-    });
-
-    const res = await endpoint.handler(request);
-    expect(res.status).toBe(200);
-    expect(res.headers.get("Content-Type")).toBe("text/event-stream");
-
-    const decoder = new TextDecoder();
-    let result = "";
-    for await (const chunk of res.body!) {
-      result += decoder.decode(chunk);
-    }
-
-    expect(result).toContain('data: {"id":"chatcmpl-');
-    expect(result).toContain('"content":"Hello');
-    expect(result).toContain('"content":" world');
-    expect(result).toContain('"finish_reason":"stop');
-    expect(result).toContain("data: [DONE]");
-  });
-
-  test("should accept reasoning and reasoning_effort parameters", async () => {
-    const request = postJson(baseUrl, {
-      model: "openai/gpt-oss-20b",
-      messages: [{ role: "user", content: "hi" }],
-      reasoning: {
-        effort: "high",
-        max_tokens: 1000,
-      },
-      reasoning_effort: "medium",
-    });
-
-    const res = await endpoint.handler(request);
-    expect(res.status).toBe(200);
-    const data = await parseResponse(res);
-    expect(data.model).toBe("openai/gpt-oss-20b");
-  });
-
-  test("should accept max_completion_tokens parameter", async () => {
-    const request = postJson(baseUrl, {
-      model: "openai/gpt-oss-20b",
-      messages: [{ role: "user", content: "hi" }],
-      max_completion_tokens: 100,
-    });
-
-    const res = await endpoint.handler(request);
-    expect(res.status).toBe(200);
-    const data = await parseResponse(res);
-    expect(data.model).toBe("openai/gpt-oss-20b");
-  });
-
-  test("should generate non-streaming structured output", async () => {
-    const request = postJson(baseUrl, {
-      model: "openai/gpt-oss-20b",
-      messages: [{ role: "user", content: "Return weather as JSON" }],
-      response_format: {
-        type: "json_schema",
-        json_schema: {
-          name: "weather",
-          schema: {
-            type: "object",
-            properties: {
-              city: { type: "string" },
-              temp_c: { type: "number" },
-            },
-            required: ["city", "temp_c"],
-            additionalProperties: false,
-          },
-          strict: true,
-        },
-      },
-    });
-
-    const res = await endpoint.handler(request);
-    expect(res.status).toBe(200);
-    const data = await parseResponse(res);
-    expect(data.choices[0].message.content).toBe('{"city":"San Francisco","temp_c":18}');
-  });
-
-  test('should accept response_format type "text"', async () => {
-    const request = postJson(baseUrl, {
-      model: "openai/gpt-oss-20b",
-      messages: [{ role: "user", content: "Say hi" }],
-      response_format: {
-        type: "text",
-      },
-    });
-
-    const res = await endpoint.handler(request);
-    expect(res.status).toBe(200);
-    const data = await parseResponse(res);
-    expect(data.choices[0].message.content).toBe("Hello from AI");
-  });
-});

package/src/endpoints/chat-completions/handler.ts

@@ -1,201 +0,0 @@
-import {
-  generateText,
-  Output,
-  streamText,
-  wrapLanguageModel,
-  type GenerateTextResult,
-  type ToolSet,
-} from "ai";
-import * as z from "zod/mini";
-
-import type {
-  AfterHookContext,
-  BeforeHookContext,
-  GatewayConfig,
-  Endpoint,
-  GatewayContext,
-  ResolveProviderHookContext,
-  ResolveModelHookContext,
-} from "../../types";
-
-import { GatewayError } from "../../errors/gateway";
-import { winterCgHandler } from "../../lifecycle";
-import { logger } from "../../logger";
-import { modelMiddlewareMatcher } from "../../middleware/matcher";
-import { resolveProvider } from "../../providers/registry";
-import {
-  getGenAiGeneralAttributes,
-  recordTimePerOutputToken,
-  recordTokenUsage,
-} from "../../telemetry/gen-ai";
-import { addSpanEvent, setSpanAttributes } from "../../telemetry/span";
-import { prepareForwardHeaders } from "../../utils/request";
-import { convertToTextCallOptions, toChatCompletions, toChatCompletionsStream } from "./converters";
-import { getChatRequestAttributes, getChatResponseAttributes } from "./otel";
-import { ChatCompletionsBodySchema, type ChatCompletionsBody } from "./schema";
-
-export const chatCompletions = (config: GatewayConfig): Endpoint => {
-  const hooks = config.hooks;
-
-  const handler = async (ctx: GatewayContext) => {
-    const start = performance.now();
-    ctx.operation = "chat";
-    addSpanEvent("hebo.handler.started");
-
-    // Guard: enforce HTTP method early.
-    if (!ctx.request || ctx.request.method !== "POST") {
-      throw new GatewayError("Method Not Allowed", 405);
-    }
-
-    // Parse + validate input.
-    try {
-      ctx.body = await ctx.request.json();
-    } catch {
-      throw new GatewayError("Invalid JSON", 400);
-    }
-    logger.trace({ requestId: ctx.requestId, body: ctx.body }, "[chat] ChatCompletionsBody");
-    addSpanEvent("hebo.request.deserialized");
-
-    const parsed = ChatCompletionsBodySchema.safeParse(ctx.body);
-    if (!parsed.success) {
-      // FUTURE: consider adding body shape to metadata
-      throw new GatewayError(z.prettifyError(parsed.error), 400, undefined, parsed.error);
-    }
-    ctx.body = parsed.data;
-    addSpanEvent("hebo.request.parsed");
-
-    if (hooks?.before) {
-      ctx.body =
-        ((await hooks.before(ctx as BeforeHookContext)) as ChatCompletionsBody) ?? ctx.body;
-      addSpanEvent("hebo.hooks.before.completed");
-    }
-
-    // Resolve model + provider (hooks may override defaults).
-    let inputs, stream;
-    ({ model: ctx.modelId, stream, ...inputs } = ctx.body);
-
-    ctx.resolvedModelId =
-      (await hooks?.resolveModelId?.(ctx as ResolveModelHookContext)) ?? ctx.modelId;
-    logger.debug(`[chat] resolved ${ctx.modelId} to ${ctx.resolvedModelId}`);
-    addSpanEvent("hebo.model.resolved");
-
-    const override = await hooks?.resolveProvider?.(ctx as ResolveProviderHookContext);
-    ctx.provider =
-      override ??
-      resolveProvider({
-        providers: ctx.providers,
-        models: ctx.models,
-        modelId: ctx.resolvedModelId,
-        operation: ctx.operation,
-      });
-
-    const languageModel = ctx.provider.languageModel(ctx.resolvedModelId);
-    ctx.resolvedProviderId = languageModel.provider;
-    logger.debug(`[chat] using ${languageModel.provider} for ${ctx.resolvedModelId}`);
-    addSpanEvent("hebo.provider.resolved");
-
-    const genAiSignalLevel = config.telemetry?.signals?.gen_ai;
-    const genAiGeneralAttrs = getGenAiGeneralAttributes(ctx, genAiSignalLevel);
-    setSpanAttributes(genAiGeneralAttrs);
-
-    // Convert inputs to AI SDK call options.
-    const textOptions = convertToTextCallOptions(inputs);
-    logger.trace(
-      {
-        requestId: ctx.requestId,
-        options: textOptions,
-      },
-      "[chat] AI SDK options",
-    );
-    addSpanEvent("hebo.options.prepared");
-    setSpanAttributes(getChatRequestAttributes(ctx.body, genAiSignalLevel));
-
-    // Build middleware chain (model -> forward params -> provider).
-    const languageModelWithMiddleware = wrapLanguageModel({
-      model: languageModel,
-      middleware: modelMiddlewareMatcher.for(ctx.resolvedModelId, languageModel.provider),
-    });
-
-    // Execute request (streaming vs. non-streaming).
-    if (stream) {
-      addSpanEvent("hebo.ai-sdk.started");
-      const result = streamText({
-        model: languageModelWithMiddleware,
-        headers: prepareForwardHeaders(ctx.request),
-        abortSignal: ctx.request.signal,
-        timeout: {
-          totalMs: 5 * 60 * 1000,
-        },
-        onAbort: () => {
-          throw new DOMException("The operation was aborted.", "AbortError");
-        },
-        onError: () => {},
-        onFinish: (res) => {
-          addSpanEvent("hebo.ai-sdk.completed");
-          const streamResult = toChatCompletions(
-            res as unknown as GenerateTextResult<ToolSet, Output.Output>,
-            ctx.resolvedModelId!,
-          );
-          logger.trace(
-            { requestId: ctx.requestId, result: streamResult },
-            "[chat] ChatCompletions",
-          );
-          addSpanEvent("hebo.result.transformed");
-
-          const genAiResponseAttrs = getChatResponseAttributes(streamResult, genAiSignalLevel);
-          setSpanAttributes(genAiResponseAttrs);
-          recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-          recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-        },
-        experimental_include: {
-          requestBody: false,
-        },
-        includeRawChunks: false,
-        ...textOptions,
-      });
-
-      ctx.result = toChatCompletionsStream(result, ctx.resolvedModelId);
-
-      if (hooks?.after) {
-        ctx.result = (await hooks.after(ctx as AfterHookContext)) ?? ctx.result;
-        addSpanEvent("hebo.hooks.after.completed");
-      }
-
-      return ctx.result;
-    }
-
-    addSpanEvent("hebo.ai-sdk.started");
-    const result = await generateText({
-      model: languageModelWithMiddleware,
-      headers: prepareForwardHeaders(ctx.request),
-      abortSignal: ctx.request.signal,
-      timeout: 5 * 60 * 1000,
-      experimental_include: {
-        requestBody: false,
-        responseBody: false,
-      },
-      ...textOptions,
-    });
-    logger.trace({ requestId: ctx.requestId, result }, "[chat] AI SDK result");
-    addSpanEvent("hebo.ai-sdk.completed");
-
-    // Transform result.
-    ctx.result = toChatCompletions(result, ctx.resolvedModelId);
-    logger.trace({ requestId: ctx.requestId, result: ctx.result }, "[chat] ChatCompletions");
-    addSpanEvent("hebo.result.transformed");
-
-    const genAiResponseAttrs = getChatResponseAttributes(ctx.result, genAiSignalLevel);
-    setSpanAttributes(genAiResponseAttrs);
-    recordTokenUsage(genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-
-    if (hooks?.after) {
-      ctx.result = (await hooks.after(ctx as AfterHookContext)) ?? ctx.result;
-      addSpanEvent("hebo.hooks.after.completed");
-    }
-
-    recordTimePerOutputToken(start, genAiResponseAttrs, genAiGeneralAttrs, genAiSignalLevel);
-    return ctx.result;
-  };
-
-  return { handler: winterCgHandler(handler, config) };
-};

package/src/endpoints/chat-completions/index.ts

@@ -1,4 +0,0 @@
-export * from "./converters";
-export * from "./handler";
-export * from "./schema";
-export * from "./otel";