@hebo-ai/gateway 0.9.2 → 0.9.3

package/dist/errors/ai-sdk.js

@@ -0,0 +1,52 @@
+import { AISDKError, APICallError, DownloadError, EmptyResponseBodyError, InvalidArgumentError, InvalidDataContentError, InvalidMessageRoleError, InvalidPromptError, InvalidResponseDataError, InvalidStreamPartError, InvalidToolApprovalError, InvalidToolInputError, JSONParseError, LoadAPIKeyError, LoadSettingError, MessageConversionError, MissingToolResultsError, NoContentGeneratedError, NoImageGeneratedError, NoObjectGeneratedError, NoOutputGeneratedError, NoSpeechGeneratedError, NoSuchModelError, NoSuchProviderError, NoSuchToolError, NoTranscriptGeneratedError, NoVideoGeneratedError, RetryError, ToolCallNotFoundForApprovalError, ToolCallRepairError, TooManyEmbeddingValuesForCallError, TypeValidationError, UIMessageStreamError, UnsupportedModelVersionError, UnsupportedFunctionalityError, } from "ai";
+import { GatewayError } from "./gateway";
+import { STATUS_CODE } from "./utils";
+export const normalizeAiSdkError = (error) => {
+    if (APICallError.isInstance(error)) {
+        const status = error.statusCode ?? (error.isRetryable ? 502 : 422);
+        const code = `UPSTREAM_${STATUS_CODE(status)}`;
+        return new GatewayError(error, status, code);
+    }
+    if (JSONParseError.isInstance(error) ||
+        InvalidResponseDataError.isInstance(error) ||
+        TypeValidationError.isInstance(error) ||
+        EmptyResponseBodyError.isInstance(error) ||
+        NoContentGeneratedError.isInstance(error) ||
+        NoOutputGeneratedError.isInstance(error) ||
+        InvalidStreamPartError.isInstance(error) ||
+        UIMessageStreamError.isInstance(error) ||
+        RetryError.isInstance(error) ||
+        DownloadError.isInstance(error) ||
+        ToolCallRepairError.isInstance(error) ||
+        NoImageGeneratedError.isInstance(error) ||
+        NoObjectGeneratedError.isInstance(error) ||
+        NoSpeechGeneratedError.isInstance(error) ||
+        NoTranscriptGeneratedError.isInstance(error) ||
+        NoVideoGeneratedError.isInstance(error)) {
+        return new GatewayError(error, 502, `UPSTREAM_${STATUS_CODE(502)}`);
+    }
+    if (InvalidArgumentError.isInstance(error) ||
+        InvalidPromptError.isInstance(error) ||
+        InvalidMessageRoleError.isInstance(error) ||
+        InvalidDataContentError.isInstance(error) ||
+        MessageConversionError.isInstance(error) ||
+        InvalidToolInputError.isInstance(error) ||
+        InvalidToolApprovalError.isInstance(error) ||
+        ToolCallNotFoundForApprovalError.isInstance(error) ||
+        MissingToolResultsError.isInstance(error) ||
+        NoSuchToolError.isInstance(error) ||
+        UnsupportedModelVersionError.isInstance(error) ||
+        UnsupportedFunctionalityError.isInstance(error) ||
+        TooManyEmbeddingValuesForCallError.isInstance(error) ||
+        NoSuchModelError.isInstance(error) ||
+        NoSuchProviderError.isInstance(error)) {
+        return new GatewayError(error, 422, `UPSTREAM_${STATUS_CODE(422)}`);
+    }
+    if (LoadSettingError.isInstance(error) || LoadAPIKeyError.isInstance(error)) {
+        return new GatewayError(error, 500);
+    }
+    if (AISDKError.isInstance(error)) {
+        return new GatewayError(error, 500);
+    }
+    return undefined;
+};

package/dist/errors/gateway.d.ts

@@ -0,0 +1,5 @@
+export declare class GatewayError extends Error {
+    readonly status: number;
+    readonly code: string;
+    constructor(error: unknown, status: number, code?: string, cause?: unknown);
+}

package/dist/errors/gateway.js

@@ -0,0 +1,13 @@
+import { STATUS_CODE } from "./utils";
+export class GatewayError extends Error {
+    status;
+    code;
+    constructor(error, status, code, cause) {
+        const isError = error instanceof Error;
+        super(isError ? error.message : String(error));
+        this.name = "GatewayError";
+        this.cause = cause ?? (isError ? error : undefined);
+        this.status = status;
+        this.code = code ?? STATUS_CODE(status);
+    }
+}

package/dist/errors/openai.d.ts

@@ -0,0 +1,15 @@
+import * as z from "zod";
+export declare const OpenAIErrorSchema: z.ZodObject<{
+    error: z.ZodObject<{
+        message: z.ZodString;
+        type: z.ZodString;
+        code: z.ZodNullable<z.ZodOptional<z.ZodString>>;
+        param: z.ZodNullable<z.ZodOptional<z.ZodString>>;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export declare class OpenAIError {
+    readonly error: z.infer<typeof OpenAIErrorSchema>["error"];
+    constructor(message: string, type?: string, code?: string, param?: string);
+}
+export declare function toOpenAIError(error: unknown): OpenAIError;
+export declare function toOpenAIErrorResponse(error: unknown, responseInit?: ResponseInit): Response;

package/dist/errors/openai.js

@@ -0,0 +1,40 @@
+import * as z from "zod";
+import { isProduction } from "../utils/env";
+import { resolveRequestId } from "../utils/headers";
+import { toResponse } from "../utils/response";
+import { getErrorMeta, STATUS_CODE } from "./utils";
+export const OpenAIErrorSchema = z.object({
+    error: z.object({
+        message: z.string(),
+        type: z.string(),
+        code: z.string().optional().nullable(),
+        param: z.string().optional().nullable(),
+    }),
+});
+export class OpenAIError {
+    error;
+    constructor(message, type = "server_error", code, param = "") {
+        this.error = { message, type, code: code?.toLowerCase(), param };
+    }
+}
+const mapType = (status) => (status < 500 ? "invalid_request_error" : "server_error");
+const maybeMaskMessage = (meta, requestId) => {
+    // FUTURE: consider masking all upstream errors, also 4xx
+    if (!(isProduction() && meta.status >= 500)) {
+        return meta.message;
+    }
+    // FUTURE: always attach requestId to errors (masked and unmasked)
+    return `${STATUS_CODE(meta.status)} (${requestId ?? "see requestId in response headers"})`;
+};
+export function toOpenAIError(error) {
+    const meta = getErrorMeta(error);
+    return new OpenAIError(maybeMaskMessage(meta), mapType(meta.status), meta.code);
+}
+export function toOpenAIErrorResponse(error, responseInit) {
+    const meta = getErrorMeta(error);
+    return toResponse(new OpenAIError(maybeMaskMessage(meta, resolveRequestId(responseInit)), mapType(meta.status), meta.code), {
+        ...responseInit,
+        status: meta.status,
+        statusText: meta.code,
+    });
+}

package/dist/errors/utils.d.ts

@@ -0,0 +1,24 @@
+export declare const STATUS_CODES: {
+    readonly 400: "BAD_REQUEST";
+    readonly 401: "UNAUTHORIZED";
+    readonly 402: "PAYMENT_REQUIRED";
+    readonly 403: "FORBIDDEN";
+    readonly 404: "NOT_FOUND";
+    readonly 405: "METHOD_NOT_ALLOWED";
+    readonly 409: "CONFLICT";
+    readonly 413: "PAYLOAD_TOO_LARGE";
+    readonly 415: "UNSUPPORTED_MEDIA_TYPE";
+    readonly 422: "UNPROCESSABLE_ENTITY";
+    readonly 429: "TOO_MANY_REQUESTS";
+    readonly 499: "CLIENT_CLOSED_REQUEST";
+    readonly 500: "INTERNAL_SERVER_ERROR";
+    readonly 502: "BAD_GATEWAY";
+    readonly 503: "SERVICE_UNAVAILABLE";
+    readonly 504: "GATEWAY_TIMEOUT";
+};
+export declare const STATUS_CODE: (status: number) => "BAD_REQUEST" | "UNAUTHORIZED" | "PAYMENT_REQUIRED" | "FORBIDDEN" | "NOT_FOUND" | "METHOD_NOT_ALLOWED" | "CONFLICT" | "PAYLOAD_TOO_LARGE" | "UNSUPPORTED_MEDIA_TYPE" | "UNPROCESSABLE_ENTITY" | "TOO_MANY_REQUESTS" | "CLIENT_CLOSED_REQUEST" | "INTERNAL_SERVER_ERROR" | "BAD_GATEWAY" | "SERVICE_UNAVAILABLE" | "GATEWAY_TIMEOUT";
+export declare function getErrorMeta(error: unknown): {
+    status: number;
+    code: string;
+    message: string;
+};

package/dist/errors/utils.js

@@ -0,0 +1,46 @@
+import { normalizeAiSdkError } from "./ai-sdk";
+import { GatewayError } from "./gateway";
+export const STATUS_CODES = {
+    400: "BAD_REQUEST",
+    401: "UNAUTHORIZED",
+    402: "PAYMENT_REQUIRED",
+    403: "FORBIDDEN",
+    404: "NOT_FOUND",
+    405: "METHOD_NOT_ALLOWED",
+    409: "CONFLICT",
+    413: "PAYLOAD_TOO_LARGE",
+    415: "UNSUPPORTED_MEDIA_TYPE",
+    422: "UNPROCESSABLE_ENTITY",
+    429: "TOO_MANY_REQUESTS",
+    499: "CLIENT_CLOSED_REQUEST",
+    500: "INTERNAL_SERVER_ERROR",
+    502: "BAD_GATEWAY",
+    503: "SERVICE_UNAVAILABLE",
+    504: "GATEWAY_TIMEOUT",
+};
+export const STATUS_CODE = (status) => {
+    const label = STATUS_CODES[status];
+    if (label)
+        return label;
+    return status >= 400 && status < 500 ? STATUS_CODES[400] : STATUS_CODES[500];
+};
+// FUTURE: always return a wrapped GatewayError?
+export function getErrorMeta(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    let status;
+    let code;
+    if (error instanceof GatewayError) {
+        ({ status, code } = error);
+    }
+    else {
+        const normalized = normalizeAiSdkError(error);
+        if (normalized) {
+            ({ status, code } = normalized);
+        }
+        else {
+            status = 500;
+            code = STATUS_CODE(status);
+        }
+    }
+    return { status, code, message };
+}

package/dist/gateway.d.ts

@@ -0,0 +1,11 @@
+import type { Endpoint, GatewayConfig } from "./types";
+export declare function gateway(config: GatewayConfig): {
+    handler: (req: Request, state?: Record<string, unknown>) => Promise<Response>;
+    routes: {
+        readonly "/chat/completions": Endpoint;
+        readonly "/embeddings": Endpoint;
+        readonly "/models": Endpoint;
+        readonly "/conversations": Endpoint;
+        readonly "/responses": Endpoint;
+    };
+};

package/dist/gateway.js

@@ -0,0 +1,44 @@
+import { parseConfig } from "./config";
+import { chatCompletions } from "./endpoints/chat-completions/handler";
+import { conversations } from "./endpoints/conversations/handler";
+import { embeddings } from "./endpoints/embeddings/handler";
+import { models } from "./endpoints/models/handler";
+import { responses } from "./endpoints/responses/handler";
+import { GatewayError } from "./errors/gateway";
+import { winterCgHandler } from "./lifecycle";
+import { logger } from "./logger";
+let inflight = 0;
+export function gateway(config) {
+    const basePath = (config.basePath ?? "").replace(/\/+$/, "");
+    const parsedConfig = parseConfig(config);
+    const notFoundHandler = winterCgHandler(() => {
+        throw new GatewayError("Not Found", 404);
+    }, parsedConfig);
+    const routes = {
+        ["/chat/completions"]: chatCompletions(parsedConfig),
+        ["/embeddings"]: embeddings(parsedConfig),
+        ["/models"]: models(parsedConfig),
+        ["/conversations"]: conversations(parsedConfig),
+        ["/responses"]: responses(parsedConfig),
+    };
+    const routeEntries = Object.entries(routes);
+    const handler = (req, state) => {
+        let pathname = new URL(req.url).pathname;
+        if (basePath && pathname.startsWith(basePath)) {
+            pathname = pathname.slice(basePath.length);
+        }
+        logger.info(`[gateway] ${req.method} ${pathname} (${++inflight})`);
+        for (const [route, endpoint] of routeEntries) {
+            if (pathname === route || pathname.startsWith(route + "/")) {
+                try {
+                    return endpoint.handler(req, state);
+                }
+                finally {
+                    inflight--;
+                }
+            }
+        }
+        return notFoundHandler(req, state);
+    };
+    return { handler, routes };
+}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+export * from "./gateway";
+export type * from "./types";
+export * from "./errors/gateway";
+export * from "./errors/openai";
+export * from "./logger";
+export * from "./middleware/common";
+export * from "./middleware/matcher";
+export * from "./models/catalog";
+export * from "./models/types";
+export * from "./providers/registry";
+export * from "./providers/types";

package/dist/index.js

@@ -0,0 +1,10 @@
+export * from "./gateway";
+export * from "./errors/gateway";
+export * from "./errors/openai";
+export * from "./logger";
+export * from "./middleware/common";
+export * from "./middleware/matcher";
+export * from "./models/catalog";
+export * from "./models/types";
+export * from "./providers/registry";
+export * from "./providers/types";

package/dist/lifecycle.d.ts

@@ -0,0 +1,3 @@
+import type { GatewayConfig, GatewayConfigParsed, GatewayContext } from "./types";
+import type { SseFrame } from "./utils/stream";
+export declare const winterCgHandler: (run: (ctx: GatewayContext, cfg: GatewayConfigParsed) => Promise<object | ReadableStream<SseFrame>>, config: GatewayConfig) => (request: Request, state?: Record<string, unknown>) => Promise<Response>;

package/dist/lifecycle.js

@@ -0,0 +1,114 @@
+import { parseConfig } from "./config";
+import { GatewayError } from "./errors/gateway";
+import { toOpenAIErrorResponse } from "./errors/openai";
+import { logger } from "./logger";
+import { getBaggageAttributes } from "./telemetry/baggage";
+import { instrumentFetch } from "./telemetry/fetch";
+import { recordRequestDuration } from "./telemetry/gen-ai";
+import { getRequestAttributes, getResponseAttributes } from "./telemetry/http";
+import { observeV8jsMemoryMetrics } from "./telemetry/memory";
+import { addSpanEvent, setSpanEventsEnabled, setSpanTracer, startSpan } from "./telemetry/span";
+import { resolveOrCreateRequestId } from "./utils/request";
+import { prepareResponseInit, toResponse } from "./utils/response";
+export const winterCgHandler = (run, config) => {
+    const parsedConfig = parseConfig(config);
+    if (parsedConfig.telemetry?.enabled) {
+        setSpanTracer(parsedConfig.telemetry?.tracer);
+        setSpanEventsEnabled(parsedConfig.telemetry?.signals?.hebo);
+        instrumentFetch(parsedConfig.telemetry?.signals?.hebo);
+        observeV8jsMemoryMetrics(parsedConfig.telemetry?.signals?.hebo);
+    }
+    return async (request, state) => {
+        const start = performance.now();
+        const ctx = {
+            request,
+            state: state ?? {},
+            otel: {},
+            providers: parsedConfig.providers,
+            models: parsedConfig.models,
+            requestId: resolveOrCreateRequestId(request),
+        };
+        const span = startSpan(ctx.request.url);
+        span.setAttributes(getBaggageAttributes(ctx.request));
+        if (!span.isExisting) {
+            span.setAttributes(getRequestAttributes(ctx.request, parsedConfig.telemetry?.signals?.http));
+            span.setAttributes({ "http.request.id": ctx.requestId });
+        }
+        const finalize = (status, reason) => {
+            if (ctx.operation) {
+                span.updateName(`${ctx.operation}${ctx.modelId ? ` ${ctx.modelId}` : ""}`);
+            }
+            if (!span.isExisting) {
+                // FUTURE add http.server.request.duration
+                span.setAttributes(getResponseAttributes(ctx.response, parsedConfig.telemetry?.signals?.http));
+            }
+            let realStatus = status;
+            if (ctx.request.signal.aborted)
+                realStatus = 499;
+            else if (status === 200 && ctx.response?.status)
+                realStatus = ctx.response.status;
+            if (realStatus !== 200) {
+                logger[realStatus >= 500 ? "error" : "warn"]({
+                    requestId: ctx.requestId,
+                    err: reason ?? ctx.request.signal.reason,
+                });
+                span.recordError(reason);
+            }
+            span.setAttributes({ "http.response.status_code_effective": realStatus });
+            if (ctx.operation === "chat" ||
+                ctx.operation === "embeddings" ||
+                ctx.operation === "responses") {
+                recordRequestDuration(performance.now() - start, realStatus, ctx, parsedConfig.telemetry?.signals?.gen_ai);
+            }
+            span.finish();
+        };
+        await span.runWithContext(async () => {
+            try {
+                if (parsedConfig.hooks?.onRequest) {
+                    const onRequest = await parsedConfig.hooks.onRequest(ctx);
+                    addSpanEvent("hebo.hooks.on_request.completed");
+                    if (onRequest instanceof Response) {
+                        ctx.response = onRequest;
+                    }
+                }
+                if (!ctx.response) {
+                    ctx.result = (await run(ctx, parsedConfig));
+                    ctx.response = toResponse(ctx.result, prepareResponseInit(ctx.requestId), {
+                        onDone: finalize,
+                    });
+                }
+                if (parsedConfig.hooks?.onResponse) {
+                    const onResponse = await parsedConfig.hooks.onResponse(ctx);
+                    addSpanEvent("hebo.hooks.on_response.completed");
+                    if (onResponse) {
+                        ctx.response = onResponse;
+                    }
+                }
+                // FUTURE: this can leak if onResponse removed wrapper from response.body
+                if (!(ctx.result instanceof ReadableStream)) {
+                    finalize(ctx.response.status);
+                }
+            }
+            catch (error) {
+                if (parsedConfig.hooks?.onError) {
+                    try {
+                        ctx.error = error;
+                        const onError = await parsedConfig.hooks.onError(ctx);
+                        addSpanEvent("hebo.hooks.on_error.completed");
+                        if (onError) {
+                            ctx.response = onError;
+                        }
+                    }
+                    catch {
+                        logger.debug("[lifecycle] onError hook threw");
+                    }
+                }
+                ctx.response ??= toOpenAIErrorResponse(ctx.request.signal.aborted
+                    ? new GatewayError(error ?? ctx.request.signal.reason, 499)
+                    : error, prepareResponseInit(ctx.requestId));
+                finalize(ctx.response.status, error);
+            }
+        });
+        return ctx.response ?? new Response("Internal Server Error", { status: 500 });
+    };
+};

package/dist/logger/default.d.ts

@@ -0,0 +1,4 @@
+import type { LogLevel, Logger } from "./index";
+export declare const createDefaultLogger: (config: {
+    level?: LogLevel;
+}) => Logger;

package/dist/logger/default.js

@@ -0,0 +1,81 @@
+import { isProduction, isTest } from "../utils/env";
+const getDefaultLogLevel = () => isTest() ? "silent" : isProduction() ? "info" : "debug";
+const noop = () => { };
+const LEVEL = {
+    trace: 5,
+    debug: 10,
+    info: 20,
+    warn: 30,
+    error: 40,
+    silent: 50,
+};
+const LEVELS = Object.keys(LEVEL);
+const isRecord = (value) => typeof value === "object" && value !== null && !(value instanceof Error);
+function serializeError(err, _seen) {
+    if (!(err instanceof Error))
+        return { message: String(err) };
+    const seen = _seen ?? new WeakSet();
+    if (seen.has(err))
+        return { name: err.name, message: err.message, circular: true };
+    seen.add(err);
+    const out = {};
+    for (const k of Object.getOwnPropertyNames(err)) {
+        if (k.startsWith("_"))
+            continue;
+        let val;
+        try {
+            val = err[k];
+        }
+        catch {
+            val = "[Unreadable]";
+        }
+        if (typeof val === "bigint")
+            val = `${val}n`;
+        // FUTURE: check for circular references within val
+        out[k] = val instanceof Error ? serializeError(val, seen) : val;
+    }
+    return out;
+}
+const buildLogObject = (level, args) => {
+    const [first, second] = args;
+    let obj;
+    let err;
+    let msg;
+    if (first instanceof Error) {
+        err = serializeError(first);
+    }
+    else if (isRecord(first)) {
+        if (first["err"] !== undefined) {
+            err = serializeError(first["err"]);
+            delete first["err"];
+        }
+        obj = first;
+    }
+    else {
+        msg = first;
+    }
+    if (second !== undefined) {
+        msg = second;
+    }
+    if (err && msg === undefined) {
+        msg = err["message"];
+    }
+    return {
+        level,
+        time: Date.now(),
+        ...(msg ? { msg } : {}),
+        ...(err ? { err } : {}),
+        ...obj,
+    };
+};
+const makeLogFn = (level, write) => (...args) => {
+    write(JSON.stringify(buildLogObject(level, args)));
+};
+export const createDefaultLogger = (config) => {
+    if (config.level === "silent" || getDefaultLogLevel() === "silent") {
+        return { trace: noop, debug: noop, info: noop, warn: noop, error: noop };
+    }
+    const threshold = LEVEL[config.level ?? getDefaultLogLevel()];
+    const enabled = (lvl) => LEVEL[lvl] >= threshold;
+    return Object.fromEntries(LEVELS.map((lvl) => [lvl, enabled(lvl) ? makeLogFn(lvl, console.log) : noop]));
+};

package/dist/logger/index.d.ts

@@ -0,0 +1,11 @@
+export type LogArgs = [msg: string] | [obj: Record<string, unknown>, msg?: string] | [err: Error, msg?: string];
+export type LogFn = (...args: LogArgs) => void;
+export type Logger = Record<"trace" | "debug" | "info" | "warn" | "error", LogFn>;
+export type LogLevel = "trace" | "debug" | "info" | "warn" | "error" | "silent";
+export type LoggerConfig = {
+    level?: LogLevel;
+};
+export declare let logger: Logger;
+export declare const isLogger: (input: Logger | LoggerConfig) => input is Logger;
+export declare function isLoggerDisabled(input?: Logger | LoggerConfig | null): boolean;
+export declare function setLoggerInstance(next: Logger): void;

package/dist/logger/index.js

@@ -0,0 +1,25 @@
+import { isTest } from "../utils/env";
+const KEY = Symbol.for("@hebo/logger");
+const g = globalThis;
+g[KEY] ??= {
+    trace: () => { },
+    debug: () => { },
+    info: () => { },
+    warn: () => { },
+    error: () => { },
+};
+export let logger = g[KEY];
+export const isLogger = (input) => typeof input === "object" && input !== null && "info" in input;
+export function isLoggerDisabled(input) {
+    if (isTest())
+        return true;
+    if (input === null)
+        return true;
+    if (!input || typeof input !== "object" || "info" in input)
+        return false;
+    return input.level === "silent";
+}
+export function setLoggerInstance(next) {
+    g[KEY] = next;
+    logger = g[KEY];
+}

package/dist/middleware/common.d.ts

@@ -0,0 +1,12 @@
+import type { EmbeddingModelMiddleware, LanguageModelMiddleware } from "ai";
+import type { ProviderId } from "../providers/types";
+/**
+ * Converts snake_case params in providerOptions to camelCase
+ * and moves all of them into providerOptions[providerName].
+ * Also snakizes values in providerMetadata for OpenAI compatibility.
+ */
+export declare function forwardLanguageParams(providerName: ProviderId): LanguageModelMiddleware;
+export declare function forwardEmbeddingParams(providerName: ProviderId): EmbeddingModelMiddleware;
+export declare function extractProviderNamespace(id: string): string;
+export declare function forwardParamsMiddleware(provider: string): LanguageModelMiddleware;
+export declare function forwardParamsEmbeddingMiddleware(provider: string): EmbeddingModelMiddleware;