@hebo-ai/gateway 0.6.2-rc0 → 0.6.2

package/src/providers/registry.ts

@@ -1,165 +0,0 @@
-import type { ProviderV3 } from "@ai-sdk/provider";
-
-import { customProvider } from "ai";
-
-import type { ModelCatalog, ModelId } from "../models/types";
-import type { ProviderRegistry } from "./types";
-
-import { GatewayError } from "../errors/gateway";
-import { logger } from "../logger";
-
-export const resolveProvider = (args: {
-  providers: ProviderRegistry;
-  models: ModelCatalog;
-  modelId: ModelId;
-  operation: "chat" | "embeddings";
-}): ProviderV3 => {
-  const { providers, models, modelId, operation } = args;
-
-  const catalogModel = models[modelId];
-
-  if (!catalogModel) {
-    throw new GatewayError(`Model '${modelId}' not found in catalog`, 422, "MODEL_NOT_FOUND");
-  }
-
-  const modality = operation === "embeddings" ? "embedding" : "text";
-  if (catalogModel.modalities && !catalogModel.modalities.output.includes(modality)) {
-    throw new GatewayError(
-      `Model '${modelId}' does not support '${modality}' output`,
-      422,
-      "MODEL_UNSUPPORTED_OPERATION",
-    );
-  }
-
-  // FUTURE: implement fallback logic [e.g. runtime config invalid]
-  const resolvedProviderId = catalogModel.providers[0];
-
-  if (!resolvedProviderId) {
-    throw new GatewayError(`No providers configured for model '${modelId}'`, 422, "NO_PROVIDERS");
-  }
-
-  const provider = providers[resolvedProviderId];
-  if (!provider) {
-    throw new GatewayError(
-      `Provider '${resolvedProviderId}' not configured`,
-      422,
-      "PROVIDER_NOT_CONFIGURED",
-    );
-  }
-
-  return provider;
-};
-
-export type CanonicalIdsOptions = {
-  mapping?: Partial<Record<ModelId, string>>;
-  options?: {
-    /** @default true */
-    stripNamespace?: boolean;
-    /** @default false */
-    normalizeDelimiters?: boolean | readonly string[];
-    prefix?: string;
-    template?: Record<string, string | undefined>;
-    postfix?: string;
-    /** @default "/" */
-    namespaceSeparator?: "/" | "." | ":";
-  };
-};
-
-export const withCanonicalIds = (
-  provider: ProviderV3,
-  config: CanonicalIdsOptions = {},
-): ProviderV3 => {
-  const {
-    mapping,
-    options: {
-      stripNamespace = true,
-      normalizeDelimiters = false,
-      template,
-      prefix,
-      postfix,
-      namespaceSeparator = "/",
-    } = {},
-  } = config;
-
-  const shouldNormalizeDelimiters = (canonicalId: string) => {
-    if (typeof normalizeDelimiters === "boolean") return normalizeDelimiters;
-    return normalizeDelimiters.some((x) => canonicalId.startsWith(`${x}/`));
-  };
-
-  const normalizeId = (canonicalId: string) => {
-    let out = canonicalId;
-
-    if (shouldNormalizeDelimiters(canonicalId)) {
-      out = out.replaceAll(".", "-");
-    }
-    if (stripNamespace) {
-      out = out.replace(/^[^/]+\//, "");
-    } else if (namespaceSeparator !== "/") {
-      out = out.replace("/", namespaceSeparator);
-    }
-
-    return out;
-  };
-
-  const applyTemplate = (input: string) => {
-    if (!template) return input;
-    return Object.entries(template).reduce((out, [k, v]) => out.replace(`{${k}}`, v ?? ""), input);
-  };
-
-  const applyFallbackAffixes = (v: string) => {
-    let out = prefix && !v.startsWith(prefix) ? `${prefix}${v}` : v;
-    if (postfix && !out.endsWith(postfix)) out = `${out}${postfix}`;
-    return out;
-  };
-
-  const needsFallbackWrap =
-    stripNamespace || normalizeDelimiters || namespaceSeparator !== "/" || !!prefix || !!postfix;
-
-  // FUTURE: use embeddingModel instead of textEmbeddingModel once voyage supports it
-  const languageModel = provider.languageModel;
-  const embeddingModel = provider.textEmbeddingModel!;
-
-  const fallbackProvider = needsFallbackWrap
-    ? ({
-        ...provider,
-        specificationVersion: "v3",
-        languageModel: (id: string) => {
-          const mapped = applyFallbackAffixes(normalizeId(id));
-          logger.debug(`[canonical] mapped ${id} to ${mapped}`);
-          return languageModel(mapped);
-        },
-        embeddingModel: (id: string) => {
-          const mapped = applyFallbackAffixes(normalizeId(id));
-          logger.debug(`[canonical] mapped ${id} to ${mapped}`);
-          return embeddingModel(mapped);
-        },
-      } satisfies ProviderV3)
-    : provider;
-
-  const mapModels = <T>(fn?: (id: string) => T) => {
-    const out = {} as Record<string, T>;
-
-    // Some providers don't have languageModel / embeddingModel
-    if (fn === undefined) return out;
-
-    for (const [k, v] of Object.entries(mapping ?? {})) {
-      if (v === undefined) continue;
-      // This is lazy so that provider is only create once called
-      Object.defineProperty(out, k, {
-        get: () => {
-          const mapped = applyTemplate(v);
-          logger.debug(`[canonical] mapped ${k} to ${mapped}`);
-          return fn(mapped);
-        },
-      });
-    }
-
-    return out;
-  };
-
-  return customProvider({
-    languageModels: mapModels(languageModel),
-    embeddingModels: mapModels(embeddingModel),
-    fallbackProvider,
-  });
-};

package/src/providers/types.ts

@@ -1,20 +0,0 @@
-import { type ProviderV3 } from "@ai-sdk/provider";
-
-export const CANONICAL_PROVIDER_IDS = [
-  "anthropic",
-  "azure",
-  "bedrock",
-  "cohere",
-  "groq",
-  "openai",
-  "vertex",
-  "voyage",
-] as const;
-
-export type CanonicalProviderId = (typeof CANONICAL_PROVIDER_IDS)[number];
-// oxlint-disable-next-line ban-types
-export type ProviderId = CanonicalProviderId | (string & {});
-
-export type ProviderRegistry = {
-  [K in ProviderId]?: ProviderV3;
-};

package/src/providers/vertex/canonical.ts

@@ -1,17 +0,0 @@
-import type { GoogleVertexProvider } from "@ai-sdk/google-vertex";
-
-import type { ModelId } from "../../models/types";
-
-import { withCanonicalIds } from "../registry";
-
-export const withCanonicalIdsForVertex = (
-  provider: GoogleVertexProvider,
-  extraMapping?: Record<ModelId, string>,
-) =>
-  withCanonicalIds(provider, {
-    mapping: extraMapping,
-    options: {
-      stripNamespace: true,
-      normalizeDelimiters: ["anthropic"],
-    },
-  });

package/src/providers/vertex/index.ts

@@ -1 +0,0 @@
-export * from "./canonical";

package/src/providers/voyage/canonical.ts

@@ -1,16 +0,0 @@
-import type { VoyageProvider } from "voyage-ai-provider";
-
-import type { ModelId } from "../../models/types";
-
-import { withCanonicalIds } from "../registry";
-
-export const withCanonicalIdsForVoyage = (
-  provider: VoyageProvider,
-  extraMapping?: Record<ModelId, string>,
-) =>
-  withCanonicalIds(provider, {
-    mapping: extraMapping,
-    options: {
-      stripNamespace: true,
-    },
-  });

package/src/providers/voyage/index.ts

@@ -1 +0,0 @@
-export * from "./canonical";

package/src/telemetry/ai-sdk.ts

@@ -1,46 +0,0 @@
-import type { LogWarningsFunction } from "ai";
-
-import type { TelemetrySignalLevel } from "../types";
-
-import { logger } from "../logger";
-import { addSpanEvent, setSpanAttributes } from "./span";
-
-type GlobalWithAiSdkWarningLogger = typeof globalThis & {
-  AI_SDK_LOG_WARNINGS?: LogWarningsFunction | false;
-};
-
-export const installAiSdkWarningLogger = (genAiSignalLevel?: TelemetrySignalLevel) => {
-  const logWarnings: LogWarningsFunction = ({ warnings, provider, model }) => {
-    if (warnings.length === 0) return;
-
-    for (const warning of warnings) {
-      logger.warn(
-        {
-          provider,
-          model,
-          warning,
-        },
-        `[ai-sdk] ${warning.type}`,
-      );
-    }
-
-    if (!(genAiSignalLevel === "recommended" || genAiSignalLevel === "full")) return;
-
-    setSpanAttributes({
-      "gen_ai.response.warning_count": warnings.length,
-    });
-
-    for (const warning of warnings) {
-      addSpanEvent("gen_ai.warning", {
-        "gen_ai.provider.name": provider,
-        "gen_ai.response.model": model,
-        "gen_ai.warning.type": warning.type,
-        "gen_ai.warning.feature": "feature" in warning ? warning.feature : undefined,
-        "gen_ai.warning.details": "details" in warning ? warning.details : undefined,
-        "gen_ai.warning.message": "message" in warning ? warning.message : undefined,
-      });
-    }
-  };
-
-  (globalThis as GlobalWithAiSdkWarningLogger).AI_SDK_LOG_WARNINGS = logWarnings;
-};

package/src/telemetry/baggage.ts

@@ -1,27 +0,0 @@
-const HEBO_BAGGAGE_PREFIX = "hebo.";
-
-export const getBaggageAttributes = (request?: Request) => {
-  const h = request?.headers.get("baggage");
-  if (!h) return {};
-
-  const attrs: Record<string, string> = {};
-
-  for (const part of h.split(",")) {
-    const [k, v] = part.trim().split("=", 2);
-    if (!k || !v) continue;
-
-    const [rawValue] = v.split(";", 1);
-    if (!rawValue) continue;
-
-    let value = rawValue;
-    try {
-      value = decodeURIComponent(rawValue);
-    } catch {}
-
-    if (k.startsWith(HEBO_BAGGAGE_PREFIX)) {
-      attrs[k.slice(HEBO_BAGGAGE_PREFIX.length)] = value;
-    }
-  }
-
-  return attrs;
-};

package/src/telemetry/fetch.ts

@@ -1,62 +0,0 @@
-import { SpanKind, type Attributes } from "@opentelemetry/api";
-
-import type { TelemetrySignalLevel } from "../types";
-
-import { setSpanAttributes, withSpan } from "./span";
-
-const ORIGINAL_FETCH_KEY = Symbol.for("@hebo/fetch/original-fetch");
-
-type GlobalFetchState = typeof globalThis & {
-  [ORIGINAL_FETCH_KEY]?: typeof fetch;
-};
-
-const g = globalThis as GlobalFetchState;
-let fetchTracingEnabled = false;
-
-const isRequest = (value: unknown): value is Request =>
-  typeof Request !== "undefined" && value instanceof Request;
-
-const getRequestAttributes = (input: RequestInfo | URL, init?: RequestInit): Attributes => {
-  const attrs: Attributes = {
-    "http.request.method": init?.method ?? (isRequest(input) ? input.method : "GET"),
-  };
-
-  if (input instanceof URL) attrs["url.full"] = input.href;
-  else if (typeof input === "string") attrs["url.full"] = input;
-  else if (isRequest(input)) attrs["url.full"] = input.url;
-
-  return attrs;
-};
-
-const shouldTraceFetch = (init?: RequestInit): boolean =>
-  typeof (init?.headers as any)?.["user-agent"] === "string" &&
-  (init!.headers as any)["user-agent"].indexOf("ai-sdk/provider-utils") !== -1;
-
-const otelFetch = (input: RequestInfo | URL, init?: RequestInit) => {
-  const original = g[ORIGINAL_FETCH_KEY]!;
-
-  if (!fetchTracingEnabled) return original(input, init);
-  if (!shouldTraceFetch(init)) return original(input, init);
-
-  return withSpan(
-    "fetch",
-    async () => {
-      const response = await original(input, init);
-      setSpanAttributes({ "http.response.status_code": response.status });
-      return response;
-    },
-    {
-      kind: SpanKind.CLIENT,
-      attributes: getRequestAttributes(input, init),
-    },
-  );
-};
-
-export const instrumentFetch = (level?: TelemetrySignalLevel) => {
-  fetchTracingEnabled = level === "full";
-  if (!fetchTracingEnabled) return;
-  if (g[ORIGINAL_FETCH_KEY]) return;
-
-  g[ORIGINAL_FETCH_KEY] = globalThis.fetch.bind(globalThis);
-  globalThis.fetch = otelFetch as typeof fetch;
-};

package/src/telemetry/gen-ai.ts

@@ -1,113 +0,0 @@
-import { metrics, type Attributes } from "@opentelemetry/api";
-
-import type { GatewayContext, TelemetrySignalLevel } from "../types";
-
-import { STATUS_CODE } from "../errors/utils";
-
-const meter = metrics.getMeter("@hebo/gateway");
-
-const requestDurationHistogram = meter.createHistogram("gen_ai.server.request.duration", {
-  description: "End-to-end gateway request duration",
-  unit: "s",
-  advice: {
-    explicitBucketBoundaries: [
-      0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 240,
-    ],
-  },
-});
-
-const timePerOutputTokenHistogram = meter.createHistogram("gen_ai.server.time_per_output_token", {
-  description: "End-to-end gateway request duration per output token",
-  unit: "s",
-  advice: {
-    explicitBucketBoundaries: [
-      0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 2.5,
-    ],
-  },
-});
-
-const tokenUsageHistogram = meter.createHistogram("gen_ai.client.token.usage", {
-  description: "Token usage reported by upstream model responses",
-  unit: "{token}",
-  advice: {
-    explicitBucketBoundaries: [
-      1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
-      524288, 1048576,
-    ],
-  },
-});
-
-export const getGenAiGeneralAttributes = (
-  ctx: GatewayContext,
-  signalLevel?: TelemetrySignalLevel,
-): Attributes => {
-  if (!signalLevel || signalLevel === "off") return {};
-
-  const requestModel = typeof ctx.body?.model === "string" ? ctx.body.model : ctx.modelId;
-
-  return {
-    "gen_ai.operation.name": ctx.operation,
-    "gen_ai.request.model": requestModel,
-    "gen_ai.response.model": ctx.resolvedModelId,
-    "gen_ai.provider.name": ctx.resolvedProviderId,
-  };
-};
-
-export const recordRequestDuration = (
-  duration: number,
-  status: number,
-  ctx: GatewayContext,
-  signalLevel?: TelemetrySignalLevel,
-) => {
-  if (!signalLevel || signalLevel === "off") return;
-
-  const attrs = getGenAiGeneralAttributes(ctx, signalLevel);
-
-  if (status !== 200) {
-    attrs["error.type"] = `${status} ${STATUS_CODE(status).toLowerCase()}`;
-  }
-
-  requestDurationHistogram.record(duration / 1000, attrs);
-};
-
-// FUTURE: record unsuccessful calls
-export const recordTimePerOutputToken = (
-  start: number,
-  tokenAttrs: Attributes,
-  metricAttrs: Attributes,
-  signalLevel?: TelemetrySignalLevel,
-) => {
-  if (!signalLevel || (signalLevel !== "recommended" && signalLevel !== "full")) return;
-
-  const outputTokens = tokenAttrs["gen_ai.usage.output_tokens"];
-  if (typeof outputTokens !== "number" || outputTokens <= 0) return;
-
-  timePerOutputTokenHistogram.record(
-    (performance.now() - start) / 1000 / outputTokens,
-    metricAttrs,
-  );
-};
-
-// FUTURE: record unsuccessful calls
-export const recordTokenUsage = (
-  tokenAttrs: Attributes,
-  metricAttrs: Attributes,
-  signalLevel?: TelemetrySignalLevel,
-) => {
-  if (!signalLevel || (signalLevel !== "recommended" && signalLevel !== "full")) return;
-
-  const record = (value: unknown, tokenType: string) => {
-    if (typeof value !== "number") return;
-    tokenUsageHistogram.record(
-      value,
-      Object.assign({}, metricAttrs, { "gen_ai.token.type": tokenType }),
-    );
-  };
-
-  record(tokenAttrs["gen_ai.usage.input_tokens"], "input");
-  record(tokenAttrs["gen_ai.usage.output_tokens"], "output");
-
-  // FUTURE: Monitor otel for emerging cached / reasoning tokens standard:
-  // https://github.com/open-telemetry/semantic-conventions/issues/1959
-  // https://github.com/open-telemetry/semantic-conventions/issues/3341
-};

package/src/telemetry/http.ts

@@ -1,62 +0,0 @@
-import { type TelemetrySignalLevel } from "../types";
-
-const headerArr = (h: Headers, k: string) => (h.has(k) ? [h.get(k)!] : undefined);
-
-export const getRequestAttributes = (request: Request, signalLevel?: TelemetrySignalLevel) => {
-  if (!signalLevel || signalLevel === "off") return {};
-
-  let url;
-  try {
-    // FUTURE: reuse URL from lifecycle
-    url = new URL(request.url);
-  } catch {}
-
-  const attrs = {
-    "http.request.method": request.method,
-    "url.full": request.url,
-    "url.path": url?.pathname,
-    "url.scheme": url?.protocol.replace(":", ""),
-    "server.address": url?.hostname,
-    "server.port": url
-      ? url.port
-        ? Number(url.port)
-        : url.protocol === "https:"
-          ? 443
-          : 80
-      : undefined,
-  };
-
-  if (signalLevel !== "required") {
-    Object.assign(attrs, {
-      "user_agent.original": request.headers.get("user-agent") ?? undefined,
-    });
-  }
-
-  if (signalLevel === "full") {
-    Object.assign(attrs, {
-      // FUTURE: "url.query"
-      "http.request.header.content-type": headerArr(request.headers, "content-type"),
-      "http.request.header.content-length": headerArr(request.headers, "content-length"),
-      // FUTURE: "client.address"
-    });
-  }
-
-  return attrs;
-};
-
-export const getResponseAttributes = (response: Response, signalLevel?: TelemetrySignalLevel) => {
-  if (!signalLevel || signalLevel === "off") return {};
-
-  const attrs = {
-    "http.response.status_code": response.status,
-  };
-
-  if (signalLevel === "full") {
-    Object.assign(attrs, {
-      "http.response.header.content-type": [headerArr(response.headers, "content-type")],
-      "http.response.header.content-length": [headerArr(response.headers, "content-length")],
-    });
-  }
-
-  return attrs;
-};

package/src/telemetry/index.ts

@@ -1 +0,0 @@
-export * from "./fetch";

package/src/telemetry/memory.ts

@@ -1,36 +0,0 @@
-import { metrics } from "@opentelemetry/api";
-
-import type { TelemetrySignalLevel } from "../types";
-
-const meter = metrics.getMeter("@hebo/gateway");
-const defaultHeapSpaceAttrs = { "v8js.heap.space.name": "total" } as const;
-
-const heapUsedCounter = meter.createUpDownCounter("v8js.memory.heap.used", {
-  description: "Used bytes in the V8 heap",
-  unit: "By",
-});
-
-const heapSpacePhysicalSizeCounter = meter.createUpDownCounter(
-  "v8js.memory.heap.space.physical_size",
-  {
-    description: "Physical bytes allocated for the V8 heap space",
-    unit: "By",
-  },
-);
-
-const isEnabled = (level?: TelemetrySignalLevel) => level === "recommended" || level === "full";
-
-export const recordV8jsMemory = (level?: TelemetrySignalLevel) => {
-  if (!isEnabled(level)) return;
-
-  let usage;
-  try {
-    usage = globalThis.process?.memoryUsage?.();
-  } catch {
-    return;
-  }
-  if (!usage) return;
-
-  heapUsedCounter.add(usage.heapUsed, defaultHeapSpaceAttrs);
-  heapSpacePhysicalSizeCounter.add(usage.rss, defaultHeapSpaceAttrs);
-};

package/src/telemetry/span.ts

@@ -1,85 +0,0 @@
-import type { Attributes, SpanOptions, Tracer } from "@opentelemetry/api";
-
-import { INVALID_SPAN_CONTEXT, SpanKind, SpanStatusCode, context, trace } from "@opentelemetry/api";
-
-import type { TelemetrySignalLevel } from "../types";
-
-const DEFAULT_TRACER_NAME = "@hebo/gateway";
-
-let spanTracer: Tracer | undefined;
-let spanEventsEnabled = false;
-
-const NOOP_SPAN = {
-  runWithContext: <T>(fn: () => Promise<T> | T) => fn(),
-  recordError: (_error: unknown) => {},
-  finish: () => {},
-  isExisting: true,
-};
-
-export const setSpanTracer = (tracer?: Tracer) => {
-  spanTracer = tracer ?? trace.getTracer(DEFAULT_TRACER_NAME);
-};
-
-export const setSpanEventsEnabled = (level?: TelemetrySignalLevel) => {
-  spanEventsEnabled = level === "recommended" || level === "full";
-};
-
-export const startSpan = (name: string, options?: SpanOptions) => {
-  if (!spanTracer) {
-    return Object.assign(trace.wrapSpanContext(INVALID_SPAN_CONTEXT), NOOP_SPAN);
-  }
-
-  const parentContext = context.active();
-  const activeSpan = trace.getActiveSpan();
-
-  const span = spanTracer.startSpan(
-    name,
-    { kind: activeSpan ? SpanKind.INTERNAL : SpanKind.SERVER, ...options },
-    parentContext,
-  );
-
-  const runWithContext = <T>(fn: () => Promise<T> | T) =>
-    context.with(trace.setSpan(parentContext, span), fn);
-
-  const recordError = (error: unknown) => {
-    const err = error instanceof Error ? error : new Error(String(error));
-    span.recordException(err);
-    span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
-  };
-
-  const finish = () => {
-    span.end();
-  };
-
-  return Object.assign(span, { runWithContext, recordError, finish, isExisting: !!activeSpan });
-};
-
-export const withSpan = async <T>(
-  name: string,
-  run: () => Promise<T> | T,
-  options?: SpanOptions,
-): Promise<T> => {
-  if (!spanTracer) {
-    return await run();
-  }
-
-  const started = startSpan(name, options);
-  try {
-    return await started.runWithContext(run);
-  } catch (error) {
-    started.recordError(error);
-    throw error;
-  } finally {
-    started.finish();
-  }
-};
-
-export const addSpanEvent = (name: string, attributes?: Attributes) => {
-  if (!spanEventsEnabled) return;
-  trace.getActiveSpan()?.addEvent(name, attributes);
-};
-
-export const setSpanAttributes = (attributes?: Attributes) => {
-  if (!attributes) return;
-  trace.getActiveSpan()?.setAttributes(attributes);
-};