@neutrome/open-ai-router 0.4.2 → 0.5.1

package/src/app/http-utils.ts

@@ -0,0 +1,14 @@
+import type { Context } from "hono";
+import { createOpenAiErrorResponse } from "../router/errors.ts";
+
+export function honoEnv(context: Context): Record<string, unknown> {
+  return (context.env ?? {}) as Record<string, unknown>;
+}
+
+export function unauthorizedResponse(): Response {
+  return createOpenAiErrorResponse(
+    "Authentication required",
+    401,
+    "auth_required",
+  );
+}

package/src/app/model-listing.ts

@@ -0,0 +1,29 @@
+import { canAccessPrincipalModel } from "../admission/access.ts";
+import type { RuntimeAuthenticator } from "../admission/types.ts";
+import { createOpenAiErrorResponse } from "../router/errors.ts";
+import { listConfiguredModels } from "../router/index.ts";
+
+export function createModelListingHandler(options: {
+  runtime: Parameters<typeof listConfiguredModels>[0];
+  auth: RuntimeAuthenticator;
+}) {
+  return async (request: Request, env: Record<string, unknown>) => {
+    const principal = await options.auth.authenticate(request, env);
+    if (!principal) {
+      return createOpenAiErrorResponse(
+        "Authentication required",
+        401,
+        "auth_required",
+      );
+    }
+    const data = listConfiguredModels(options.runtime)
+      .filter((id) => canAccessPrincipalModel(principal, id))
+      .map((id) => ({
+        id,
+        object: "model" as const,
+        created: 0,
+        owned_by: id.split("/")[0] || "unknown",
+      }));
+    return { object: "list" as const, data };
+  };
+}

package/src/app/response-lifecycle.ts

@@ -0,0 +1,66 @@
+import type { Context } from "hono";
+
+export function isEventStreamResponse(response: Response): boolean {
+  const contentType = response.headers.get("content-type") ?? "";
+  return contentType.startsWith("text/event-stream");
+}
+
+export function runBackgroundTask(
+  context: Context,
+  task: () => unknown,
+  label: string,
+): void {
+  const tracked = Promise.resolve()
+    .then(task)
+    .catch((error) => {
+      console.error(`[open-ai-router] ${label} failed:`, error);
+    });
+  let executionContext:
+    | { waitUntil?(promise: Promise<unknown>): void }
+    | undefined;
+  try {
+    executionContext = context.executionCtx as
+      | { waitUntil?(promise: Promise<unknown>): void }
+      | undefined;
+  } catch {
+    executionContext = undefined;
+  }
+  if (executionContext?.waitUntil) {
+    executionContext.waitUntil(tracked);
+    return;
+  }
+  void tracked;
+}
+
+export function finalizeWhenStreamCloses(
+  response: Response,
+  finalize: () => void,
+): Response {
+  if (!response.body) return response;
+  const reader = response.body.getReader();
+  const body = new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      try {
+        const { done, value } = await reader.read();
+        if (done) {
+          finalize();
+          controller.close();
+          return;
+        }
+        controller.enqueue(value);
+      } catch (error) {
+        finalize();
+        controller.error(error);
+      }
+    },
+    async cancel(reason) {
+      finalize();
+      await reader.cancel(reason).catch(() => {});
+    },
+  });
+  return new Response(body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers: new Headers(response.headers),
+  });
+}

package/src/app/types.ts

@@ -0,0 +1,38 @@
+import type { Executor, ProgramTransform } from "@neutrome/lilsdk-ts";
+import type {
+  AuthPrincipal,
+  RuntimeAuthenticator,
+} from "../admission/types.ts";
+import type { ExecutionSummary } from "../telemetry.ts";
+import type { RouterConfig } from "../router/config.ts";
+import type { ExecutionRuntimeOptions } from "../router/execution-types.ts";
+import type { RemoteMcpClientFactory } from "../router/mcp.ts";
+
+export type RouterAppAuth = RuntimeAuthenticator;
+
+export type RouterAppHooks = {
+  beforeRequest?(ctx: {
+    request: Request;
+    model: string | null;
+    principal: AuthPrincipal;
+    env: Record<string, unknown>;
+  }): Promise<Response | void>;
+  onExecutionSummary?(
+    summary: ExecutionSummary,
+    env: Record<string, unknown>,
+  ): void | Promise<void>;
+};
+
+export type RouterAppOptions = {
+  config: RouterConfig;
+  executorImplementations: Readonly<Record<string, Executor>>;
+  transforms?: Readonly<Record<string, ProgramTransform>>;
+  auth: RouterAppAuth;
+  hooks?: RouterAppHooks;
+  fetchImpl?: typeof fetch;
+  observe?: ExecutionRuntimeOptions["observe"];
+  requestIdFactory?: ExecutionRuntimeOptions["requestIdFactory"];
+  executionIdFactory?: ExecutionRuntimeOptions["executionIdFactory"];
+  upstreamTimeoutMs?: number;
+  remoteMcpClientFactory?: RemoteMcpClientFactory;
+};

package/src/index.ts

@@ -1,24 +1,21 @@
 export { createRouterApp } from "./app/factory.ts";
-export type { RouterAppAuth, RouterAppHooks, RouterAppOptions } from "./app/factory.ts";
-export {
-  createTraceCollector,
-  renderProgramAsLil,
-} from "./observer.ts";
 export type {
-  ExecutionTrace,
-  TraceSpan,
-  TraceSpanKind,
-} from "./observer.ts";
-
-export { createApp } from "./example.ts";
-export type { CreateAppOptions } from "./example.ts";
+  RouterAppAuth,
+  RouterAppHooks,
+  RouterAppOptions,
+} from "./app/types.ts";
+export { createRouterTelemetry } from "./telemetry.ts";
+export type {
+  ExecutionSummary,
+  ExecutionSummarySpan,
+  RouterTelemetry,
+} from "./telemetry.ts";
 
 export {
   createFetchProviderInvoker,
   createExecutionRuntime,
   createRouterExecutionRuntime,
   createRouterRuntime,
-  createAuditEvent,
   createInMemoryAuditSink,
   ExecutionError,
   handleAnthropicMessages,
@@ -36,8 +33,6 @@ export {
   executionTargetId,
 } from "./router/index.ts";
 export type {
-  AuditEvent,
-  AuditEventInput,
   CreateRouterOptions,
   ExecutionErrorDetails,
   ExecutionErrorKind,
@@ -93,32 +88,5 @@ export type {
 export { buildUpstreamAuthChain } from "./upstream-auth/registry.ts";
 export { envAuthDriver } from "./upstream-auth/env.ts";
 export { proxyAuthDriver } from "./upstream-auth/proxy.ts";
-export {
-  createRuntimeJwtAuthenticator,
-  DEFAULT_RUNTIME_AUTH_AUDIENCE,
-  DEFAULT_RUNTIME_AUTH_ISSUER,
-  readRuntimeToken,
-  verifyRuntimeJwt,
-} from "./admission/jwt.ts";
 export { canAccessPrincipalModel } from "./admission/access.ts";
-export type {
-  AuthPrincipal,
-  RuntimeAuthenticator,
-} from "./admission/types.ts";
-export type { RuntimeJwtAuthConfig } from "./admission/jwt.ts";
-
-export { cors } from "./app/cors.ts";
-export { healthHandler } from "./app/health.ts";
-export {
-  anthropicMessagesHandler,
-  chatCompletionsHandler,
-  googleGenAIHandler,
-  listModelsHandler,
-  responsesHandler,
-} from "./app/handlers.ts";
-export type {
-  AnthropicMessagesHandlerOptions,
-  ChatCompletionsHandlerOptions,
-  GoogleGenAIHandlerOptions,
-  ResponsesHandlerOptions,
-} from "./app/handlers.ts";
+export type { AuthPrincipal, RuntimeAuthenticator } from "./admission/types.ts";

package/src/otlp.ts

@@ -0,0 +1,155 @@
+export type OtlpSpan = {
+  traceId: string;
+  spanId: string;
+  parentSpanId?: string;
+  name: string;
+  startTimeUnixNano: string;
+  endTimeUnixNano: string;
+  attributes: OtlpAttribute[];
+  status?: { code: number; message?: string };
+};
+
+type Environment = Record<string, unknown>;
+type OtlpAttribute = {
+  key: string;
+  value: { stringValue?: string; intValue?: string; boolValue?: boolean };
+};
+
+export async function exportOtlp(
+  env: Environment,
+  signal: "traces" | "errors",
+  spans: OtlpSpan[],
+): Promise<void> {
+  if (spans.length === 0) return;
+  const endpoint = endpointFor(env, signal);
+  if (!endpoint) return;
+  try {
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        ...headersFor(env, signal),
+      },
+      body: JSON.stringify({
+        resourceSpans: [
+          {
+            resource: {
+              attributes: attributes({
+                "service.name":
+                  stringEnv(env, "OTEL_SERVICE_NAME") ?? "neutrome-router",
+              }),
+            },
+            scopeSpans: [
+              { scope: { name: "@neutrome/open-ai-router" }, spans },
+            ],
+          },
+        ],
+      }),
+    });
+    if (!response.ok) {
+      console.error("[telemetry] OTLP export rejected", {
+        signal,
+        status: response.status,
+        statusText: response.statusText,
+      });
+    }
+  } catch (error) {
+    console.error("[telemetry] OTLP export failed", error);
+  }
+}
+
+export function rootOtlpSpan(input: {
+  traceId: string;
+  spanId: string;
+  startedAt: string;
+  finishedAt: string;
+  error?: string;
+}): OtlpSpan {
+  return {
+    traceId: input.traceId,
+    spanId: input.spanId,
+    name: "router.request",
+    startTimeUnixNano: unixNanos(input.startedAt),
+    endTimeUnixNano: unixNanos(input.finishedAt),
+    attributes: [],
+    ...(input.error ? { status: { code: 2, message: input.error } } : {}),
+  };
+}
+
+export function attributes(values: Record<string, unknown>): OtlpAttribute[] {
+  const result: OtlpAttribute[] = [];
+  for (const [key, value] of Object.entries(values)) {
+    if (typeof value === "string")
+      result.push({ key, value: { stringValue: value } });
+    else if (typeof value === "boolean")
+      result.push({ key, value: { boolValue: value } });
+    else if (typeof value === "number" && Number.isFinite(value))
+      result.push({ key, value: { intValue: String(Math.trunc(value)) } });
+  }
+  return result;
+}
+
+export function unixNanos(timestamp: string): string {
+  return `${Date.parse(timestamp)}000000`;
+}
+export function randomHex(bytes: number): string {
+  return [...crypto.getRandomValues(new Uint8Array(bytes))]
+    .map((byte) => byte.toString(16).padStart(2, "0"))
+    .join("");
+}
+
+function endpointFor(
+  env: Environment,
+  signal: "traces" | "errors",
+): string | undefined {
+  const endpoint = stringEnv(
+    env,
+    signal === "traces"
+      ? "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
+      : "OTEL_EXPORTER_OTLP_ERRORS_ENDPOINT",
+  );
+  if (endpoint) return endpoint;
+  const base = stringEnv(env, "OTEL_EXPORTER_OTLP_ENDPOINT");
+  return base
+    ? new URL(
+        "v1/traces",
+        `${base.endsWith("/") ? base : `${base}/`}`,
+      ).toString()
+    : undefined;
+}
+
+function headersFor(
+  env: Environment,
+  signal: "traces" | "errors",
+): Record<string, string> {
+  return parseHeaders(
+    stringEnv(
+      env,
+      signal === "traces"
+        ? "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
+        : "OTEL_EXPORTER_OTLP_ERRORS_HEADERS",
+    ) ?? stringEnv(env, "OTEL_EXPORTER_OTLP_HEADERS"),
+  );
+}
+
+function parseHeaders(value: string | undefined): Record<string, string> {
+  if (!value) return {};
+  return Object.fromEntries(
+    value.split(",").flatMap((entry) => {
+      const index = entry.indexOf("=");
+      return index < 1
+        ? []
+        : [
+            [
+              entry.slice(0, index).trim(),
+              decodeURIComponent(entry.slice(index + 1).trim()),
+            ],
+          ];
+    }),
+  );
+}
+
+function stringEnv(env: Environment, key: string): string | undefined {
+  const value = env[key];
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}

package/src/router/audit.ts

@@ -1,20 +1,13 @@
-import type { AuditEvent, AuditEventInput } from "./execution-types.ts";
-
-export function createAuditEvent(input: AuditEventInput): AuditEvent {
-  return {
-    ...input,
-    timestamp: input.timestamp ?? new Date().toISOString(),
-  };
-}
+import type { ExecutionEvent } from "@neutrome/lilsdk-ts";
 
 export function createInMemoryAuditSink(): {
-  events: AuditEvent[];
-  observe(event: AuditEvent): void;
+  events: ExecutionEvent[];
+  observe(event: ExecutionEvent): void;
 } {
-  const events: AuditEvent[] = [];
+  const events: ExecutionEvent[] = [];
   return {
     events,
-    observe(event: AuditEvent) {
+    observe(event: ExecutionEvent) {
       events.push(event);
     },
   };

package/src/router/config.ts

@@ -6,9 +6,9 @@ export type ProviderApiStyle = ProviderStyle;
 export type UpstreamAuthConfig = UpstreamAuthConfigShape;
 
 export type ProviderTargetConfig = {
-  api_base_url: string;
+  apiBaseUrl: string;
   style: ProviderApiStyle;
-  exports?: string[] | null;
+  exports?: string[];
   headers?: Record<string, string>;
 };
 
@@ -27,7 +27,7 @@ export type ModelRouteConfig =
     };
 
 export type ModelNamespaceConfig = {
-  exports?: string[] | null;
+  exports?: string[];
   models: Record<string, ModelRouteConfig>;
 };
 
@@ -45,7 +45,7 @@ export type RemoteMcpServerConfig = {
 };
 
 export type RouterConfig = {
-  trace?: boolean;
+  modelSuffixes?: string[];
   upstreamAuth?: UpstreamAuthConfig;
   providers: Record<string, ProviderTargetConfig>;
   modelNamespaces: Record<string, ModelNamespaceConfig>;

package/src/router/error-codec.ts

@@ -0,0 +1,170 @@
+import {
+  appendErrorBlock,
+  createProgram,
+  emitProviderError,
+  firstError,
+  parseProviderError,
+  type Program,
+  type ProviderStyle,
+} from "@neutrome/lil-engine";
+import { ExecutionError, ProviderHttpError, RouterError } from "./errors.ts";
+import type { ProviderRuntime } from "./runtime.ts";
+
+export async function toProviderError(
+  provider: ProviderRuntime,
+  response: Response,
+): Promise<ProviderHttpError> {
+  const body = new Uint8Array(await response.arrayBuffer());
+  const program = parseProviderError(provider.style, body, {
+    provider: provider.name,
+    status: response.status,
+    contentType: response.headers.get("content-type"),
+  });
+  const summary =
+    firstError(program)?.message ||
+    `Provider ${provider.name} returned ${response.status}`;
+  return new ProviderHttpError(summary, response.status, program);
+}
+
+export function errorResponse(
+  error: unknown,
+  responseStyle: ProviderStyle,
+): Response {
+  if (error instanceof ProviderHttpError) {
+    return new Response(
+      toBody(emitProviderError(responseStyle, error.program)),
+      {
+        status: error.status,
+        headers: {
+          "content-type": "application/json; charset=utf-8",
+        },
+      },
+    );
+  }
+
+  if (error instanceof RouterError) {
+    return emitErrorProgramResponse(
+      routerErrorProgram(error),
+      error.status,
+      responseStyle,
+    );
+  }
+
+  if (error instanceof ExecutionError) {
+    const providerError = unwrapCause(error.cause, ProviderHttpError);
+    if (providerError) {
+      return errorResponse(providerError, responseStyle);
+    }
+    const nestedRouterError = unwrapCause(error.cause, RouterError);
+    if (nestedRouterError) {
+      return errorResponse(nestedRouterError, responseStyle);
+    }
+    const routerError = routerErrorFromExecutionError(error);
+    return emitErrorProgramResponse(
+      routerErrorProgram(routerError),
+      routerError.status,
+      responseStyle,
+    );
+  }
+
+  console.error("[open-ai-router] unhandled error:", error);
+  return emitErrorProgramResponse(
+    appendErrorBlock(createProgram(), {
+      source: "router.internal",
+      kind: "invalid_request_error",
+      status: 500,
+      message: "Internal router error",
+      data: { code: "internal_error", param: null },
+    }),
+    500,
+    responseStyle,
+  );
+}
+
+function emitErrorProgramResponse(
+  program: Program,
+  status: number,
+  responseStyle: ProviderStyle,
+): Response {
+  return new Response(toBody(emitProviderError(responseStyle, program)), {
+    status,
+    headers: { "content-type": "application/json; charset=utf-8" },
+  });
+}
+
+function routerErrorProgram(error: RouterError): Program {
+  return appendErrorBlock(createProgram(), {
+    source: "router",
+    kind: error.type,
+    status: error.status,
+    message: error.message,
+    data: { code: error.code, param: null },
+  });
+}
+
+export function errorProgramFromUnknown(error: unknown): Program {
+  if (error instanceof ProviderHttpError) {
+    return error.program;
+  }
+  if (error instanceof RouterError) {
+    return routerErrorProgram(error);
+  }
+  if (error instanceof ExecutionError) {
+    const providerError = unwrapCause(error.cause, ProviderHttpError);
+    if (providerError) return providerError.program;
+    const nestedRouterError = unwrapCause(error.cause, RouterError);
+    if (nestedRouterError) return routerErrorProgram(nestedRouterError);
+    return routerErrorProgram(routerErrorFromExecutionError(error));
+  }
+  return appendErrorBlock(createProgram(), {
+    source: "router.stream",
+    kind: "stream_error",
+    status: 500,
+    message: error instanceof Error ? error.message : String(error),
+  });
+}
+
+function routerErrorFromExecutionError(error: ExecutionError): RouterError {
+  switch (error.kind) {
+    case "validation":
+      return new RouterError(error.message, 400, "invalid_program");
+    case "resolution":
+      return new RouterError(error.message, 400, "target_resolution_failed");
+    case "cancellation":
+      return new RouterError(error.message, 499, "request_cancelled");
+    case "provider":
+      return new RouterError(error.message, 502, "provider_execution_failed");
+    case "transform":
+      return new RouterError(error.message, 500, "transform_failed");
+    case "executor":
+      return new RouterError(error.message, 500, "executor_failed");
+    case "stream":
+      return new RouterError(error.message, 500, "stream_failed");
+    case "recursion_limit":
+      return new RouterError(error.message, 500, "recursion_limit");
+  }
+}
+
+function unwrapCause<T extends Error>(
+  error: unknown,
+  ctor: abstract new (...args: never[]) => T,
+): T | null {
+  let current = error;
+  while (current && typeof current === "object") {
+    if (current instanceof ctor) {
+      return current;
+    }
+    if (!("cause" in current)) {
+      return null;
+    }
+    current = (current as { cause?: unknown }).cause;
+  }
+  return null;
+}
+
+export function toBody(bytes: Uint8Array): ArrayBuffer {
+  return bytes.buffer.slice(
+    bytes.byteOffset,
+    bytes.byteOffset + bytes.byteLength,
+  ) as ArrayBuffer;
+}

package/src/router/errors.ts

@@ -44,7 +44,10 @@ export class ExecutionError extends Error {
       cause?: unknown;
     } = {},
   ) {
-    super(message, options.cause === undefined ? undefined : { cause: options.cause });
+    super(
+      message,
+      options.cause === undefined ? undefined : { cause: options.cause },
+    );
     this.kind = kind;
     if (options.stage !== undefined) {
       this.stage = options.stage;
@@ -55,6 +58,40 @@ export class ExecutionError extends Error {
   }
 }
 
+export class RouterError extends Error {
+  constructor(
+    message: string,
+    readonly status: number,
+    readonly code: string,
+    readonly type = "invalid_request_error",
+  ) {
+    super(message);
+  }
+}
+
+export class ProviderHttpError extends Error {
+  constructor(
+    message: string,
+    readonly status: number,
+    readonly program: import("@neutrome/lil-engine").Program,
+  ) {
+    super(message);
+  }
+}
+
 export function isExecutionError(error: unknown): error is ExecutionError {
   return error instanceof ExecutionError;
 }
+
+export function createOpenAiErrorResponse(
+  message: string,
+  status: number,
+  code: string,
+): Response {
+  return new Response(
+    JSON.stringify({
+      error: { message, type: "invalid_request_error", param: null, code },
+    }),
+    { status, headers: { "content-type": "application/json; charset=utf-8" } },
+  );
+}