omnigateway 0.9.1 → 0.9.2

package/gateway.js

@@ -29520,6 +29520,19 @@ function loadConfig(env) {
   const bodyLoggingAllowed = TRUTHY.has((env.OMNI_BODY_LOGGING_ALLOWED ?? "").trim().toLowerCase());
   const rawLogLevel = env.OMNI_LOG_LEVEL?.trim();
   const logLevel = parseLogLevel(rawLogLevel);
+  const metricsToken = env.OMNI_METRICS_TOKEN?.trim() || null;
+  const rawMaxSeries = env.OMNI_METRICS_MAX_SERIES ?? "5000";
+  const metricsMaxSeries = Number(rawMaxSeries);
+  if (!DECIMAL_INTEGER.test(rawMaxSeries) || metricsMaxSeries < 1) {
+    throw new Error(`OMNI_METRICS_MAX_SERIES must be a positive integer, got "${rawMaxSeries}"`);
+  }
+  const otlpEndpoint = env.OMNI_OTLP_ENDPOINT?.trim().replace(/\/+$/, "") || null;
+  const otlpHeaders = env.OMNI_OTLP_HEADERS?.trim() || null;
+  const rawTraceSample = env.OMNI_TRACE_SAMPLE ?? "1.0";
+  const traceSample = Number(rawTraceSample);
+  if (!Number.isFinite(traceSample) || traceSample < 0 || traceSample > 1) {
+    throw new Error(`OMNI_TRACE_SAMPLE must be between 0 and 1, got "${rawTraceSample}"`);
+  }
   return {
     logLevel: logLevel ?? "info",
     logLevelFallbackFrom: logLevel === null && rawLogLevel ? rawLogLevel : null,
@@ -29533,7 +29546,12 @@ function loadConfig(env) {
     logFile: logFile === undefined || logFile.length === 0 ? null : logFile,
     clusterMode,
     databaseUrl: clusterMode ? databaseUrl ?? null : null,
-    redisUrl: clusterMode ? redisUrl ?? null : null
+    redisUrl: clusterMode ? redisUrl ?? null : null,
+    metricsToken,
+    metricsMaxSeries,
+    otlpEndpoint,
+    otlpHeaders,
+    traceSample
   };
 }
 // packages/control/src/oauth/pending.ts
@@ -29732,7 +29750,8 @@ function codecAdapter(id, capabilities2, codec2, origins) {
       const res = await req.http({
         provider: id,
         ...sendable,
-        signal: req.signal
+        signal: req.signal,
+        ...req.requestId === undefined ? {} : { requestId: req.requestId }
       });
       if (res.status < 200 || res.status >= 300) {
         const text = await res.text().catch(() => "");
@@ -33117,15 +33136,28 @@ function nodeHttpClient(options = {}) {
     const startedAt = now();
     let traced = false;
     const trace = (status, failed = false) => {
-      if (traced || !logger2.enabled("debug"))
+      if (traced)
         return;
       traced = true;
+      const durationMs = now() - startedAt;
+      try {
+        options.onResponseHead?.({
+          provider: req.provider,
+          host: url2.host,
+          path: url2.pathname,
+          ...status === undefined ? {} : { status },
+          durationMs,
+          ...req.requestId === undefined ? {} : { requestId: req.requestId }
+        });
+      } catch {}
+      if (!logger2.enabled("debug"))
+        return;
       logger2.debug("upstream http", {
         provider: req.provider,
         status,
         host: url2.host,
         path: url2.pathname,
-        durationMs: now() - startedAt,
+        durationMs,
         reason: failed ? "transport error" : undefined
       });
     };
@@ -36847,6 +36879,7 @@ var WINDOW_MS2 = {
   "1w": 7 * 24 * 60 * 60 * 1000
 };
 // apps/gateway/src/app.ts
+import { timingSafeEqual } from "crypto";
 import { existsSync, realpathSync as realpathSync3 } from "fs";
 import { resolve as resolve3, sep as sep3 } from "path";
 
@@ -51324,11 +51357,13 @@ class ApiKeyRateLimiter {
   store;
   now;
   logger;
+  onRejected;
   constructor(deps) {
     this.store = deps.store;
     this.now = deps.now;
     this.logger = deps.logger ?? noopLogger;
     this.coord = deps.coord ?? memoryCoord();
+    this.onRejected = deps.onRejected;
   }
   async admit(keyId, limits, requestId) {
     if (!anyLimit(limits))
@@ -51403,6 +51438,9 @@ class ApiKeyRateLimiter {
     const exact = await this.exactReset(keyId, window2, now, requestId);
     const resolved = exact === null ? violation : { ...violation, resetAt: exact };
     const headroom = exact === null || window2 === null ? decision.headroom : withReset(decision.headroom, window2, exact);
+    try {
+      this.onRejected?.(violation.dimension, violation.window);
+    } catch {}
     throw new RateLimitExceeded(retryAfterMs(resolved, now), headroom);
   }
   async exactReset(keyId, window2, now, requestId) {
@@ -51501,6 +51539,61 @@ function withReset(headroom, window2, resetAt) {
   return corrected;
 }
 
+// apps/gateway/src/dispatch/loadRegistry.ts
+var PREFIX6 = "load:";
+var SLOT_TTL_MS = 300000;
+function createLoadRegistry(coord = memoryCoord()) {
+  const local = new Map;
+  const providers = new Map;
+  let remote = new Map;
+  return {
+    acquire(credentialId, model, provider) {
+      const key = healthKey(credentialId, model);
+      local.set(key, (local.get(key) ?? 0) + 1);
+      if (provider !== undefined)
+        providers.set(provider, (providers.get(provider) ?? 0) + 1);
+      coord.gauge.acquire(PREFIX6 + key, SLOT_TTL_MS);
+      let released = false;
+      return () => {
+        if (released)
+          return;
+        released = true;
+        const next = (local.get(key) ?? 0) - 1;
+        if (next > 0)
+          local.set(key, next);
+        else
+          local.delete(key);
+        if (provider !== undefined) {
+          const providerCount = (providers.get(provider) ?? 0) - 1;
+          if (providerCount > 0)
+            providers.set(provider, providerCount);
+          else
+            providers.delete(provider);
+        }
+        coord.gauge.release(PREFIX6 + key);
+      };
+    },
+    counts() {
+      const out = new Map(local);
+      for (const [key, count] of remote) {
+        if (count > (out.get(key) ?? 0))
+          out.set(key, count);
+      }
+      return out;
+    },
+    localCounts() {
+      return new Map(providers);
+    },
+    async refresh() {
+      const held = await coord.gauge.snapshot(PREFIX6);
+      const sample = new Map;
+      for (const [key, count] of held)
+        sample.set(key.slice(PREFIX6.length), count);
+      remote = sample;
+    }
+  };
+}
+
 // apps/gateway/src/dispatch/snapshotCache.ts
 function createRoutingSnapshotCache(store, logger2 = noopLogger) {
   let snapshot = null;
@@ -55473,7 +55566,7 @@ async function routeLog(store, requestId, target, logger2 = noopLogger, broadcas
     }
   }
 }
-async function finishLog(store, log, keyId, logger2 = noopLogger, bodies, debit, emit, broadcast) {
+async function finishLog(store, log, keyId, logger2 = noopLogger, bodies, debit, emit, broadcast, telemetry, trace) {
   let appended = true;
   try {
     await store.usage.append({ ...log, apiKeyId: keyId });
@@ -55517,15 +55610,108 @@ async function finishLog(store, log, keyId, logger2 = noopLogger, bodies, debit,
       report(logger2, "failed to publish a resource invalidation", log.id, error61);
     }
   }
-  if (bodies === undefined)
-    return;
+  if (bodies !== undefined) {
+    try {
+      await bodies(log);
+    } catch (error61) {
+      report(logger2, "failed to persist request bodies", log.id, error61);
+    }
+  }
   try {
-    await bodies(log);
+    telemetry?.record(log, keyId, trace ?? null);
   } catch (error61) {
-    report(logger2, "failed to persist request bodies", log.id, error61);
+    report(logger2, "failed to record request telemetry", log.id, error61);
+  }
+  try {
+    telemetry?.flush(log.id, trace ?? null);
+  } catch (error61) {
+    report(logger2, "failed to export request telemetry", log.id, error61);
   }
 }
 
+// apps/gateway/src/telemetry/spans.ts
+var TRACEPARENT = /^[\da-f]{2}-([\da-f]{32})-([\da-f]{16})-([\da-f]{2})(?:-[\da-f]+)*$/i;
+var ZERO_TRACE = "0".repeat(32);
+var ZERO_SPAN = "0".repeat(16);
+function parseTraceparent(value) {
+  if (value === null)
+    return null;
+  const match = TRACEPARENT.exec(value);
+  const traceId = match?.[1]?.toLowerCase();
+  const parentSpanId = match?.[2]?.toLowerCase();
+  const flags = match?.[3];
+  if (traceId === undefined || parentSpanId === undefined || flags === undefined)
+    return null;
+  if (traceId === ZERO_TRACE || parentSpanId === ZERO_SPAN)
+    return null;
+  return { traceId, parentSpanId, sampled: (Number.parseInt(flags, 16) & 1) === 1 };
+}
+function randomHex(bytes) {
+  const out = new Uint8Array(bytes);
+  crypto.getRandomValues(out);
+  return [...out].map((byte2) => byte2.toString(16).padStart(2, "0")).join("");
+}
+function createTrace(opts) {
+  const id = opts.id ?? (() => randomHex(8));
+  return {
+    startedAt: opts.startedAt,
+    traceId: opts.traceparent?.traceId ?? `${id()}${id()}`.slice(0, 32),
+    parentSpanId: opts.traceparent?.parentSpanId ?? null,
+    spanIds: [],
+    spans: [],
+    activeAttempt: null,
+    routeSpan: null,
+    nextId: id
+  };
+}
+function addSpan(trace, name, parent, startMs, endMs, attrs) {
+  try {
+    const id = trace.spans.length;
+    trace.spanIds.push(trace.nextId());
+    trace.spans.push({ id, parent, name, startMs, endMs, attrs });
+    return id;
+  } catch {
+    return -1;
+  }
+}
+function otlpAttrs(attrs) {
+  const out = [];
+  for (const [key, value] of Object.entries(attrs)) {
+    if (value === undefined)
+      continue;
+    out.push(typeof value === "number" ? { key, value: { intValue: String(value) } } : { key, value: { stringValue: value } });
+  }
+  return out;
+}
+function encodeTrace(trace, serviceVersion) {
+  const spans = trace.spans.map((span) => {
+    const parentSpanId = span.parent === null ? trace.parentSpanId : trace.spanIds[span.parent] ?? trace.parentSpanId;
+    return {
+      traceId: trace.traceId,
+      spanId: trace.spanIds[span.id] ?? trace.nextId(),
+      ...parentSpanId === null ? {} : { parentSpanId },
+      name: span.name,
+      startTimeUnixNano: String(BigInt(Math.round(trace.startedAt + span.startMs)) * 1000000n),
+      endTimeUnixNano: String(BigInt(Math.round(trace.startedAt + span.endMs)) * 1000000n),
+      attributes: otlpAttrs(span.attrs),
+      status: { code: span.attrs.code === undefined ? 0 : 2 }
+    };
+  });
+  return {
+    resourceSpans: [
+      {
+        resource: {
+          attributes: [
+            { key: "service.name", value: { stringValue: "omnigateway" } },
+            { key: "service.version", value: { stringValue: serviceVersion } }
+          ]
+        },
+        scopeSpans: [{ spans }]
+      }
+    ]
+  };
+}
+
 // apps/gateway/src/dispatch/attempt.ts
 async function attempt(opts) {
   const { candidate, adapter, http, now, signal, refresh, refreshLeadMs } = opts;
@@ -55708,6 +55894,7 @@ async function dispatch(request2, deps, signal, requestId) {
     return fail(code, describeError(error61, "unresolvable model"));
   }
   await deps.loadRegistry.refresh();
+  const routeStartedAt = deps.now();
   const { candidates, excluded } = rank({
     request: dispatchRequest,
     model,
@@ -55717,6 +55904,15 @@ async function dispatch(request2, deps, signal, requestId) {
     load: deps.loadRegistry.counts(),
     providers: deps.providers
   });
+  if (deps.trace !== null && deps.trace !== undefined) {
+    deps.trace.routeSpan = addSpan(deps.trace, "dispatch.route", 0, routeStartedAt - startedAt, deps.now() - startedAt, {
+      candidates: candidates.length,
+      ...candidates[0] === undefined ? {} : {
+        chosen_provider: candidates[0].target.provider,
+        chosen_model: candidates[0].target.model
+      }
+    });
+  }
   logger2.debug("routing candidates ranked", {
     requestId,
     requestedModel: request2.model,
@@ -55744,7 +55940,7 @@ async function dispatch(request2, deps, signal, requestId) {
   const eager = {
     credentialId: head.credential.id,
     model: head.target.model,
-    release: deps.loadRegistry.acquire(head.credential.id, head.target.model)
+    release: deps.loadRegistry.acquire(head.credential.id, head.target.model, head.target.provider)
   };
   let eagerHeld = true;
   releaseOnAbort = () => eager.release();
@@ -55755,7 +55951,14 @@ async function dispatch(request2, deps, signal, requestId) {
   const persistHealth = async (candidate, transition) => {
     const credentialId = candidate.credential.id;
     const { model: model2 } = candidate.target;
-    await deps.store.credentials.updateHealth(credentialId, model2, (current) => transition(current ?? blankHealth(credentialId, model2)));
+    await deps.store.credentials.updateHealth(credentialId, model2, (current) => {
+      const before = current ?? blankHealth(credentialId, model2);
+      const after = transition(before);
+      try {
+        deps.telemetry?.breaker(candidate.target.provider, before.breakerState, after.breakerState);
+      } catch {}
+      return after;
+    });
   };
   async function* run() {
     try {
@@ -55774,7 +55977,17 @@ async function dispatch(request2, deps, signal, requestId) {
           const adoptable = eagerHeld && eager.credentialId === candidate.credential.id && eager.model === candidate.target.model;
           if (adoptable)
             eagerHeld = false;
-          const releaseSlot = adoptable ? eager.release : deps.loadRegistry.acquire(candidate.credential.id, candidate.target.model);
+          const releaseSlot = adoptable ? eager.release : deps.loadRegistry.acquire(candidate.credential.id, candidate.target.model, candidate.target.provider);
+          const attemptStartedAt = deps.now();
+          let attemptCode;
+          if (deps.trace !== null && deps.trace !== undefined) {
+            deps.trace.activeAttempt = addSpan(deps.trace, "dispatch.attempt", 0, attemptStartedAt - startedAt, attemptStartedAt - startedAt, {
+              attempt: i + 1,
+              provider: candidate.target.provider,
+              model: candidate.target.model,
+              credential_id: candidate.credential.id
+            });
+          }
           try {
             log.attempts = i + 1;
             log.credentialId = candidate.credential.id;
@@ -55838,6 +56051,9 @@ async function dispatch(request2, deps, signal, requestId) {
                   if (event.type === "blockDelta" && !committed) {
                     committed = true;
                     log.ttftMs = deps.now() - startedAt;
+                    if (deps.trace !== null && deps.trace !== undefined && deps.trace.activeAttempt !== null) {
+                      addSpan(deps.trace, "stream.commit", deps.trace.activeAttempt, log.ttftMs, log.ttftMs, { provider: candidate.target.provider, model: candidate.target.model });
+                    }
                     logger2.debug("stream committed", {
                       requestId,
                       provider: candidate.target.provider,
@@ -55920,6 +56136,7 @@ async function dispatch(request2, deps, signal, requestId) {
                   throw signal.reason;
                 const classifiedError = deadlineAt !== null && dispatchSignal.aborted ? { code: "TIMEOUT" } : classify(error61);
                 const { code: code2 } = classifiedError;
+                attemptCode = code2;
                 const message2 = describeError(error61, "attempt failed");
                 lastError = rewrap(classifiedError, message2);
                 if (error61 instanceof GatewayError)
@@ -55993,6 +56210,15 @@ async function dispatch(request2, deps, signal, requestId) {
               }
             }
           } finally {
+            if (deps.trace !== null && deps.trace !== undefined && deps.trace.activeAttempt !== null) {
+              const span = deps.trace.spans[deps.trace.activeAttempt];
+              if (span !== undefined) {
+                span.endMs = deps.now() - startedAt;
+                if (attemptCode !== undefined)
+                  span.attrs = { ...span.attrs, code: attemptCode };
+              }
+              deps.trace.activeAttempt = null;
+            }
             releaseSlot();
           }
         }
@@ -56064,48 +56290,6 @@ function waitForCancellation(promise2, signal) {
   });
 }
 
-// apps/gateway/src/dispatch/loadRegistry.ts
-var PREFIX6 = "load:";
-var SLOT_TTL_MS = 300000;
-function createLoadRegistry(coord = memoryCoord()) {
-  const local = new Map;
-  let remote = new Map;
-  return {
-    acquire(credentialId, model) {
-      const key = healthKey(credentialId, model);
-      local.set(key, (local.get(key) ?? 0) + 1);
-      coord.gauge.acquire(PREFIX6 + key, SLOT_TTL_MS);
-      let released = false;
-      return () => {
-        if (released)
-          return;
-        released = true;
-        const next = (local.get(key) ?? 0) - 1;
-        if (next > 0)
-          local.set(key, next);
-        else
-          local.delete(key);
-        coord.gauge.release(PREFIX6 + key);
-      };
-    },
-    counts() {
-      const out = new Map(local);
-      for (const [key, count] of remote) {
-        if (count > (out.get(key) ?? 0))
-          out.set(key, count);
-      }
-      return out;
-    },
-    async refresh() {
-      const held = await coord.gauge.snapshot(PREFIX6);
-      const sample = new Map;
-      for (const [key, count] of held)
-        sample.set(key.slice(PREFIX6.length), count);
-      remote = sample;
-    }
-  };
-}
-
 // apps/gateway/src/ingress/anthropicTools.ts
 var TOOL_FIELDS = {
   cache_control: cacheControlSchema.nullable(),
@@ -57330,6 +57514,7 @@ async function bodyCollectorFor(deps, key) {
 async function handle(deps, rateLimiter, surface, request2) {
   const requestId = deps.requestId();
   const startedAt = deps.now();
+  const trace = deps.telemetry?.startRequest(requestId, startedAt, request2.headers.get("traceparent"), surface, deps.rand()) ?? null;
   let keyId = null;
   let requestedModel = "";
   let outcome = null;
@@ -57411,6 +57596,7 @@ async function handle(deps, rateLimiter, surface, request2) {
     const dispatched = await dispatch(chatRequest, {
       ...deps,
       ...captured === null ? {} : { http: captured.wrap(deps.http) },
+      trace,
       async onRoute(target) {
         if (began) {
           await routeLog(deps.store, requestId, target, deps.logger, deps.broadcaster);
@@ -57448,7 +57634,7 @@ async function handle(deps, rateLimiter, surface, request2) {
         reportRejection(deps.logger, requestId, completed, gatewayError, surface);
       }
       logged = true;
-      await finishLog(deps.store, completed, keyId, deps.logger, writeBodies, debit, deps.emit, deps.broadcaster);
+      await finishLog(deps.store, completed, keyId, deps.logger, writeBodies, debit, deps.emit, deps.broadcaster, deps.telemetry, trace);
     };
     if (chatRequest.stream) {
       const frames = surface === "anthropic" ? anthropicStream(dispatched.events, requestId) : surface === "responses" ? responsesStream(dispatched.events, render2()) : openaiStream(dispatched.events, requestId, Math.floor(deps.now() / 1000));
@@ -57489,7 +57675,7 @@ async function handle(deps, rateLimiter, surface, request2) {
     if (collector !== null && !logged)
       collector.client.response = rejection;
     if (!logged)
-      await finishLog(deps.store, completed, keyId, deps.logger, writeBodies, debit, deps.emit, deps.broadcaster);
+      await finishLog(deps.store, completed, keyId, deps.logger, writeBodies, debit, deps.emit, deps.broadcaster, deps.telemetry, trace);
     if (!cancelled)
       reportRejection(deps.logger, requestId, completed, gatewayError, surface);
     return jsonResponse(rejection, HTTP_STATUS[gatewayError.code], {
@@ -58457,7 +58643,7 @@ function createRing(limits) {
 }
 
 // apps/gateway/src/version.ts
-var VERSION = "0.9.1";
+var VERSION = "0.9.2";
 
 // apps/gateway/src/app.ts
 var ADMIN_SESSION_TTL_MS = 12 * 60 * 60 * 1000;
@@ -58495,10 +58681,22 @@ function createApp(deps) {
   const nodeId = deps.nodeId ?? crypto.randomUUID();
   const logger2 = deps.logger ?? noopLogger;
   const rand = deps.rand ?? Math.random;
-  const http = deps.http ?? nodeHttpClient({ logger: logger2, now });
+  const http = deps.http ?? nodeHttpClient({
+    logger: logger2,
+    now,
+    ...deps.telemetry === undefined ? {} : { onResponseHead: deps.telemetry.httpHead }
+  });
   const adapters = deps.adapters ?? ADAPTERS;
   const requestId = deps.requestId ?? (() => `req_${crypto.randomUUID()}`);
-  const rateLimiter = deps.rateLimiter ?? new ApiKeyRateLimiter({ store: deps.store, now, logger: logger2, coord });
+  const loadRegistry = deps.loadRegistry ?? createLoadRegistry(coord);
+  const telemetry = deps.telemetry;
+  const rateLimiter = deps.rateLimiter ?? new ApiKeyRateLimiter({
+    store: deps.store,
+    now,
+    logger: logger2,
+    coord,
+    ...telemetry === undefined ? {} : { onRejected: telemetry.rateLimit }
+  });
   const snapshots = createRoutingSnapshotCache(deps.store, logger2);
   const ROUTING_TOPIC = "routing";
   deps.store.routing.subscribe((change) => {
@@ -58563,7 +58761,17 @@ function createApp(deps) {
     ok: true,
     mode: deps.mode ?? "single",
     nodeId,
-    coord: "healthy" in coord && !coord.healthy() ? "fallback" : "ok"
+    coord: deps.coordHealthy?.() === false ? "fallback" : "ok"
+  })).use(deps.metricsToken === undefined || telemetry === undefined ? new Elysia : new Elysia().get("/metrics", ({ request: request2 }) => {
+    const supplied = request2.headers.get("authorization");
+    const expected = `Bearer ${deps.metricsToken}`;
+    const actualBytes = Buffer.from(supplied ?? "");
+    const expectedBytes = Buffer.from(expected);
+    if (actualBytes.length !== expectedBytes.length || !timingSafeEqual(actualBytes, expectedBytes)) {
+      return new Response(null, { status: 401 });
+    }
+    const coordFallback = deps.coordHealthy?.() === false;
+    return new Response(telemetry.scrape(loadRegistry.localCounts?.() ?? new Map, registry2.stats(), coordFallback), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" } });
   })).use(proxyRoutes({
     store: deps.store,
     snapshots,
@@ -58576,9 +58784,10 @@ function createApp(deps) {
     rateLimiter,
     logger: logger2,
     coord,
-    ...deps.loadRegistry === undefined ? {} : { loadRegistry: deps.loadRegistry },
+    loadRegistry,
     bodyLoggingAllowed: deps.bodyLoggingAllowed === true,
     ...deps.emit === undefined ? {} : { emit: deps.emit },
+    ...telemetry === undefined ? {} : { telemetry },
     broadcaster
   })).use(adminRoutes({
     store: deps.store,
@@ -59847,6 +60056,340 @@ function startJournalPoll(deps, source, logger2) {
   };
 }
 
+// apps/gateway/src/telemetry/otlp.ts
+function createOtlpExporter(opts) {
+  const queue = [];
+  const send = opts.fetch ?? fetch;
+  const schedule = opts.schedule ?? ((run, ms) => {
+    const timer = setInterval(run, ms);
+    timer.unref?.();
+    return () => clearInterval(timer);
+  });
+  let reporting = false;
+  const exporter = {
+    enqueue(trace) {
+      try {
+        const encoded = JSON.stringify(encodeTrace(trace, opts.version ?? "unknown"));
+        if (queue.length >= opts.capacity) {
+          opts.registry.add("omni_otlp_spans_dropped_total", { reason: "queue_full" });
+          return;
+        }
+        queue.push(encoded);
+      } catch {
+        opts.registry.add("omni_otlp_spans_dropped_total", { reason: "encode" });
+      }
+    },
+    async flush() {
+      if (queue.length === 0)
+        return;
+      const encoded = queue.splice(0, opts.batchMax);
+      try {
+        const bodies = encoded.map((body2) => JSON.parse(body2));
+        const resourceSpans = bodies.flatMap((body2) => body2.resourceSpans);
+        const response = await send(`${opts.endpoint.replace(/\/+$/, "")}/v1/traces`, {
+          method: "POST",
+          headers: { "content-type": "application/json", ...opts.headers },
+          body: JSON.stringify({ resourceSpans })
+        });
+        if (!response.ok)
+          throw new Error(`collector returned ${response.status}`);
+        reporting = false;
+      } catch {
+        if (!reporting)
+          opts.logger?.warn("trace export failed", { reason: "collector unavailable" });
+        reporting = true;
+      }
+    },
+    queued: () => queue.length,
+    stop() {
+      cancel();
+    }
+  };
+  const cancel = schedule(() => void exporter.flush(), opts.intervalMs);
+  return exporter;
+}
+
+// apps/gateway/src/telemetry/registry.ts
+var DEFINITIONS = {
+  omni_requests_total: { help: "Requests completed by this process.", type: "counter" },
+  omni_request_duration_seconds: { help: "Request duration in seconds.", type: "histogram" },
+  omni_ttft_seconds: { help: "Streaming time to first token in seconds.", type: "histogram" },
+  omni_tokens_total: { help: "Billable tokens completed by this process.", type: "counter" },
+  omni_cost_usd_total: { help: "Request cost in US dollars.", type: "counter" },
+  omni_upstream_duration_seconds: {
+    help: "Provider response-head duration in seconds.",
+    type: "histogram"
+  },
+  omni_inflight: { help: "Provider requests in flight in this process.", type: "gauge" },
+  omni_breaker_open: { help: "Credential breakers observed open in this process.", type: "gauge" },
+  omni_ratelimit_rejected_total: { help: "API-key rate-limit refusals.", type: "counter" },
+  omni_stream_connections: { help: "Live stream connections in this process.", type: "gauge" },
+  omni_stream_queued: { help: "Stream frames queued in this process.", type: "gauge" },
+  omni_stream_dropped_total: { help: "Stream frames dropped in this process.", type: "counter" },
+  omni_coord_fallback: {
+    help: "Whether this process is using coordination fallback.",
+    type: "gauge"
+  },
+  omni_metrics_series_folded_total: {
+    help: "Metric series folded into api_key_id other.",
+    type: "counter"
+  },
+  omni_otlp_spans_dropped_total: { help: "Spans dropped before OTLP export.", type: "counter" },
+  omni_build_info: { help: "Gateway build information.", type: "gauge" }
+};
+var HISTOGRAM_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
+function labelKey(labels) {
+  return Object.entries(labels).sort(([a], [b]) => a.localeCompare(b)).map(([key, value]) => `${key.length}:${key}${value.length}:${value}`).join("");
+}
+function createMetricRegistry(opts) {
+  const series = new Map;
+  const rawSeries = new Set;
+  const folded = new Set;
+  const normalize = (name, labels) => {
+    const copy = { ...labels };
+    const raw = `${name}:${labelKey(copy)}`;
+    if (rawSeries.has(raw))
+      return copy;
+    if (rawSeries.size < opts.maxSeries || copy.api_key_id === undefined) {
+      rawSeries.add(raw);
+      return copy;
+    }
+    copy.api_key_id = "other";
+    if (!folded.has(raw)) {
+      folded.add(raw);
+      change("omni_metrics_series_folded_total", {}, 1, false);
+    }
+    return copy;
+  };
+  const change = (name, labels, value, replace) => {
+    const normalized = normalize(name, labels);
+    const key = labelKey(normalized);
+    const table = series.get(name) ?? new Map;
+    const current = table.get(key) ?? { labels: normalized, value: 0 };
+    current.value = replace ? value : current.value + value;
+    table.set(key, current);
+    series.set(name, table);
+    return current;
+  };
+  return {
+    add(name, labels, value = 1) {
+      change(name, labels, value, false);
+    },
+    set(name, labels, value) {
+      change(name, labels, value, true);
+    },
+    observe(name, value, labels) {
+      const current = change(name, labels, value, false);
+      current.count = (current.count ?? 0) + 1;
+      const buckets = current.buckets ?? HISTOGRAM_BUCKETS.map(() => 0);
+      for (let i = 0;i < HISTOGRAM_BUCKETS.length; i++) {
+        if (value <= HISTOGRAM_BUCKETS[i])
+          buckets[i] = (buckets[i] ?? 0) + 1;
+      }
+      current.buckets = buckets;
+    },
+    value(name, labels) {
+      return series.get(name)?.get(labelKey(labels))?.value ?? 0;
+    },
+    snapshot() {
+      return {
+        at: opts.now(),
+        metrics: Object.entries(DEFINITIONS).map(([name, definition]) => ({
+          name,
+          ...definition,
+          samples: [...series.get(name)?.values() ?? []].map((sample) => definition.type === "histogram" ? {
+            labels: sample.labels,
+            value: sample.value,
+            sum: sample.value,
+            count: sample.count ?? 0,
+            buckets: sample.buckets ?? HISTOGRAM_BUCKETS.map(() => 0)
+          } : { labels: sample.labels, value: sample.value })
+        })).filter((metric) => metric.samples.length > 0)
+      };
+    }
+  };
+}
+
+// apps/gateway/src/telemetry/render.ts
+function escapeLabel(value) {
+  return value.replaceAll("\\", "\\\\").replaceAll(`
+`, "\\n").replaceAll('"', "\\\"");
+}
+function labels(labels2, extra) {
+  const entries = Object.entries(labels2);
+  if (extra !== undefined)
+    entries.push([extra[0], extra[1]]);
+  if (entries.length === 0)
+    return "";
+  return `{${entries.map(([key, value]) => `${key}="${escapeLabel(value)}"`).join(",")}}`;
+}
+function number4(value) {
+  if (value === Number.POSITIVE_INFINITY)
+    return "+Inf";
+  if (value === Number.NEGATIVE_INFINITY)
+    return "-Inf";
+  if (Number.isNaN(value))
+    return "NaN";
+  return String(value);
+}
+function renderPrometheus(snapshot) {
+  const out = [];
+  for (const metric of snapshot.metrics) {
+    out.push(`# HELP ${metric.name} ${metric.help}`, `# TYPE ${metric.name} ${metric.type}`);
+    for (const sample of metric.samples) {
+      if (metric.type !== "histogram" || !("buckets" in sample)) {
+        out.push(`${metric.name}${labels(sample.labels)} ${number4(sample.value)}`);
+        continue;
+      }
+      for (let i = 0;i < HISTOGRAM_BUCKETS.length; i++) {
+        out.push(`${metric.name}_bucket${labels(sample.labels, ["le", String(HISTOGRAM_BUCKETS[i])])} ${number4(sample.buckets[i] ?? 0)}`);
+      }
+      out.push(`${metric.name}_bucket${labels(sample.labels, ["le", "+Inf"])} ${sample.count}`, `${metric.name}_sum${labels(sample.labels)} ${number4(sample.sum)}`, `${metric.name}_count${labels(sample.labels)} ${sample.count}`);
+    }
+  }
+  return `${out.join(`
+`)}
+`;
+}
+
+// apps/gateway/src/telemetry/index.ts
+function parseOtlpHeaders(value) {
+  const headers = {};
+  if (value === null)
+    return headers;
+  for (const entry of value.split(",")) {
+    const at = entry.indexOf("=");
+    if (at <= 0)
+      continue;
+    const key = entry.slice(0, at).trim();
+    if (key.length > 0)
+      headers[key] = entry.slice(at + 1).trim();
+  }
+  return headers;
+}
+function codeAttr(value) {
+  return value === "interrupted" || /^[A-Z_]{1,32}$/.test(value) ? { code: value } : {};
+}
+function createTelemetry(opts) {
+  const registry2 = createMetricRegistry({ maxSeries: opts.maxSeries, now: opts.now });
+  const traces = new Map;
+  const openBreakers = new Map;
+  const exporter = opts.exporter ?? (opts.otlpEndpoint === null ? null : createOtlpExporter({
+    endpoint: opts.otlpEndpoint,
+    headers: opts.otlpHeaders,
+    capacity: 1e4,
+    batchMax: 512,
+    intervalMs: 5000,
+    registry: registry2,
+    ...opts.logger === undefined ? {} : { logger: opts.logger },
+    version: opts.version
+  }));
+  if (opts.metricsEnabled)
+    registry2.set("omni_build_info", { version: opts.version }, 1);
+  return {
+    metricsEnabled: opts.metricsEnabled,
+    tracingEnabled: exporter !== null,
+    registry: registry2,
+    startRequest(requestId, startedAt, traceparent, surface, rand) {
+      if (exporter === null)
+        return null;
+      const inbound = parseTraceparent(traceparent);
+      if (inbound !== null ? !inbound.sampled : rand >= opts.traceSample)
+        return null;
+      const trace = createTrace({ startedAt, traceparent: inbound });
+      addSpan(trace, "gateway.request", null, 0, 0, { surface });
+      traces.set(requestId, trace);
+      return trace;
+    },
+    record(log, keyId, trace) {
+      if (opts.metricsEnabled) {
+        const provider = log.resolvedProvider ?? "";
+        const model = log.resolvedModel ?? log.requestedModel;
+        const key = keyId ?? "";
+        registry2.add("omni_requests_total", {
+          provider,
+          model,
+          status: String(log.status),
+          code: log.errorCode ?? "",
+          api_key_id: key
+        });
+        registry2.observe("omni_request_duration_seconds", log.durationMs / 1000, {
+          provider,
+          model
+        });
+        if (log.ttftMs !== null)
+          registry2.observe("omni_ttft_seconds", log.ttftMs / 1000, { provider, model });
+        for (const [kind, value] of [
+          ["input", log.inputTokens],
+          ["output", log.outputTokens],
+          ["cache_read", log.cacheReadTokens],
+          ["cache_write", log.cacheWriteTokens]
+        ]) {
+          registry2.add("omni_tokens_total", { provider, model, api_key_id: key, kind }, value);
+        }
+        registry2.add("omni_cost_usd_total", { provider, model, api_key_id: key }, log.costUsd);
+      }
+      if (trace !== null) {
+        const root = trace.spans[0];
+        if (root !== undefined) {
+          root.endMs = log.durationMs;
+          root.attrs = {
+            ...root.attrs,
+            requested_model: log.requestedModel,
+            api_key_id: keyId ?? "",
+            status: log.status,
+            ...log.errorCode === null ? {} : codeAttr(log.errorCode)
+          };
+        }
+      }
+    },
+    flush(requestId, trace) {
+      traces.delete(requestId);
+      if (trace !== null)
+        exporter?.enqueue(trace);
+    },
+    httpHead(event) {
+      if (opts.metricsEnabled)
+        registry2.observe("omni_upstream_duration_seconds", event.durationMs / 1000, {
+          provider: event.provider
+        });
+      const trace = event.requestId === undefined ? undefined : traces.get(event.requestId);
+      if (trace === undefined)
+        return;
+      const endMs = Math.max(0, opts.now() - trace.startedAt);
+      addSpan(trace, "provider.http", trace.activeAttempt ?? null, Math.max(0, endMs - event.durationMs), endMs, {
+        provider: event.provider,
+        host: event.host,
+        path: event.path,
+        ...event.status === undefined ? {} : { status: event.status }
+      });
+    },
+    breaker(provider, before, after) {
+      if (!opts.metricsEnabled || before === after)
+        return;
+      const count = Math.max(0, (openBreakers.get(provider) ?? 0) + (after === "open" ? 1 : before === "open" ? -1 : 0));
+      openBreakers.set(provider, count);
+      registry2.set("omni_breaker_open", { provider }, count);
+    },
+    rateLimit(dimension, window2) {
+      if (opts.metricsEnabled)
+        registry2.add("omni_ratelimit_rejected_total", { dimension, window: window2 ?? "none" });
+    },
+    scrape(localInflight, streams, coordFallback) {
+      for (const [provider, value] of localInflight)
+        registry2.set("omni_inflight", { provider }, value);
+      registry2.set("omni_stream_connections", {}, streams.connections);
+      registry2.set("omni_stream_queued", {}, streams.queued);
+      registry2.set("omni_stream_dropped_total", {}, streams.dropped);
+      registry2.set("omni_coord_fallback", {}, coordFallback ? 1 : 0);
+      return renderPrometheus(registry2.snapshot());
+    },
+    stop() {
+      exporter?.stop();
+    }
+  };
+}
+
 // apps/gateway/src/index.ts
 function stdoutLogger(level) {
   return createLogger({
@@ -59915,7 +60458,17 @@ async function main() {
     logger2.info("retired interrupted requests", { count: swept });
   const console2 = consoleSource(config2.logFile);
   logger2.info("console log source resolved", { reason: console2.source.kind });
-  const http = nodeHttpClient({ logger: logger2, now });
+  const telemetry = createTelemetry({
+    metricsEnabled: config2.metricsToken !== null,
+    maxSeries: config2.metricsMaxSeries,
+    otlpEndpoint: config2.otlpEndpoint,
+    otlpHeaders: parseOtlpHeaders(config2.otlpHeaders),
+    traceSample: config2.traceSample,
+    now,
+    version: VERSION,
+    logger: logger2
+  });
+  const http = nodeHttpClient({ logger: logger2, now, onResponseHead: telemetry.httpHead });
   const shared = config2.clusterMode && config2.redisUrl !== null ? redisCoord({ url: config2.redisUrl, logger: logger2, now }) : null;
   const coord = shared ?? memoryCoord({ now });
   const lease = { coord, nodeId };
@@ -59986,6 +60539,8 @@ async function main() {
     broadcaster,
     ring: streamRing,
     channels: pluginChannels,
+    telemetry,
+    ...config2.metricsToken === null ? {} : { metricsToken: config2.metricsToken },
     bodyLoggingAllowed: config2.bodyLoggingAllowed,
     plugins: loadedPlugins.plugins.map((plugin) => ({ id: plugin.id, routes: plugin.routes })),
     pluginUi: loadedPlugins.plugins,
@@ -60043,6 +60598,7 @@ async function main() {
         streamRegistry.closeAll(1001, "restart");
         streamRegistry.stop();
         broadcaster.stop();
+        telemetry.stop();
         pluginChannels.stop();
       }
     ],