@neutrome/open-ai-router 0.1.11 → 0.1.12

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@neutrome/open-ai-router",
-  "version": "0.1.11",
+  "version": "0.1.12",
   "type": "module",
   "exports": {
     ".": "./src/index.ts"
   },
   "dependencies": {
     "hono": "^4.12.18",
-    "@neutrome/lil-engine": "0.1.3"
+    "@neutrome/lil-engine": "0.1.4"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",

package/src/app/factory.ts

@@ -3,6 +3,7 @@ import type { Executor, ProgramTransform } from "@neutrome/lil-engine";
 import type { ExecutionRuntimeOptions } from "@neutrome/lil-engine";
 import { cors } from "./cors.ts";
 import { healthHandler } from "./health.ts";
+import { createTraceCollector, type ExecutionTrace } from "../observer.ts";
 import {
   createRouterRuntime,
   handleAnthropicMessages,
@@ -14,7 +15,6 @@ import {
 } from "../router/index.ts";
 import type { RequestContext } from "../auth/types.ts";
 import type { RouterConfig } from "../router/config.ts";
-import type { UsageReport, ProviderCallReport } from "../router/execute.ts";
 
 export type RouterAppAuth = {
   authenticate(request: Request, env: Record<string, unknown>): Promise<object | null>;
@@ -23,9 +23,7 @@ export type RouterAppAuth = {
 
 export type RouterAppHooks = {
   beforeRequest?(ctx: { request: Request; model: string | null; env: Record<string, unknown> }): Promise<Response | void>;
-  onUsage?(report: UsageReport, env: Record<string, unknown>): void;
-  onProviderCall?(report: ProviderCallReport, env: Record<string, unknown>): void;
-  onToolCall?(report: { toolName: string }, env: Record<string, unknown>): void;
+  onTrace?(trace: ExecutionTrace, env: Record<string, unknown>): void | Promise<void>;
 };
 
 export type RouterAppOptions = {
@@ -40,7 +38,6 @@ export type RouterAppOptions = {
   requestIdFactory?: ExecutionRuntimeOptions["requestIdFactory"];
   executionIdFactory?: ExecutionRuntimeOptions["executionIdFactory"];
   upstreamTimeoutMs?: number;
-  suppressUsageToClient?: boolean;
 };
 
 export function createRouterApp(options: RouterAppOptions) {
@@ -114,25 +111,39 @@ export function createRouterApp(options: RouterAppOptions) {
       driver.collectIncoming?.(ctx);
     }
 
-    const onUsage = options.hooks?.onUsage
-      ? (report: UsageReport) => options.hooks!.onUsage!(report, env)
-      : undefined;
-    const onProviderCall = options.hooks?.onProviderCall
-      ? (report: ProviderCallReport) => options.hooks!.onProviderCall!(report, env)
-      : undefined;
-
-    return handler({
-      runtime,
-      ctx,
-      executors: options.executors,
-      ...(options.transforms ? { transforms: options.transforms } : {}),
-      ...(options.observe ? { observe: options.observe } : {}),
-      ...(options.upstreamTimeoutMs !== undefined ? { upstreamTimeoutMs: options.upstreamTimeoutMs } : {}),
-      ...(options.suppressUsageToClient ? { suppressUsageToClient: true } : {}),
-      ...(onUsage ? { onUsage } : {}),
-      ...(onProviderCall ? { onProviderCall } : {}),
-      ...(forceStreaming ? { forceStreaming: true } : {}),
-    });
+    const collector = createTraceCollector();
+    const observe = (event: Parameters<NonNullable<ExecutionRuntimeOptions["observe"]>>[0]) => {
+      collector.observe(event);
+      options.observe?.(event);
+    };
+    let traceFinalized = false;
+    const finalizeTrace = () => {
+      if (traceFinalized) return;
+      traceFinalized = true;
+      Promise.resolve(options.hooks?.onTrace?.(collector.getTrace(), env)).catch((error) => {
+        console.error("[open-ai-router] onTrace hook failed:", error);
+      });
+    };
+
+    try {
+      const response = await handler({
+        runtime,
+        ctx,
+        executors: options.executors,
+        ...(options.transforms ? { transforms: options.transforms } : {}),
+        observe,
+        ...(options.upstreamTimeoutMs !== undefined ? { upstreamTimeoutMs: options.upstreamTimeoutMs } : {}),
+        ...(forceStreaming ? { forceStreaming: true } : {}),
+      });
+      if (!isEventStreamResponse(response) || !response.body) {
+        finalizeTrace();
+        return response;
+      }
+      return wrapResponseWithFinalize(response, finalizeTrace);
+    } catch (error) {
+      finalizeTrace();
+      throw error;
+    }
   }
 
   return app;
@@ -191,3 +202,37 @@ function errorResponse(message: string, status: number, code: string): Response
     { status, headers: { "content-type": "application/json; charset=utf-8" } },
   );
 }
+
+function isEventStreamResponse(response: Response): boolean {
+  const contentType = response.headers.get("content-type") ?? "";
+  return contentType.startsWith("text/event-stream");
+}
+
+function wrapResponseWithFinalize(response: Response, finalize: () => void): Response {
+  const reader = response.body!.getReader();
+  const body = new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      try {
+        const { done, value } = await reader.read();
+        if (done) {
+          finalize();
+          controller.close();
+          return;
+        }
+        controller.enqueue(value);
+      } catch (error) {
+        finalize();
+        controller.error(error);
+      }
+    },
+    async cancel(reason) {
+      finalize();
+      await reader.cancel(reason).catch(() => {});
+    },
+  });
+  return new Response(body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers: new Headers(response.headers),
+  });
+}

package/src/index.ts

@@ -1,5 +1,14 @@
 export { createRouterApp } from "./app/factory.ts";
 export type { RouterAppAuth, RouterAppHooks, RouterAppOptions } from "./app/factory.ts";
+export {
+  createTraceCollector,
+  renderProgramAsLil,
+} from "./observer.ts";
+export type {
+  ExecutionTrace,
+  TraceSpan,
+  TraceSpanKind,
+} from "./observer.ts";
 
 export { createApp } from "./example.ts";
 export type { CreateAppOptions } from "./example.ts";
@@ -32,15 +41,12 @@ export type {
   HandleResponsesOptions,
   ProviderApiStyle,
   ProviderApiStyle as ProviderStyle,
-  ProviderCallReport,
-  ProviderErrorReport,
   ProviderRuntime,
   ProviderTargetConfig,
   ResolvedTarget,
   RouterConfig,
   RouterExecutionOptions,
   RouterRuntime,
-  UsageReport,
 } from "./router/index.ts";
 
 export type {

package/src/observer.ts

@@ -0,0 +1,250 @@
+import { opcodeName, type AuditEvent, type Program } from "@neutrome/lil-engine";
+
+const decoder = new TextDecoder();
+
+export type TraceSpanKind = "request" | "provider" | "tool" | "executor";
+
+export type TraceSpan = {
+  kind: TraceSpanKind;
+  event: string;
+  requestId: string;
+  executionId: string;
+  parentExecutionId?: string;
+  model?: string;
+  provider?: string;
+  toolName?: string;
+  executor?: string;
+  startedAt: string;
+  finishedAt?: string;
+  tokensInput: number;
+  tokensOutput: number;
+  cachedTokensInput: number;
+  requestBytes: number;
+  responseBytes: number;
+  error?: string;
+};
+
+export type ExecutionTrace = {
+  spanId: string;
+  startedAt: string;
+  finishedAt?: string;
+  spans: TraceSpan[];
+  totalTokensInput: number;
+  totalTokensOutput: number;
+  totalCachedTokensInput: number;
+  rawTrace: string;
+  error?: string;
+};
+
+export function createTraceCollector(spanId = crypto.randomUUID()): {
+  observe: (event: AuditEvent) => void;
+  getTrace: () => ExecutionTrace;
+} {
+  const spans: TraceSpan[] = [];
+  const providerSpans = new Map<string, TraceSpan>();
+  const executorSpans = new Map<string, TraceSpan>();
+  const rawSegments: string[] = [];
+  const startedAt = new Date().toISOString();
+  let finishedAt: string | undefined;
+  let error: string | undefined;
+
+  return {
+    observe(event) {
+      appendLilSegment(event);
+
+      switch (event.kind) {
+        case "request.received":
+          spans.push(createSpan("request", event, {
+            finishedAt: event.timestamp,
+          }));
+          break;
+        case "provider.started":
+        case "provider.stream_started": {
+          const span = createSpan("provider", event, {
+            event: event.kind,
+            ...withMaybeString("model", asString(event.data?.model)),
+            ...withMaybeString("provider", asString(event.data?.provider)),
+          });
+          spans.push(span);
+          providerSpans.set(event.executionId, span);
+          break;
+        }
+        case "provider.usage": {
+          const span = providerSpans.get(event.executionId);
+          if (!span) break;
+          span.tokensInput = asNumber(event.data?.tokensInput);
+          span.tokensOutput = asNumber(event.data?.tokensOutput);
+          span.cachedTokensInput = asNumber(event.data?.cachedTokensInput);
+          span.requestBytes = asNumber(event.data?.requestBytes);
+          span.responseBytes = asNumber(event.data?.responseBytes);
+          const provider = asString(event.data?.provider);
+          if (!span.provider && provider) {
+            span.provider = provider;
+          }
+          const model = asString(event.data?.model);
+          if (!span.model && model) {
+            span.model = model;
+          }
+          break;
+        }
+        case "provider.finished":
+        case "provider.stream_finished": {
+          const span = providerSpans.get(event.executionId);
+          if (!span) break;
+          span.finishedAt = event.timestamp;
+          break;
+        }
+        case "executor.started":
+        case "executor.stream_started": {
+          const span = createSpan("executor", event, {
+            event: event.kind,
+            ...withMaybeString("executor", asString(event.data?.executor) || asString(event.target?.id)),
+          });
+          spans.push(span);
+          executorSpans.set(event.executionId, span);
+          break;
+        }
+        case "executor.finished":
+        case "executor.stream_finished": {
+          const span = executorSpans.get(event.executionId);
+          if (!span) break;
+          span.finishedAt = event.timestamp;
+          break;
+        }
+        case "tool.executed":
+          spans.push(createSpan("tool", event, {
+            finishedAt: event.timestamp,
+            ...withMaybeString("toolName", asString(event.data?.toolName)),
+          }));
+          break;
+        case "execution.failed": {
+          const message = asString(event.data?.message) || `Execution failed (${event.errorKind ?? "unknown"})`;
+          error = error ?? message;
+          const span = providerSpans.get(event.executionId) ?? executorSpans.get(event.executionId);
+          if (span) {
+            span.error = message;
+            span.finishedAt = span.finishedAt ?? event.timestamp;
+          }
+          break;
+        }
+      }
+    },
+
+    getTrace() {
+      const totalTokensInput = spans.reduce((sum, span) => sum + span.tokensInput, 0);
+      const totalTokensOutput = spans.reduce((sum, span) => sum + span.tokensOutput, 0);
+      const totalCachedTokensInput = spans.reduce((sum, span) => sum + span.cachedTokensInput, 0);
+      finishedAt ??= new Date().toISOString();
+      return {
+        spanId,
+        startedAt,
+        finishedAt,
+        spans,
+        totalTokensInput,
+        totalTokensOutput,
+        totalCachedTokensInput,
+        rawTrace: rawSegments.join("\n\n"),
+        ...(error ? { error } : {}),
+      };
+    },
+  };
+
+  function appendLilSegment(event: AuditEvent): void {
+    const lilText = asString(event.data?.lilText)?.trim();
+    if (!lilText) return;
+    rawSegments.push(lilText);
+  }
+}
+
+export function renderProgramAsLil(program: Program): string {
+  return program.code
+    .map((instruction) => {
+      const name = opcodeName(instruction.opcode) ?? `OP_${instruction.opcode}`;
+      const value = formatInstructionValue(program, instruction.value);
+      return value ? `${name} ${value}` : name;
+    })
+    .join("\n");
+}
+
+function createSpan(
+  kind: TraceSpanKind,
+  event: AuditEvent,
+  input: Partial<Omit<TraceSpan, "kind" | "requestId" | "executionId" | "startedAt">> = {},
+): TraceSpan {
+  return {
+    kind,
+    event: input.event ?? event.kind,
+    requestId: event.requestId,
+    executionId: event.executionId,
+    ...(event.parentExecutionId ? { parentExecutionId: event.parentExecutionId } : {}),
+    startedAt: event.timestamp,
+    ...withMaybeString("model", input.model),
+    ...withMaybeString("provider", input.provider),
+    ...withMaybeString("toolName", input.toolName),
+    ...withMaybeString("executor", input.executor),
+    ...(input.finishedAt ? { finishedAt: input.finishedAt } : {}),
+    tokensInput: input.tokensInput ?? 0,
+    tokensOutput: input.tokensOutput ?? 0,
+    cachedTokensInput: input.cachedTokensInput ?? 0,
+    requestBytes: input.requestBytes ?? 0,
+    responseBytes: input.responseBytes ?? 0,
+    ...withMaybeString("error", input.error),
+  };
+}
+
+function formatInstructionValue(program: Program, value: Program["code"][number]["value"]): string {
+  switch (value.kind) {
+    case "none":
+      return "";
+    case "string":
+      return JSON.stringify(value.value);
+    case "float":
+    case "int":
+      return String(value.value);
+    case "json":
+      return formatBytes(value.value);
+    case "ref": {
+      const bytes = program.buffers[value.value];
+      if (!bytes) {
+        return `@${value.value}`;
+      }
+      return `@${value.value}=${formatBytes(bytes)}`;
+    }
+    case "key_string":
+      return `${JSON.stringify(value.key)}=${JSON.stringify(value.value)}`;
+    case "key_json":
+      return `${JSON.stringify(value.key)}=${formatBytes(value.value)}`;
+  }
+}
+
+function formatBytes(bytes: Uint8Array): string {
+  const text = decoder.decode(bytes);
+  if (text.includes("\uFFFD")) {
+    return `base64:${toBase64(bytes)}`;
+  }
+  const trimmed = text.trim();
+  if (trimmed.startsWith("{") || trimmed.startsWith("[") || trimmed.startsWith("\"")) {
+    return trimmed;
+  }
+  return JSON.stringify(text);
+}
+
+function toBase64(bytes: Uint8Array): string {
+  let binary = "";
+  for (const byte of bytes) {
+    binary += String.fromCharCode(byte);
+  }
+  return btoa(binary);
+}
+
+function asString(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function asNumber(value: unknown): number {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0;
+}
+
+function withMaybeString<K extends string>(key: K, value: string | undefined): Partial<Record<K, string>> {
+  return value ? { [key]: value } as Record<K, string> : {};
+}

package/src/router/execute.ts

@@ -1,4 +1,5 @@
 import {
+  createAuditEvent,
   emitChatCompletionsResponse,
   emitProviderRequest,
   emitProviderResponse,
@@ -27,6 +28,7 @@ import {
 import type { RequestContext, TargetAuthContext } from "../auth/types.ts";
 import { cloneForwardHeaders, isSse } from "../util/headers.ts";
 import { eventDataLines, splitSseEvents } from "../util/sse.ts";
+import { renderProgramAsLil } from "../observer.ts";
 import { resolveInvocationTarget, type ResolvedTarget } from "./resolve.ts";
 import type { ProviderRuntime, RouterRuntime } from "./runtime.ts";
 
@@ -34,25 +36,6 @@ const encoder = new TextEncoder();
 const decoder = new TextDecoder();
 const DEFAULT_UPSTREAM_TIMEOUT_MS = 120_000;
 
-export type UsageReport = {
-  model: string;
-  provider: string;
-  tokensInput: number;
-  tokensOutput: number;
-  cachedTokensInput: number;
-};
-
-export type ProviderErrorReport = {
-  model: string;
-  provider: string;
-  status: number;
-  message: string;
-};
-
-export type ProviderCallReport =
-  | { kind: "usage"; requestBytes: number; responseBytes: number } & UsageReport
-  | { kind: "error"; } & ProviderErrorReport;
-
 export type RouterExecutionOptions = Pick<
   ExecutionRuntimeOptions,
   "observe" | "requestIdFactory" | "executionIdFactory" | "resolveExecutor"
@@ -60,9 +43,6 @@ export type RouterExecutionOptions = Pick<
   executors?: Readonly<Record<string, Executor>>;
   transforms?: Readonly<Record<string, ProgramTransform>>;
   upstreamTimeoutMs?: number;
-  suppressUsageToClient?: boolean;
-  onUsage?: (report: UsageReport) => void;
-  onProviderCall?: (report: ProviderCallReport) => void;
 };
 
 export type HandleChatCompletionsOptions = RouterExecutionOptions & {
@@ -122,7 +102,6 @@ export function createRouterExecutionRuntime(
       runtime,
       ctx,
       options.upstreamTimeoutMs,
-      options.onProviderCall,
     ),
   };
 
@@ -183,6 +162,11 @@ export async function handleChatCompletions(
   try {
     const requestBody = new Uint8Array(await options.ctx.request.arrayBuffer());
     const request = parseChatCompletionsRequest(requestBody);
+    const incomingExecutionId = `incoming_${crypto.randomUUID()}`;
+    emitObservedProgram(options.observe, "request.received", request, {
+      requestId: incomingExecutionId,
+      executionId: incomingExecutionId,
+    });
     const resolved = resolveRequestTarget(options.runtime, request);
     const execution = createRouterExecutionRuntime(
       options.runtime,
@@ -192,15 +176,12 @@ export async function handleChatCompletions(
 
     const startTime = Date.now();
     if (isStreaming(request)) {
-      const userWantsUsage = options.suppressUsageToClient ? false : requestedStreamUsage(requestBody);
       const chunks = execution.stream(request, { target: resolved.target });
       return await streamExecutionResponse(
         "chat-completions",
         chunks,
         resolved,
         startTime,
-        options.onUsage,
-        userWantsUsage,
       );
     }
 
@@ -208,7 +189,6 @@ export async function handleChatCompletions(
       target: resolved.target,
     });
     const userTime = Date.now() - startTime;
-    reportUsage(response, resolved, options.onUsage);
     return jsonExecutionResponse(
       emitChatCompletionsResponse(response),
       resolved,
@@ -254,6 +234,11 @@ async function handleProviderStyle(
     const request = mutateRequest
       ? mutateRequest(parseProviderRequest(style, requestBody))
       : parseProviderRequest(style, requestBody);
+    const incomingExecutionId = `incoming_${crypto.randomUUID()}`;
+    emitObservedProgram(options.observe, "request.received", request, {
+      requestId: incomingExecutionId,
+      executionId: incomingExecutionId,
+    });
     const resolved = resolveRequestTarget(options.runtime, request);
     const execution = createRouterExecutionRuntime(
       options.runtime,
@@ -269,8 +254,6 @@ async function handleProviderStyle(
         chunks,
         resolved,
         startTime,
-        options.onUsage,
-        !options.suppressUsageToClient,
       );
     }
 
@@ -278,7 +261,6 @@ async function handleProviderStyle(
       target: resolved.target,
     });
     const userTime = Date.now() - startTime;
-    reportUsage(response, resolved, options.onUsage);
     return jsonExecutionResponse(
       emitProviderResponse(style, response),
       resolved,
@@ -313,7 +295,6 @@ export function createFetchProviderInvoker(
   runtime: RouterRuntime,
   ctx: RequestContext,
   upstreamTimeoutMs = DEFAULT_UPSTREAM_TIMEOUT_MS,
-  onProviderCall?: (report: ProviderCallReport) => void,
 ): ProviderInvoker {
   return {
     async execute(request, providerCtx) {
@@ -321,24 +302,37 @@ export function createFetchProviderInvoker(
       const timed = createUpstreamSignal(providerCtx.signal, upstreamTimeoutMs);
 
       try {
-        const reqBody = emitProviderRequest(provider.style, setModel(request, providerCtx.target.model));
+        const resolvedRequest = setModel(request, providerCtx.target.model);
+        emitObservedProgram(providerCtx.observe, "provider.request", resolvedRequest, {
+          requestId: providerCtx.requestId,
+          executionId: providerCtx.executionId,
+          ...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
+          provider: provider.name,
+          model: providerCtx.target.model,
+        });
+        const reqBody = emitProviderRequest(provider.style, resolvedRequest);
         const reqBytes = reqBody.byteLength;
         const upstream = await fetchProvider(
           runtime,
           ctx,
           provider,
-          setModel(request, providerCtx.target.model),
+          resolvedRequest,
           timed.signal,
         );
         if (!upstream.ok) {
-          const err = await toProviderError(provider.name, upstream);
-          reportProviderError(onProviderCall, providerCtx.target, err);
-          throw err;
+          throw await toProviderError(provider.name, upstream);
         }
 
         const bytes = new Uint8Array(await upstream.arrayBuffer());
         const response = parseProviderResponse(provider.style, bytes);
-        reportProviderUsage(onProviderCall, response, providerCtx.target, reqBytes, bytes.byteLength);
+        emitObservedProgram(providerCtx.observe, "provider.response", response, {
+          requestId: providerCtx.requestId,
+          executionId: providerCtx.executionId,
+          ...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
+          provider: provider.name,
+          model: providerCtx.target.model,
+        });
+        emitProviderUsage(providerCtx, response, provider.name, reqBytes, bytes.byteLength);
         return response;
       } catch (error) {
         throw normalizeUpstreamAbort(error, timed.signal);
@@ -354,19 +348,25 @@ export function createFetchProviderInvoker(
       let buffer = "";
 
       try {
-        const reqBody = emitProviderRequest(provider.style, setModel(request, providerCtx.target.model));
+        const resolvedRequest = setModel(request, providerCtx.target.model);
+        emitObservedProgram(providerCtx.observe, "provider.request", resolvedRequest, {
+          requestId: providerCtx.requestId,
+          executionId: providerCtx.executionId,
+          ...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
+          provider: provider.name,
+          model: providerCtx.target.model,
+        });
+        const reqBody = emitProviderRequest(provider.style, resolvedRequest);
         const reqBytes = reqBody.byteLength;
         const upstream = await fetchProvider(
           runtime,
           ctx,
           provider,
-          setModel(request, providerCtx.target.model),
+          resolvedRequest,
           timed.signal,
         );
         if (!upstream.ok) {
-          const err = await toProviderError(provider.name, upstream);
-          reportProviderError(onProviderCall, providerCtx.target, err);
-          throw err;
+          throw await toProviderError(provider.name, upstream);
         }
         if (!isSse(upstream.headers) || !upstream.body) {
           throw new RouterError(
@@ -391,7 +391,7 @@ export function createFetchProviderInvoker(
           for (const event of split.events) {
             for (const data of eventDataLines(event)) {
               if (data === "[DONE]") {
-                reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target, reqBytes, respBytes);
+                emitProviderUsage(providerCtx, lastChunkWithUsage, provider.name, reqBytes, respBytes);
                 return;
               }
               if (data[0] !== "{") continue;
@@ -400,6 +400,13 @@ export function createFetchProviderInvoker(
                 encoder.encode(data),
               );
               if (usageObject(chunk) !== undefined) lastChunkWithUsage = chunk;
+              emitObservedProgram(providerCtx.observe, "provider.response_chunk", chunk, {
+                requestId: providerCtx.requestId,
+                executionId: providerCtx.executionId,
+                ...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
+                provider: provider.name,
+                model: providerCtx.target.model,
+              });
               yield chunk;
             }
           }
@@ -408,7 +415,7 @@ export function createFetchProviderInvoker(
         if (buffer.length > 0) {
           for (const data of eventDataLines(buffer)) {
             if (data === "[DONE]") {
-              reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target, reqBytes, respBytes);
+              emitProviderUsage(providerCtx, lastChunkWithUsage, provider.name, reqBytes, respBytes);
               return;
             }
             if (data[0] !== "{") continue;
@@ -417,10 +424,17 @@ export function createFetchProviderInvoker(
               encoder.encode(data),
             );
             if (usageObject(chunk) !== undefined) lastChunkWithUsage = chunk;
+            emitObservedProgram(providerCtx.observe, "provider.response_chunk", chunk, {
+              requestId: providerCtx.requestId,
+              executionId: providerCtx.executionId,
+              ...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
+              provider: provider.name,
+              model: providerCtx.target.model,
+            });
             yield chunk;
           }
         }
-        reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target, reqBytes, respBytes);
+        emitProviderUsage(providerCtx, lastChunkWithUsage, provider.name, reqBytes, respBytes);
       } catch (error) {
         throw normalizeUpstreamAbort(error, timed.signal);
       } finally {
@@ -437,8 +451,6 @@ async function streamExecutionResponse(
   chunks: AsyncIterable<Program>,
   resolved: ResolvedTarget,
   startTime: number,
-  onUsage?: (report: UsageReport) => void,
-  emitUsageToClient = true,
 ): Promise<Response> {
   const iterator = chunks[Symbol.asyncIterator]();
 
@@ -448,12 +460,7 @@ async function streamExecutionResponse(
   const stream = new ReadableStream<Uint8Array>({
     async start(controller) {
       try {
-        let lastChunkWithUsage: Program | null = null;
-
         function emitChunk(chunk: Program): void {
-          const hasUsage = usageObject(chunk) !== undefined;
-          if (hasUsage) lastChunkWithUsage = chunk;
-          if (hasUsage && !emitUsageToClient) return;
           const bytes = emitProviderStreamChunk(responseStyle, chunk);
           controller.enqueue(
             encoder.encode(`data: ${decoder.decode(bytes)}\n\n`),
@@ -474,9 +481,6 @@ async function streamExecutionResponse(
         if (responseStyle === "chat-completions") {
           controller.enqueue(encoder.encode("data: [DONE]\n\n"));
         }
-        if (lastChunkWithUsage) {
-          reportUsage(lastChunkWithUsage, resolved, onUsage);
-        }
         controller.close();
       } catch (error) {
         controller.enqueue(
@@ -749,91 +753,40 @@ function createUpstreamSignal(
   };
 }
 
-function reportUsage(
-  response: Program,
-  resolved: ResolvedTarget,
-  onUsage?: (report: UsageReport) => void,
-): void {
-  if (!onUsage) return;
-  const usage = usageObject(response);
-  if (!usage || typeof usage !== "object") return;
-  const u = usage as Record<string, unknown>;
-  const tokensInput = asNum(u.prompt_tokens) || asNum(u.input_tokens);
-  const tokensOutput = asNum(u.completion_tokens) || asNum(u.output_tokens);
-  if (tokensInput === 0 && tokensOutput === 0) return;
-  const details = u.prompt_tokens_details as Record<string, unknown> | undefined;
-  const cachedTokensInput = details ? asNum(details.cached_tokens) : 0;
-  const model = resolved.target.kind === "provider" ? resolved.target.model : resolved.target.alias;
-  const provider = resolved.target.kind === "provider" ? (resolved.target.provider ?? "") : "";
-  try {
-    onUsage({ model, provider, tokensInput, tokensOutput, cachedTokensInput });
-  } catch {}
-}
-
-function reportProviderUsage(
-  onProviderCall: ((report: ProviderCallReport) => void) | undefined,
+function emitProviderUsage(
+  providerCtx: import("@neutrome/lil-engine").ProviderInvocationContext,
   response: Program | null,
-  target: Extract<import("@neutrome/lil-engine").ExecutionTarget, { kind: "provider" }>,
+  provider: string,
   requestBytes: number,
   responseBytes: number,
 ): void {
-  if (!onProviderCall) return;
   const usage = response ? usageObject(response) : undefined;
   const u = (usage && typeof usage === "object" ? usage : {}) as Record<string, unknown>;
   const tokensInput = asNum(u.prompt_tokens) || asNum(u.input_tokens);
   const tokensOutput = asNum(u.completion_tokens) || asNum(u.output_tokens);
   const details = u.prompt_tokens_details as Record<string, unknown> | undefined;
   const cachedTokensInput = details ? asNum(details.cached_tokens) : 0;
-  try {
-    onProviderCall({
-      kind: "usage",
-      model: target.model,
-      provider: target.provider ?? "",
+  providerCtx.observe(createAuditEvent({
+    kind: "provider.usage",
+    requestId: providerCtx.requestId,
+    executionId: providerCtx.executionId,
+    ...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
+    data: {
+      provider,
+      model: providerCtx.target.model,
       tokensInput,
       tokensOutput,
       cachedTokensInput,
       requestBytes,
       responseBytes,
-    });
-  } catch {}
-}
-
-function reportProviderError(
-  onProviderCall: ((report: ProviderCallReport) => void) | undefined,
-  target: Extract<import("@neutrome/lil-engine").ExecutionTarget, { kind: "provider" }>,
-  error: ProviderHttpError,
-): void {
-  if (!onProviderCall) return;
-  let message = `HTTP ${error.status}`;
-  try {
-    const text = decoder.decode(error.body).slice(0, 500);
-    message = text || message;
-  } catch {}
-  try {
-    onProviderCall({
-      kind: "error",
-      model: target.model,
-      provider: target.provider ?? "",
-      status: error.status,
-      message,
-    });
-  } catch {}
+    },
+  }));
 }
 
 function asNum(v: unknown): number {
   return typeof v === "number" && v >= 0 ? v : 0;
 }
 
-function requestedStreamUsage(body: Uint8Array): boolean {
-  try {
-    const obj = JSON.parse(decoder.decode(body)) as Record<string, unknown>;
-    const opts = obj.stream_options as Record<string, unknown> | undefined;
-    return opts?.include_usage === true;
-  } catch {
-    return false;
-  }
-}
-
 function injectStreamUsage(body: Uint8Array): Uint8Array {
   try {
     const obj = JSON.parse(decoder.decode(body)) as Record<string, unknown>;
@@ -850,3 +803,29 @@ function normalizeUpstreamAbort(error: unknown, signal: AbortSignal): unknown {
   }
   return error;
 }
+
+function emitObservedProgram(
+  observe: ExecutionRuntimeOptions["observe"],
+  kind: string,
+  program: Program,
+  input: {
+    requestId: string;
+    executionId: string;
+    parentExecutionId?: string;
+    provider?: string;
+    model?: string;
+  },
+): void {
+  if (!observe) return;
+  observe(createAuditEvent({
+    kind,
+    requestId: input.requestId,
+    executionId: input.executionId,
+    ...(input.parentExecutionId ? { parentExecutionId: input.parentExecutionId } : {}),
+    data: {
+      ...(input.provider ? { provider: input.provider } : {}),
+      ...(input.model ? { model: input.model } : {}),
+      lilText: renderProgramAsLil(program),
+    },
+  }));
+}

package/src/router/index.ts

@@ -32,8 +32,5 @@ export type {
   HandleGoogleGenAIOptions,
   HandleResponsesOptions,
   RouterExecutionOptions,
-  UsageReport,
-  ProviderCallReport,
-  ProviderErrorReport,
 } from "./execute.ts";
 export type { ResolvedTarget } from "./resolve.ts";

package/src/worker.ts

@@ -38,6 +38,6 @@ export default {
       throw error;
     });
     const app = await appPromise;
-    return app.fetch(request, env, ctx);
+    return app.fetch(request, env, ctx as never);
   },
 };