@neutrome/open-ai-router 0.6.9 → 0.8.0

package/README.md

@@ -30,4 +30,4 @@ Notes:
 
 - `RouterConfig` is the canonical router config surface.
 - `/v1/models` lists configured executor aliases.
-- Executor contracts and `connectTools` come from `@neutrome/lilsdk`.
+- Executor contracts and `createToolsExecutor` come from `@neutrome/lilsdk`.

package/package.json

@@ -1,18 +1,23 @@
 {
   "name": "@neutrome/open-ai-router",
-  "version": "0.6.9",
+  "version": "0.8.0",
   "type": "module",
+  "files": [
+    "src",
+    "README.md"
+  ],
   "exports": {
     ".": "./src/index.ts"
   },
   "dependencies": {
-    "@posthog/ai": "^8.3.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
+    "@posthog/ai": "^8.3.0",
+    "gpt-tokenizer": "4.0.0",
     "hono": "^4.12.18",
     "posthog-node": "^5.41.0",
     "zod": "^4.2.1",
-    "@neutrome/lil-engine": "0.4.6",
-    "@neutrome/lilsdk": "0.4.6"
+    "@neutrome/lil-engine": "0.6.0",
+    "@neutrome/lilsdk": "0.6.0"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",

package/src/app/factory.test.ts

@@ -100,6 +100,7 @@ describe("createRouterApp trace finalization", () => {
   it("keeps non-stream trace hooks alive with waitUntil", async () => {
     const waited: Promise<unknown>[] = [];
     const onExecutionSummary = vi.fn(async (_summary: ExecutionSummary) => {});
+    const onUserUsage = vi.fn(async () => {});
     const app = createRouterApp({
       config,
       executorImplementations: {},
@@ -108,7 +109,7 @@ describe("createRouterApp trace finalization", () => {
           return principal();
         },
       },
-      hooks: { onExecutionSummary },
+      hooks: { onExecutionSummary, onUserUsage },
       fetchImpl: vi.fn(
         async () =>
           new Response(
@@ -145,11 +146,21 @@ describe("createRouterApp trace finalization", () => {
     expect(waited).toHaveLength(1);
     await waited[0];
     expect(onExecutionSummary).toHaveBeenCalledTimes(1);
+    expect(onUserUsage).toHaveBeenCalledWith(
+      expect.objectContaining({
+        requestedModel: "gpt-4o",
+        outcome: "ok",
+        inputTokens: expect.any(Number),
+        outputTokens: expect.any(Number),
+      }),
+      {},
+    );
   });
 
   it("keeps stream trace hooks alive until the stream closes", async () => {
     const waited: Promise<unknown>[] = [];
     const onExecutionSummary = vi.fn(async (_summary: ExecutionSummary) => {});
+    const onUserUsage = vi.fn(async () => {});
     const app = createRouterApp({
       config,
       executorImplementations: {},
@@ -158,7 +169,7 @@ describe("createRouterApp trace finalization", () => {
           return principal();
         },
       },
-      hooks: { onExecutionSummary },
+      hooks: { onExecutionSummary, onUserUsage },
       fetchImpl: vi.fn(async () => {
         const stream = new ReadableStream<Uint8Array>({
           start(controller) {
@@ -217,6 +228,7 @@ describe("createRouterApp trace finalization", () => {
     expect(waited).toHaveLength(1);
     await waited[0];
     expect(onExecutionSummary).toHaveBeenCalledTimes(1);
+    expect(onUserUsage).toHaveBeenCalledTimes(1);
     const events = onExecutionSummary.mock.calls[0]?.[0].spans.flatMap((span) =>
       span.timeline.map((entry) => entry.event),
     );

package/src/app/factory.ts

@@ -1,9 +1,9 @@
 import { Hono, type Context } from "hono";
 import {
   getModel,
-  setStreaming,
   type Program,
   type ProviderStyle,
+  setStreaming,
 } from "@neutrome/lil-engine";
 import { cors } from "./cors.ts";
 import { healthHandler } from "./health.ts";
@@ -35,6 +35,10 @@ import {
 import { createModelListingHandler } from "./model-listing.ts";
 import { honoEnv, unauthorizedResponse } from "./http-utils.ts";
 import type { RouterAppOptions } from "./types.ts";
+import {
+  createUserUsageCollector,
+  responseResult,
+} from "./user-usage-collector.ts";
 
 type ProtocolHandler = (
   options: Parameters<typeof handleChatCompletions>[0],
@@ -100,6 +104,13 @@ export function createRouterApp(options: RouterAppOptions) {
         : {}),
     });
     const onExecutionSummary = options.hooks?.onExecutionSummary;
+    const onUserUsage = options.hooks?.onUserUsage;
+    const requestStartedMs = Date.now();
+    const usage = createUserUsageCollector({
+      requestId: incomingExecutionId,
+      protocol: style,
+      startedAtMs: requestStartedMs,
+    });
     const observe = (
       event: Parameters<NonNullable<ExecutionRuntimeOptions["observe"]>>[0],
     ) => {
@@ -114,13 +125,14 @@ export function createRouterApp(options: RouterAppOptions) {
         c,
         async () => {
           await telemetry.flush();
-          await onExecutionSummary?.(telemetry.summary(), env);
+          const summary = telemetry.summary();
+          await onExecutionSummary?.(summary, env);
+          await onUserUsage?.(usage.report(summary), env);
         },
         "onExecutionSummary hook",
       );
     };
 
-    const requestStartedMs = Date.now();
     observeTimedExecution({
       observe,
       kind: "request.started",
@@ -142,6 +154,7 @@ export function createRouterApp(options: RouterAppOptions) {
       data: { authenticated: !!principal },
     });
     if (!principal) {
+      usage.setResult(responseResult(401, "Authentication failed"));
       telemetry.recordError({
         message: "Authentication failed",
         requestId: incomingExecutionId,
@@ -166,6 +179,7 @@ export function createRouterApp(options: RouterAppOptions) {
         data: { style, streaming: forceStreaming === true },
       });
     } catch {
+      usage.setResult(responseResult(400, "Invalid request body"));
       telemetry.recordError({
         message: "Invalid request body",
         requestId: incomingExecutionId,
@@ -189,6 +203,7 @@ export function createRouterApp(options: RouterAppOptions) {
     }
 
     const model = getModel(program) || null;
+    usage.setRequest(program, model);
     const accessStartedMs = Date.now();
     const hasModelAccess = !model || canAccessPrincipalModel(principal, model);
     observeTimedExecution({
@@ -201,6 +216,7 @@ export function createRouterApp(options: RouterAppOptions) {
       data: { model: model ?? undefined, allowed: hasModelAccess },
     });
     if (!hasModelAccess) {
+      usage.setResult(responseResult(404, "Model access denied"));
       telemetry.recordError({
         message: "Model access denied",
         requestId: incomingExecutionId,
@@ -232,6 +248,12 @@ export function createRouterApp(options: RouterAppOptions) {
         data: { returnedResponse: !!hookResponse },
       });
       if (hookResponse) {
+        usage.setResult(
+          responseResult(
+            hookResponse.status,
+            hookResponse.status >= 400 ? "Request rejected" : null,
+          ),
+        );
         if (hookResponse.status >= 400) {
           telemetry.recordError({
             message: `Request hook returned ${hookResponse.status}`,
@@ -245,6 +267,7 @@ export function createRouterApp(options: RouterAppOptions) {
     }
 
     if (style === "responses" && (await hasPreviousResponseId(request))) {
+      usage.setResult(responseResult(400, "Unsupported response state"));
       finalizeTrace();
       return errorResponse(
         new RouterError(
@@ -270,6 +293,10 @@ export function createRouterApp(options: RouterAppOptions) {
         ...(options.transforms ? { transforms: options.transforms } : {}),
         observe,
         incomingExecutionId,
+        userUsage: { inputTokens: usage.inputTokens() },
+        onUserResponse(result) {
+          usage.setResult(result);
+        },
         ...(options.upstreamTimeoutMs !== undefined
           ? { upstreamTimeoutMs: options.upstreamTimeoutMs }
           : {}),
@@ -296,6 +323,7 @@ export function createRouterApp(options: RouterAppOptions) {
       }
       return finalizeWhenStreamCloses(response, finalizeTrace);
     } catch (error) {
+      usage.setResult(responseResult(500, "Request execution failed"));
       telemetry.recordError({
         message:
           error instanceof Error ? error.message : "Request execution failed",

package/src/app/response-lifecycle.ts

@@ -54,8 +54,8 @@ export function finalizeWhenStreamCloses(
       }
     },
     async cancel(reason) {
-      finalize();
       await reader.cancel(reason).catch(() => {});
+      finalize();
     },
   });
   return new Response(body, {

package/src/app/types.ts

@@ -7,6 +7,7 @@ import type { ExecutionSummary } from "../telemetry/index.ts";
 import type { RouterConfig } from "../router/config.ts";
 import type { ExecutionRuntimeOptions } from "../router/execution-types.ts";
 import type { RemoteMcpClientFactory } from "../router/mcp.ts";
+import type { UserUsageReport } from "../usage/user-usage.ts";
 
 export type RouterAppAuth = RuntimeAuthenticator;
 
@@ -21,6 +22,10 @@ export type RouterAppHooks = {
     summary: ExecutionSummary,
     env: Record<string, unknown>,
   ): void | Promise<void>;
+  onUserUsage?(
+    report: UserUsageReport,
+    env: Record<string, unknown>,
+  ): void | Promise<void>;
 };
 
 export type RouterAppOptions = {

package/src/app/user-usage-collector.ts

@@ -0,0 +1,68 @@
+import type { Program, ProviderStyle } from "@neutrome/lil-engine";
+import type { ExecutionSummary } from "../telemetry/index.ts";
+import {
+  countUserProgramTokens,
+  toolUsageFromEvents,
+  type UserUsageReport,
+} from "../usage/user-usage.ts";
+
+export type UserResponseResult = Pick<
+  UserUsageReport,
+  "outputTokens" | "outcome" | "httpStatus" | "error"
+>;
+
+export function createUserUsageCollector(input: {
+  requestId: string;
+  protocol: ProviderStyle;
+  startedAtMs: number;
+}) {
+  let inputTokens = 0;
+  let requestedModel: string | null = null;
+  let result: UserResponseResult | null = null;
+  return {
+    setRequest(program: Program, model: string | null) {
+      inputTokens = countUserProgramTokens(program);
+      requestedModel = model;
+    },
+    setResult(next: UserResponseResult) {
+      result = next;
+    },
+    inputTokens() {
+      return inputTokens;
+    },
+    report(summary: ExecutionSummary): UserUsageReport {
+      const finishedAt = new Date().toISOString();
+      return {
+        requestId: input.requestId,
+        protocol: input.protocol,
+        requestedModel,
+        inputTokens,
+        ...(result ??
+          responseResult(null, "Request did not produce a response")),
+        tools: toolUsageFromEvents(
+          summary.spans
+            .filter((span) => span.kind === "tool" && span.toolName)
+            .map((span) => ({
+              name: span.toolName!,
+              error: span.status === "error",
+            })),
+        ),
+        startedAt: new Date(input.startedAtMs).toISOString(),
+        finishedAt,
+        durationMs: Math.max(0, Date.parse(finishedAt) - input.startedAtMs),
+      };
+    },
+  };
+}
+
+export function responseResult(
+  status: number | null,
+  error: string | null,
+): UserResponseResult {
+  return {
+    outputTokens: 0,
+    outcome: status !== null && status < 400 ? "ok" : "error",
+    httpStatus: status,
+    error,
+  };
+}

package/src/index.ts

@@ -9,6 +9,7 @@ export {
   PosthogErrorTelemetry,
   PosthogTraceTelemetry,
 } from "./telemetry/index.ts";
+export type { UserUsageReport, UserToolUsage } from "./usage/user-usage.ts";
 export type {
   ErrorTelemetryDriver,
   ExecutionSummary,

package/src/router/error-codec.ts

@@ -2,7 +2,7 @@ import {
   appendErrorBlock,
   createProgram,
   emitProviderError,
-  firstError,
+  getFirstErrorBlock,
   parseProviderError,
   type Program,
   type ProviderStyle,
@@ -21,7 +21,7 @@ export async function toProviderError(
     contentType: response.headers.get("content-type"),
   });
   const summary =
-    firstError(program)?.message ||
+    getFirstErrorBlock(program)?.message ||
     `Provider ${provider.name} returned ${response.status}`;
   return new ProviderHttpError(
     summary,

package/src/router/execute.test.ts

@@ -1,4 +1,4 @@
-import { appendAssistantMessage } from "@neutrome/lilsdk/synthetic";
+import { appendAssistantMessage, createProgram } from "@neutrome/lil-engine";
 import { describe, expect, it, vi } from "vitest";
 import type { RequestContext } from "../upstream-auth/types.ts";
 import { handleChatCompletions } from "./execute.ts";
@@ -88,6 +88,7 @@ describe("router execution", () => {
       object: "chat.completion",
       id: "resp_1",
       model: "gpt-4o",
+      usage: { prompt_tokens: 0, completion_tokens: 27, total_tokens: 27 },
       choices: [
         {
           index: 0,
@@ -334,6 +335,7 @@ describe("router execution", () => {
       object: "chat.completion",
       id: "resp_1",
       model: "gpt-5",
+      usage: { prompt_tokens: 0, completion_tokens: 27, total_tokens: 27 },
       choices: [
         {
           index: 0,
@@ -411,9 +413,9 @@ describe("router execution", () => {
       id: "msg_1",
       model: "claude-sonnet-4",
       usage: {
-        prompt_tokens: 10,
-        completion_tokens: 5,
-        total_tokens: 15,
+        prompt_tokens: 0,
+        completion_tokens: 27,
+        total_tokens: 27,
       },
       choices: [
         {
@@ -494,6 +496,7 @@ describe("router execution", () => {
     expect(await response.json()).toEqual({
       object: "chat.completion",
       model: "gemini-2.5-flash",
+      usage: { prompt_tokens: 0, completion_tokens: 27, total_tokens: 27 },
       choices: [
         {
           index: 0,
@@ -543,10 +546,10 @@ describe("router execution", () => {
       executorImplementations: {
         "enei-1": {
           async execute(request) {
-            return appendAssistantMessage(request, "done");
+            return appendAssistantMessage(createProgram(), "done");
           },
           async *stream(request) {
-            yield appendAssistantMessage(request, "done");
+            yield appendAssistantMessage(createProgram(), "done");
           },
         },
       },
@@ -556,6 +559,7 @@ describe("router execution", () => {
     expect(response.headers.get("x-openairouter-executor-id")).toBe("enei-1");
     expect(await response.json()).toEqual({
       object: "chat.completion",
+      usage: { prompt_tokens: 0, completion_tokens: 27, total_tokens: 27 },
       choices: [
         {
           index: 0,

package/src/router/execute.ts

@@ -1,20 +1,19 @@
 import {
-  emitChatCompletionsResponse,
   emitProviderError,
   emitProviderRequest,
   emitProviderResponse,
   emitProviderStreamChunk,
   getModel,
+  getUsage,
   isStreaming,
-  parseChatCompletionsRequest,
   parseProviderRequest,
   parseProviderResponse,
   parseProviderStreamChunk,
-  setModel,
-  setStreaming,
   type Program,
   type ProviderStyle,
-  usageObject,
+  setModel,
+  setStreaming,
+  replaceUsage,
 } from "@neutrome/lil-engine";
 import { createExecutionRuntime } from "./execution-runtime.ts";
 import { createFetchProviderInvoker } from "./provider-invoker.ts";
@@ -63,6 +62,7 @@ import {
 import type { RemoteMcpClientFactory } from "./mcp.ts";
 import type { ProviderRuntime, RouterRuntime } from "./runtime.ts";
 import { observeTimedExecution } from "./execution-events.ts";
+import { countUserProgramTokens, protocolUsage } from "../usage/user-usage.ts";
 
 const encoder = new TextEncoder();
 const decoder = new TextDecoder();
@@ -81,6 +81,13 @@ export type RouterExecutionOptions = Pick<
   upstreamTimeoutMs?: number;
   remoteMcpClientFactory?: RemoteMcpClientFactory;
   incomingExecutionId?: string;
+  userUsage?: { inputTokens: number };
+  onUserResponse?: (result: {
+    outputTokens: number;
+    outcome: "ok" | "error" | "cancelled";
+    httpStatus: number | null;
+    error: string | null;
+  }) => void | Promise<void>;
 };
 
 export type HandleChatCompletionsOptions = RouterExecutionOptions & {
@@ -165,10 +172,8 @@ export function createRouterExecutionRuntime(
 export async function handleChatCompletions(
   options: HandleChatCompletionsOptions,
 ): Promise<Response> {
-  return handleRequestStyle(
-    "chat-completions",
-    options,
-    emitChatCompletionsResponse,
+  return handleRequestStyle("chat-completions", options, (program) =>
+    emitProviderResponse("chat-completions", program),
   );
 }
 
@@ -244,6 +249,10 @@ async function handleRequestStyle(
             requestId: incomingExecutionId,
             executionId: incomingExecutionId,
             startedAtMs: responseStartedMs,
+            inputTokens: options.userUsage?.inputTokens ?? 0,
+            onFinish: async (result) => {
+              await options.onUserResponse?.({ ...result, httpStatus: 200 });
+            },
           },
           remoteMcp.close,
         );
@@ -259,12 +268,38 @@ async function handleRequestStyle(
         target: remoteMcp.target,
       });
       const userTime = Date.now() - startTime;
-      return jsonExecutionResponse(emitResponse(response), resolved, userTime);
+      const outputTokens = countUserProgramTokens(response);
+      await options.onUserResponse?.({
+        outputTokens,
+        outcome: "ok",
+        httpStatus: 200,
+        error: null,
+      });
+      return jsonExecutionResponse(
+        emitResponse(
+          replaceUsage(
+            response,
+            protocolUsage(
+              style,
+              options.userUsage?.inputTokens ?? 0,
+              outputTokens,
+            ),
+          ),
+        ),
+        resolved,
+        userTime,
+      );
     } finally {
       await remoteMcp.close();
     }
   } catch (error) {
     const response = errorResponse(error, style);
+    await options.onUserResponse?.({
+      outputTokens: 0,
+      outcome: "error",
+      httpStatus: response.status,
+      error: "Request execution failed",
+    });
     return resolved ? applyRoutingHeaders(response, resolved) : response;
   }
 }

package/src/router/execution-resolution.ts

@@ -1,4 +1,4 @@
-import { firstError, type Program } from "@neutrome/lil-engine";
+import { getFirstErrorBlock, type Program } from "@neutrome/lil-engine";
 import {
   ExecutionError,
   isExecutionError,
@@ -100,7 +100,7 @@ export function normalizeExecutionError(
 ): ExecutionError {
   if (isExecutionError(error)) return error;
   if (error instanceof ProviderHttpError) {
-    const upstream = firstError(error.program);
+    const upstream = getFirstErrorBlock(error.program);
     return new ExecutionError("provider", error.message, {
       stage: fallbackStage,
       details: {

package/src/router/provider-stream.ts

@@ -1,10 +1,10 @@
 import {
   emitProviderRequest,
+  getUsage,
   isStreaming,
   parseProviderStreamChunk,
-  setModel,
   type Program,
-  usageObject,
+  setModel,
 } from "@neutrome/lil-engine";
 import { RouterError } from "./errors.ts";
 import { toProviderError } from "./error-codec.ts";
@@ -155,7 +155,7 @@ export async function* streamProvider(
             encoder.encode(data),
           );
           responseChunks.push(chunk);
-          if (usageObject(chunk) !== undefined) lastChunkWithUsage = chunk;
+          if (getUsage(chunk) !== undefined) lastChunkWithUsage = chunk;
           yield chunk;
         }
       }
@@ -181,7 +181,7 @@ export async function* streamProvider(
           encoder.encode(data),
         );
         responseChunks.push(chunk);
-        if (usageObject(chunk) !== undefined) lastChunkWithUsage = chunk;
+        if (getUsage(chunk) !== undefined) lastChunkWithUsage = chunk;
         yield chunk;
       }
     }

package/src/router/provider-telemetry.ts

@@ -1,4 +1,4 @@
-import { usageObject, type Program } from "@neutrome/lil-engine";
+import { getUsage, type Program } from "@neutrome/lil-engine";
 import { createExecutionEvent } from "@neutrome/lilsdk";
 import { z } from "zod";
 import {
@@ -21,7 +21,7 @@ export function emitProviderUsage(
   responseBytes: number,
   streamChunks?: readonly Program[],
 ): void {
-  const usage = response ? usageObject(response) : undefined;
+  const usage = response ? getUsage(response) : undefined;
   const u = (usage && typeof usage === "object" ? usage : {}) as Record<
     string,
     unknown

package/src/router/remote-mcp-execution.ts

@@ -1,4 +1,4 @@
-import { connectTools } from "@neutrome/lilsdk/tools";
+import { createToolsExecutor } from "@neutrome/lilsdk/tools";
 import type {
   ExecutionTarget,
   Executor,
@@ -49,7 +49,10 @@ export function prepareRemoteMcpExecution(
       ...options,
       executorImplementations: {
         ...(options.executorImplementations ?? {}),
-        [executorName]: connectTools(remoteMcp.tools, resolved.requestedModel),
+        [executorName]: createToolsExecutor(
+          resolved.requestedModel,
+          remoteMcp.tools,
+        ),
       },
     },
     close: remoteMcp.close,

package/src/router/request-program.ts

@@ -1,5 +1,4 @@
 import {
-  parseChatCompletionsRequest,
   parseProviderRequest,
   type Program,
   type ProviderStyle,
@@ -11,6 +10,6 @@ export async function parseRequestProgram(
 ): Promise<Program> {
   const body = new Uint8Array(await request.arrayBuffer());
   return style === "chat-completions"
-    ? parseChatCompletionsRequest(body)
+    ? parseProviderRequest("chat-completions", body)
     : parseProviderRequest(style, body);
 }