@neutrome/open-ai-router 0.6.10 → 0.8.0

package/package.json

@@ -1,18 +1,23 @@
 {
   "name": "@neutrome/open-ai-router",
-  "version": "0.6.10",
+  "version": "0.8.0",
   "type": "module",
+  "files": [
+    "src",
+    "README.md"
+  ],
   "exports": {
     ".": "./src/index.ts"
   },
   "dependencies": {
-    "@posthog/ai": "^8.3.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
+    "@posthog/ai": "^8.3.0",
+    "gpt-tokenizer": "4.0.0",
     "hono": "^4.12.18",
     "posthog-node": "^5.41.0",
     "zod": "^4.2.1",
-    "@neutrome/lil-engine": "0.5.1",
-    "@neutrome/lilsdk": "0.5.1"
+    "@neutrome/lil-engine": "0.6.0",
+    "@neutrome/lilsdk": "0.6.0"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",

package/src/app/factory.test.ts

@@ -100,6 +100,7 @@ describe("createRouterApp trace finalization", () => {
   it("keeps non-stream trace hooks alive with waitUntil", async () => {
     const waited: Promise<unknown>[] = [];
     const onExecutionSummary = vi.fn(async (_summary: ExecutionSummary) => {});
+    const onUserUsage = vi.fn(async () => {});
     const app = createRouterApp({
       config,
       executorImplementations: {},
@@ -108,7 +109,7 @@ describe("createRouterApp trace finalization", () => {
           return principal();
         },
       },
-      hooks: { onExecutionSummary },
+      hooks: { onExecutionSummary, onUserUsage },
       fetchImpl: vi.fn(
         async () =>
           new Response(
@@ -145,11 +146,21 @@ describe("createRouterApp trace finalization", () => {
     expect(waited).toHaveLength(1);
     await waited[0];
     expect(onExecutionSummary).toHaveBeenCalledTimes(1);
+    expect(onUserUsage).toHaveBeenCalledWith(
+      expect.objectContaining({
+        requestedModel: "gpt-4o",
+        outcome: "ok",
+        inputTokens: expect.any(Number),
+        outputTokens: expect.any(Number),
+      }),
+      {},
+    );
   });
 
   it("keeps stream trace hooks alive until the stream closes", async () => {
     const waited: Promise<unknown>[] = [];
     const onExecutionSummary = vi.fn(async (_summary: ExecutionSummary) => {});
+    const onUserUsage = vi.fn(async () => {});
     const app = createRouterApp({
       config,
       executorImplementations: {},
@@ -158,7 +169,7 @@ describe("createRouterApp trace finalization", () => {
           return principal();
         },
       },
-      hooks: { onExecutionSummary },
+      hooks: { onExecutionSummary, onUserUsage },
       fetchImpl: vi.fn(async () => {
         const stream = new ReadableStream<Uint8Array>({
           start(controller) {
@@ -217,6 +228,7 @@ describe("createRouterApp trace finalization", () => {
     expect(waited).toHaveLength(1);
     await waited[0];
     expect(onExecutionSummary).toHaveBeenCalledTimes(1);
+    expect(onUserUsage).toHaveBeenCalledTimes(1);
     const events = onExecutionSummary.mock.calls[0]?.[0].spans.flatMap((span) =>
       span.timeline.map((entry) => entry.event),
     );

package/src/app/factory.ts

@@ -35,6 +35,10 @@ import {
 import { createModelListingHandler } from "./model-listing.ts";
 import { honoEnv, unauthorizedResponse } from "./http-utils.ts";
 import type { RouterAppOptions } from "./types.ts";
+import {
+  createUserUsageCollector,
+  responseResult,
+} from "./user-usage-collector.ts";
 
 type ProtocolHandler = (
   options: Parameters<typeof handleChatCompletions>[0],
@@ -100,6 +104,13 @@ export function createRouterApp(options: RouterAppOptions) {
         : {}),
     });
     const onExecutionSummary = options.hooks?.onExecutionSummary;
+    const onUserUsage = options.hooks?.onUserUsage;
+    const requestStartedMs = Date.now();
+    const usage = createUserUsageCollector({
+      requestId: incomingExecutionId,
+      protocol: style,
+      startedAtMs: requestStartedMs,
+    });
     const observe = (
       event: Parameters<NonNullable<ExecutionRuntimeOptions["observe"]>>[0],
     ) => {
@@ -114,13 +125,14 @@ export function createRouterApp(options: RouterAppOptions) {
         c,
         async () => {
           await telemetry.flush();
-          await onExecutionSummary?.(telemetry.summary(), env);
+          const summary = telemetry.summary();
+          await onExecutionSummary?.(summary, env);
+          await onUserUsage?.(usage.report(summary), env);
         },
         "onExecutionSummary hook",
       );
     };
 
-    const requestStartedMs = Date.now();
     observeTimedExecution({
       observe,
       kind: "request.started",
@@ -142,6 +154,7 @@ export function createRouterApp(options: RouterAppOptions) {
       data: { authenticated: !!principal },
     });
     if (!principal) {
+      usage.setResult(responseResult(401, "Authentication failed"));
       telemetry.recordError({
         message: "Authentication failed",
         requestId: incomingExecutionId,
@@ -166,6 +179,7 @@ export function createRouterApp(options: RouterAppOptions) {
         data: { style, streaming: forceStreaming === true },
       });
     } catch {
+      usage.setResult(responseResult(400, "Invalid request body"));
       telemetry.recordError({
         message: "Invalid request body",
         requestId: incomingExecutionId,
@@ -189,6 +203,7 @@ export function createRouterApp(options: RouterAppOptions) {
     }
 
     const model = getModel(program) || null;
+    usage.setRequest(program, model);
     const accessStartedMs = Date.now();
     const hasModelAccess = !model || canAccessPrincipalModel(principal, model);
     observeTimedExecution({
@@ -201,6 +216,7 @@ export function createRouterApp(options: RouterAppOptions) {
       data: { model: model ?? undefined, allowed: hasModelAccess },
     });
     if (!hasModelAccess) {
+      usage.setResult(responseResult(404, "Model access denied"));
       telemetry.recordError({
         message: "Model access denied",
         requestId: incomingExecutionId,
@@ -232,6 +248,12 @@ export function createRouterApp(options: RouterAppOptions) {
         data: { returnedResponse: !!hookResponse },
       });
       if (hookResponse) {
+        usage.setResult(
+          responseResult(
+            hookResponse.status,
+            hookResponse.status >= 400 ? "Request rejected" : null,
+          ),
+        );
         if (hookResponse.status >= 400) {
           telemetry.recordError({
             message: `Request hook returned ${hookResponse.status}`,
@@ -245,6 +267,7 @@ export function createRouterApp(options: RouterAppOptions) {
     }
 
     if (style === "responses" && (await hasPreviousResponseId(request))) {
+      usage.setResult(responseResult(400, "Unsupported response state"));
       finalizeTrace();
       return errorResponse(
         new RouterError(
@@ -270,6 +293,10 @@ export function createRouterApp(options: RouterAppOptions) {
         ...(options.transforms ? { transforms: options.transforms } : {}),
         observe,
         incomingExecutionId,
+        userUsage: { inputTokens: usage.inputTokens() },
+        onUserResponse(result) {
+          usage.setResult(result);
+        },
         ...(options.upstreamTimeoutMs !== undefined
           ? { upstreamTimeoutMs: options.upstreamTimeoutMs }
           : {}),
@@ -296,6 +323,7 @@ export function createRouterApp(options: RouterAppOptions) {
       }
       return finalizeWhenStreamCloses(response, finalizeTrace);
     } catch (error) {
+      usage.setResult(responseResult(500, "Request execution failed"));
       telemetry.recordError({
         message:
           error instanceof Error ? error.message : "Request execution failed",

package/src/app/response-lifecycle.ts

@@ -54,8 +54,8 @@ export function finalizeWhenStreamCloses(
       }
     },
     async cancel(reason) {
-      finalize();
       await reader.cancel(reason).catch(() => {});
+      finalize();
     },
   });
   return new Response(body, {

package/src/app/types.ts

@@ -7,6 +7,7 @@ import type { ExecutionSummary } from "../telemetry/index.ts";
 import type { RouterConfig } from "../router/config.ts";
 import type { ExecutionRuntimeOptions } from "../router/execution-types.ts";
 import type { RemoteMcpClientFactory } from "../router/mcp.ts";
+import type { UserUsageReport } from "../usage/user-usage.ts";
 
 export type RouterAppAuth = RuntimeAuthenticator;
 
@@ -21,6 +22,10 @@ export type RouterAppHooks = {
     summary: ExecutionSummary,
     env: Record<string, unknown>,
   ): void | Promise<void>;
+  onUserUsage?(
+    report: UserUsageReport,
+    env: Record<string, unknown>,
+  ): void | Promise<void>;
 };
 
 export type RouterAppOptions = {

package/src/app/user-usage-collector.ts

@@ -0,0 +1,68 @@
+import type { Program, ProviderStyle } from "@neutrome/lil-engine";
+import type { ExecutionSummary } from "../telemetry/index.ts";
+import {
+  countUserProgramTokens,
+  toolUsageFromEvents,
+  type UserUsageReport,
+} from "../usage/user-usage.ts";
+
+export type UserResponseResult = Pick<
+  UserUsageReport,
+  "outputTokens" | "outcome" | "httpStatus" | "error"
+>;
+
+export function createUserUsageCollector(input: {
+  requestId: string;
+  protocol: ProviderStyle;
+  startedAtMs: number;
+}) {
+  let inputTokens = 0;
+  let requestedModel: string | null = null;
+  let result: UserResponseResult | null = null;
+  return {
+    setRequest(program: Program, model: string | null) {
+      inputTokens = countUserProgramTokens(program);
+      requestedModel = model;
+    },
+    setResult(next: UserResponseResult) {
+      result = next;
+    },
+    inputTokens() {
+      return inputTokens;
+    },
+    report(summary: ExecutionSummary): UserUsageReport {
+      const finishedAt = new Date().toISOString();
+      return {
+        requestId: input.requestId,
+        protocol: input.protocol,
+        requestedModel,
+        inputTokens,
+        ...(result ??
+          responseResult(null, "Request did not produce a response")),
+        tools: toolUsageFromEvents(
+          summary.spans
+            .filter((span) => span.kind === "tool" && span.toolName)
+            .map((span) => ({
+              name: span.toolName!,
+              error: span.status === "error",
+            })),
+        ),
+        startedAt: new Date(input.startedAtMs).toISOString(),
+        finishedAt,
+        durationMs: Math.max(0, Date.parse(finishedAt) - input.startedAtMs),
+      };
+    },
+  };
+}
+
+export function responseResult(
+  status: number | null,
+  error: string | null,
+): UserResponseResult {
+  return {
+    outputTokens: 0,
+    outcome: status !== null && status < 400 ? "ok" : "error",
+    httpStatus: status,
+    error,
+  };
+}

package/src/index.ts

@@ -9,6 +9,7 @@ export {
   PosthogErrorTelemetry,
   PosthogTraceTelemetry,
 } from "./telemetry/index.ts";
+export type { UserUsageReport, UserToolUsage } from "./usage/user-usage.ts";
 export type {
   ErrorTelemetryDriver,
   ExecutionSummary,

package/src/router/execute.test.ts

@@ -1,4 +1,4 @@
-import { appendAssistantMessage } from "@neutrome/lil-engine";
+import { appendAssistantMessage, createProgram } from "@neutrome/lil-engine";
 import { describe, expect, it, vi } from "vitest";
 import type { RequestContext } from "../upstream-auth/types.ts";
 import { handleChatCompletions } from "./execute.ts";
@@ -88,6 +88,7 @@ describe("router execution", () => {
       object: "chat.completion",
       id: "resp_1",
       model: "gpt-4o",
+      usage: { prompt_tokens: 0, completion_tokens: 27, total_tokens: 27 },
       choices: [
         {
           index: 0,
@@ -334,6 +335,7 @@ describe("router execution", () => {
       object: "chat.completion",
       id: "resp_1",
       model: "gpt-5",
+      usage: { prompt_tokens: 0, completion_tokens: 27, total_tokens: 27 },
       choices: [
         {
           index: 0,
@@ -411,9 +413,9 @@ describe("router execution", () => {
       id: "msg_1",
       model: "claude-sonnet-4",
       usage: {
-        prompt_tokens: 10,
-        completion_tokens: 5,
-        total_tokens: 15,
+        prompt_tokens: 0,
+        completion_tokens: 27,
+        total_tokens: 27,
       },
       choices: [
         {
@@ -494,6 +496,7 @@ describe("router execution", () => {
     expect(await response.json()).toEqual({
       object: "chat.completion",
       model: "gemini-2.5-flash",
+      usage: { prompt_tokens: 0, completion_tokens: 27, total_tokens: 27 },
       choices: [
         {
           index: 0,
@@ -543,10 +546,10 @@ describe("router execution", () => {
       executorImplementations: {
         "enei-1": {
           async execute(request) {
-            return appendAssistantMessage(request, "done");
+            return appendAssistantMessage(createProgram(), "done");
           },
           async *stream(request) {
-            yield appendAssistantMessage(request, "done");
+            yield appendAssistantMessage(createProgram(), "done");
           },
         },
       },
@@ -556,6 +559,7 @@ describe("router execution", () => {
     expect(response.headers.get("x-openairouter-executor-id")).toBe("enei-1");
     expect(await response.json()).toEqual({
       object: "chat.completion",
+      usage: { prompt_tokens: 0, completion_tokens: 27, total_tokens: 27 },
       choices: [
         {
           index: 0,

package/src/router/execute.ts

@@ -13,6 +13,7 @@ import {
   type ProviderStyle,
   setModel,
   setStreaming,
+  replaceUsage,
 } from "@neutrome/lil-engine";
 import { createExecutionRuntime } from "./execution-runtime.ts";
 import { createFetchProviderInvoker } from "./provider-invoker.ts";
@@ -61,6 +62,7 @@ import {
 import type { RemoteMcpClientFactory } from "./mcp.ts";
 import type { ProviderRuntime, RouterRuntime } from "./runtime.ts";
 import { observeTimedExecution } from "./execution-events.ts";
+import { countUserProgramTokens, protocolUsage } from "../usage/user-usage.ts";
 
 const encoder = new TextEncoder();
 const decoder = new TextDecoder();
@@ -79,6 +81,13 @@ export type RouterExecutionOptions = Pick<
   upstreamTimeoutMs?: number;
   remoteMcpClientFactory?: RemoteMcpClientFactory;
   incomingExecutionId?: string;
+  userUsage?: { inputTokens: number };
+  onUserResponse?: (result: {
+    outputTokens: number;
+    outcome: "ok" | "error" | "cancelled";
+    httpStatus: number | null;
+    error: string | null;
+  }) => void | Promise<void>;
 };
 
 export type HandleChatCompletionsOptions = RouterExecutionOptions & {
@@ -240,6 +249,10 @@ async function handleRequestStyle(
             requestId: incomingExecutionId,
             executionId: incomingExecutionId,
             startedAtMs: responseStartedMs,
+            inputTokens: options.userUsage?.inputTokens ?? 0,
+            onFinish: async (result) => {
+              await options.onUserResponse?.({ ...result, httpStatus: 200 });
+            },
           },
           remoteMcp.close,
         );
@@ -255,12 +268,38 @@ async function handleRequestStyle(
         target: remoteMcp.target,
       });
       const userTime = Date.now() - startTime;
-      return jsonExecutionResponse(emitResponse(response), resolved, userTime);
+      const outputTokens = countUserProgramTokens(response);
+      await options.onUserResponse?.({
+        outputTokens,
+        outcome: "ok",
+        httpStatus: 200,
+        error: null,
+      });
+      return jsonExecutionResponse(
+        emitResponse(
+          replaceUsage(
+            response,
+            protocolUsage(
+              style,
+              options.userUsage?.inputTokens ?? 0,
+              outputTokens,
+            ),
+          ),
+        ),
+        resolved,
+        userTime,
+      );
     } finally {
       await remoteMcp.close();
     }
   } catch (error) {
     const response = errorResponse(error, style);
+    await options.onUserResponse?.({
+      outputTokens: 0,
+      outcome: "error",
+      httpStatus: response.status,
+      error: "Request execution failed",
+    });
     return resolved ? applyRoutingHeaders(response, resolved) : response;
   }
 }

package/src/router/response-delivery.test.ts

@@ -0,0 +1,150 @@
+import { describe, expect, it } from "vitest";
+import { Opcode, createProgram, type Program } from "@neutrome/lil-engine";
+import { streamExecutionResponse } from "./response-delivery.ts";
+import type { ResolvedTarget } from "./resolve.ts";
+
+const resolved: ResolvedTarget = {
+  requestedModel: "main/assistant",
+  namespace: "main",
+  source: "provider",
+  suffixes: [],
+  mcps: [],
+  target: {
+    kind: "provider",
+    provider: "core",
+    model: "actual",
+    transforms: [],
+  },
+};
+
+describe("user stream boundary", () => {
+  for (const [style, usageKey] of [
+    ["chat-completions", "prompt_tokens"],
+    ["responses", "input_tokens"],
+    ["anthropic-messages", "input_tokens"],
+    ["google-genai", "promptTokenCount"],
+  ] as const) {
+    it(`emits one internal usage block for ${style}`, async () => {
+      const response = await streamExecutionResponse(
+        style,
+        successfulChunks(),
+        resolved,
+        undefined,
+        {
+          requestId: "r1",
+          executionId: "r1",
+          startedAtMs: Date.now(),
+          inputTokens: 7,
+        },
+      );
+      const text = await response.text();
+      expect(text, text).toContain(usageKey);
+      expect(text.match(new RegExp(usageKey, "g"))).toHaveLength(1);
+      expect(text).not.toContain("999");
+    });
+  }
+
+  it("emits no usage after a failed partial stream", async () => {
+    async function* failed(): AsyncGenerator<Program> {
+      yield chunk(Opcode.STREAM_DELTA, { kind: "string", value: "partial" });
+      throw new Error("broken");
+    }
+    const response = await streamExecutionResponse(
+      "chat-completions",
+      failed(),
+      resolved,
+      undefined,
+      {
+        requestId: "r1",
+        executionId: "r1",
+        startedAtMs: Date.now(),
+        inputTokens: 7,
+      },
+    );
+    expect(await response.text()).not.toContain("usage");
+  });
+
+  it("does not trust a terminal chunk until the stream actually closes", async () => {
+    async function* failedAfterTerminal(): AsyncGenerator<Program> {
+      yield createProgram({
+        code: [
+          {
+            opcode: Opcode.RESP_DONE,
+            value: { kind: "string", value: "stop" },
+          },
+          { opcode: Opcode.STREAM_END, value: { kind: "none" } },
+        ],
+      });
+      throw new Error("late failure");
+    }
+    const response = await streamExecutionResponse(
+      "chat-completions",
+      failedAfterTerminal(),
+      resolved,
+      undefined,
+      {
+        requestId: "r1",
+        executionId: "r1",
+        startedAtMs: Date.now(),
+        inputTokens: 7,
+      },
+    );
+    expect(await response.text()).not.toContain("usage");
+  });
+
+  it("reports cancellation with partial internal tokens and no usage block", async () => {
+    let outcome: unknown;
+    async function* chunks(): AsyncGenerator<Program> {
+      yield chunk(Opcode.STREAM_START, { kind: "none" });
+      yield chunk(Opcode.STREAM_DELTA, { kind: "string", value: "partial" });
+    }
+    const response = await streamExecutionResponse(
+      "chat-completions",
+      chunks(),
+      resolved,
+      undefined,
+      {
+        requestId: "r1",
+        executionId: "r1",
+        startedAtMs: Date.now(),
+        inputTokens: 7,
+        onFinish(result) {
+          outcome = result;
+        },
+      },
+    );
+    const reader = response.body!.getReader();
+    const first = await reader.read();
+    expect(new TextDecoder().decode(first.value)).not.toContain("usage");
+    await reader.cancel();
+    expect(outcome).toMatchObject({
+      outcome: "cancelled",
+      outputTokens: expect.any(Number),
+    });
+  });
+});
+
+async function* successfulChunks(): AsyncGenerator<Program> {
+  yield chunk(Opcode.STREAM_START, { kind: "none" });
+  yield chunk(Opcode.STREAM_DELTA, { kind: "string", value: "hello" });
+  yield createProgram({
+    code: [
+      {
+        opcode: Opcode.USAGE,
+        value: {
+          kind: "json",
+          value: new TextEncoder().encode('{"prompt_tokens":999}'),
+        },
+      },
+      { opcode: Opcode.RESP_DONE, value: { kind: "string", value: "stop" } },
+      { opcode: Opcode.STREAM_END, value: { kind: "none" } },
+    ],
+  });
+}
+
+function chunk(
+  opcode: Opcode,
+  value: Program["code"][number]["value"],
+): Program {
+  return createProgram({ code: [{ opcode, value }] });
+}