@neutrome/open-ai-router 0.1.8 → 0.1.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neutrome/open-ai-router",
-  "version": "0.1.8",
+  "version": "0.1.10",
   "type": "module",
   "exports": {
     ".": "./src/index.ts"

package/src/app/factory.ts

@@ -14,7 +14,7 @@ import {
 } from "../router/index.ts";
 import type { RequestContext } from "../auth/types.ts";
 import type { RouterConfig } from "../router/config.ts";
-import type { UsageReport } from "../router/execute.ts";
+import type { UsageReport, ProviderCallReport } from "../router/execute.ts";
 
 export type RouterAppAuth = {
   authenticate(request: Request, env: Record<string, unknown>): Promise<object | null>;
@@ -24,6 +24,7 @@ export type RouterAppAuth = {
 export type RouterAppHooks = {
   beforeRequest?(ctx: { request: Request; model: string | null; env: Record<string, unknown> }): Promise<Response | void>;
   onUsage?(report: UsageReport, env: Record<string, unknown>): void;
+  onProviderCall?(report: ProviderCallReport, env: Record<string, unknown>): void;
   onToolCall?(report: { toolName: string }, env: Record<string, unknown>): void;
 };
 
@@ -115,6 +116,9 @@ export function createRouterApp(options: RouterAppOptions) {
     const onUsage = options.hooks?.onUsage
       ? (report: UsageReport) => options.hooks!.onUsage!(report, env)
       : undefined;
+    const onProviderCall = options.hooks?.onProviderCall
+      ? (report: ProviderCallReport) => options.hooks!.onProviderCall!(report, env)
+      : undefined;
 
     return handler({
       runtime,
@@ -124,6 +128,7 @@ export function createRouterApp(options: RouterAppOptions) {
       ...(options.observe ? { observe: options.observe } : {}),
       ...(options.upstreamTimeoutMs !== undefined ? { upstreamTimeoutMs: options.upstreamTimeoutMs } : {}),
       ...(onUsage ? { onUsage } : {}),
+      ...(onProviderCall ? { onProviderCall } : {}),
       ...(forceStreaming ? { forceStreaming: true } : {}),
     });
   }

package/src/index.ts

@@ -32,6 +32,8 @@ export type {
   HandleResponsesOptions,
   ProviderApiStyle,
   ProviderApiStyle as ProviderStyle,
+  ProviderCallReport,
+  ProviderErrorReport,
   ProviderRuntime,
   ProviderTargetConfig,
   ResolvedTarget,

package/src/router/execute.ts

@@ -36,11 +36,23 @@ const DEFAULT_UPSTREAM_TIMEOUT_MS = 120_000;
 
 export type UsageReport = {
   model: string;
+  provider: string;
   tokensInput: number;
   tokensOutput: number;
   cachedTokensInput: number;
 };
 
+export type ProviderErrorReport = {
+  model: string;
+  provider: string;
+  status: number;
+  message: string;
+};
+
+export type ProviderCallReport =
+  | { kind: "usage"; } & UsageReport
+  | { kind: "error"; } & ProviderErrorReport;
+
 export type RouterExecutionOptions = Pick<
   ExecutionRuntimeOptions,
   "observe" | "requestIdFactory" | "executionIdFactory" | "resolveExecutor"
@@ -49,6 +61,7 @@ export type RouterExecutionOptions = Pick<
   transforms?: Readonly<Record<string, ProgramTransform>>;
   upstreamTimeoutMs?: number;
   onUsage?: (report: UsageReport) => void;
+  onProviderCall?: (report: ProviderCallReport) => void;
 };
 
 export type HandleChatCompletionsOptions = RouterExecutionOptions & {
@@ -108,6 +121,7 @@ export function createRouterExecutionRuntime(
       runtime,
       ctx,
       options.upstreamTimeoutMs,
+      options.onProviderCall,
     ),
   };
 
@@ -297,6 +311,7 @@ export function createFetchProviderInvoker(
   runtime: RouterRuntime,
   ctx: RequestContext,
   upstreamTimeoutMs = DEFAULT_UPSTREAM_TIMEOUT_MS,
+  onProviderCall?: (report: ProviderCallReport) => void,
 ): ProviderInvoker {
   return {
     async execute(request, providerCtx) {
@@ -312,11 +327,15 @@ export function createFetchProviderInvoker(
           timed.signal,
         );
         if (!upstream.ok) {
-          throw await toProviderError(provider.name, upstream);
+          const err = await toProviderError(provider.name, upstream);
+          reportProviderError(onProviderCall, providerCtx.target, err);
+          throw err;
         }
 
         const bytes = new Uint8Array(await upstream.arrayBuffer());
-        return parseProviderResponse(provider.style, bytes);
+        const response = parseProviderResponse(provider.style, bytes);
+        reportProviderUsage(onProviderCall, response, providerCtx.target);
+        return response;
       } catch (error) {
         throw normalizeUpstreamAbort(error, timed.signal);
       } finally {
@@ -339,7 +358,9 @@ export function createFetchProviderInvoker(
           timed.signal,
         );
         if (!upstream.ok) {
-          throw await toProviderError(provider.name, upstream);
+          const err = await toProviderError(provider.name, upstream);
+          reportProviderError(onProviderCall, providerCtx.target, err);
+          throw err;
         }
         if (!isSse(upstream.headers) || !upstream.body) {
           throw new RouterError(
@@ -350,6 +371,7 @@ export function createFetchProviderInvoker(
         }
 
         reader = upstream.body.getReader();
+        let lastChunkWithUsage: Program | null = null;
         while (true) {
           const { done, value } = await reader.read();
           if (done) {
@@ -361,13 +383,16 @@ export function createFetchProviderInvoker(
           for (const event of split.events) {
             for (const data of eventDataLines(event)) {
               if (data === "[DONE]") {
+                reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target);
                 return;
               }
               if (data[0] !== "{") continue;
-              yield parseProviderStreamChunk(
+              const chunk = parseProviderStreamChunk(
                 provider.style,
                 encoder.encode(data),
               );
+              if (usageObject(chunk) !== undefined) lastChunkWithUsage = chunk;
+              yield chunk;
             }
           }
         }
@@ -375,15 +400,19 @@ export function createFetchProviderInvoker(
         if (buffer.length > 0) {
           for (const data of eventDataLines(buffer)) {
             if (data === "[DONE]") {
+              reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target);
               return;
             }
             if (data[0] !== "{") continue;
-            yield parseProviderStreamChunk(
+            const chunk = parseProviderStreamChunk(
               provider.style,
               encoder.encode(data),
             );
+            if (usageObject(chunk) !== undefined) lastChunkWithUsage = chunk;
+            yield chunk;
           }
         }
+        reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target);
       } catch (error) {
         throw normalizeUpstreamAbort(error, timed.signal);
       } finally {
@@ -727,8 +756,55 @@ function reportUsage(
   const details = u.prompt_tokens_details as Record<string, unknown> | undefined;
   const cachedTokensInput = details ? asNum(details.cached_tokens) : 0;
   const model = resolved.target.kind === "provider" ? resolved.target.model : resolved.target.alias;
+  const provider = resolved.target.kind === "provider" ? (resolved.target.provider ?? "") : "";
+  try {
+    onUsage({ model, provider, tokensInput, tokensOutput, cachedTokensInput });
+  } catch {}
+}
+
+function reportProviderUsage(
+  onProviderCall: ((report: ProviderCallReport) => void) | undefined,
+  response: Program | null,
+  target: Extract<import("@neutrome/lil-engine").ExecutionTarget, { kind: "provider" }>,
+): void {
+  if (!onProviderCall) return;
+  const usage = response ? usageObject(response) : undefined;
+  const u = (usage && typeof usage === "object" ? usage : {}) as Record<string, unknown>;
+  const tokensInput = asNum(u.prompt_tokens) || asNum(u.input_tokens);
+  const tokensOutput = asNum(u.completion_tokens) || asNum(u.output_tokens);
+  const details = u.prompt_tokens_details as Record<string, unknown> | undefined;
+  const cachedTokensInput = details ? asNum(details.cached_tokens) : 0;
+  try {
+    onProviderCall({
+      kind: "usage",
+      model: target.model,
+      provider: target.provider ?? "",
+      tokensInput,
+      tokensOutput,
+      cachedTokensInput,
+    });
+  } catch {}
+}
+
+function reportProviderError(
+  onProviderCall: ((report: ProviderCallReport) => void) | undefined,
+  target: Extract<import("@neutrome/lil-engine").ExecutionTarget, { kind: "provider" }>,
+  error: ProviderHttpError,
+): void {
+  if (!onProviderCall) return;
+  let message = `HTTP ${error.status}`;
+  try {
+    const text = decoder.decode(error.body).slice(0, 500);
+    message = text || message;
+  } catch {}
   try {
-    onUsage({ model, tokensInput, tokensOutput, cachedTokensInput });
+    onProviderCall({
+      kind: "error",
+      model: target.model,
+      provider: target.provider ?? "",
+      status: error.status,
+      message,
+    });
   } catch {}
 }
 

package/src/router/index.ts

@@ -33,5 +33,7 @@ export type {
   HandleResponsesOptions,
   RouterExecutionOptions,
   UsageReport,
+  ProviderCallReport,
+  ProviderErrorReport,
 } from "./execute.ts";
 export type { ResolvedTarget } from "./resolve.ts";