@neutrome/open-ai-router 0.1.10 → 0.1.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neutrome/open-ai-router",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "type": "module",
   "exports": {
     ".": "./src/index.ts"

package/src/app/factory.ts

@@ -40,6 +40,7 @@ export type RouterAppOptions = {
   requestIdFactory?: ExecutionRuntimeOptions["requestIdFactory"];
   executionIdFactory?: ExecutionRuntimeOptions["executionIdFactory"];
   upstreamTimeoutMs?: number;
+  suppressUsageToClient?: boolean;
 };
 
 export function createRouterApp(options: RouterAppOptions) {
@@ -127,6 +128,7 @@ export function createRouterApp(options: RouterAppOptions) {
       ...(options.transforms ? { transforms: options.transforms } : {}),
       ...(options.observe ? { observe: options.observe } : {}),
       ...(options.upstreamTimeoutMs !== undefined ? { upstreamTimeoutMs: options.upstreamTimeoutMs } : {}),
+      ...(options.suppressUsageToClient ? { suppressUsageToClient: true } : {}),
       ...(onUsage ? { onUsage } : {}),
       ...(onProviderCall ? { onProviderCall } : {}),
       ...(forceStreaming ? { forceStreaming: true } : {}),

package/src/router/execute.ts

@@ -50,7 +50,7 @@ export type ProviderErrorReport = {
 };
 
 export type ProviderCallReport =
-  | { kind: "usage"; } & UsageReport
+  | { kind: "usage"; requestBytes: number; responseBytes: number } & UsageReport
   | { kind: "error"; } & ProviderErrorReport;
 
 export type RouterExecutionOptions = Pick<
@@ -60,6 +60,7 @@ export type RouterExecutionOptions = Pick<
   executors?: Readonly<Record<string, Executor>>;
   transforms?: Readonly<Record<string, ProgramTransform>>;
   upstreamTimeoutMs?: number;
+  suppressUsageToClient?: boolean;
   onUsage?: (report: UsageReport) => void;
   onProviderCall?: (report: ProviderCallReport) => void;
 };
@@ -191,7 +192,7 @@ export async function handleChatCompletions(
 
     const startTime = Date.now();
     if (isStreaming(request)) {
-      const userWantsUsage = requestedStreamUsage(requestBody);
+      const userWantsUsage = options.suppressUsageToClient ? false : requestedStreamUsage(requestBody);
       const chunks = execution.stream(request, { target: resolved.target });
       return await streamExecutionResponse(
         "chat-completions",
@@ -269,6 +270,7 @@ async function handleProviderStyle(
         resolved,
         startTime,
         options.onUsage,
+        !options.suppressUsageToClient,
       );
     }
 
@@ -319,6 +321,8 @@ export function createFetchProviderInvoker(
       const timed = createUpstreamSignal(providerCtx.signal, upstreamTimeoutMs);
 
       try {
+        const reqBody = emitProviderRequest(provider.style, setModel(request, providerCtx.target.model));
+        const reqBytes = reqBody.byteLength;
         const upstream = await fetchProvider(
           runtime,
           ctx,
@@ -334,7 +338,7 @@ export function createFetchProviderInvoker(
 
         const bytes = new Uint8Array(await upstream.arrayBuffer());
         const response = parseProviderResponse(provider.style, bytes);
-        reportProviderUsage(onProviderCall, response, providerCtx.target);
+        reportProviderUsage(onProviderCall, response, providerCtx.target, reqBytes, bytes.byteLength);
         return response;
       } catch (error) {
         throw normalizeUpstreamAbort(error, timed.signal);
@@ -350,6 +354,8 @@ export function createFetchProviderInvoker(
       let buffer = "";
 
       try {
+        const reqBody = emitProviderRequest(provider.style, setModel(request, providerCtx.target.model));
+        const reqBytes = reqBody.byteLength;
         const upstream = await fetchProvider(
           runtime,
           ctx,
@@ -372,18 +378,20 @@ export function createFetchProviderInvoker(
 
         reader = upstream.body.getReader();
         let lastChunkWithUsage: Program | null = null;
+        let respBytes = 0;
         while (true) {
           const { done, value } = await reader.read();
           if (done) {
             break;
           }
+          respBytes += value.byteLength;
           buffer += decoder.decode(value, { stream: true });
           const split = splitSseEvents(buffer);
           buffer = split.remainder;
           for (const event of split.events) {
             for (const data of eventDataLines(event)) {
               if (data === "[DONE]") {
-                reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target);
+                reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target, reqBytes, respBytes);
                 return;
               }
               if (data[0] !== "{") continue;
@@ -400,7 +408,7 @@ export function createFetchProviderInvoker(
         if (buffer.length > 0) {
           for (const data of eventDataLines(buffer)) {
             if (data === "[DONE]") {
-              reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target);
+              reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target, reqBytes, respBytes);
               return;
             }
             if (data[0] !== "{") continue;
@@ -412,7 +420,7 @@ export function createFetchProviderInvoker(
             yield chunk;
           }
         }
-        reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target);
+        reportProviderUsage(onProviderCall, lastChunkWithUsage, providerCtx.target, reqBytes, respBytes);
       } catch (error) {
         throw normalizeUpstreamAbort(error, timed.signal);
       } finally {
@@ -766,6 +774,8 @@ function reportProviderUsage(
   onProviderCall: ((report: ProviderCallReport) => void) | undefined,
   response: Program | null,
   target: Extract<import("@neutrome/lil-engine").ExecutionTarget, { kind: "provider" }>,
+  requestBytes: number,
+  responseBytes: number,
 ): void {
   if (!onProviderCall) return;
   const usage = response ? usageObject(response) : undefined;
@@ -782,6 +792,8 @@ function reportProviderUsage(
       tokensInput,
       tokensOutput,
       cachedTokensInput,
+      requestBytes,
+      responseBytes,
     });
   } catch {}
 }