@neutrome/open-ai-router 0.3.0 → 0.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neutrome/open-ai-router",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "type": "module",
   "exports": {
     ".": "./src/index.ts"
@@ -8,8 +8,8 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",
     "hono": "^4.12.18",
-    "@neutrome/lil-engine": "0.2.0",
-    "@neutrome/lilsdk-ts": "0.2.0"
+    "@neutrome/lilsdk-ts": "0.2.1",
+    "@neutrome/lil-engine": "0.2.0"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",

package/src/router/execute.ts

@@ -184,7 +184,6 @@ export async function handleChatCompletions(
     const remoteMcp = prepareRemoteMcpExecution(options.runtime, resolved, options);
     const execution = createRouterExecutionRuntime(options.runtime, options.ctx, remoteMcp.options);
 
-    const startTime = Date.now();
     if (isStreaming(request)) {
       try {
         const chunks = execution.stream(request, { target: remoteMcp.target });
@@ -192,7 +191,6 @@ export async function handleChatCompletions(
           "chat-completions",
           chunks,
           resolved,
-          startTime,
           remoteMcp.close,
         );
       } catch (error) {
@@ -201,6 +199,7 @@ export async function handleChatCompletions(
       }
     }
 
+    const startTime = Date.now();
     try {
       const response = await execution.execute(request, {
         target: remoteMcp.target,
@@ -262,7 +261,6 @@ async function handleProviderStyle(
     const remoteMcp = prepareRemoteMcpExecution(options.runtime, resolved, options);
     const execution = createRouterExecutionRuntime(options.runtime, options.ctx, remoteMcp.options);
 
-    const startTime = Date.now();
     if (isStreaming(request)) {
       try {
         const chunks = execution.stream(request, { target: remoteMcp.target });
@@ -270,7 +268,6 @@ async function handleProviderStyle(
           style,
           chunks,
           resolved,
-          startTime,
           remoteMcp.close,
         );
       } catch (error) {
@@ -279,6 +276,7 @@ async function handleProviderStyle(
       }
     }
 
+    const startTime = Date.now();
     try {
       const response = await execution.execute(request, {
         target: remoteMcp.target,
@@ -346,13 +344,16 @@ export function createFetchProviderInvoker(
           provider: provider.name,
           model: providerCtx.target.model,
         });
-        const reqBody = emitProviderRequest(provider.style, resolvedRequest);
+        let reqBody = emitProviderRequest(provider.style, resolvedRequest);
+        if (provider.style === "chat-completions" && isStreaming(resolvedRequest)) {
+          reqBody = injectStreamUsage(reqBody);
+        }
         const reqBytes = reqBody.byteLength;
         const upstream = await fetchProvider(
           runtime,
           ctx,
           provider,
-          resolvedRequest,
+          reqBody,
           timed.signal,
         );
         if (!upstream.ok) {
@@ -392,13 +393,16 @@ export function createFetchProviderInvoker(
           provider: provider.name,
           model: providerCtx.target.model,
         });
-        const reqBody = emitProviderRequest(provider.style, resolvedRequest);
+        let reqBody = emitProviderRequest(provider.style, resolvedRequest);
+        if (provider.style === "chat-completions" && isStreaming(resolvedRequest)) {
+          reqBody = injectStreamUsage(reqBody);
+        }
         const reqBytes = reqBody.byteLength;
         const upstream = await fetchProvider(
           runtime,
           ctx,
           provider,
-          resolvedRequest,
+          reqBody,
           timed.signal,
         );
         if (!upstream.ok) {
@@ -486,41 +490,52 @@ async function streamExecutionResponse(
   responseStyle: ProviderStyle,
   chunks: AsyncIterable<Program>,
   resolved: ResolvedTarget,
-  startTime: number,
   cleanup?: () => Promise<void>,
 ): Promise<Response> {
   const iterator = chunks[Symbol.asyncIterator]();
 
   const first = await iterator.next();
-  const userTime = Date.now() - startTime;
+  if (first.done) {
+    await cleanup?.();
+    const stream = new ReadableStream<Uint8Array>({
+      start(controller) {
+        if (responseStyle === "chat-completions") {
+          controller.enqueue(encoder.encode("data: [DONE]\n\n"));
+        }
+        controller.close();
+      },
+    });
+    return new Response(stream, {
+      headers: {
+        "content-type": "text/event-stream; charset=utf-8",
+        ...routingHeaders(resolved),
+      },
+    });
+  }
 
+  let done = false;
   const stream = new ReadableStream<Uint8Array>({
-    async start(controller) {
+    start(controller) {
+      const bytes = emitProviderStreamChunk(responseStyle, first.value);
+      controller.enqueue(encoder.encode(`data: ${decoder.decode(bytes)}\n\n`));
+    },
+    async pull(controller) {
+      if (done) return;
       try {
-        function emitChunk(chunk: Program): void {
-          const bytes = emitProviderStreamChunk(responseStyle, chunk);
-          controller.enqueue(
-            encoder.encode(`data: ${decoder.decode(bytes)}\n\n`),
-          );
-        }
-
-        if (!first.done) {
-          emitChunk(first.value);
-        }
-
-        while (true) {
-          const { done, value } = await iterator.next();
-          if (done) {
-            break;
+        const result = await iterator.next();
+        if (result.done) {
+          done = true;
+          if (responseStyle === "chat-completions") {
+            controller.enqueue(encoder.encode("data: [DONE]\n\n"));
           }
-          emitChunk(value);
+          await cleanup?.();
+          controller.close();
+          return;
         }
-        if (responseStyle === "chat-completions") {
-          controller.enqueue(encoder.encode("data: [DONE]\n\n"));
-        }
-        await cleanup?.();
-        controller.close();
+        const bytes = emitProviderStreamChunk(responseStyle, result.value);
+        controller.enqueue(encoder.encode(`data: ${decoder.decode(bytes)}\n\n`));
       } catch (error) {
+        done = true;
         await cleanup?.();
         controller.enqueue(
           encoder.encode(
@@ -540,7 +555,6 @@ async function streamExecutionResponse(
     headers: {
       "content-type": "text/event-stream; charset=utf-8",
       ...routingHeaders(resolved),
-      ...timingHeaders(userTime),
     },
   });
 }
@@ -628,7 +642,7 @@ async function fetchProvider(
   runtime: RouterRuntime,
   ctx: RequestContext,
   provider: ProviderRuntime,
-  request: Program,
+  body: Uint8Array,
   signal: AbortSignal,
 ): Promise<Response> {
   const headers = cloneForwardHeaders(ctx.request.headers);
@@ -647,10 +661,6 @@ async function fetchProvider(
     break;
   }
 
-  let body = emitProviderRequest(provider.style, request);
-  if (provider.style === "chat-completions" && isStreaming(request)) {
-    body = injectStreamUsage(body);
-  }
   return runtime.fetchImpl(`${provider.apiBaseUrl}${provider.endpointPath}`, {
     method: "POST",
     headers,