ai-speedometer-headless 2.1.5 → 2.1.7

package/dist/ai-speedometer-headless

@@ -542,7 +542,7 @@ var getAllAvailableProviders = async (includeAllProviders = false) => {
 var TEST_PROMPT = `make a 300 word story`;
 
 // ../core/src/benchmark.ts
-async function benchmarkSingleModelRest(model) {
+async function benchmarkSingleModelRest(model, logger) {
   try {
     if (!model.providerConfig || !model.providerConfig.apiKey) {
       throw new Error(`Missing API key for provider ${model.providerName}`);
@@ -559,6 +559,7 @@ async function benchmarkSingleModelRest(model) {
       actualModelId = model.name;
     }
     actualModelId = actualModelId.trim();
+    await logger?.logHeader(model.name, model.providerName, model.providerConfig.apiKey);
     const startTime = Date.now();
     let firstTokenTime = null;
     let streamedText = "";
@@ -592,7 +593,8 @@ async function benchmarkSingleModelRest(model) {
       messages: [{ role: "user", content: TEST_PROMPT }],
       max_tokens: 500,
       temperature: 0.7,
-      stream: true
+      stream: true,
+      stream_options: { include_usage: true }
     };
     if (model.providerType === "google") {
       body["contents"] = [{ parts: [{ text: TEST_PROMPT }] }];
@@ -600,6 +602,9 @@ async function benchmarkSingleModelRest(model) {
       delete body["messages"];
       delete body["max_tokens"];
       delete body["stream"];
+      delete body["stream_options"];
+    } else if (model.providerType === "anthropic") {
+      delete body["stream_options"];
     }
     const response = await fetch(url, {
       method: "POST",
@@ -627,21 +632,21 @@ async function benchmarkSingleModelRest(model) {
     const reader = response.body.getReader();
     const decoder = new TextDecoder;
     let buffer = "";
-    let isFirstChunk = true;
+    let firstParsedTokenTime = null;
     while (true) {
       const { done, value } = await reader.read();
       if (done)
         break;
-      if (isFirstChunk && !firstTokenTime) {
+      if (!firstTokenTime)
         firstTokenTime = Date.now();
-        isFirstChunk = false;
-      }
       buffer += decoder.decode(value, { stream: true });
       const lines = buffer.split(`
 `);
       buffer = lines.pop() || "";
       for (const line of lines) {
         const trimmedLine = line.trim();
+        if (trimmedLine)
+          await logger?.logRaw(trimmedLine);
         if (!trimmedLine)
           continue;
         try {
@@ -649,10 +654,12 @@ async function benchmarkSingleModelRest(model) {
             if (trimmedLine.startsWith("data: ")) {
               const jsonStr = trimmedLine.slice(6);
               if (jsonStr === "[DONE]")
-                break;
+                continue;
               const chunk = JSON.parse(jsonStr);
               const chunkTyped = chunk;
               if (chunkTyped.type === "content_block_delta" && chunkTyped.delta?.text) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunkTyped.delta.text;
               } else if (chunkTyped.type === "message_start" && chunkTyped.message?.usage) {
                 inputTokens = chunkTyped.message.usage.input_tokens || 0;
@@ -667,6 +674,8 @@ async function benchmarkSingleModelRest(model) {
             } else {
               const chunk = JSON.parse(trimmedLine);
               if (chunk.type === "content_block_delta" && chunk.delta?.text) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunk.delta.text;
               } else if (chunk.type === "message_start" && chunk.message?.usage) {
                 inputTokens = chunk.message.usage.input_tokens || 0;
@@ -680,6 +689,8 @@ async function benchmarkSingleModelRest(model) {
           } else if (model.providerType === "google") {
             const chunk = JSON.parse(trimmedLine);
             if (chunk.candidates?.[0]?.content?.parts?.[0]?.text) {
+              if (!firstParsedTokenTime)
+                firstParsedTokenTime = Date.now();
               streamedText += chunk.candidates[0].content.parts[0].text;
             }
             if (chunk.usageMetadata?.promptTokenCount)
@@ -690,12 +701,17 @@ async function benchmarkSingleModelRest(model) {
             if (trimmedLine.startsWith("data: ")) {
               const jsonStr = trimmedLine.slice(6);
               if (jsonStr === "[DONE]")
-                break;
+                continue;
               const chunk = JSON.parse(jsonStr);
-              if (chunk.choices?.[0]?.delta?.content)
+              if (chunk.choices?.[0]?.delta?.content) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunk.choices[0].delta.content;
-              else if (chunk.choices?.[0]?.delta?.reasoning)
+              } else if (chunk.choices?.[0]?.delta?.reasoning) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunk.choices[0].delta.reasoning;
+              }
               if (chunk.usage?.prompt_tokens)
                 inputTokens = chunk.usage.prompt_tokens;
               if (chunk.usage?.completion_tokens)
@@ -707,15 +723,18 @@ async function benchmarkSingleModelRest(model) {
         }
       }
     }
+    await logger?.flush();
     const endTime = Date.now();
     const totalTime = endTime - startTime;
-    const timeToFirstToken = firstTokenTime ? firstTokenTime - startTime : totalTime;
+    const effectiveFirstToken = firstParsedTokenTime ?? firstTokenTime;
+    const timeToFirstToken = effectiveFirstToken ? effectiveFirstToken - startTime : totalTime;
+    const generationTime = totalTime - timeToFirstToken;
     const usedEstimateForOutput = !outputTokens;
     const usedEstimateForInput = !inputTokens;
     const finalOutputTokens = outputTokens || Math.round(streamedText.length / 4);
     const finalInputTokens = inputTokens || Math.round(TEST_PROMPT.length / 4);
     const totalTokens = finalInputTokens + finalOutputTokens;
-    const tokensPerSecond = totalTime > 0 ? finalOutputTokens / totalTime * 1000 : 0;
+    const tokensPerSecond = generationTime > 0 ? finalOutputTokens / generationTime * 1000 : 0;
     return {
       model: model.name,
       provider: model.providerName,
@@ -730,6 +749,7 @@ async function benchmarkSingleModelRest(model) {
       success: true
     };
   } catch (error) {
+    await logger?.flush();
     return {
       model: model.name,
       provider: model.providerName,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-speedometer-headless",
-  "version": "2.1.5",
+  "version": "2.1.7",
   "description": "Headless CLI for benchmarking AI models — runs on Node.js and Bun, no TUI dependencies",
   "bin": {
     "ai-speedometer-headless": "dist/ai-speedometer-headless"