ai-speedometer-headless 2.1.6 → 2.1.7

package/dist/ai-speedometer-headless

@@ -542,7 +542,7 @@ var getAllAvailableProviders = async (includeAllProviders = false) => {
 var TEST_PROMPT = `make a 300 word story`;
 
 // ../core/src/benchmark.ts
-async function benchmarkSingleModelRest(model) {
+async function benchmarkSingleModelRest(model, logger) {
   try {
     if (!model.providerConfig || !model.providerConfig.apiKey) {
       throw new Error(`Missing API key for provider ${model.providerName}`);
@@ -559,6 +559,7 @@ async function benchmarkSingleModelRest(model) {
       actualModelId = model.name;
     }
     actualModelId = actualModelId.trim();
+    await logger?.logHeader(model.name, model.providerName, model.providerConfig.apiKey);
     const startTime = Date.now();
     let firstTokenTime = null;
     let streamedText = "";
@@ -631,21 +632,21 @@ async function benchmarkSingleModelRest(model) {
     const reader = response.body.getReader();
     const decoder = new TextDecoder;
     let buffer = "";
-    let isFirstChunk = true;
+    let firstParsedTokenTime = null;
     while (true) {
       const { done, value } = await reader.read();
       if (done)
         break;
-      if (isFirstChunk && !firstTokenTime) {
+      if (!firstTokenTime)
         firstTokenTime = Date.now();
-        isFirstChunk = false;
-      }
       buffer += decoder.decode(value, { stream: true });
       const lines = buffer.split(`
 `);
       buffer = lines.pop() || "";
       for (const line of lines) {
         const trimmedLine = line.trim();
+        if (trimmedLine)
+          await logger?.logRaw(trimmedLine);
         if (!trimmedLine)
           continue;
         try {
@@ -657,6 +658,8 @@ async function benchmarkSingleModelRest(model) {
               const chunk = JSON.parse(jsonStr);
               const chunkTyped = chunk;
               if (chunkTyped.type === "content_block_delta" && chunkTyped.delta?.text) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunkTyped.delta.text;
               } else if (chunkTyped.type === "message_start" && chunkTyped.message?.usage) {
                 inputTokens = chunkTyped.message.usage.input_tokens || 0;
@@ -671,6 +674,8 @@ async function benchmarkSingleModelRest(model) {
             } else {
               const chunk = JSON.parse(trimmedLine);
               if (chunk.type === "content_block_delta" && chunk.delta?.text) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunk.delta.text;
               } else if (chunk.type === "message_start" && chunk.message?.usage) {
                 inputTokens = chunk.message.usage.input_tokens || 0;
@@ -684,6 +689,8 @@ async function benchmarkSingleModelRest(model) {
           } else if (model.providerType === "google") {
             const chunk = JSON.parse(trimmedLine);
             if (chunk.candidates?.[0]?.content?.parts?.[0]?.text) {
+              if (!firstParsedTokenTime)
+                firstParsedTokenTime = Date.now();
               streamedText += chunk.candidates[0].content.parts[0].text;
             }
             if (chunk.usageMetadata?.promptTokenCount)
@@ -696,10 +703,15 @@ async function benchmarkSingleModelRest(model) {
               if (jsonStr === "[DONE]")
                 continue;
               const chunk = JSON.parse(jsonStr);
-              if (chunk.choices?.[0]?.delta?.content)
+              if (chunk.choices?.[0]?.delta?.content) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunk.choices[0].delta.content;
-              else if (chunk.choices?.[0]?.delta?.reasoning)
+              } else if (chunk.choices?.[0]?.delta?.reasoning) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunk.choices[0].delta.reasoning;
+              }
               if (chunk.usage?.prompt_tokens)
                 inputTokens = chunk.usage.prompt_tokens;
               if (chunk.usage?.completion_tokens)
@@ -711,15 +723,18 @@ async function benchmarkSingleModelRest(model) {
         }
       }
     }
+    await logger?.flush();
     const endTime = Date.now();
     const totalTime = endTime - startTime;
-    const timeToFirstToken = firstTokenTime ? firstTokenTime - startTime : totalTime;
+    const effectiveFirstToken = firstParsedTokenTime ?? firstTokenTime;
+    const timeToFirstToken = effectiveFirstToken ? effectiveFirstToken - startTime : totalTime;
+    const generationTime = totalTime - timeToFirstToken;
     const usedEstimateForOutput = !outputTokens;
     const usedEstimateForInput = !inputTokens;
     const finalOutputTokens = outputTokens || Math.round(streamedText.length / 4);
     const finalInputTokens = inputTokens || Math.round(TEST_PROMPT.length / 4);
     const totalTokens = finalInputTokens + finalOutputTokens;
-    const tokensPerSecond = totalTime > 0 ? finalOutputTokens / totalTime * 1000 : 0;
+    const tokensPerSecond = generationTime > 0 ? finalOutputTokens / generationTime * 1000 : 0;
     return {
       model: model.name,
       provider: model.providerName,
@@ -734,6 +749,7 @@ async function benchmarkSingleModelRest(model) {
       success: true
     };
   } catch (error) {
+    await logger?.flush();
     return {
       model: model.name,
       provider: model.providerName,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-speedometer-headless",
-  "version": "2.1.6",
+  "version": "2.1.7",
   "description": "Headless CLI for benchmarking AI models — runs on Node.js and Bun, no TUI dependencies",
   "bin": {
     "ai-speedometer-headless": "dist/ai-speedometer-headless"