ai-speedometer-headless 2.1.6 → 2.2.0

package/dist/ai-speedometer-headless

@@ -542,7 +542,7 @@ var getAllAvailableProviders = async (includeAllProviders = false) => {
 var TEST_PROMPT = `make a 300 word story`;
 
 // ../core/src/benchmark.ts
-async function benchmarkSingleModelRest(model) {
+async function benchmarkSingleModelRest(model, logger) {
   try {
     if (!model.providerConfig || !model.providerConfig.apiKey) {
       throw new Error(`Missing API key for provider ${model.providerName}`);
@@ -559,6 +559,7 @@ async function benchmarkSingleModelRest(model) {
       actualModelId = model.name;
     }
     actualModelId = actualModelId.trim();
+    await logger?.logHeader(model.name, model.providerName, model.providerConfig.apiKey);
     const startTime = Date.now();
     let firstTokenTime = null;
     let streamedText = "";
@@ -631,33 +632,36 @@ async function benchmarkSingleModelRest(model) {
     const reader = response.body.getReader();
     const decoder = new TextDecoder;
     let buffer = "";
-    let isFirstChunk = true;
+    let firstParsedTokenTime = null;
     while (true) {
       const { done, value } = await reader.read();
       if (done)
         break;
-      if (isFirstChunk && !firstTokenTime) {
+      if (!firstTokenTime)
         firstTokenTime = Date.now();
-        isFirstChunk = false;
-      }
       buffer += decoder.decode(value, { stream: true });
       const lines = buffer.split(`
 `);
       buffer = lines.pop() || "";
       for (const line of lines) {
         const trimmedLine = line.trim();
+        if (trimmedLine)
+          await logger?.logRaw(trimmedLine);
         if (!trimmedLine)
           continue;
         try {
           if (model.providerType === "anthropic") {
-            if (trimmedLine.startsWith("data: ")) {
-              const jsonStr = trimmedLine.slice(6);
+            const anthropicDataPrefix = trimmedLine.startsWith("data: ") ? 6 : trimmedLine.startsWith("data:") ? 5 : -1;
+            if (anthropicDataPrefix !== -1) {
+              const jsonStr = trimmedLine.slice(anthropicDataPrefix);
               if (jsonStr === "[DONE]")
                 continue;
               const chunk = JSON.parse(jsonStr);
               const chunkTyped = chunk;
-              if (chunkTyped.type === "content_block_delta" && chunkTyped.delta?.text) {
-                streamedText += chunkTyped.delta.text;
+              if (chunkTyped.type === "content_block_delta" && (chunkTyped.delta?.text || chunkTyped.delta?.thinking)) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
+                streamedText += chunkTyped.delta?.text || chunkTyped.delta?.thinking || "";
               } else if (chunkTyped.type === "message_start" && chunkTyped.message?.usage) {
                 inputTokens = chunkTyped.message.usage.input_tokens || 0;
               } else if (chunkTyped.type === "message_delta") {
@@ -670,8 +674,10 @@ async function benchmarkSingleModelRest(model) {
               continue;
             } else {
               const chunk = JSON.parse(trimmedLine);
-              if (chunk.type === "content_block_delta" && chunk.delta?.text) {
-                streamedText += chunk.delta.text;
+              if (chunk.type === "content_block_delta" && (chunk.delta?.text || chunk.delta?.thinking)) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
+                streamedText += chunk.delta?.text || chunk.delta?.thinking || "";
               } else if (chunk.type === "message_start" && chunk.message?.usage) {
                 inputTokens = chunk.message.usage.input_tokens || 0;
               } else if (chunk.type === "message_delta") {
@@ -684,6 +690,8 @@ async function benchmarkSingleModelRest(model) {
           } else if (model.providerType === "google") {
             const chunk = JSON.parse(trimmedLine);
             if (chunk.candidates?.[0]?.content?.parts?.[0]?.text) {
+              if (!firstParsedTokenTime)
+                firstParsedTokenTime = Date.now();
               streamedText += chunk.candidates[0].content.parts[0].text;
             }
             if (chunk.usageMetadata?.promptTokenCount)
@@ -691,19 +699,42 @@ async function benchmarkSingleModelRest(model) {
             if (chunk.usageMetadata?.candidatesTokenCount)
               outputTokens = chunk.usageMetadata.candidatesTokenCount;
           } else {
-            if (trimmedLine.startsWith("data: ")) {
-              const jsonStr = trimmedLine.slice(6);
-              if (jsonStr === "[DONE]")
-                continue;
-              const chunk = JSON.parse(jsonStr);
-              if (chunk.choices?.[0]?.delta?.content)
-                streamedText += chunk.choices[0].delta.content;
-              else if (chunk.choices?.[0]?.delta?.reasoning)
-                streamedText += chunk.choices[0].delta.reasoning;
-              if (chunk.usage?.prompt_tokens)
-                inputTokens = chunk.usage.prompt_tokens;
-              if (chunk.usage?.completion_tokens)
-                outputTokens = chunk.usage.completion_tokens;
+            const dataPrefix = trimmedLine.startsWith("data: ") ? 6 : trimmedLine.startsWith("data:") ? 5 : -1;
+            if (dataPrefix === -1)
+              continue;
+            const jsonStr = trimmedLine.slice(dataPrefix);
+            if (jsonStr === "[DONE]")
+              continue;
+            const chunk = JSON.parse(jsonStr);
+            if (chunk.choices?.[0]?.delta?.content) {
+              if (!firstParsedTokenTime)
+                firstParsedTokenTime = Date.now();
+              streamedText += chunk.choices[0].delta.content;
+            } else if (chunk.choices?.[0]?.delta?.reasoning) {
+              if (!firstParsedTokenTime)
+                firstParsedTokenTime = Date.now();
+              streamedText += chunk.choices[0].delta.reasoning;
+            } else if (chunk.choices?.[0]?.delta?.reasoning_content) {
+              if (!firstParsedTokenTime)
+                firstParsedTokenTime = Date.now();
+              streamedText += chunk.choices[0].delta.reasoning_content;
+            } else if (chunk.type === "content_block_delta" && chunk.delta?.text) {
+              if (!firstParsedTokenTime)
+                firstParsedTokenTime = Date.now();
+              streamedText += chunk.delta.text;
+            } else if (chunk.type === "content_block_delta" && chunk.delta?.thinking) {
+              if (!firstParsedTokenTime)
+                firstParsedTokenTime = Date.now();
+              streamedText += chunk.delta.thinking;
+            }
+            if (chunk.usage?.prompt_tokens)
+              inputTokens = chunk.usage.prompt_tokens;
+            if (chunk.usage?.completion_tokens)
+              outputTokens = chunk.usage.completion_tokens;
+            if (chunk.type === "message_start" && chunk.message?.usage?.input_tokens)
+              inputTokens = chunk.message.usage.input_tokens;
+            if (chunk.type === "message_delta" && chunk.usage?.output_tokens) {
+              outputTokens = chunk.usage.output_tokens;
             }
           }
         } catch {
@@ -711,15 +742,18 @@ async function benchmarkSingleModelRest(model) {
         }
       }
     }
+    await logger?.flush();
     const endTime = Date.now();
     const totalTime = endTime - startTime;
-    const timeToFirstToken = firstTokenTime ? firstTokenTime - startTime : totalTime;
+    const effectiveFirstToken = firstParsedTokenTime ?? firstTokenTime;
+    const timeToFirstToken = effectiveFirstToken ? effectiveFirstToken - startTime : totalTime;
+    const generationTime = totalTime - timeToFirstToken;
     const usedEstimateForOutput = !outputTokens;
     const usedEstimateForInput = !inputTokens;
     const finalOutputTokens = outputTokens || Math.round(streamedText.length / 4);
     const finalInputTokens = inputTokens || Math.round(TEST_PROMPT.length / 4);
     const totalTokens = finalInputTokens + finalOutputTokens;
-    const tokensPerSecond = totalTime > 0 ? finalOutputTokens / totalTime * 1000 : 0;
+    const tokensPerSecond = generationTime > 0 ? finalOutputTokens / generationTime * 1000 : 0;
     return {
       model: model.name,
       provider: model.providerName,
@@ -734,6 +768,7 @@ async function benchmarkSingleModelRest(model) {
       success: true
     };
   } catch (error) {
+    await logger?.flush();
     return {
       model: model.name,
       provider: model.providerName,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-speedometer-headless",
-  "version": "2.1.6",
+  "version": "2.2.0",
   "description": "Headless CLI for benchmarking AI models — runs on Node.js and Bun, no TUI dependencies",
   "bin": {
     "ai-speedometer-headless": "dist/ai-speedometer-headless"