ai-speedometer 2.1.5 → 2.1.7

package/dist/ai-speedometer

@@ -1712,7 +1712,7 @@ var exports_benchmark = {};
 __export(exports_benchmark, {
   benchmarkSingleModelRest: () => benchmarkSingleModelRest
 });
-async function benchmarkSingleModelRest(model) {
+async function benchmarkSingleModelRest(model, logger) {
   try {
     if (!model.providerConfig || !model.providerConfig.apiKey) {
       throw new Error(`Missing API key for provider ${model.providerName}`);
@@ -1729,6 +1729,7 @@ async function benchmarkSingleModelRest(model) {
       actualModelId = model.name;
     }
     actualModelId = actualModelId.trim();
+    await logger?.logHeader(model.name, model.providerName, model.providerConfig.apiKey);
     const startTime = Date.now();
     let firstTokenTime = null;
     let streamedText = "";
@@ -1762,7 +1763,8 @@ async function benchmarkSingleModelRest(model) {
       messages: [{ role: "user", content: TEST_PROMPT }],
       max_tokens: 500,
       temperature: 0.7,
-      stream: true
+      stream: true,
+      stream_options: { include_usage: true }
     };
     if (model.providerType === "google") {
       body["contents"] = [{ parts: [{ text: TEST_PROMPT }] }];
@@ -1770,6 +1772,9 @@ async function benchmarkSingleModelRest(model) {
       delete body["messages"];
       delete body["max_tokens"];
       delete body["stream"];
+      delete body["stream_options"];
+    } else if (model.providerType === "anthropic") {
+      delete body["stream_options"];
     }
     const response = await fetch(url, {
       method: "POST",
@@ -1797,21 +1802,21 @@ async function benchmarkSingleModelRest(model) {
     const reader = response.body.getReader();
     const decoder = new TextDecoder;
     let buffer = "";
-    let isFirstChunk = true;
+    let firstParsedTokenTime = null;
     while (true) {
       const { done, value } = await reader.read();
       if (done)
         break;
-      if (isFirstChunk && !firstTokenTime) {
+      if (!firstTokenTime)
         firstTokenTime = Date.now();
-        isFirstChunk = false;
-      }
       buffer += decoder.decode(value, { stream: true });
       const lines = buffer.split(`
 `);
       buffer = lines.pop() || "";
       for (const line of lines) {
         const trimmedLine = line.trim();
+        if (trimmedLine)
+          await logger?.logRaw(trimmedLine);
         if (!trimmedLine)
           continue;
         try {
@@ -1819,10 +1824,12 @@ async function benchmarkSingleModelRest(model) {
             if (trimmedLine.startsWith("data: ")) {
               const jsonStr = trimmedLine.slice(6);
               if (jsonStr === "[DONE]")
-                break;
+                continue;
               const chunk = JSON.parse(jsonStr);
               const chunkTyped = chunk;
               if (chunkTyped.type === "content_block_delta" && chunkTyped.delta?.text) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunkTyped.delta.text;
               } else if (chunkTyped.type === "message_start" && chunkTyped.message?.usage) {
                 inputTokens = chunkTyped.message.usage.input_tokens || 0;
@@ -1837,6 +1844,8 @@ async function benchmarkSingleModelRest(model) {
             } else {
               const chunk = JSON.parse(trimmedLine);
               if (chunk.type === "content_block_delta" && chunk.delta?.text) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunk.delta.text;
               } else if (chunk.type === "message_start" && chunk.message?.usage) {
                 inputTokens = chunk.message.usage.input_tokens || 0;
@@ -1850,6 +1859,8 @@ async function benchmarkSingleModelRest(model) {
           } else if (model.providerType === "google") {
             const chunk = JSON.parse(trimmedLine);
             if (chunk.candidates?.[0]?.content?.parts?.[0]?.text) {
+              if (!firstParsedTokenTime)
+                firstParsedTokenTime = Date.now();
               streamedText += chunk.candidates[0].content.parts[0].text;
             }
             if (chunk.usageMetadata?.promptTokenCount)
@@ -1860,12 +1871,17 @@ async function benchmarkSingleModelRest(model) {
             if (trimmedLine.startsWith("data: ")) {
               const jsonStr = trimmedLine.slice(6);
               if (jsonStr === "[DONE]")
-                break;
+                continue;
               const chunk = JSON.parse(jsonStr);
-              if (chunk.choices?.[0]?.delta?.content)
+              if (chunk.choices?.[0]?.delta?.content) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunk.choices[0].delta.content;
-              else if (chunk.choices?.[0]?.delta?.reasoning)
+              } else if (chunk.choices?.[0]?.delta?.reasoning) {
+                if (!firstParsedTokenTime)
+                  firstParsedTokenTime = Date.now();
                 streamedText += chunk.choices[0].delta.reasoning;
+              }
               if (chunk.usage?.prompt_tokens)
                 inputTokens = chunk.usage.prompt_tokens;
               if (chunk.usage?.completion_tokens)
@@ -1877,15 +1893,18 @@ async function benchmarkSingleModelRest(model) {
         }
       }
     }
+    await logger?.flush();
     const endTime = Date.now();
     const totalTime = endTime - startTime;
-    const timeToFirstToken = firstTokenTime ? firstTokenTime - startTime : totalTime;
+    const effectiveFirstToken = firstParsedTokenTime ?? firstTokenTime;
+    const timeToFirstToken = effectiveFirstToken ? effectiveFirstToken - startTime : totalTime;
+    const generationTime = totalTime - timeToFirstToken;
     const usedEstimateForOutput = !outputTokens;
     const usedEstimateForInput = !inputTokens;
     const finalOutputTokens = outputTokens || Math.round(streamedText.length / 4);
     const finalInputTokens = inputTokens || Math.round(TEST_PROMPT.length / 4);
     const totalTokens = finalInputTokens + finalOutputTokens;
-    const tokensPerSecond = totalTime > 0 ? finalOutputTokens / totalTime * 1000 : 0;
+    const tokensPerSecond = generationTime > 0 ? finalOutputTokens / generationTime * 1000 : 0;
     return {
       model: model.name,
       provider: model.providerName,
@@ -1900,6 +1919,7 @@ async function benchmarkSingleModelRest(model) {
       success: true
     };
   } catch (error) {
+    await logger?.flush();
     return {
       model: model.name,
       provider: model.providerName,
@@ -2081,6 +2101,66 @@ var init_headless = __esm(() => {
   init_benchmark();
 });
 
+// ../core/src/logger.ts
+var exports_logger = {};
+__export(exports_logger, {
+  getLogPath: () => getLogPath,
+  createRunId: () => createRunId,
+  createBenchLogger: () => createBenchLogger
+});
+import { mkdir, appendFile } from "fs/promises";
+import { homedir as homedir4 } from "os";
+import { join } from "path";
+function generateRunId() {
+  const now = new Date;
+  const date = now.toISOString().slice(0, 10);
+  const time = now.toTimeString().slice(0, 8).replace(/:/g, "");
+  const rand = Math.random().toString(16).slice(2, 6);
+  return `${date}_${time}_${rand}`;
+}
+function redactSecrets(line, apiKey) {
+  if (!apiKey)
+    return line;
+  return line.split(apiKey).join("[REDACTED]");
+}
+function createRunId() {
+  return generateRunId();
+}
+function getLogPath(runId) {
+  return join(homedir4(), ".local", "share", "ai-speedometer", "logs", `${runId}.log`);
+}
+async function createBenchLogger(runId) {
+  const logPath = getLogPath(runId);
+  const logDir = join(homedir4(), ".local", "share", "ai-speedometer", "logs");
+  await mkdir(logDir, { recursive: true });
+  let currentApiKey = "";
+  let buffer = "";
+  return {
+    logPath,
+    runId,
+    logHeader: async (modelName, providerName, apiKey = "") => {
+      currentApiKey = apiKey;
+      const ts = new Date().toISOString();
+      buffer = `
+=== ${modelName} | ${providerName} | ${ts} ===
+`;
+    },
+    logRaw: async (line) => {
+      const safe = redactSecrets(line, currentApiKey);
+      buffer += safe + `
+`;
+    },
+    flush: async () => {
+      buffer += `
+` + "=".repeat(60) + `
+`;
+      await appendFile(logPath, buffer, "utf8");
+      buffer = "";
+    }
+  };
+}
+var init_logger = () => {};
+
 // src/tui/context/AppContext.tsx
 import { createContext, useContext, useReducer, useEffect } from "react";
 import { jsxDEV } from "@opentui/react/jsx-dev-runtime";
@@ -2117,12 +2197,23 @@ function appReducer(state, action) {
       };
     case "BENCH_RESET":
       return { ...state, benchResults: [], selectedModels: [] };
+    case "SET_LOG_INFO":
+      return { ...state, logMode: action.logMode, logPath: action.logPath, runId: action.runId };
     default:
       return state;
   }
 }
-function AppProvider({ children }) {
+function AppProvider({ children, logMode = false }) {
   const [state, dispatch] = useReducer(appReducer, initialState);
+  useEffect(() => {
+    if (logMode) {
+      Promise.resolve().then(() => (init_logger(), exports_logger)).then(({ createRunId: createRunId2, getLogPath: getLogPath2 }) => {
+        const runId = createRunId2();
+        const logPath = getLogPath2(runId);
+        dispatch({ type: "SET_LOG_INFO", logMode: true, logPath, runId });
+      });
+    }
+  }, [logMode]);
   useEffect(() => {
     let cancelled = false;
     async function loadConfig2() {
@@ -2165,7 +2256,10 @@ var init_AppContext = __esm(() => {
     config: null,
     selectedModels: [],
     benchResults: [],
-    isLoadingConfig: true
+    isLoadingConfig: true,
+    logMode: false,
+    logPath: null,
+    runId: null
   };
   AppContext = createContext(null);
 });
@@ -2175,7 +2269,7 @@ var package_default;
 var init_package = __esm(() => {
   package_default = {
     name: "ai-speedometer",
-    version: "2.1.5",
+    version: "2.1.7",
     description: "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
     bin: {
       "ai-speedometer": "dist/ai-speedometer",
@@ -2672,15 +2766,15 @@ function ModelSelectScreen() {
       }
       return;
     }
-    if (!searchQuery && (key.sequence === "A" || key.sequence === "a")) {
+    if (!searchQuery && key.sequence === "A") {
       setSelected(new Set(orderedModels.map((m) => m.key)));
       return;
     }
-    if (!searchQuery && (key.sequence === "N" || key.sequence === "n")) {
+    if (!searchQuery && key.sequence === "N") {
       setSelected(new Set);
       return;
     }
-    if (!searchQuery && recentCount > 0 && (key.sequence === "R" || key.sequence === "r")) {
+    if (!searchQuery && recentCount > 0 && key.sequence === "R") {
       launchBench(orderedModels.slice(0, recentCount));
       return;
     }
@@ -3001,9 +3095,12 @@ function BenchmarkScreen() {
     setModelStates((prev) => prev.map((s) => ({ ...s, status: "running", startedAt: Date.now() })));
     async function runAll() {
       const { benchmarkSingleModelRest: benchmarkSingleModelRest2 } = await Promise.resolve().then(() => (init_benchmark(), exports_benchmark));
+      const logEnabled = state.logMode && !!state.runId;
+      const { createBenchLogger: createBenchLogger2 } = logEnabled ? await Promise.resolve().then(() => (init_logger(), exports_logger)) : { createBenchLogger: null };
       const promises = models.map(async (model) => {
+        const logger = logEnabled && createBenchLogger2 ? await createBenchLogger2(state.runId) : undefined;
         try {
-          const result = await benchmarkSingleModelRest2(model);
+          const result = await benchmarkSingleModelRest2(model, logger);
           if (!result.success) {
             const errMsg = result.error ?? "Request failed";
             setModelStates((prev) => prev.map((s) => s.model.id === model.id && s.model.providerId === model.providerId ? { ...s, status: "error", error: errMsg } : s));
@@ -3424,10 +3521,22 @@ function BenchmarkScreen() {
       navigate("main-menu");
     }
   });
-  const statusLine = allDone ? /* @__PURE__ */ jsxDEV10("text", {
-    fg: "#9ece6a",
-    children: "All done!  [Enter]/[q] return  [\u2191\u2193/PgUp/PgDn/wheel] scroll"
-  }, undefined, false, undefined, this) : /* @__PURE__ */ jsxDEV10("box", {
+  const statusLine = allDone ? /* @__PURE__ */ jsxDEV10("box", {
+    flexDirection: "row",
+    children: [
+      /* @__PURE__ */ jsxDEV10("text", {
+        fg: "#9ece6a",
+        children: "All done!  [Enter]/[Q] return  [\u2191\u2193/PgUp/PgDn/wheel] scroll"
+      }, undefined, false, undefined, this),
+      state.logMode && state.logPath && /* @__PURE__ */ jsxDEV10("text", {
+        fg: "#565f89",
+        children: [
+          "  log: ",
+          state.logPath
+        ]
+      }, undefined, true, undefined, this)
+    ]
+  }, undefined, true, undefined, this) : /* @__PURE__ */ jsxDEV10("box", {
     flexDirection: "row",
     children: [
       running.length > 0 && /* @__PURE__ */ jsxDEV10("text", {
@@ -5028,23 +5137,23 @@ function getHints(screen, benchResults) {
     case "main-menu":
       return ["[\u2191\u2193] navigate", "[Enter] select", "[Ctrl+C] quit"];
     case "model-menu":
-      return ["[\u2191\u2193] navigate", "[Enter] select", "[q] back"];
+      return ["[\u2191\u2193] navigate", "[Enter] select", "[Q] back"];
     case "model-select":
       return ["[\u2191\u2193] navigate", "[Tab] select", "[Enter] run", "[A] all", "[N] none", "[R] recent", "[Esc] back"];
     case "benchmark": {
       const allDone = benchResults.length > 0 && benchResults.every((r) => r.status === "done" || r.status === "error");
-      return allDone ? ["[Enter] back to menu", "[q] back to menu"] : ["Benchmark in progress..."];
+      return allDone ? ["[Enter] back to menu", "[Q] back to menu"] : ["Benchmark in progress..."];
     }
     case "list-providers":
-      return ["[\u2191\u2193] scroll", "[q] back"];
+      return ["[\u2191\u2193] scroll", "[Q] back"];
     case "add-verified":
-      return ["[\u2191\u2193] navigate", "[Enter] select", "[q] back"];
+      return ["[\u2191\u2193] navigate", "[Enter] select", "[Q] back"];
     case "add-custom":
       return ["[\u2191\u2193] navigate", "[Enter] confirm", "[Esc] back"];
     case "add-models":
       return ["[\u2191\u2193] navigate", "[Enter] add / finish", "[Esc] back"];
     default:
-      return ["[q] back"];
+      return ["[Q] back"];
   }
 }
 function ActiveScreen() {
@@ -5096,8 +5205,9 @@ function Shell() {
     ]
   }, undefined, true, undefined, this);
 }
-function App() {
+function App({ logMode = false }) {
   return /* @__PURE__ */ jsxDEV15(AppProvider, {
+    logMode,
     children: /* @__PURE__ */ jsxDEV15(Shell, {}, undefined, false, undefined, this)
   }, undefined, false, undefined, this);
 }
@@ -5123,7 +5233,7 @@ __export(exports_tui, {
 import { createCliRenderer } from "@opentui/core";
 import { createRoot } from "@opentui/react";
 import { jsxDEV as jsxDEV16 } from "@opentui/react/jsx-dev-runtime";
-async function startTui() {
+async function startTui(logMode = false) {
   const renderer = await createCliRenderer({
     exitOnCtrlC: false
   });
@@ -5137,7 +5247,9 @@ async function startTui() {
     renderer.destroy();
     process.exit(0);
   });
-  createRoot(renderer).render(/* @__PURE__ */ jsxDEV16(App, {}, undefined, false, undefined, this));
+  createRoot(renderer).render(/* @__PURE__ */ jsxDEV16(App, {
+    logMode
+  }, undefined, false, undefined, this));
 }
 var ENABLE_BRACKETED_PASTE = "\x1B[?2004h", DISABLE_BRACKETED_PASTE = "\x1B[?2004l";
 var init_tui = __esm(() => {
@@ -5149,6 +5261,7 @@ function parseCliArgs() {
   const args = process.argv.slice(2);
   const parsed = {
     debug: false,
+    log: false,
     bench: null,
     benchCustom: null,
     apiKey: null,
@@ -5161,6 +5274,8 @@ function parseCliArgs() {
     const arg = args[i];
     if (arg === "--debug")
       parsed.debug = true;
+    else if (arg === "--log")
+      parsed.log = true;
     else if (arg === "--bench")
       parsed.bench = args[++i] ?? null;
     else if (arg === "--bench-custom")
@@ -5193,6 +5308,7 @@ function showHelp() {
   console.log("  --api-key <key>                 API key for custom provider");
   console.log("  --endpoint-format <format>      Endpoint format (default: chat/completions)");
   console.log("  --formatted                     Format JSON output for human readability");
+  console.log("  --log                           Log raw SSE streams to ~/.local/share/ai-speedometer/logs/");
   console.log("  --debug                         Enable debug logging");
   console.log("  --help, -h                      Show this help message");
   console.log("");
@@ -5210,5 +5326,5 @@ if (cliArgs.help) {
   await runHeadlessBenchmark2(cliArgs);
 } else {
   const { startTui: startTui2 } = await Promise.resolve().then(() => (init_tui(), exports_tui));
-  await startTui2();
+  await startTui2(cliArgs.log);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-speedometer",
-  "version": "2.1.5",
+  "version": "2.1.7",
   "description": "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
   "bin": {
     "ai-speedometer": "dist/ai-speedometer",