ai-speedometer 1.3.1 → 1.3.3

package/README.md

@@ -17,6 +17,12 @@ npm install -g ai-speedometer
 - **Tokens/Second** - Real-time throughput
 - **Token Counts** - Input, output, and total tokens used
 
+## New Features
+
+- **REST API Default** - REST API benchmarking is now the default method for better compatibility
+- **Headless Mode** - Run benchmarks without interactive CLI using command-line arguments
+- **Streaming Support** - Full streaming support now available in REST API benchmarks
+
 ## Quick Setup
 
 1. **Set Model**
@@ -41,7 +47,8 @@ npm install -g ai-speedometer
 4. **Run Benchmark**
    ```bash
    ai-speedometer
-   # Select "Run Benchmark (AI SDK)" → Choose models → Press ENTER
+   # Select "Run Benchmark (REST API)" → Choose models → Press ENTER
+   # Note: REST API is now the default benchmark method
    ```
 
 ## Usage
@@ -55,6 +62,13 @@ aispeed
 
 # Debug mode
 ai-speedometer --debug
+
+# Headless mode - run benchmark directly
+ai-speedometer --bench openai:gpt-4
+# With custom API key
+ai-speedometer --bench openai:gpt-4 --api-key "sk-your-key"
+# Use AI SDK instead of REST API
+ai-speedometer --bench openai:gpt-4 --ai-sdk
 ```
 
 ## Configuration Files

package/cli.js

@@ -36,6 +36,7 @@ function parseCliArgs() {
     bench: null,
     apiKey: null,
     useAiSdk: false,
+    formatted: false,
     help: false
   };
 
@@ -50,6 +51,8 @@ function parseCliArgs() {
       parsed.apiKey = args[++i];
     } else if (arg === '--ai-sdk') {
       parsed.useAiSdk = true;
+    } else if (arg === '--formatted') {
+      parsed.formatted = true;
     } else if (arg === '--help' || arg === '-h') {
       parsed.help = true;
     }
@@ -69,6 +72,7 @@ function showHelp() {
   console.log('  --bench <provider:model>    ' + colorText('Run benchmark in headless mode', 'dim'));
   console.log('  --api-key <key>             ' + colorText('Override API key (optional)', 'dim'));
   console.log('  --ai-sdk                    ' + colorText('Use AI SDK instead of REST API', 'dim'));
+  console.log('  --formatted                 ' + colorText('Format JSON output for human readability', 'dim'));
   console.log('  --debug                     ' + colorText('Enable debug logging', 'dim'));
   console.log('  --help, -h                  ' + colorText('Show this help message', 'dim'));
   console.log('');
@@ -76,6 +80,7 @@ function showHelp() {
   console.log('  ai-speedometer --bench openai:gpt-4');
   console.log('  ai-speedometer --bench anthropic:claude-3-opus --api-key "sk-..."');
   console.log('  ai-speedometer --bench openai:gpt-4 --ai-sdk');
+  console.log('  ai-speedometer --bench openai:gpt-4 --formatted');
   console.log('');
 }
 
@@ -179,7 +184,7 @@ function showHeader() {
 }
 
 // Configuration management - now using ai-benchmark-config.json for custom providers
-async function loadConfig() {
+async function loadConfig(includeAllProviders = false) {
   try {
     // Check if we need to migrate from old config
     const oldConfigFile = 'ai-benchmark-config.json';
@@ -212,7 +217,8 @@ async function loadConfig() {
     }
     
     // Load providers from both auth.json (verified) and ai-benchmark-config.json (custom)
-    const providers = await getAllAvailableProviders();
+    // Include all models.dev providers if includeAllProviders is true (for headless mode)
+    const providers = await getAllAvailableProviders(includeAllProviders);
     
     return {
       providers,
@@ -1826,9 +1832,11 @@ async function benchmarkSingleModelRest(model) {
         if (isFirstChunk && !firstTokenTime) {
           firstTokenTime = Date.now();
           isFirstChunk = false;
-          // Show live TTFT result
+          // Show live TTFT result (only in interactive mode, not headless)
           const ttftSeconds = ((firstTokenTime - startTime) / 1000).toFixed(2);
-          console.log(colorText(`TTFT received at ${ttftSeconds}s for ${model.name}`, 'green'));
+          if (!cliArgs.bench) {
+            console.log(colorText(`TTFT received at ${ttftSeconds}s for ${model.name}`, 'green'));
+          }
         }
 
         buffer += decoder.decode(value, { stream: true });
@@ -2175,8 +2183,8 @@ async function runHeadlessBenchmark(benchSpec, apiKey, useAiSdk) {
       process.exit(1);
     }
 
-    // Load all available providers
-    const config = await loadConfig();
+    // Load all available providers (include all models.dev providers for headless mode)
+    const config = await loadConfig(true);
     
     // Find the provider (case-insensitive search)
     const provider = config.providers.find(p => 
@@ -2281,10 +2289,11 @@ async function runHeadlessBenchmark(benchSpec, apiKey, useAiSdk) {
       outputTokens: result.tokenCount,
       promptTokens: result.promptTokens,
       totalTokens: result.totalTokens,
+      is_estimated: !!(result.usedEstimateForOutput || result.usedEstimateForInput),
       error: result.error || null
     };
 
-    console.log(JSON.stringify(jsonOutput, null, 2));
+    console.log(JSON.stringify(jsonOutput, null, cliArgs.formatted ? 2 : 0));
     process.exit(result.success ? 0 : 1);
   } catch (error) {
     console.error(colorText('Error: ' + error.message, 'red'));
@@ -2296,7 +2305,7 @@ async function runHeadlessBenchmark(benchSpec, apiKey, useAiSdk) {
 }
 
 // Start the CLI
-if (require.main === module) {
+if (import.meta.url === `file://${process.argv[1]}`) {
   // Check if help flag
   if (cliArgs.help) {
     showHelp();