ai-speedometer 1.4.0 → 1.4.2

package/cli.js

@@ -1786,9 +1786,16 @@ async function benchmarkSingleModelRest(model) {
       }
       
       // Extract the actual model ID for API calls (moved before usage)
-      let actualModelId = model.name;
+      let actualModelId;
       if (model.id && model.id.includes('_')) {
+        // Strip provider prefix (e.g., "provider_model" -> "model")
         actualModelId = model.id.split('_')[1];
+      } else if (model.id) {
+        // Use raw model ID directly (e.g., "zai-org/GLM-4.5-FP8")
+        actualModelId = model.id;
+      } else {
+        // Fallback to model name
+        actualModelId = model.name;
       }
       actualModelId = actualModelId.trim();
       
@@ -2273,7 +2280,7 @@ async function runHeadlessBenchmark(benchSpec, apiKey, useAiSdk, cliArgs = null)
         error: result.error || null
       };
 
-      console.log(JSON.stringify(jsonOutput, null, cliArgs.formatted ? 2 : 0));
+    console.log(JSON.stringify(jsonOutput, null, cliArgs?.formatted ? 2 : 0));
       process.exit(result.success ? 0 : 1);
     }
 
@@ -2414,7 +2421,7 @@ async function runHeadlessBenchmark(benchSpec, apiKey, useAiSdk, cliArgs = null)
       error: result.error || null
     };
 
-    console.log(JSON.stringify(jsonOutput, null, cliArgs.formatted ? 2 : 0));
+    console.log(JSON.stringify(jsonOutput, null, cliArgs?.formatted ? 2 : 0));
     process.exit(result.success ? 0 : 1);
   } catch (error) {
     console.error(colorText('Error: ' + error.message, 'red'));

package/dist/ai-speedometer

@@ -134,7 +134,7 @@ Press Enter to continue...`,"yellow"))}async function yO(){je(),Ct(),console.log
 `,i+=`
 `,e.forEach((u,l)=>{let c=l===t,p=c?m("\u25CF","green"):m("\u25CB","dim"),h=c?m(u.name,"bright"):m(u.name,"yellow"),_=c?m(`(${u.type})`,"cyan"):m(`(${u.type})`,"dim");i+=`${p} ${h} ${_}
 `}),je(),console.log(i);let a=await Cr();if(a==="\x1B[A")t=(t-1+e.length)%e.length;else if(a==="\x1B[B")t=(t+1)%e.length;else{if(a==="\r")break;a===""&&process.exit(0)}}let n=e[t];console.log(""),console.log(m("Selected provider: ","cyan")+m(n.name,"white")),console.log(""),console.log(m("Do you want to add multiple models?","cyan")),console.log(m("1. Add single model","yellow")),console.log(m("2. Add multiple models","yellow"));let o=await ce(m("Enter choice (1 or 2): ","cyan")),r=0;if(o==="2")for(console.log(""),console.log(m("Enter model names (one per line, empty line to finish):","cyan")),console.log(m("Examples: gpt-4, gpt-4-turbo, gpt-3.5-turbo","dim")),console.log("");;){let i=await ce(m("Model name: ","cyan"));if(!i.trim())break;let a=i.trim().toLowerCase().replace(/[^a-z0-9-]/g,"-"),u={name:i.trim(),id:a};await Fa(n.id,u)?(r++,console.log(m(`\u2713 Added model: ${i.trim()}`,"green"))):console.log(m(`\u2717 Failed to add model: ${i.trim()}`,"red"))}else{let i=await ce(m("Enter model name: ","cyan"));if(i.trim()){let a=i.trim().toLowerCase().replace(/[^a-z0-9-]/g,"-"),u={name:i.trim(),id:a};await Fa(n.id,u)?(r=1,console.log(m(`\u2713 Added model: ${i.trim()}`,"green"))):console.log(m(`\u2717 Failed to add model: ${i.trim()}`,"red"))}}r>0?(console.log(""),console.log(m(`Successfully added ${r} model(s) to ${n.name}`,"green"))):console.log(m("No models were added.","yellow")),await ce(m(`
-Press Enter to continue...`,"yellow"))}async function wp(e){try{if(!e.providerConfig||!e.providerConfig.apiKey)throw new Error(`Missing API key for provider ${e.providerName}`);if(!e.providerConfig.baseUrl)throw new Error(`Missing base URL for provider ${e.providerName}`);let t=e.name;e.id&&e.id.includes("_")&&(t=e.id.split("_")[1]),t=t.trim();let n=Date.now(),o=null,r="",i=0,a;e.providerConfig.endpointFormat?a="/"+e.providerConfig.endpointFormat:e.providerType==="anthropic"?a="/messages":e.providerType==="google"?a="/models/"+t+":streamGenerateContent":a="/chat/completions";let l=`${e.providerConfig.baseUrl.replace(/\/$/,"")}${a}`,c={"Content-Type":"application/json",Authorization:`Bearer ${e.providerConfig.apiKey}`};e.providerType==="anthropic"?(c["x-api-key"]=e.providerConfig.apiKey,c["anthropic-version"]="2023-06-01"):e.providerType==="google"&&(delete c.Authorization,c["x-goog-api-key"]=e.providerConfig.apiKey);let p={model:t,messages:[{role:"user",content:bn}],max_tokens:500,temperature:.7,stream:!0};e.providerType==="anthropic"?(p.max_tokens=500,p.stream=!0):e.providerType==="google"&&(p.contents=[{parts:[{text:bn}]}],p.generationConfig={maxOutputTokens:500,temperature:.7},delete p.messages,delete p.max_tokens,delete p.stream);let h=await fetch(l,{method:"POST",headers:c,body:JSON.stringify(p)});if(!h.ok){let ie=await h.text();throw new Error(`API request failed: ${h.status} ${h.statusText}`)}let _=h.body.getReader(),f=new TextDecoder,d="",g=0,y=0,x=!0;for(;;){let{done:ie,value:W}=await _.read();if(ie)break;if(x&&!o){o=Date.now(),x=!1;let $=((o-n)/1e3).toFixed(2);!ot.bench&&!ot.benchCustom&&console.log(m(`TTFT received at ${$}s for ${e.name} (${e.providerName})`,"green"))}d+=f.decode(W,{stream:!0});let U=d.split(`
+Press Enter to continue...`,"yellow"))}async function wp(e){try{if(!e.providerConfig||!e.providerConfig.apiKey)throw new Error(`Missing API key for provider ${e.providerName}`);if(!e.providerConfig.baseUrl)throw new Error(`Missing base URL for provider ${e.providerName}`);let t;e.id&&e.id.includes("_")?t=e.id.split("_")[1]:e.id?t=e.id:t=e.name,t=t.trim();let n=Date.now(),o=null,r="",i=0,a;e.providerConfig.endpointFormat?a="/"+e.providerConfig.endpointFormat:e.providerType==="anthropic"?a="/messages":e.providerType==="google"?a="/models/"+t+":streamGenerateContent":a="/chat/completions";let l=`${e.providerConfig.baseUrl.replace(/\/$/,"")}${a}`,c={"Content-Type":"application/json",Authorization:`Bearer ${e.providerConfig.apiKey}`};e.providerType==="anthropic"?(c["x-api-key"]=e.providerConfig.apiKey,c["anthropic-version"]="2023-06-01"):e.providerType==="google"&&(delete c.Authorization,c["x-goog-api-key"]=e.providerConfig.apiKey);let p={model:t,messages:[{role:"user",content:bn}],max_tokens:500,temperature:.7,stream:!0};e.providerType==="anthropic"?(p.max_tokens=500,p.stream=!0):e.providerType==="google"&&(p.contents=[{parts:[{text:bn}]}],p.generationConfig={maxOutputTokens:500,temperature:.7},delete p.messages,delete p.max_tokens,delete p.stream);let h=await fetch(l,{method:"POST",headers:c,body:JSON.stringify(p)});if(!h.ok){let ie=await h.text();throw new Error(`API request failed: ${h.status} ${h.statusText}`)}let _=h.body.getReader(),f=new TextDecoder,d="",g=0,y=0,x=!0;for(;;){let{done:ie,value:W}=await _.read();if(ie)break;if(x&&!o){o=Date.now(),x=!1;let $=((o-n)/1e3).toFixed(2);!ot.bench&&!ot.benchCustom&&console.log(m(`TTFT received at ${$}s for ${e.name} (${e.providerName})`,"green"))}d+=f.decode(W,{stream:!0});let U=d.split(`
 `);d=U.pop()||"";for(let $ of U){let K=$.trim();if(K)try{if(e.providerType==="anthropic")if(K.startsWith("data: ")){let I=K.slice(6);if(I==="[DONE]")break;let q=JSON.parse(I);q.type==="content_block_delta"&&q.delta?.text?r+=q.delta.text:q.type==="message_start"&&q.message?.usage?g=q.message.usage.input_tokens||0:q.type==="message_delta"&&(q.usage?.output_tokens&&(y=q.usage.output_tokens),q.usage?.input_tokens&&!g&&(g=q.usage.input_tokens))}else{if(K.startsWith("event: "))continue;{let I=JSON.parse(K);I.type==="content_block_delta"&&I.delta?.text?r+=I.delta.text:I.type==="message_start"&&I.message?.usage?g=I.message.usage.input_tokens||0:I.type==="message_delta"&&(I.usage?.output_tokens&&(y=I.usage.output_tokens),I.usage?.input_tokens&&!g&&(g=I.usage.input_tokens))}}else if(e.providerType==="google"){let I=JSON.parse(K);if(I.candidates?.[0]?.content?.parts?.[0]?.text){let q=I.candidates[0].content.parts[0].text;r+=q}I.usageMetadata?.promptTokenCount&&(g=I.usageMetadata.promptTokenCount),I.usageMetadata?.candidatesTokenCount&&(y=I.usageMetadata.candidatesTokenCount)}else if(K.startsWith("data: ")){let I=K.slice(6);if(I==="[DONE]")break;let q=JSON.parse(I);q.choices?.[0]?.delta?.content?r+=q.choices[0].delta.content:q.choices?.[0]?.delta?.reasoning&&(r+=q.choices[0].delta.reasoning),q.usage?.prompt_tokens&&(g=q.usage.prompt_tokens),q.usage?.completion_tokens&&(y=q.usage.completion_tokens)}}catch{continue}}}let D=Date.now()-n,b=o?o-n:D,P=!y,N=!g,A=y||Math.round(r.length/4),F=g||Math.round(bn.length/4),j=F+A,z=D>0?j/D*1e3:0;return{model:e.name,provider:e.providerName,totalTime:D,timeToFirstToken:b,tokenCount:A,tokensPerSecond:z,promptTokens:F,totalTokens:j,usedEstimateForOutput:P,usedEstimateForInput:N,success:!0}}catch(t){return{model:e.name,provider:e.providerName,totalTime:0,timeToFirstToken:0,tokenCount:0,tokensPerSecond:0,promptTokens:0,totalTokens:0,success:!1,error:t.message}}}async function xO(e){if(e.length===0){console.log(m("No models selected for benchmarking.","red"));return}je(),Ct(),console.log(m("Running REST API Benchmark with Streaming...","green")),console.log(m(`Running ${e.length} models in parallel...`,"cyan")),console.log(m("Note: This uses direct REST API calls with streaming support","dim")),console.log(""),console.log(m("Starting parallel REST API benchmark execution...","cyan"));let t=e.map(r=>(console.log(m(`Testing ${r.name} (${r.providerName}) via REST API with streaming...`,"yellow")),wp(r))),n=await Promise.all(t);n.forEach((r,i)=>{r.success?(console.log(m(`\u2713 ${r.model} (${r.provider}) completed!`,"green")),console.log(m(`  Total Time: ${(r.totalTime/1e3).toFixed(2)}s`,"cyan")),console.log(m(`  TTFT: ${(r.timeToFirstToken/1e3).toFixed(2)}s`,"cyan")),console.log(m(`  Tokens/Sec: ${r.tokensPerSecond.toFixed(1)}`,"cyan"))):console.log(m(`\u2717 ${r.model} (${r.provider}) failed: `,"red")+r.error)}),console.log(""),console.log(m("All REST API benchmarks completed!","green")),await W_(n,"REST API (Streaming)",e);let o=n.filter(r=>r.success).map(r=>{let i=e.find(a=>a.name===r.model&&a.providerName===r.provider);return{modelId:i?i.id:r.model,modelName:r.model,providerName:r.provider}});o.length>0&&await Ra(o)}async function Dp(){let e=[{id:1,text:"Set Model",action:()=>wO()},{id:2,text:"Run Benchmark (REST API)",action:async()=>{let n=await xp();n.length>0&&await xO(n)}},{id:3,text:"Run Benchmark (AI SDK - Legacy)",action:async()=>{let n=await xp();n.length>0&&await G_(n)}},{id:4,text:"Exit",action:()=>{console.log(m("Goodbye!","green")),qa.close(),process.exit(0)}}],t=0;for(;;){let n="";n+=m("Ai-speedometer","cyan")+`
 `,n+=m("=============================","cyan")+`
 `,n+=`
@@ -153,4 +153,4 @@ Press Enter to continue...`,"yellow"))}async function wp(e){try{if(!e.providerCo
 `,e.forEach((r,i)=>{let a=i===t,u=a?m("\u25CF","green"):m("\u25CB","dim"),l=a?m(r.text,"bright"):m(r.text,"yellow");n+=`${u} ${l}
 `}),je(),console.log(n);let o=await Cr();if(o==="\x1B[A")t=(t-1+e.length)%e.length;else if(o==="\x1B[B")t=(t+1)%e.length;else if(o==="\r"){if(await e[t].action()==="back")return}else o===""&&process.exit(0)}}process.on("SIGINT",()=>{console.log(m(`
 
-CLI interrupted by user`,"yellow")),qa.close(),process.exit(0)});async function V_(e,t,n,o=null){try{if(o&&o.benchCustom){let d=pO(o),g={...d.models[0],providerName:d.name,providerType:d.type,providerId:d.id,providerConfig:{baseUrl:d.baseUrl,apiKey:d.apiKey,endpointFormat:d.endpointFormat},selected:!0},y=await wp(g),x={provider:d.name,providerId:d.id,model:d.models[0].name,modelId:d.models[0].id,method:"rest-api",success:y.success,totalTime:y.totalTime,totalTimeSeconds:y.totalTime/1e3,timeToFirstToken:y.timeToFirstToken,timeToFirstTokenSeconds:y.timeToFirstToken/1e3,tokensPerSecond:y.tokensPerSecond,outputTokens:y.tokenCount,promptTokens:y.promptTokens,totalTokens:y.totalTokens,is_estimated:!!(y.usedEstimateForOutput||y.usedEstimateForInput),error:y.error||null};console.log(JSON.stringify(x,null,o.formatted?2:0)),process.exit(y.success?0:1)}let r,i,a=e.indexOf(":");a===-1&&(console.error(m("Error: Invalid --bench format. Use: provider:model","red")),console.error(m("Example: --bench zai-code-anth:glm-4.6","yellow")),process.exit(1)),r=e.substring(0,a),i=e.substring(a+1),(i.startsWith('"')&&i.endsWith('"')||i.startsWith("'")&&i.endsWith("'"))&&(i=i.slice(1,-1)),(!r||!i)&&(console.error(m("Error: Invalid --bench format. Use: provider:model","red")),console.error(m("Example: --bench zai-code-anth:glm-4.6","yellow")),process.exit(1));let u=await Ja(!0),l=u.providers.find(d=>d.id?.toLowerCase()===r.toLowerCase()||d.name?.toLowerCase()===r.toLowerCase());l||(console.error(m(`Error: Provider '${r}' not found`,"red")),console.error(m("Available providers:","yellow")),u.providers.forEach(d=>{console.error(m(`  - ${d.id||d.name}`,"cyan"))}),process.exit(1));let c=l.models.find(d=>{let g=d.id?.toLowerCase()||"",y=d.name?.toLowerCase()||"",x=i.toLowerCase();return g===x||(g.includes("_")?g.split("_").slice(1).join("_"):g)===x||y===x});c||(console.error(m(`Error: Model '${i}' not found in provider '${l.name}'`,"red")),console.error(m("Available models:","yellow")),l.models.forEach(d=>{let g=d.id?.includes("_")?d.id.split("_").slice(1).join("_"):d.id;console.error(m(`  - ${d.name} (id: ${g})`,"cyan"))}),process.exit(1));let p=t||l.apiKey;p||(console.error(m(`Error: No API key found for provider '${l.name}'`,"red")),console.error(m("Please provide --api-key flag or configure the provider first","yellow")),process.exit(1));let h={...c,providerName:l.name,providerType:l.type,providerId:l.id,providerConfig:{...l,apiKey:p,baseUrl:l.baseUrl||""},selected:!0},_;n?(console.error(m("AI SDK headless mode not yet implemented","red")),process.exit(1)):_=await wp(h);let f={provider:l.name,providerId:l.id,model:c.name,modelId:c.id,method:n?"ai-sdk":"rest-api",success:_.success,totalTime:_.totalTime,totalTimeSeconds:_.totalTime/1e3,timeToFirstToken:_.timeToFirstToken,timeToFirstTokenSeconds:_.timeToFirstToken/1e3,tokensPerSecond:_.tokensPerSecond,outputTokens:_.tokenCount,promptTokens:_.promptTokens,totalTokens:_.totalTokens,is_estimated:!!(_.usedEstimateForOutput||_.usedEstimateForInput),error:_.error||null};console.log(JSON.stringify(f,null,o.formatted?2:0)),process.exit(_.success?0:1)}catch(r){console.error(m("Error: "+r.message,"red")),Va&&console.error(r.stack),process.exit(1)}}typeof require<"u"&&require.main===module&&(ot.help&&(cO(),process.exit(0)),ot.benchCustom?V_(ot.benchCustom,ot.apiKey,ot.useAiSdk,ot):ot.bench?V_(ot.bench,ot.apiKey,ot.useAiSdk,null):Ma().then(()=>{Dp()}).catch(()=>{Dp()}));0&&(module.exports={listProviders,loadConfig,runStreamingBenchmark,saveConfig,selectModelsCircular,showMainMenu});
+CLI interrupted by user`,"yellow")),qa.close(),process.exit(0)});async function V_(e,t,n,o=null){try{if(o&&o.benchCustom){let d=pO(o),g={...d.models[0],providerName:d.name,providerType:d.type,providerId:d.id,providerConfig:{baseUrl:d.baseUrl,apiKey:d.apiKey,endpointFormat:d.endpointFormat},selected:!0},y=await wp(g),x={provider:d.name,providerId:d.id,model:d.models[0].name,modelId:d.models[0].id,method:"rest-api",success:y.success,totalTime:y.totalTime,totalTimeSeconds:y.totalTime/1e3,timeToFirstToken:y.timeToFirstToken,timeToFirstTokenSeconds:y.timeToFirstToken/1e3,tokensPerSecond:y.tokensPerSecond,outputTokens:y.tokenCount,promptTokens:y.promptTokens,totalTokens:y.totalTokens,is_estimated:!!(y.usedEstimateForOutput||y.usedEstimateForInput),error:y.error||null};console.log(JSON.stringify(x,null,o?.formatted?2:0)),process.exit(y.success?0:1)}let r,i,a=e.indexOf(":");a===-1&&(console.error(m("Error: Invalid --bench format. Use: provider:model","red")),console.error(m("Example: --bench zai-code-anth:glm-4.6","yellow")),process.exit(1)),r=e.substring(0,a),i=e.substring(a+1),(i.startsWith('"')&&i.endsWith('"')||i.startsWith("'")&&i.endsWith("'"))&&(i=i.slice(1,-1)),(!r||!i)&&(console.error(m("Error: Invalid --bench format. Use: provider:model","red")),console.error(m("Example: --bench zai-code-anth:glm-4.6","yellow")),process.exit(1));let u=await Ja(!0),l=u.providers.find(d=>d.id?.toLowerCase()===r.toLowerCase()||d.name?.toLowerCase()===r.toLowerCase());l||(console.error(m(`Error: Provider '${r}' not found`,"red")),console.error(m("Available providers:","yellow")),u.providers.forEach(d=>{console.error(m(`  - ${d.id||d.name}`,"cyan"))}),process.exit(1));let c=l.models.find(d=>{let g=d.id?.toLowerCase()||"",y=d.name?.toLowerCase()||"",x=i.toLowerCase();return g===x||(g.includes("_")?g.split("_").slice(1).join("_"):g)===x||y===x});c||(console.error(m(`Error: Model '${i}' not found in provider '${l.name}'`,"red")),console.error(m("Available models:","yellow")),l.models.forEach(d=>{let g=d.id?.includes("_")?d.id.split("_").slice(1).join("_"):d.id;console.error(m(`  - ${d.name} (id: ${g})`,"cyan"))}),process.exit(1));let p=t||l.apiKey;p||(console.error(m(`Error: No API key found for provider '${l.name}'`,"red")),console.error(m("Please provide --api-key flag or configure the provider first","yellow")),process.exit(1));let h={...c,providerName:l.name,providerType:l.type,providerId:l.id,providerConfig:{...l,apiKey:p,baseUrl:l.baseUrl||""},selected:!0},_;n?(console.error(m("AI SDK headless mode not yet implemented","red")),process.exit(1)):_=await wp(h);let f={provider:l.name,providerId:l.id,model:c.name,modelId:c.id,method:n?"ai-sdk":"rest-api",success:_.success,totalTime:_.totalTime,totalTimeSeconds:_.totalTime/1e3,timeToFirstToken:_.timeToFirstToken,timeToFirstTokenSeconds:_.timeToFirstToken/1e3,tokensPerSecond:_.tokensPerSecond,outputTokens:_.tokenCount,promptTokens:_.promptTokens,totalTokens:_.totalTokens,is_estimated:!!(_.usedEstimateForOutput||_.usedEstimateForInput),error:_.error||null};console.log(JSON.stringify(f,null,o?.formatted?2:0)),process.exit(_.success?0:1)}catch(r){console.error(m("Error: "+r.message,"red")),Va&&console.error(r.stack),process.exit(1)}}typeof require<"u"&&require.main===module&&(ot.help&&(cO(),process.exit(0)),ot.benchCustom?V_(ot.benchCustom,ot.apiKey,ot.useAiSdk,ot):ot.bench?V_(ot.bench,ot.apiKey,ot.useAiSdk,null):Ma().then(()=>{Dp()}).catch(()=>{Dp()}));0&&(module.exports={listProviders,loadConfig,runStreamingBenchmark,saveConfig,selectModelsCircular,showMainMenu});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-speedometer",
-  "version": "1.4.0",
+  "version": "1.4.2",
   "description": "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
   "main": "cli.js",
   "bin": {