ai-speedometer 1.4.1 → 1.4.2

package/cli.js

@@ -1786,9 +1786,16 @@ async function benchmarkSingleModelRest(model) {
       }
       
       // Extract the actual model ID for API calls (moved before usage)
-      let actualModelId = model.name;
+      let actualModelId;
       if (model.id && model.id.includes('_')) {
+        // Strip provider prefix (e.g., "provider_model" -> "model")
         actualModelId = model.id.split('_')[1];
+      } else if (model.id) {
+        // Use raw model ID directly (e.g., "zai-org/GLM-4.5-FP8")
+        actualModelId = model.id;
+      } else {
+        // Fallback to model name
+        actualModelId = model.name;
       }
       actualModelId = actualModelId.trim();
       

package/dist/ai-speedometer

@@ -134,7 +134,7 @@ Press Enter to continue...`,"yellow"))}async function yO(){je(),Ct(),console.log
 `,i+=`
 `,e.forEach((u,l)=>{let c=l===t,p=c?m("\u25CF","green"):m("\u25CB","dim"),h=c?m(u.name,"bright"):m(u.name,"yellow"),_=c?m(`(${u.type})`,"cyan"):m(`(${u.type})`,"dim");i+=`${p} ${h} ${_}
 `}),je(),console.log(i);let a=await Cr();if(a==="\x1B[A")t=(t-1+e.length)%e.length;else if(a==="\x1B[B")t=(t+1)%e.length;else{if(a==="\r")break;a===""&&process.exit(0)}}let n=e[t];console.log(""),console.log(m("Selected provider: ","cyan")+m(n.name,"white")),console.log(""),console.log(m("Do you want to add multiple models?","cyan")),console.log(m("1. Add single model","yellow")),console.log(m("2. Add multiple models","yellow"));let o=await ce(m("Enter choice (1 or 2): ","cyan")),r=0;if(o==="2")for(console.log(""),console.log(m("Enter model names (one per line, empty line to finish):","cyan")),console.log(m("Examples: gpt-4, gpt-4-turbo, gpt-3.5-turbo","dim")),console.log("");;){let i=await ce(m("Model name: ","cyan"));if(!i.trim())break;let a=i.trim().toLowerCase().replace(/[^a-z0-9-]/g,"-"),u={name:i.trim(),id:a};await Fa(n.id,u)?(r++,console.log(m(`\u2713 Added model: ${i.trim()}`,"green"))):console.log(m(`\u2717 Failed to add model: ${i.trim()}`,"red"))}else{let i=await ce(m("Enter model name: ","cyan"));if(i.trim()){let a=i.trim().toLowerCase().replace(/[^a-z0-9-]/g,"-"),u={name:i.trim(),id:a};await Fa(n.id,u)?(r=1,console.log(m(`\u2713 Added model: ${i.trim()}`,"green"))):console.log(m(`\u2717 Failed to add model: ${i.trim()}`,"red"))}}r>0?(console.log(""),console.log(m(`Successfully added ${r} model(s) to ${n.name}`,"green"))):console.log(m("No models were added.","yellow")),await ce(m(`
-Press Enter to continue...`,"yellow"))}async function wp(e){try{if(!e.providerConfig||!e.providerConfig.apiKey)throw new Error(`Missing API key for provider ${e.providerName}`);if(!e.providerConfig.baseUrl)throw new Error(`Missing base URL for provider ${e.providerName}`);let t=e.name;e.id&&e.id.includes("_")&&(t=e.id.split("_")[1]),t=t.trim();let n=Date.now(),o=null,r="",i=0,a;e.providerConfig.endpointFormat?a="/"+e.providerConfig.endpointFormat:e.providerType==="anthropic"?a="/messages":e.providerType==="google"?a="/models/"+t+":streamGenerateContent":a="/chat/completions";let l=`${e.providerConfig.baseUrl.replace(/\/$/,"")}${a}`,c={"Content-Type":"application/json",Authorization:`Bearer ${e.providerConfig.apiKey}`};e.providerType==="anthropic"?(c["x-api-key"]=e.providerConfig.apiKey,c["anthropic-version"]="2023-06-01"):e.providerType==="google"&&(delete c.Authorization,c["x-goog-api-key"]=e.providerConfig.apiKey);let p={model:t,messages:[{role:"user",content:bn}],max_tokens:500,temperature:.7,stream:!0};e.providerType==="anthropic"?(p.max_tokens=500,p.stream=!0):e.providerType==="google"&&(p.contents=[{parts:[{text:bn}]}],p.generationConfig={maxOutputTokens:500,temperature:.7},delete p.messages,delete p.max_tokens,delete p.stream);let h=await fetch(l,{method:"POST",headers:c,body:JSON.stringify(p)});if(!h.ok){let ie=await h.text();throw new Error(`API request failed: ${h.status} ${h.statusText}`)}let _=h.body.getReader(),f=new TextDecoder,d="",g=0,y=0,x=!0;for(;;){let{done:ie,value:W}=await _.read();if(ie)break;if(x&&!o){o=Date.now(),x=!1;let $=((o-n)/1e3).toFixed(2);!ot.bench&&!ot.benchCustom&&console.log(m(`TTFT received at ${$}s for ${e.name} (${e.providerName})`,"green"))}d+=f.decode(W,{stream:!0});let U=d.split(`
+Press Enter to continue...`,"yellow"))}async function wp(e){try{if(!e.providerConfig||!e.providerConfig.apiKey)throw new Error(`Missing API key for provider ${e.providerName}`);if(!e.providerConfig.baseUrl)throw new Error(`Missing base URL for provider ${e.providerName}`);let t;e.id&&e.id.includes("_")?t=e.id.split("_")[1]:e.id?t=e.id:t=e.name,t=t.trim();let n=Date.now(),o=null,r="",i=0,a;e.providerConfig.endpointFormat?a="/"+e.providerConfig.endpointFormat:e.providerType==="anthropic"?a="/messages":e.providerType==="google"?a="/models/"+t+":streamGenerateContent":a="/chat/completions";let l=`${e.providerConfig.baseUrl.replace(/\/$/,"")}${a}`,c={"Content-Type":"application/json",Authorization:`Bearer ${e.providerConfig.apiKey}`};e.providerType==="anthropic"?(c["x-api-key"]=e.providerConfig.apiKey,c["anthropic-version"]="2023-06-01"):e.providerType==="google"&&(delete c.Authorization,c["x-goog-api-key"]=e.providerConfig.apiKey);let p={model:t,messages:[{role:"user",content:bn}],max_tokens:500,temperature:.7,stream:!0};e.providerType==="anthropic"?(p.max_tokens=500,p.stream=!0):e.providerType==="google"&&(p.contents=[{parts:[{text:bn}]}],p.generationConfig={maxOutputTokens:500,temperature:.7},delete p.messages,delete p.max_tokens,delete p.stream);let h=await fetch(l,{method:"POST",headers:c,body:JSON.stringify(p)});if(!h.ok){let ie=await h.text();throw new Error(`API request failed: ${h.status} ${h.statusText}`)}let _=h.body.getReader(),f=new TextDecoder,d="",g=0,y=0,x=!0;for(;;){let{done:ie,value:W}=await _.read();if(ie)break;if(x&&!o){o=Date.now(),x=!1;let $=((o-n)/1e3).toFixed(2);!ot.bench&&!ot.benchCustom&&console.log(m(`TTFT received at ${$}s for ${e.name} (${e.providerName})`,"green"))}d+=f.decode(W,{stream:!0});let U=d.split(`
 `);d=U.pop()||"";for(let $ of U){let K=$.trim();if(K)try{if(e.providerType==="anthropic")if(K.startsWith("data: ")){let I=K.slice(6);if(I==="[DONE]")break;let q=JSON.parse(I);q.type==="content_block_delta"&&q.delta?.text?r+=q.delta.text:q.type==="message_start"&&q.message?.usage?g=q.message.usage.input_tokens||0:q.type==="message_delta"&&(q.usage?.output_tokens&&(y=q.usage.output_tokens),q.usage?.input_tokens&&!g&&(g=q.usage.input_tokens))}else{if(K.startsWith("event: "))continue;{let I=JSON.parse(K);I.type==="content_block_delta"&&I.delta?.text?r+=I.delta.text:I.type==="message_start"&&I.message?.usage?g=I.message.usage.input_tokens||0:I.type==="message_delta"&&(I.usage?.output_tokens&&(y=I.usage.output_tokens),I.usage?.input_tokens&&!g&&(g=I.usage.input_tokens))}}else if(e.providerType==="google"){let I=JSON.parse(K);if(I.candidates?.[0]?.content?.parts?.[0]?.text){let q=I.candidates[0].content.parts[0].text;r+=q}I.usageMetadata?.promptTokenCount&&(g=I.usageMetadata.promptTokenCount),I.usageMetadata?.candidatesTokenCount&&(y=I.usageMetadata.candidatesTokenCount)}else if(K.startsWith("data: ")){let I=K.slice(6);if(I==="[DONE]")break;let q=JSON.parse(I);q.choices?.[0]?.delta?.content?r+=q.choices[0].delta.content:q.choices?.[0]?.delta?.reasoning&&(r+=q.choices[0].delta.reasoning),q.usage?.prompt_tokens&&(g=q.usage.prompt_tokens),q.usage?.completion_tokens&&(y=q.usage.completion_tokens)}}catch{continue}}}let D=Date.now()-n,b=o?o-n:D,P=!y,N=!g,A=y||Math.round(r.length/4),F=g||Math.round(bn.length/4),j=F+A,z=D>0?j/D*1e3:0;return{model:e.name,provider:e.providerName,totalTime:D,timeToFirstToken:b,tokenCount:A,tokensPerSecond:z,promptTokens:F,totalTokens:j,usedEstimateForOutput:P,usedEstimateForInput:N,success:!0}}catch(t){return{model:e.name,provider:e.providerName,totalTime:0,timeToFirstToken:0,tokenCount:0,tokensPerSecond:0,promptTokens:0,totalTokens:0,success:!1,error:t.message}}}async function xO(e){if(e.length===0){console.log(m("No models selected for benchmarking.","red"));return}je(),Ct(),console.log(m("Running REST API Benchmark with Streaming...","green")),console.log(m(`Running ${e.length} models in parallel...`,"cyan")),console.log(m("Note: This uses direct REST API calls with streaming support","dim")),console.log(""),console.log(m("Starting parallel REST API benchmark execution...","cyan"));let t=e.map(r=>(console.log(m(`Testing ${r.name} (${r.providerName}) via REST API with streaming...`,"yellow")),wp(r))),n=await Promise.all(t);n.forEach((r,i)=>{r.success?(console.log(m(`\u2713 ${r.model} (${r.provider}) completed!`,"green")),console.log(m(`  Total Time: ${(r.totalTime/1e3).toFixed(2)}s`,"cyan")),console.log(m(`  TTFT: ${(r.timeToFirstToken/1e3).toFixed(2)}s`,"cyan")),console.log(m(`  Tokens/Sec: ${r.tokensPerSecond.toFixed(1)}`,"cyan"))):console.log(m(`\u2717 ${r.model} (${r.provider}) failed: `,"red")+r.error)}),console.log(""),console.log(m("All REST API benchmarks completed!","green")),await W_(n,"REST API (Streaming)",e);let o=n.filter(r=>r.success).map(r=>{let i=e.find(a=>a.name===r.model&&a.providerName===r.provider);return{modelId:i?i.id:r.model,modelName:r.model,providerName:r.provider}});o.length>0&&await Ra(o)}async function Dp(){let e=[{id:1,text:"Set Model",action:()=>wO()},{id:2,text:"Run Benchmark (REST API)",action:async()=>{let n=await xp();n.length>0&&await xO(n)}},{id:3,text:"Run Benchmark (AI SDK - Legacy)",action:async()=>{let n=await xp();n.length>0&&await G_(n)}},{id:4,text:"Exit",action:()=>{console.log(m("Goodbye!","green")),qa.close(),process.exit(0)}}],t=0;for(;;){let n="";n+=m("Ai-speedometer","cyan")+`
 `,n+=m("=============================","cyan")+`
 `,n+=`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-speedometer",
-  "version": "1.4.1",
+  "version": "1.4.2",
   "description": "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
   "main": "cli.js",
   "bin": {