ai-speedometer 2.1.0 → 2.1.2

package/dist/ai-speedometer

@@ -1431,7 +1431,7 @@ var getXDGPaths = () => ({
     const errors = [];
     const parsed = parse2(data, errors, { allowTrailingComma: true });
     if (errors.length > 0) {
-      console.warn("Warning: JSONC parsing errors in auth.json");
+      console.warn("Warning: JSONC parsing errors in auth.json:", errors.map((e) => e.error).join(", "));
       return parsed || {};
     }
     return parsed;
@@ -1475,7 +1475,9 @@ var getXDGPaths = () => ({
           provider: { ...merged.provider ?? {}, ...parsed.provider ?? {} }
         };
       }
-    } catch {}
+    } catch (error) {
+      console.warn(`Warning: Could not read opencode config file ${filename}:`, error.message);
+    }
   }
   return merged;
 }, getOpencodeGlobalConfigProviders = async () => {
@@ -1938,7 +1940,8 @@ async function loadConfig(includeAll) {
   try {
     const providers = await getAllAvailableProviders(includeAll);
     return { providers };
-  } catch {
+  } catch (error) {
+    console.error("Error: Failed to load providers:", error.message);
     return { providers: [] };
   }
 }
@@ -1981,6 +1984,8 @@ async function runHeadlessBenchmark(cliArgs) {
         }
       };
       const result2 = await benchmarkSingleModelRest(modelConfig2);
+      if (!result2.success && result2.error)
+        console.error(`Error: Benchmark failed: ${result2.error}`);
       console.log(buildJsonOutput(customProvider.name, customProvider.id, modelDef.name, modelDef.id, result2, cliArgs.formatted));
       process.exit(result2.success ? 0 : 1);
     }
@@ -2048,6 +2053,8 @@ async function runHeadlessBenchmark(cliArgs) {
       }
     };
     const result = await benchmarkSingleModelRest(modelConfig);
+    if (!result.success && result.error)
+      console.error(`Error: Benchmark failed: ${result.error}`);
     console.log(buildJsonOutput(provider.name, provider.id, model.name, model.id, result, cliArgs.formatted));
     process.exit(result.success ? 0 : 1);
   } catch (error) {
@@ -2154,11 +2161,12 @@ var package_default;
 var init_package = __esm(() => {
   package_default = {
     name: "ai-speedometer",
-    version: "2.1.0",
+    version: "2.1.2",
     description: "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
     bin: {
       "ai-speedometer": "dist/ai-speedometer",
-      aispeed: "dist/ai-speedometer"
+      aispeed: "dist/ai-speedometer",
+      "ai-speedometer-headless": "dist/ai-speedometer-headless"
     },
     engines: {
       bun: ">=1.0.0",
@@ -2171,7 +2179,7 @@ var init_package = __esm(() => {
       "test:watch": "bun test --watch",
       "test:update": "bun test --update-snapshots",
       typecheck: "bun tsc --noEmit",
-      build: "bun build src/index.ts --outdir dist --target bun --external '@opentui/core' --external '@opentui/react' --external 'react' --external 'react-reconciler' && cat scripts/shebang dist/index.js > dist/ai-speedometer && chmod +x dist/ai-speedometer && rm dist/index.js",
+      build: "bun build src/index.ts --outdir dist --target bun --external '@opentui/core' --external '@opentui/react' --external 'react' --external 'react-reconciler' && cat scripts/shebang dist/index.js > dist/ai-speedometer && chmod +x dist/ai-speedometer && rm dist/index.js && bun build src/headless-entry.ts --outdir dist --target node --external 'jsonc-parser' && mv dist/headless-entry.js dist/ai-speedometer-headless && chmod +x dist/ai-speedometer-headless",
       prepublishOnly: "bun run build"
     },
     keywords: [

package/dist/ai-speedometer-headless

@@ -0,0 +1,941 @@
+var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
+
+// src/ai-config.ts
+import fs2 from "fs";
+import path2 from "path";
+import { homedir as homedir2 } from "os";
+var getAIConfigPaths = () => {
+  const aiConfigDir = process.env.XDG_CONFIG_HOME || path2.join(homedir2(), ".config");
+  const aiSpeedometerConfigDir = path2.join(aiConfigDir, "ai-speedometer");
+  return {
+    configDir: aiSpeedometerConfigDir,
+    configJson: path2.join(aiSpeedometerConfigDir, "ai-benchmark-config.json"),
+    recentModelsCache: path2.join(aiSpeedometerConfigDir, "recent-models.json")
+  };
+}, readAIConfig = async () => {
+  const { configJson } = getAIConfigPaths();
+  try {
+    if (!fs2.existsSync(configJson)) {
+      return { verifiedProviders: {}, customProviders: [] };
+    }
+    const data = fs2.readFileSync(configJson, "utf8");
+    return JSON.parse(data);
+  } catch (error) {
+    console.warn("Warning: Could not read ai-benchmark-config.json:", error.message);
+    return { verifiedProviders: {}, customProviders: [] };
+  }
+}, getCustomProvidersFromConfig = async () => {
+  const config = await readAIConfig();
+  return config.customProviders || [];
+};
+var init_ai_config = () => {};
+
+// src/opencode-integration.ts
+import fs3 from "fs";
+import path3 from "path";
+import { homedir as homedir3 } from "os";
+
+// src/models-dev.ts
+import fs from "fs";
+import path from "path";
+import { homedir } from "os";
+
+// custom-verified-providers.json
+var custom_verified_providers_default = {
+  "custom-verified-providers": {
+    "zai-code-anth": {
+      id: "zai-code-anth",
+      name: "zai-code-anth",
+      baseUrl: "https://api.z.ai/api/anthropic/v1",
+      type: "anthropic",
+      models: {
+        "glm-4-5": {
+          id: "glm-4.5",
+          name: "GLM-4.5-anth"
+        },
+        "glm-4-5-air": {
+          id: "glm-4.5-air",
+          name: "GLM-4.5-air-anth"
+        },
+        "glm-4-6": {
+          id: "glm-4.6",
+          name: "GLM-4.6-anth"
+        }
+      }
+    },
+    "zai-china-anth-base": {
+      id: "zai-china-anth-base",
+      name: "zai-china-anth-base",
+      baseUrl: "https://open.bigmodel.cn/api/anthropic/v1",
+      type: "anthropic",
+      models: {
+        "glm-4-5": {
+          id: "glm-4.5",
+          name: "GLM-4.5-china-anth"
+        },
+        "glm-4-5-air": {
+          id: "glm-4.5-air",
+          name: "GLM-4.5-air-china-anth"
+        },
+        "glm-4-6": {
+          id: "glm-4.6",
+          name: "GLM-4.6-china-anth"
+        }
+      }
+    },
+    "extra-models-dev": {
+      chutes: {
+        "deepseek-ai/DeepSeek-V3.1-Terminus": {
+          id: "deepseek-ai/DeepSeek-V3.1-Terminus",
+          name: "DeepSeek V3.1 Terminus"
+        },
+        "meituan-longcat/LongCat-Flash-Thinking-FP8": {
+          id: "meituan-longcat/LongCat-Flash-Thinking-FP8",
+          name: "LongCat Flash Thinking FP8"
+        }
+      }
+    },
+    "nanogpt-plan": {
+      id: "nanogpt-plan",
+      name: "nanogpt-plan",
+      baseUrl: "https://nano-gpt.com/api/v1",
+      type: "openai-compatible",
+      models: {
+        "deepseek-ai/DeepSeek-V3.1-Terminus": {
+          id: "deepseek-ai/DeepSeek-V3.1-Terminus",
+          name: "DeepSeek V3.1 Terminus"
+        },
+        "deepseek-ai/DeepSeek-V3.1": {
+          id: "deepseek-ai/DeepSeek-V3.1",
+          name: "DeepSeek V3.1"
+        },
+        "deepseek-ai/DeepSeek-V3.1-Terminus:thinking": {
+          id: "deepseek-ai/DeepSeek-V3.1-Terminus:thinking",
+          name: "DeepSeek V3.1 Terminus Thinking"
+        },
+        "zai-org/GLM-4.5-FP8": {
+          id: "zai-org/GLM-4.5-FP8",
+          name: "GLM 4.5 FP8"
+        },
+        "zai-org/GLM-4.5-FP8:thinking": {
+          id: "zai-org/GLM-4.5-FP8:thinking",
+          name: "GLM 4.5 FP8 Thinking"
+        },
+        "zai-org/GLM-4.5-Air": {
+          id: "zai-org/GLM-4.5-Air",
+          name: "GLM 4.5 Air"
+        },
+        "moonshotai/Kimi-K2-Instruct": {
+          id: "moonshotai/Kimi-K2-Instruct",
+          name: "Kimi K2 Instruct"
+        },
+        "moonshotai/Kimi-K2-Instruct-0905": {
+          id: "moonshotai/Kimi-K2-Instruct-0905",
+          name: "Kimi K2 Instruct 0905"
+        },
+        "moonshotai/kimi-k2-thinking": {
+          id: "moonshotai/kimi-k2-thinking",
+          name: "Kimi K2 Thinking"
+        },
+        "deepseek-ai/deepseek-v3.2-exp": {
+          id: "deepseek-ai/deepseek-v3.2-exp",
+          name: "DeepSeek V3.2 Exp"
+        },
+        "z-ai/glm-4.6": {
+          id: "z-ai/glm-4.6",
+          name: "GLM 4.6"
+        },
+        "z-ai/glm-4.6:thinking": {
+          id: "z-ai/glm-4.6:thinking",
+          name: "GLM 4.6 Thinking"
+        },
+        "qwen3-vl-235b-a22b-instruct": {
+          id: "qwen3-vl-235b-a22b-instruct",
+          name: "Qwen3 VL 235B A22B Instruct"
+        },
+        "MiniMax-M2": {
+          id: "MiniMax-M2",
+          name: "MiniMax-M2"
+        }
+      }
+    }
+  }
+};
+
+// src/models-dev.ts
+var CACHE_DIR = path.join(homedir(), ".cache", "ai-speedometer");
+var CACHE_FILE = path.join(CACHE_DIR, "models.json");
+var FALLBACK_PROVIDERS = [
+  {
+    id: "openai",
+    name: "OpenAI",
+    baseUrl: "https://api.openai.com/v1",
+    type: "openai-compatible",
+    models: [
+      { id: "gpt-4o", name: "GPT-4o" },
+      { id: "gpt-4o-mini", name: "GPT-4o Mini" },
+      { id: "gpt-4-turbo", name: "GPT-4 Turbo" },
+      { id: "gpt-3.5-turbo", name: "GPT-3.5 Turbo" }
+    ]
+  },
+  {
+    id: "anthropic",
+    name: "Anthropic",
+    baseUrl: "https://api.anthropic.com",
+    type: "anthropic",
+    models: [
+      { id: "claude-3-5-sonnet-20241022", name: "Claude 3.5 Sonnet" },
+      { id: "claude-3-5-haiku-20241022", name: "Claude 3.5 Haiku" },
+      { id: "claude-3-opus-20240229", name: "Claude 3 Opus" }
+    ]
+  },
+  {
+    id: "openrouter",
+    name: "OpenRouter",
+    baseUrl: "https://openrouter.ai/api/v1",
+    type: "openai-compatible",
+    models: [
+      { id: "anthropic/claude-3.5-sonnet", name: "Claude 3.5 Sonnet" },
+      { id: "openai/gpt-4o", name: "GPT-4o" },
+      { id: "openai/gpt-4o-mini", name: "GPT-4o Mini" }
+    ]
+  }
+];
+function ensureCacheDir() {
+  try {
+    if (!fs.existsSync(CACHE_DIR)) {
+      fs.mkdirSync(CACHE_DIR, { recursive: true });
+    }
+  } catch (error) {
+    console.warn("Warning: Could not create cache directory:", error.message);
+  }
+}
+function isCacheExpired(cacheData) {
+  if (!cacheData.timestamp)
+    return true;
+  return Date.now() - cacheData.timestamp > 60 * 60 * 1000;
+}
+function loadCache() {
+  try {
+    if (fs.existsSync(CACHE_FILE)) {
+      const data = fs.readFileSync(CACHE_FILE, "utf8");
+      const parsed = JSON.parse(data);
+      if (isCacheExpired(parsed))
+        return null;
+      return parsed;
+    }
+  } catch (error) {
+    console.warn("Warning: Could not load cache:", error.message);
+  }
+  return null;
+}
+function saveCache(data) {
+  try {
+    ensureCacheDir();
+    const cacheData = { ...data, timestamp: Date.now() };
+    fs.writeFileSync(CACHE_FILE, JSON.stringify(cacheData, null, 2));
+  } catch (error) {
+    console.warn("Warning: Could not save cache:", error.message);
+  }
+}
+async function fetchFromAPI() {
+  try {
+    const response = await fetch("https://models.dev/api.json");
+    if (!response.ok)
+      throw new Error(`HTTP error! status: ${response.status}`);
+    return await response.json();
+  } catch (error) {
+    console.warn("Warning: Could not fetch from models.dev API:", error.message);
+    return null;
+  }
+}
+function getCustomProvidersJson() {
+  let data = custom_verified_providers_default;
+  const customProvidersPath = path.join(process.cwd(), "custom-verified-providers.json");
+  if (fs.existsSync(customProvidersPath)) {
+    try {
+      data = JSON.parse(fs.readFileSync(customProvidersPath, "utf8"));
+    } catch (fileError) {
+      console.warn("Warning: Could not load custom providers from file, using embedded data:", fileError.message);
+    }
+  }
+  return data;
+}
+function loadCustomVerifiedProviders() {
+  try {
+    const customProviders = getCustomProvidersJson();
+    const providers = [];
+    if (customProviders["custom-verified-providers"]) {
+      for (const [, providerData] of Object.entries(customProviders["custom-verified-providers"])) {
+        const entry = providerData;
+        if (entry.id === "extra-models-dev")
+          continue;
+        if (entry.id && entry.name && entry.models) {
+          providers.push({
+            id: entry.id,
+            name: entry.name,
+            baseUrl: entry.baseUrl || "",
+            type: entry.type || "openai-compatible",
+            models: Object.values(entry.models).map((m) => ({ id: m.id, name: m.name }))
+          });
+        }
+      }
+    }
+    return providers;
+  } catch (error) {
+    console.warn("Warning: Could not load custom verified providers:", error.message);
+    return [];
+  }
+}
+function loadExtraModels() {
+  try {
+    const customProviders = getCustomProvidersJson();
+    const extraModels = {};
+    const section = customProviders["custom-verified-providers"]?.["extra-models-dev"];
+    if (section) {
+      for (const [providerId, models] of Object.entries(section)) {
+        extraModels[providerId] = Object.values(models).map((m) => ({ id: m.id, name: m.name }));
+      }
+    }
+    return extraModels;
+  } catch (error) {
+    console.warn("Warning: Could not load extra models:", error.message);
+    return {};
+  }
+}
+function transformModelsDevData(apiData) {
+  const providers = [];
+  const customProviders = loadCustomVerifiedProviders();
+  providers.push(...customProviders);
+  const extraModels = loadExtraModels();
+  if (apiData) {
+    for (const [, providerData] of Object.entries(apiData)) {
+      if (providerData.id && providerData.name && providerData.models) {
+        const models = Object.values(providerData.models).map((m) => ({ id: m.id, name: m.name }));
+        if (extraModels[providerData.id]) {
+          models.push(...extraModels[providerData.id]);
+        }
+        providers.push({
+          id: providerData.id,
+          name: providerData.name,
+          baseUrl: providerData.api || providerData.baseUrl || "",
+          type: providerData.npm ? providerData.npm.includes("anthropic") ? "anthropic" : "openai-compatible" : "openai-compatible",
+          models
+        });
+      }
+    }
+  }
+  return providers.length === 0 ? FALLBACK_PROVIDERS : providers;
+}
+async function getAllProviders() {
+  const cachedData = loadCache();
+  if (cachedData?.providers) {
+    const customVerifiedProviders = loadCustomVerifiedProviders();
+    const existingIds = new Set(cachedData.providers.map((p) => p.id));
+    const missing = customVerifiedProviders.filter((p) => !existingIds.has(p.id));
+    if (missing.length > 0)
+      cachedData.providers.push(...missing);
+    return cachedData.providers;
+  }
+  const apiData = await fetchFromAPI();
+  if (apiData) {
+    const transformed = transformModelsDevData(apiData);
+    saveCache({ providers: transformed });
+    return transformed;
+  }
+  const fallback = [...FALLBACK_PROVIDERS];
+  const extraModels = loadExtraModels();
+  const customVerified = loadCustomVerifiedProviders();
+  fallback.forEach((p) => {
+    if (extraModels[p.id])
+      p.models.push(...extraModels[p.id]);
+  });
+  fallback.push(...customVerified);
+  return fallback;
+}
+
+// src/opencode-integration.ts
+init_ai_config();
+import { parse as parseJsonc } from "jsonc-parser";
+var getXDGPaths = () => ({
+  data: path3.join(process.env.XDG_DATA_HOME || path3.join(homedir3(), ".local", "share"), "opencode"),
+  config: path3.join(process.env.XDG_CONFIG_HOME || path3.join(homedir3(), ".config"), "opencode")
+});
+var getFilePaths = () => {
+  const paths = getXDGPaths();
+  return {
+    authJson: path3.join(paths.data, "auth.json"),
+    opencodeJson: path3.join(paths.config, "opencode.json")
+  };
+};
+var readAuthJson = async () => {
+  const { authJson } = getFilePaths();
+  try {
+    if (!fs3.existsSync(authJson))
+      return {};
+    const data = fs3.readFileSync(authJson, "utf8");
+    const errors = [];
+    const parsed = parseJsonc(data, errors, { allowTrailingComma: true });
+    if (errors.length > 0) {
+      console.warn("Warning: JSONC parsing errors in auth.json:", errors.map((e) => e.error).join(", "));
+      return parsed || {};
+    }
+    return parsed;
+  } catch (error) {
+    console.warn("Warning: Could not read auth.json:", error.message);
+    return {};
+  }
+};
+var readOpencodeGlobalConfig = () => {
+  const configDir = path3.join(process.env.XDG_CONFIG_HOME || path3.join(homedir3(), ".config"), "opencode");
+  const candidates = ["config.json", "opencode.json", "opencode.jsonc"];
+  let merged = {};
+  for (const filename of candidates) {
+    const filePath = path3.join(configDir, filename);
+    try {
+      if (!fs3.existsSync(filePath))
+        continue;
+      const text = fs3.readFileSync(filePath, "utf8");
+      const errors = [];
+      const parsed = parseJsonc(text, errors, { allowTrailingComma: true });
+      if (parsed && typeof parsed === "object") {
+        merged = {
+          ...merged,
+          ...parsed,
+          provider: { ...merged.provider ?? {}, ...parsed.provider ?? {} }
+        };
+      }
+    } catch (error) {
+      console.warn(`Warning: Could not read opencode config file ${filename}:`, error.message);
+    }
+  }
+  return merged;
+};
+var getAuthenticatedProviders = async () => {
+  try {
+    const [allModelsDevProviders, authData, globalConfig] = await Promise.all([
+      getAllProviders(),
+      readAuthJson(),
+      Promise.resolve(readOpencodeGlobalConfig())
+    ]);
+    const database = new Map;
+    for (const mdProvider of allModelsDevProviders) {
+      const modelMap = new Map;
+      for (const m of mdProvider.models) {
+        modelMap.set(`${mdProvider.id}_${m.id}`, { id: `${mdProvider.id}_${m.id}`, name: m.name });
+      }
+      database.set(mdProvider.id, {
+        id: mdProvider.id,
+        name: mdProvider.name,
+        type: mdProvider.type,
+        baseUrl: mdProvider.baseUrl,
+        models: modelMap
+      });
+    }
+    for (const [providerID, entry] of Object.entries(globalConfig.provider ?? {})) {
+      const existing = database.get(providerID);
+      const modelMap = existing ? new Map(existing.models) : new Map;
+      for (const [modelKey, m] of Object.entries(entry.models ?? {})) {
+        const resolvedId = `${providerID}_${m.id ?? modelKey}`;
+        modelMap.set(resolvedId, { id: resolvedId, name: m.name ?? m.id ?? modelKey });
+      }
+      database.set(providerID, {
+        id: providerID,
+        name: entry.name ?? existing?.name ?? providerID,
+        type: existing?.type ?? "openai-compatible",
+        baseUrl: entry.options?.baseURL ?? entry.api ?? existing?.baseUrl ?? "",
+        models: modelMap,
+        npm: entry.npm
+      });
+    }
+    const providerMap = new Map;
+    for (const [providerID, authInfo] of Object.entries(authData)) {
+      if (authInfo.type !== "api" || !authInfo.key)
+        continue;
+      const dbEntry = database.get(providerID);
+      if (!dbEntry)
+        continue;
+      const configEntry = globalConfig.provider?.[providerID];
+      const npm = dbEntry.npm ?? configEntry?.npm;
+      const type = npm?.includes("anthropic") ? "anthropic" : dbEntry.type;
+      providerMap.set(providerID, {
+        id: providerID,
+        name: dbEntry.name,
+        type,
+        baseUrl: dbEntry.baseUrl,
+        apiKey: authInfo.key,
+        models: Array.from(dbEntry.models.values())
+      });
+    }
+    for (const [providerID, entry] of Object.entries(globalConfig.provider ?? {})) {
+      if (!entry.options?.apiKey)
+        continue;
+      const dbEntry = database.get(providerID);
+      if (!dbEntry)
+        continue;
+      const npm = dbEntry.npm ?? entry.npm;
+      const type = npm?.includes("anthropic") ? "anthropic" : dbEntry.type;
+      providerMap.set(providerID, {
+        id: providerID,
+        name: dbEntry.name,
+        type,
+        baseUrl: dbEntry.baseUrl,
+        apiKey: entry.options.apiKey,
+        models: Array.from(dbEntry.models.values())
+      });
+    }
+    return Array.from(providerMap.values());
+  } catch (error) {
+    console.warn("Warning: Could not load providers:", error.message);
+    return [];
+  }
+};
+var getAllAvailableProviders = async (includeAllProviders = false) => {
+  const [opencodeProviders, customProvidersFromConfig, customVerifiedProviders] = await Promise.all([
+    getAuthenticatedProviders(),
+    (async () => {
+      try {
+        return await getCustomProvidersFromConfig();
+      } catch (error) {
+        console.warn("Warning: Could not load custom providers:", error.message);
+        return [];
+      }
+    })(),
+    (async () => {
+      try {
+        return loadCustomVerifiedProviders();
+      } catch (error) {
+        console.warn("Warning: Could not load custom verified providers:", error.message);
+        return [];
+      }
+    })()
+  ]);
+  const providerMap = new Map;
+  customVerifiedProviders.forEach((p) => providerMap.set(p.id, p));
+  opencodeProviders.forEach((p) => providerMap.set(p.id, p));
+  customProvidersFromConfig.forEach((p) => providerMap.set(p.id, p));
+  if (includeAllProviders) {
+    try {
+      const allModelsDevProviders = await getAllProviders();
+      const authenticatedIds = new Set(opencodeProviders.map((p) => p.id));
+      const customIds = new Set(customProvidersFromConfig.map((p) => p.id));
+      const customVerifiedIds = new Set(customVerifiedProviders.map((p) => p.id));
+      allModelsDevProviders.forEach((provider) => {
+        if (!authenticatedIds.has(provider.id) && !customIds.has(provider.id) && !customVerifiedIds.has(provider.id)) {
+          providerMap.set(provider.id, {
+            ...provider,
+            type: provider.type,
+            apiKey: "",
+            models: provider.models.map((m) => ({ ...m, id: `${provider.id}_${m.id}` }))
+          });
+        }
+      });
+    } catch (error) {
+      console.warn("Warning: Could not load all models.dev providers:", error.message);
+    }
+  }
+  return Array.from(providerMap.values());
+};
+
+// src/constants.ts
+var TEST_PROMPT = `make a 300 word story`;
+
+// src/benchmark.ts
+async function benchmarkSingleModelRest(model) {
+  try {
+    if (!model.providerConfig || !model.providerConfig.apiKey) {
+      throw new Error(`Missing API key for provider ${model.providerName}`);
+    }
+    if (!model.providerConfig.baseUrl) {
+      throw new Error(`Missing base URL for provider ${model.providerName}`);
+    }
+    let actualModelId;
+    if (model.id && model.id.includes("_")) {
+      actualModelId = model.id.split("_")[1];
+    } else if (model.id) {
+      actualModelId = model.id;
+    } else {
+      actualModelId = model.name;
+    }
+    actualModelId = actualModelId.trim();
+    const startTime = Date.now();
+    let firstTokenTime = null;
+    let streamedText = "";
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let endpoint;
+    if (model.providerConfig.endpointFormat) {
+      endpoint = "/" + model.providerConfig.endpointFormat;
+    } else if (model.providerType === "anthropic") {
+      endpoint = "/messages";
+    } else if (model.providerType === "google") {
+      endpoint = "/models/" + actualModelId + ":streamGenerateContent";
+    } else {
+      endpoint = "/chat/completions";
+    }
+    const baseUrl = model.providerConfig.baseUrl.replace(/\/$/, "");
+    const url = `${baseUrl}${endpoint}`;
+    const headers = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${model.providerConfig.apiKey}`
+    };
+    if (model.providerType === "anthropic") {
+      headers["x-api-key"] = model.providerConfig.apiKey;
+      headers["anthropic-version"] = "2023-06-01";
+    } else if (model.providerType === "google") {
+      delete headers["Authorization"];
+      headers["x-goog-api-key"] = model.providerConfig.apiKey;
+    }
+    const body = {
+      model: actualModelId,
+      messages: [{ role: "user", content: TEST_PROMPT }],
+      max_tokens: 500,
+      temperature: 0.7,
+      stream: true
+    };
+    if (model.providerType === "google") {
+      body["contents"] = [{ parts: [{ text: TEST_PROMPT }] }];
+      body["generationConfig"] = { maxOutputTokens: 500, temperature: 0.7 };
+      delete body["messages"];
+      delete body["max_tokens"];
+      delete body["stream"];
+    }
+    const response = await fetch(url, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(body)
+    });
+    if (!response.ok) {
+      await response.text();
+      throw new Error(`API request failed: ${response.status} ${response.statusText}`);
+    }
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder;
+    let buffer = "";
+    let isFirstChunk = true;
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done)
+        break;
+      if (isFirstChunk && !firstTokenTime) {
+        firstTokenTime = Date.now();
+        isFirstChunk = false;
+      }
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split(`
+`);
+      buffer = lines.pop() || "";
+      for (const line of lines) {
+        const trimmedLine = line.trim();
+        if (!trimmedLine)
+          continue;
+        try {
+          if (model.providerType === "anthropic") {
+            if (trimmedLine.startsWith("data: ")) {
+              const jsonStr = trimmedLine.slice(6);
+              if (jsonStr === "[DONE]")
+                break;
+              const chunk = JSON.parse(jsonStr);
+              const chunkTyped = chunk;
+              if (chunkTyped.type === "content_block_delta" && chunkTyped.delta?.text) {
+                streamedText += chunkTyped.delta.text;
+              } else if (chunkTyped.type === "message_start" && chunkTyped.message?.usage) {
+                inputTokens = chunkTyped.message.usage.input_tokens || 0;
+              } else if (chunkTyped.type === "message_delta") {
+                if (chunkTyped.usage?.output_tokens)
+                  outputTokens = chunkTyped.usage.output_tokens;
+                if (chunkTyped.usage?.input_tokens && !inputTokens)
+                  inputTokens = chunkTyped.usage.input_tokens;
+              }
+            } else if (trimmedLine.startsWith("event: ")) {
+              continue;
+            } else {
+              const chunk = JSON.parse(trimmedLine);
+              if (chunk.type === "content_block_delta" && chunk.delta?.text) {
+                streamedText += chunk.delta.text;
+              } else if (chunk.type === "message_start" && chunk.message?.usage) {
+                inputTokens = chunk.message.usage.input_tokens || 0;
+              } else if (chunk.type === "message_delta") {
+                if (chunk.usage?.output_tokens)
+                  outputTokens = chunk.usage.output_tokens;
+                if (chunk.usage?.input_tokens && !inputTokens)
+                  inputTokens = chunk.usage.input_tokens;
+              }
+            }
+          } else if (model.providerType === "google") {
+            const chunk = JSON.parse(trimmedLine);
+            if (chunk.candidates?.[0]?.content?.parts?.[0]?.text) {
+              streamedText += chunk.candidates[0].content.parts[0].text;
+            }
+            if (chunk.usageMetadata?.promptTokenCount)
+              inputTokens = chunk.usageMetadata.promptTokenCount;
+            if (chunk.usageMetadata?.candidatesTokenCount)
+              outputTokens = chunk.usageMetadata.candidatesTokenCount;
+          } else {
+            if (trimmedLine.startsWith("data: ")) {
+              const jsonStr = trimmedLine.slice(6);
+              if (jsonStr === "[DONE]")
+                break;
+              const chunk = JSON.parse(jsonStr);
+              if (chunk.choices?.[0]?.delta?.content)
+                streamedText += chunk.choices[0].delta.content;
+              else if (chunk.choices?.[0]?.delta?.reasoning)
+                streamedText += chunk.choices[0].delta.reasoning;
+              if (chunk.usage?.prompt_tokens)
+                inputTokens = chunk.usage.prompt_tokens;
+              if (chunk.usage?.completion_tokens)
+                outputTokens = chunk.usage.completion_tokens;
+            }
+          }
+        } catch {
+          continue;
+        }
+      }
+    }
+    const endTime = Date.now();
+    const totalTime = endTime - startTime;
+    const timeToFirstToken = firstTokenTime ? firstTokenTime - startTime : totalTime;
+    const usedEstimateForOutput = !outputTokens;
+    const usedEstimateForInput = !inputTokens;
+    const finalOutputTokens = outputTokens || Math.round(streamedText.length / 4);
+    const finalInputTokens = inputTokens || Math.round(TEST_PROMPT.length / 4);
+    const totalTokens = finalInputTokens + finalOutputTokens;
+    const tokensPerSecond = totalTime > 0 ? finalOutputTokens / totalTime * 1000 : 0;
+    return {
+      model: model.name,
+      provider: model.providerName,
+      totalTime,
+      timeToFirstToken,
+      tokenCount: finalOutputTokens,
+      tokensPerSecond,
+      promptTokens: finalInputTokens,
+      totalTokens,
+      usedEstimateForOutput,
+      usedEstimateForInput,
+      success: true
+    };
+  } catch (error) {
+    return {
+      model: model.name,
+      provider: model.providerName,
+      totalTime: 0,
+      timeToFirstToken: 0,
+      tokenCount: 0,
+      tokensPerSecond: 0,
+      promptTokens: 0,
+      totalTokens: 0,
+      usedEstimateForOutput: true,
+      usedEstimateForInput: true,
+      success: false,
+      error: error.message
+    };
+  }
+}
+
+// src/headless.ts
+function parseProviderModel(arg) {
+  const firstColonIndex = arg.indexOf(":");
+  if (firstColonIndex === -1) {
+    throw new Error(`Invalid format. Use provider:model (e.g., openai:gpt-4)`);
+  }
+  return {
+    provider: arg.substring(0, firstColonIndex),
+    model: arg.substring(firstColonIndex + 1)
+  };
+}
+function createCustomProviderFromCli(cliArgs) {
+  const { provider, model } = parseProviderModel(cliArgs.benchCustom);
+  if (!cliArgs.baseUrl)
+    throw new Error("--base-url is required for custom provider benchmarking");
+  if (!cliArgs.apiKey)
+    throw new Error("--api-key is required for custom provider benchmarking");
+  const endpointFormat = cliArgs.endpointFormat || "chat/completions";
+  return {
+    id: provider,
+    name: provider,
+    type: "openai-compatible",
+    baseUrl: cliArgs.baseUrl,
+    apiKey: cliArgs.apiKey,
+    endpointFormat,
+    models: [{ name: model, id: model }]
+  };
+}
+async function loadConfig(includeAll) {
+  try {
+    const providers = await getAllAvailableProviders(includeAll);
+    return { providers };
+  } catch (error) {
+    console.error("Error: Failed to load providers:", error.message);
+    return { providers: [] };
+  }
+}
+function buildJsonOutput(providerName, providerId, modelName, modelId, result, formatted) {
+  const jsonOutput = {
+    provider: providerName,
+    providerId,
+    model: modelName,
+    modelId,
+    method: "rest-api",
+    success: result.success,
+    totalTime: result.totalTime,
+    totalTimeSeconds: result.totalTime / 1000,
+    timeToFirstToken: result.timeToFirstToken,
+    timeToFirstTokenSeconds: result.timeToFirstToken / 1000,
+    tokensPerSecond: result.tokensPerSecond,
+    outputTokens: result.tokenCount,
+    promptTokens: result.promptTokens,
+    totalTokens: result.totalTokens,
+    is_estimated: !!(result.usedEstimateForOutput || result.usedEstimateForInput),
+    error: result.error || null
+  };
+  return JSON.stringify(jsonOutput, null, formatted ? 2 : 0);
+}
+async function runHeadlessBenchmark(cliArgs) {
+  try {
+    if (cliArgs.benchCustom) {
+      const customProvider = createCustomProviderFromCli(cliArgs);
+      const modelDef = customProvider.models[0];
+      const modelConfig2 = {
+        id: modelDef.id,
+        name: modelDef.name,
+        providerName: customProvider.name,
+        providerType: customProvider.type,
+        providerId: customProvider.id,
+        providerConfig: {
+          baseUrl: customProvider.baseUrl,
+          apiKey: customProvider.apiKey,
+          endpointFormat: customProvider.endpointFormat
+        }
+      };
+      const result2 = await benchmarkSingleModelRest(modelConfig2);
+      if (!result2.success && result2.error)
+        console.error(`Error: Benchmark failed: ${result2.error}`);
+      console.log(buildJsonOutput(customProvider.name, customProvider.id, modelDef.name, modelDef.id, result2, cliArgs.formatted));
+      process.exit(result2.success ? 0 : 1);
+    }
+    const benchSpec = cliArgs.bench;
+    const colonIndex = benchSpec.indexOf(":");
+    if (colonIndex === -1) {
+      console.error("Error: Invalid --bench format. Use: provider:model");
+      process.exit(1);
+    }
+    const providerSpec = benchSpec.substring(0, colonIndex);
+    let modelName = benchSpec.substring(colonIndex + 1);
+    if (modelName.startsWith('"') && modelName.endsWith('"') || modelName.startsWith("'") && modelName.endsWith("'")) {
+      modelName = modelName.slice(1, -1);
+    }
+    if (!providerSpec || !modelName) {
+      console.error("Error: Invalid --bench format. Use: provider:model");
+      process.exit(1);
+    }
+    const config = await loadConfig(true);
+    const provider = config.providers.find((p) => p.id?.toLowerCase() === providerSpec.toLowerCase() || p.name?.toLowerCase() === providerSpec.toLowerCase());
+    if (!provider) {
+      console.error(`Error: Provider '${providerSpec}' not found`);
+      console.error("Available providers:");
+      config.providers.forEach((p) => console.error(`  - ${p.id || p.name}`));
+      process.exit(1);
+    }
+    const model = provider.models.find((m) => {
+      const modelIdLower = m.id?.toLowerCase() || "";
+      const modelNameLower = m.name?.toLowerCase() || "";
+      const searchLower = modelName.toLowerCase();
+      if (modelIdLower === searchLower)
+        return true;
+      const idWithoutPrefix = modelIdLower.includes("_") ? modelIdLower.split("_").slice(1).join("_") : modelIdLower;
+      if (idWithoutPrefix === searchLower)
+        return true;
+      if (modelNameLower === searchLower)
+        return true;
+      return false;
+    });
+    if (!model) {
+      console.error(`Error: Model '${modelName}' not found in provider '${provider.name}'`);
+      console.error("Available models:");
+      provider.models.forEach((m) => {
+        const idWithoutPrefix = m.id?.includes("_") ? m.id.split("_").slice(1).join("_") : m.id;
+        console.error(`  - ${m.name} (id: ${idWithoutPrefix})`);
+      });
+      process.exit(1);
+    }
+    const finalApiKey = cliArgs.apiKey || provider.apiKey;
+    if (!finalApiKey) {
+      console.error(`Error: No API key found for provider '${provider.name}'`);
+      console.error("Please provide --api-key flag or configure the provider first");
+      process.exit(1);
+    }
+    const modelConfig = {
+      id: model.id,
+      name: model.name,
+      providerName: provider.name,
+      providerType: provider.type,
+      providerId: provider.id,
+      providerConfig: {
+        ...provider,
+        apiKey: finalApiKey,
+        baseUrl: provider.baseUrl || ""
+      }
+    };
+    const result = await benchmarkSingleModelRest(modelConfig);
+    if (!result.success && result.error)
+      console.error(`Error: Benchmark failed: ${result.error}`);
+    console.log(buildJsonOutput(provider.name, provider.id, model.name, model.id, result, cliArgs.formatted));
+    process.exit(result.success ? 0 : 1);
+  } catch (error) {
+    console.error("Error: " + error.message);
+    process.exit(1);
+  }
+}
+
+// src/headless-entry.ts
+var args = process.argv.slice(2);
+var parsed = {
+  debug: false,
+  bench: null,
+  benchCustom: null,
+  apiKey: null,
+  baseUrl: null,
+  endpointFormat: null,
+  formatted: false,
+  help: false
+};
+for (let i = 0;i < args.length; i++) {
+  const arg = args[i];
+  if (arg === "--debug")
+    parsed.debug = true;
+  else if (arg === "--bench")
+    parsed.bench = args[++i] ?? null;
+  else if (arg === "--bench-custom")
+    parsed.benchCustom = args[++i] ?? null;
+  else if (arg === "--api-key")
+    parsed.apiKey = args[++i] ?? null;
+  else if (arg === "--base-url")
+    parsed.baseUrl = args[++i] ?? null;
+  else if (arg === "--endpoint-format")
+    parsed.endpointFormat = args[++i] ?? null;
+  else if (arg === "--formatted")
+    parsed.formatted = true;
+  else if (arg === "--help" || arg === "-h") {
+    console.log("ai-speedometer headless - Benchmark AI models (Node.js/Bun compatible)");
+    console.log("");
+    console.log("Usage:");
+    console.log("  ai-speedometer-headless --bench <provider:model>");
+    console.log("  ai-speedometer-headless --bench-custom <provider:model> --base-url <url> --api-key <key>");
+    console.log("");
+    console.log("Options:");
+    console.log("  --bench <provider:model>        Run benchmark in headless mode");
+    console.log("  --bench-custom <provider:model> Run custom provider benchmark");
+    console.log("  --base-url <url>                Base URL for custom provider");
+    console.log("  --api-key <key>                 API key");
+    console.log("  --endpoint-format <format>      Endpoint format (default: chat/completions)");
+    console.log("  --formatted                     Pretty-print JSON output");
+    console.log("  --help, -h                      Show this help message");
+    process.exit(0);
+  }
+}
+if (!parsed.bench && !parsed.benchCustom) {
+  console.error("Error: --bench or --bench-custom is required");
+  console.error("Run with --help for usage");
+  process.exit(1);
+}
+await runHeadlessBenchmark(parsed);

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "ai-speedometer",
-  "version": "2.1.0",
+  "version": "2.1.2",
   "description": "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
   "bin": {
     "ai-speedometer": "dist/ai-speedometer",
-    "aispeed": "dist/ai-speedometer"
+    "aispeed": "dist/ai-speedometer",
+    "ai-speedometer-headless": "dist/ai-speedometer-headless"
   },
   "engines": {
     "bun": ">=1.0.0",
@@ -17,7 +18,7 @@
     "test:watch": "bun test --watch",
     "test:update": "bun test --update-snapshots",
     "typecheck": "bun tsc --noEmit",
-    "build": "bun build src/index.ts --outdir dist --target bun --external '@opentui/core' --external '@opentui/react' --external 'react' --external 'react-reconciler' && cat scripts/shebang dist/index.js > dist/ai-speedometer && chmod +x dist/ai-speedometer && rm dist/index.js",
+    "build": "bun build src/index.ts --outdir dist --target bun --external '@opentui/core' --external '@opentui/react' --external 'react' --external 'react-reconciler' && cat scripts/shebang dist/index.js > dist/ai-speedometer && chmod +x dist/ai-speedometer && rm dist/index.js && bun build src/headless-entry.ts --outdir dist --target node --external 'jsonc-parser' && mv dist/headless-entry.js dist/ai-speedometer-headless && chmod +x dist/ai-speedometer-headless",
     "prepublishOnly": "bun run build"
   },
   "keywords": [