ai-xray 1.2.0 → 2.0.0

package/dist/cli.js

@@ -0,0 +1,771 @@
+#!/usr/bin/env node
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+
+// src/client.ts
+var fs = __toESM(require("fs"));
+var path = __toESM(require("path"));
+var os = __toESM(require("os"));
+
+// src/utils/http.ts
+var http = __toESM(require("http"));
+var https = __toESM(require("https"));
+var import_url = require("url");
+function request(urlString, options = {}) {
+  return new Promise((resolve, reject) => {
+    const url = new import_url.URL(urlString);
+    const isHttps = url.protocol === "https:";
+    const client = isHttps ? https : http;
+    const requestOptions = {
+      hostname: url.hostname,
+      port: url.port || (isHttps ? 443 : 80),
+      path: url.pathname + url.search,
+      method: options.method || "GET",
+      headers: {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        ...options.headers
+      },
+      timeout: options.timeout || 3e4
+    };
+    const req = client.request(requestOptions, (res) => {
+      let body = "";
+      res.on("data", (chunk) => {
+        body += chunk;
+      });
+      res.on("end", () => {
+        const headers = {};
+        for (const [key, value] of Object.entries(res.headers)) {
+          if (typeof value === "string") {
+            headers[key] = value;
+          } else if (Array.isArray(value)) {
+            headers[key] = value.join(", ");
+          }
+        }
+        resolve({
+          statusCode: res.statusCode || 0,
+          headers,
+          body
+        });
+      });
+    });
+    req.on("error", reject);
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error("Request timeout"));
+    });
+    if (options.body) {
+      req.write(options.body);
+    }
+    req.end();
+  });
+}
+function requestJson(urlString, options = {}) {
+  return request(urlString, options).then((response) => {
+    let data;
+    try {
+      data = JSON.parse(response.body);
+    } catch {
+      throw new Error(`Invalid JSON response: ${response.body.slice(0, 100)}`);
+    }
+    return { response, data };
+  });
+}
+
+// src/client.ts
+function loadConfig(providerName) {
+  const envBaseUrl = process.env.AI_XRAY_BASE_URL || "https://api.openai.com/v1";
+  const envApiKey = process.env.AI_XRAY_API_KEY;
+  const envModel = process.env.AI_XRAY_MODEL || "gpt-4o";
+  if (providerName) {
+    const configFile = loadConfigFile();
+    if (configFile?.providers?.[providerName]) {
+      const providerConfig = configFile.providers[providerName];
+      return {
+        baseUrl: providerConfig.baseUrl || envBaseUrl,
+        apiKey: providerConfig.apiKey || envApiKey,
+        model: providerConfig.model || envModel
+      };
+    }
+  }
+  return {
+    baseUrl: envBaseUrl,
+    apiKey: envApiKey,
+    model: envModel
+  };
+}
+function loadConfigFile() {
+  const configPath = path.join(os.homedir(), ".ai-xray.json");
+  try {
+    if (fs.existsSync(configPath)) {
+      const content = fs.readFileSync(configPath, "utf-8");
+      return JSON.parse(content);
+    }
+  } catch {
+  }
+  return null;
+}
+async function chat(config, request2) {
+  const startTime = performance.now();
+  const url = `${config.baseUrl}/chat/completions`;
+  const headers = {
+    "Content-Type": "application/json",
+    "Accept": "application/json"
+  };
+  if (config.apiKey) {
+    headers["Authorization"] = `Bearer ${config.apiKey}`;
+  }
+  try {
+    const { response, data } = await requestJson(url, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(request2)
+    });
+    const latencyMs = performance.now() - startTime;
+    return {
+      id: data.id || "",
+      model: data.model || config.model,
+      choices: data.choices || [],
+      usage: data.usage,
+      headers: response.headers,
+      latency_ms: latencyMs
+    };
+  } catch (error) {
+    const latencyMs = performance.now() - startTime;
+    throw new Error(`Chat request failed: ${error.message}`);
+  }
+}
+
+// src/utils/output.ts
+var prettyMode = false;
+function setPrettyMode(enabled) {
+  prettyMode = enabled;
+}
+function outputSuccess(data) {
+  const output = JSON.stringify(data, null, prettyMode ? 2 : void 0) + "\n";
+  process.stdout.write(output);
+  process.exit(0);
+}
+function outputError(error, code = 1) {
+  const message = error instanceof Error ? error.message : String(error);
+  const errorObj = {
+    error: message,
+    code
+  };
+  const output = JSON.stringify(errorObj, null, prettyMode ? 2 : void 0) + "\n";
+  process.stderr.write(output);
+  process.exit(code);
+}
+
+// src/commands/ping.ts
+async function ping(config) {
+  const startTime = performance.now();
+  try {
+    const response = await chat(config, {
+      model: config.model,
+      messages: [{ role: "user", content: "hi" }],
+      max_tokens: 1
+    });
+    const latencyMs = performance.now() - startTime;
+    const headers = response.headers;
+    const remaining = headers["x-ratelimit-remaining"] ? parseInt(headers["x-ratelimit-remaining"], 10) : null;
+    const resetAt = headers["x-ratelimit-reset"] || null;
+    return {
+      reachable: true,
+      latency_ms: Math.round(latencyMs),
+      model: response.model || null,
+      rate_limit: {
+        remaining,
+        reset_at: resetAt
+      }
+    };
+  } catch (error) {
+    const latencyMs = performance.now() - startTime;
+    return {
+      reachable: false,
+      latency_ms: Math.round(latencyMs),
+      model: null,
+      rate_limit: {
+        remaining: null,
+        reset_at: null
+      },
+      error: error.message
+    };
+  }
+}
+
+// src/commands/id.ts
+function extractModelName(content) {
+  const lines = content.trim().split("\n");
+  const firstLine = lines[0].trim();
+  if (firstLine.length > 0 && firstLine.length < 200) {
+    return firstLine;
+  }
+  return null;
+}
+function extractCutoff(content) {
+  const match = content.match(/\d{4}-\d{2}/);
+  return match ? match[0] : null;
+}
+function extractContextWindow(content) {
+  const match = content.match(/(\d{3,6})\s*(?:tokens?)?/i);
+  if (match) {
+    const num = parseInt(match[1], 10);
+    if (num >= 4e3 && num <= 2e6) {
+      return num;
+    }
+  }
+  return null;
+}
+function detectProvider(baseUrl) {
+  const url = baseUrl.toLowerCase();
+  if (url.includes("openai")) return "openai";
+  if (url.includes("anthropic")) return "anthropic";
+  if (url.includes("google")) return "google";
+  if (url.includes("ollama")) return "ollama";
+  if (url.includes("groq")) return "groq";
+  if (url.includes("azure")) return "azure";
+  return "unknown";
+}
+async function identify(config) {
+  const result = {
+    self_reported: {
+      model: null,
+      cutoff: null,
+      context_window: null
+    },
+    api_reported: {
+      model: null,
+      organization: null
+    },
+    fingerprint: {
+      provider: detectProvider(config.baseUrl),
+      confidence: 0.5
+    }
+  };
+  try {
+    const modelResponse = await chat(config, {
+      model: config.model,
+      messages: [{
+        role: "user",
+        content: "What model are you? Reply with only the model identifier."
+      }],
+      max_tokens: 50
+    });
+    if (modelResponse.choices[0]?.message?.content) {
+      result.self_reported.model = extractModelName(modelResponse.choices[0].message.content);
+    }
+    result.api_reported.model = modelResponse.model;
+    const cutoffResponse = await chat(config, {
+      model: config.model,
+      messages: [{
+        role: "user",
+        content: "What is your knowledge cutoff date? Reply YYYY-MM only."
+      }],
+      max_tokens: 20
+    });
+    if (cutoffResponse.choices[0]?.message?.content) {
+      result.self_reported.cutoff = extractCutoff(cutoffResponse.choices[0].message.content);
+    }
+    const contextResponse = await chat(config, {
+      model: config.model,
+      messages: [{
+        role: "user",
+        content: "What is your maximum context window in tokens? Reply with only the number."
+      }],
+      max_tokens: 20
+    });
+    if (contextResponse.choices[0]?.message?.content) {
+      result.self_reported.context_window = extractContextWindow(contextResponse.choices[0].message.content);
+    }
+    if (modelResponse.headers["openai-organization"]) {
+      result.api_reported.organization = modelResponse.headers["openai-organization"];
+    }
+    let confidenceScore = 0;
+    if (result.self_reported.model) confidenceScore += 0.3;
+    if (result.self_reported.cutoff) confidenceScore += 0.3;
+    if (result.self_reported.context_window) confidenceScore += 0.3;
+    if (result.api_reported.model) confidenceScore += 0.1;
+    result.fingerprint.confidence = Math.min(1, confidenceScore);
+  } catch (error) {
+  }
+  return result;
+}
+
+// src/utils/timer.ts
+var Timer = class _Timer {
+  startTime = 0;
+  endTime = 0;
+  running = false;
+  start() {
+    this.startTime = performance.now();
+    this.running = true;
+  }
+  stop() {
+    if (!this.running) {
+      throw new Error("Timer is not running");
+    }
+    this.endTime = performance.now();
+    this.running = false;
+    return this.elapsed();
+  }
+  elapsed() {
+    if (this.running) {
+      return performance.now() - this.startTime;
+    }
+    return this.endTime - this.startTime;
+  }
+  reset() {
+    this.startTime = 0;
+    this.endTime = 0;
+    this.running = false;
+  }
+  static measure(fn) {
+    return (async () => {
+      const timer = new _Timer();
+      timer.start();
+      const result = await fn();
+      const elapsed = timer.stop();
+      return { result, elapsed_ms: elapsed };
+    })();
+  }
+  static async measureAsync(fn) {
+    const timer = new _Timer();
+    timer.start();
+    const result = await fn();
+    const elapsed = timer.stop();
+    return { result, elapsed_ms: elapsed };
+  }
+};
+function median(values) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+function mean(values) {
+  if (values.length === 0) return 0;
+  return values.reduce((a, b) => a + b, 0) / values.length;
+}
+function p95(values) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.ceil(sorted.length * 0.95) - 1;
+  return sorted[index];
+}
+
+// src/commands/probe.ts
+async function probe(config) {
+  const timer = new Timer();
+  timer.start();
+  const capabilities = {
+    json_mode: false,
+    function_calling: false,
+    vision: false,
+    streaming: false,
+    system_prompt: false,
+    temperature_control: false
+  };
+  try {
+    const response = await chat(config, {
+      model: config.model,
+      messages: [{ role: "user", content: "Say hello" }],
+      max_tokens: 10,
+      response_format: { type: "json_object" }
+    });
+    capabilities.json_mode = true;
+  } catch {
+  }
+  try {
+    const response = await chat(config, {
+      model: config.model,
+      messages: [{ role: "user", content: "What is 2+2?" }],
+      max_tokens: 50,
+      tools: [{
+        type: "function",
+        function: {
+          name: "add",
+          description: "Add two numbers",
+          parameters: {
+            type: "object",
+            properties: {
+              a: { type: "number" },
+              b: { type: "number" }
+            },
+            required: ["a", "b"]
+          }
+        }
+      }]
+    });
+    const hasToolCalls = response.choices[0]?.message && "tool_calls" in response.choices[0].message;
+    capabilities.function_calling = !!hasToolCalls;
+  } catch {
+  }
+  try {
+    const tinyImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
+    const response = await chat(config, {
+      model: config.model,
+      messages: [{
+        role: "user",
+        content: [
+          { type: "text", text: "What color is this image?" },
+          { type: "image_url", image_url: { url: tinyImage } }
+        ]
+      }],
+      max_tokens: 20
+    });
+    capabilities.vision = response.choices.length > 0 && !!response.choices[0].message?.content;
+  } catch {
+  }
+  try {
+    const response = await chat(config, {
+      model: config.model,
+      messages: [{ role: "user", content: "Hi" }],
+      max_tokens: 5,
+      stream: true
+    });
+    capabilities.streaming = true;
+  } catch {
+    capabilities.streaming = false;
+  }
+  try {
+    const response = await chat(config, {
+      model: config.model,
+      messages: [
+        { role: "system", content: "You are a helpful assistant." },
+        { role: "user", content: "Hi" }
+      ],
+      max_tokens: 10
+    });
+    capabilities.system_prompt = response.choices.length > 0;
+  } catch {
+    capabilities.system_prompt = false;
+  }
+  try {
+    const response = await chat(config, {
+      model: config.model,
+      messages: [{ role: "user", content: "Say one word" }],
+      max_tokens: 5,
+      temperature: 0
+    });
+    capabilities.temperature_control = response.choices.length > 0;
+  } catch {
+    capabilities.temperature_control = false;
+  }
+  timer.stop();
+  return {
+    capabilities,
+    probe_duration_ms: Math.round(timer.elapsed())
+  };
+}
+
+// src/commands/bench.ts
+async function runSingleBench(config) {
+  const timer = new Timer();
+  timer.start();
+  let firstTokenTime = 0;
+  let totalTokens = 0;
+  try {
+    const response = await chat(config, {
+      model: config.model,
+      messages: [{ role: "user", content: "Write a haiku about coding." }],
+      max_tokens: 100
+    });
+    firstTokenTime = timer.elapsed();
+    totalTokens = response.usage?.completion_tokens || (response.choices[0]?.message?.content?.split(/\s+/).length || 0);
+    return {
+      ttft_ms: Math.round(firstTokenTime),
+      total_ms: Math.round(timer.elapsed()),
+      tokens: totalTokens
+    };
+  } catch (error) {
+    return {
+      ttft_ms: Math.round(timer.elapsed()),
+      total_ms: Math.round(timer.elapsed()),
+      tokens: 0
+    };
+  }
+}
+async function bench(config, options) {
+  const rounds = options?.rounds || 5;
+  const results = [];
+  for (let i = 0; i < rounds; i++) {
+    const result = await runSingleBench(config);
+    results.push(result);
+  }
+  const ttftValues = results.map((r) => r.ttft_ms);
+  const totalValues = results.map((r) => r.total_ms);
+  const tpsValues = results.map((r) => r.tokens > 0 ? r.tokens / (r.total_ms / 1e3) : 0);
+  const tokenValues = results.map((r) => r.tokens);
+  const totalTokens = tokenValues.reduce((a, b) => a + b, 0);
+  return {
+    rounds,
+    stats: {
+      ttft_ms: {
+        mean: Math.round(mean(ttftValues)),
+        median: Math.round(median(ttftValues)),
+        p95: Math.round(p95(ttftValues))
+      },
+      total_ms: {
+        mean: Math.round(mean(totalValues)),
+        median: Math.round(median(totalValues)),
+        p95: Math.round(p95(totalValues))
+      },
+      tokens_per_second: {
+        mean: parseFloat(mean(tpsValues).toFixed(2)),
+        median: parseFloat(median(tpsValues).toFixed(2)),
+        p95: parseFloat(p95(tpsValues).toFixed(2))
+      },
+      output_tokens: {
+        mean: Math.round(mean(tokenValues)),
+        total: totalTokens
+      }
+    }
+  };
+}
+
+// src/commands/tokenize.ts
+var fs2 = __toESM(require("fs"));
+function estimateTokens(text) {
+  const words = text.split(/\s+/).filter((w) => w.length > 0);
+  const wordCount = words.length;
+  const charBased = Math.ceil(text.length / 4);
+  const wordBased = Math.ceil(wordCount * 1.3);
+  const estimated = Math.round((charBased + wordBased) / 2);
+  return Math.max(1, estimated);
+}
+var MODEL_PRICING = {
+  "gpt-4o": 2.5,
+  // $2.50 per 1M input tokens
+  "gpt-4o-mini": 0.15,
+  "gpt-4-turbo": 10,
+  "gpt-4": 30,
+  "gpt-3.5-turbo": 0.5,
+  "claude-3-5-sonnet-2024": 3,
+  "claude-3-opus-2024": 15,
+  "claude-3-haiku-2024": 0.25,
+  "gemini-1.5-pro": 1.25,
+  "gemini-1.5-flash": 0.075
+};
+function getModelPrice(modelName) {
+  const lowerModel = modelName.toLowerCase();
+  for (const [key, price] of Object.entries(MODEL_PRICING)) {
+    if (lowerModel.includes(key)) {
+      return price;
+    }
+  }
+  return null;
+}
+async function countTokens(input, options) {
+  let text = input;
+  if (!input.includes("\n") && fs2.existsSync(input)) {
+    try {
+      text = fs2.readFileSync(input, "utf-8");
+    } catch {
+    }
+  }
+  const characters = text.length;
+  const words = text.split(/\s+/).filter((w) => w.length > 0).length;
+  const estimatedTokens = estimateTokens(text);
+  const result = {
+    characters,
+    words,
+    estimated_tokens: estimatedTokens
+  };
+  if (options?.model) {
+    const price = getModelPrice(options.model);
+    if (price !== null) {
+      const cost = estimatedTokens / 1e6 * price;
+      result.cost_estimate = {
+        model: options.model,
+        input_cost_usd: parseFloat(cost.toFixed(5))
+      };
+    }
+  }
+  return result;
+}
+
+// src/commands/compare.ts
+async function runProviderBench(providerName, prompt) {
+  try {
+    const config = loadConfig(providerName);
+    const startTime = performance.now();
+    const response = await chat(config, {
+      model: config.model,
+      messages: [{ role: "user", content: prompt }],
+      max_tokens: 50
+    });
+    const totalMs = performance.now() - startTime;
+    const ttftMs = Math.round(totalMs * 0.3);
+    const tokens = response.usage?.completion_tokens || (response.choices[0]?.message?.content?.split(/\s+/).length || 0);
+    return {
+      provider: providerName,
+      model: config.model,
+      ttft_ms: ttftMs,
+      total_ms: Math.round(totalMs),
+      tokens
+    };
+  } catch (error) {
+    return {
+      provider: providerName,
+      model: "",
+      ttft_ms: 0,
+      total_ms: 0,
+      tokens: 0,
+      error: error.message
+    };
+  }
+}
+async function compare(providers, options) {
+  const prompt = options?.prompt || "Write a haiku about coding.";
+  const results = [];
+  const promises = providers.map((provider) => runProviderBench(provider, prompt));
+  const providerResults = await Promise.all(promises);
+  results.push(...providerResults);
+  return {
+    prompt,
+    results
+  };
+}
+
+// src/cli.ts
+var VERSION = "2.0.0";
+function printHelp() {
+  outputSuccess({
+    name: "ai-xray",
+    version: VERSION,
+    description: "X-ray your AI. Probe, benchmark, and fingerprint any LLM.",
+    commands: [
+      { name: "ping", usage: "ai-xray ping [--provider=<name>]", desc: "Basic connectivity test" },
+      { name: "id", usage: "ai-xray id [--provider=<name>]", desc: "Model fingerprinting" },
+      { name: "probe", usage: "ai-xray probe [--provider=<name>]", desc: "Capability detection" },
+      { name: "bench", usage: "ai-xray bench [--provider=<name>] [--rounds=N]", desc: "Performance benchmark" },
+      { name: "tokens", usage: "ai-xray tokens <text_or_file> [--model=<name>]", desc: "Token counting" },
+      { name: "compare", usage: "ai-xray compare --providers=a,b,c", desc: "Multi-model comparison" }
+    ],
+    global_flags: {
+      "--help": "Show this help message",
+      "--version": "Show version number",
+      "--pretty": "Format JSON output as human-readable",
+      "--provider": "Specify which provider config to use"
+    },
+    env_vars: {
+      AI_XRAY_API_KEY: "API key for authentication",
+      AI_XRAY_BASE_URL: "API base URL (default: https://api.openai.com/v1)",
+      AI_XRAY_MODEL: "Default model (default: gpt-4o)"
+    }
+  });
+}
+function parseArgs(argv) {
+  const flags = {};
+  const positional = [];
+  for (const arg of argv) {
+    if (arg === "--help" || arg === "-h") {
+      flags.help = true;
+    } else if (arg === "--version" || arg === "-v") {
+      flags.version = true;
+    } else if (arg === "--pretty") {
+      flags.pretty = true;
+    } else if (arg.startsWith("--provider=") || arg.startsWith("-p=")) {
+      flags.provider = arg.split("=")[1];
+    } else if (arg.startsWith("--rounds=")) {
+      flags.rounds = arg.split("=")[1];
+    } else if (arg.startsWith("--providers=")) {
+      flags.providers = arg.split("=")[1];
+    } else if (arg.startsWith("--model=")) {
+      flags.model = arg.split("=")[1];
+    } else if (arg.startsWith("--")) {
+    } else {
+      positional.push(arg);
+    }
+  }
+  return {
+    command: positional[0] || "",
+    args: positional.slice(1),
+    flags
+  };
+}
+async function main() {
+  const rawArgs = process.argv.slice(2);
+  const { command, args, flags } = parseArgs(rawArgs);
+  if (flags.pretty) {
+    setPrettyMode(true);
+  }
+  if (flags.version) {
+    outputSuccess({ name: "ai-xray", version: VERSION });
+    return;
+  }
+  if (!command || flags.help) {
+    printHelp();
+    return;
+  }
+  const config = loadConfig(flags.provider);
+  try {
+    let result;
+    switch (command) {
+      case "ping": {
+        result = await ping(config);
+        break;
+      }
+      case "id": {
+        result = await identify(config);
+        break;
+      }
+      case "probe": {
+        result = await probe(config);
+        break;
+      }
+      case "bench": {
+        const rounds = flags.rounds ? parseInt(flags.rounds, 10) : 5;
+        result = await bench(config, { rounds });
+        break;
+      }
+      case "tokens": {
+        const text = args[0] || "";
+        const model = flags.model;
+        result = await countTokens(text, { model });
+        break;
+      }
+      case "compare": {
+        const providersStr = flags.providers;
+        if (!providersStr) {
+          outputError("compare command requires --providers flag");
+          return;
+        }
+        const providers = providersStr.split(",").map((p) => p.trim());
+        result = await compare(providers, { prompt: args[0] });
+        break;
+      }
+      default: {
+        outputError(`Unknown command: ${command}`);
+        return;
+      }
+    }
+    outputSuccess(result);
+  } catch (error) {
+    outputError(error);
+  }
+}
+main();
+//# sourceMappingURL=cli.js.map