llmtester 1.0.7 → 1.0.8

package/dist/config.d.ts

@@ -0,0 +1,17 @@
+import { ProviderType } from './client.js';
+export interface Config {
+    provider: ProviderType;
+    mode?: 'openai' | 'anthropic';
+    apiKey: string;
+    baseUrl: string;
+    modelName: string;
+    judgeProvider?: ProviderType;
+    judgeApiKey?: string;
+    judgeBaseUrl?: string;
+    judgeModelName?: string;
+}
+export declare function getConfigPath(): string;
+export declare function loadConfig(): Promise<Partial<Config> | null>;
+export declare function saveConfig(config: Config): Promise<void>;
+export declare function prompt(message: string): Promise<string>;
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,MAAM,WAAW,MAAM;IACrB,QAAQ,EAAE,YAAY,CAAC;IACvB,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,YAAY,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,aAAa,IAAI,MAAM,CAEtC;AAED,wBAAsB,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAUlE;AAED,wBAAsB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAS9D;AAED,wBAAsB,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAmB7D"}

package/dist/config.js

@@ -0,0 +1,58 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getConfigPath = getConfigPath;
+exports.loadConfig = loadConfig;
+exports.saveConfig = saveConfig;
+exports.prompt = prompt;
+const chalk_1 = __importDefault(require("chalk"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+const readline_1 = __importDefault(require("readline"));
+const paths_js_1 = require("./paths.js");
+function getConfigPath() {
+    return path_1.default.join((0, paths_js_1.getConfigDir)(), 'config.json');
+}
+async function loadConfig() {
+    const configPath = getConfigPath();
+    try {
+        if (await fs_extra_1.default.pathExists(configPath)) {
+            return await fs_extra_1.default.readJson(configPath);
+        }
+    }
+    catch (e) {
+        console.log(chalk_1.default.yellow(`Warning: Could not load config: ${e}`));
+    }
+    return null;
+}
+async function saveConfig(config) {
+    const configPath = getConfigPath();
+    try {
+        await fs_extra_1.default.ensureDir(path_1.default.dirname(configPath));
+        await fs_extra_1.default.writeJson(configPath, config, { spaces: 2 });
+        console.log(chalk_1.default.green(`Config saved to: ${configPath}`));
+    }
+    catch (e) {
+        console.log(chalk_1.default.yellow(`Warning: Could not save config: ${e}`));
+    }
+}
+async function prompt(message) {
+    return new Promise((resolve) => {
+        process.stdin.removeAllListeners('keypress');
+        process.stdin.setRawMode?.(false);
+        const rl = readline_1.default.createInterface({
+            input: process.stdin,
+            output: process.stdout,
+        });
+        rl.question(message, (answer) => {
+            rl.close();
+            resolve(answer);
+        });
+        if (process.stdin.isTTY) {
+            process.stdin.setRawMode(false);
+        }
+    });
+}
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";;;;;AAmBA,sCAEC;AAED,gCAUC;AAED,gCASC;AAED,wBAmBC;AAjED,kDAA0B;AAC1B,wDAA0B;AAC1B,gDAAwB;AACxB,wDAAgC;AAChC,yCAA0C;AAe1C,SAAgB,aAAa;IAC3B,OAAO,cAAI,CAAC,IAAI,CAAC,IAAA,uBAAY,GAAE,EAAE,aAAa,CAAC,CAAC;AAClD,CAAC;AAEM,KAAK,UAAU,UAAU;IAC9B,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,IAAI,MAAM,kBAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACpC,OAAO,MAAM,kBAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,MAAM,CAAC,mCAAmC,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAEM,KAAK,UAAU,UAAU,CAAC,MAAc;IAC7C,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,kBAAE,CAAC,SAAS,CAAC,cAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QAC7C,MAAM,kBAAE,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,MAAM,CAAC,mCAAmC,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,MAAM,CAAC,OAAe;IAC1C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC7C,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC;QAElC,MAAM,EAAE,GAAG,kBAAQ,CAAC,eAAe,CAAC;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAC;QAEH,EAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAc,EAAE,EAAE;YACtC,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACxB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/index.d.ts

@@ -1,3 +1,3 @@
-#!/usr/bin/env node
+#!/usr/bin//env node
 export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,572 +1,24 @@
-#!/usr/bin/env node
+#!/usr/bin//env node
 "use strict";
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const dotenv_1 = __importDefault(require("dotenv"));
-const chalk_1 = __importDefault(require("chalk"));
-const fs_extra_1 = __importDefault(require("fs-extra"));
-const path_1 = __importDefault(require("path"));
-const readline_1 = __importDefault(require("readline"));
 const child_process_1 = require("child_process");
-const client_js_1 = require("./client.js");
-const benchmarks_js_1 = require("./benchmarks.js");
-const evaluator_js_1 = require("./evaluator.js");
-const progress_js_1 = require("./progress.js");
-const logger_js_1 = require("./logger.js");
-const paths_js_1 = require("./paths.js");
-dotenv_1.default.config();
-function getConfigPath() {
-    return path_1.default.join((0, paths_js_1.getConfigDir)(), 'config.json');
-}
-async function loadConfig() {
-    const configPath = getConfigPath();
-    try {
-        if (await fs_extra_1.default.pathExists(configPath)) {
-            return await fs_extra_1.default.readJson(configPath);
-        }
-    }
-    catch (e) {
-        console.log(chalk_1.default.yellow(`Warning: Could not load config: ${e}`));
-    }
-    return null;
-}
-async function saveConfig(config) {
-    const configPath = getConfigPath();
-    try {
-        await fs_extra_1.default.ensureDir(path_1.default.dirname(configPath));
-        await fs_extra_1.default.writeJson(configPath, config, { spaces: 2 });
-        console.log(chalk_1.default.green(`Config saved to: ${configPath}`));
-    }
-    catch (e) {
-        console.log(chalk_1.default.yellow(`Warning: Could not save config: ${e}`));
-    }
-}
-const PROVIDERS = [
-    { id: 'openai', name: 'OpenAI', defaultUrl: 'https://api.openai.com/v1' },
-    { id: 'anthropic', name: 'Anthropic', defaultUrl: 'https://api.anthropic.com' },
-    { id: 'custom', name: 'Custom (OpenAI-compatible)', defaultUrl: '' },
-];
-const COMMON_OPENAI_ENDPOINTS = [
-    { name: 'OpenAI', url: 'https://api.openai.com/v1' },
-    { name: 'Together.ai', url: 'https://api.together.xyz/v1' },
-    { name: 'Groq', url: 'https://api.groq.com/openai/v1' },
-    { name: 'Fireworks AI', url: 'https://api.fireworks.ai/inference/v1' },
-    { name: 'Perplexity', url: 'https://api.perplexity.ai' },
-    { name: 'OpenRouter', url: 'https://openrouter.ai/api/v1' },
-    { name: 'Custom URL', url: '' },
-];
-async function prompt(message) {
-    return new Promise((resolve) => {
-        process.stdin.removeAllListeners('keypress');
-        process.stdin.setRawMode?.(false);
-        const rl = readline_1.default.createInterface({
-            input: process.stdin,
-            output: process.stdout,
-        });
-        rl.question(message, (answer) => {
-            rl.close();
-            resolve(answer);
-        });
-        if (process.stdin.isTTY) {
-            process.stdin.setRawMode(false);
-        }
-    });
-}
-async function selectOption(items, message) {
-    // Remove all existing keypress listeners first
-    process.stdin.removeAllListeners('keypress');
-    readline_1.default.emitKeypressEvents(process.stdin);
-    if (process.stdin.isTTY) {
-        process.stdin.setRawMode(true);
-    }
-    let cursor = 0;
-    const render = () => {
-        process.stdout.write('\x1b[H\x1b[2J');
-        console.log(chalk_1.default.cyan(`${message}\n`));
-        items.forEach((item, i) => {
-            const prefix = i === cursor ? chalk_1.default.cyan('> ') : '  ';
-            console.log(`${prefix}${item.name}`);
-        });
-        console.log(chalk_1.default.gray('\nUse arrow keys, Enter to select'));
-    };
-    render();
-    return new Promise((resolve) => {
-        const handleKeypress = (str, key) => {
-            if (key.ctrl && key.name === 'c') {
-                if (process.stdin.isTTY)
-                    process.stdin.setRawMode(false);
-                process.exit(0);
-            }
-            if (key.name === 'up') {
-                cursor = Math.max(0, cursor - 1);
-                render();
-            }
-            else if (key.name === 'down') {
-                cursor = Math.min(items.length - 1, cursor + 1);
-                render();
-            }
-            else if (key.name === 'return') {
-                process.stdin.removeListener('keypress', handleKeypress);
-                if (process.stdin.isTTY)
-                    process.stdin.setRawMode(false);
-                resolve(items[cursor]);
-            }
-        };
-        process.stdin.on('keypress', handleKeypress);
-    });
-}
-async function selectMulti(items, message, formatItem) {
-    // Remove all existing keypress listeners first
-    process.stdin.removeAllListeners('keypress');
-    readline_1.default.emitKeypressEvents(process.stdin);
-    if (process.stdin.isTTY) {
-        process.stdin.setRawMode(true);
-    }
-    process.stdout.write('\x1b[H\x1b[2J');
-    console.log(chalk_1.default.cyan(`\n${message}`));
-    console.log(chalk_1.default.gray('Space to toggle, Enter to confirm\n'));
-    const selected = [];
-    let cursor = 0;
-    const render = () => {
-        process.stdout.write('\x1b[H\x1b[2J');
-        console.log(chalk_1.default.cyan(`${message}\n`));
-        items.forEach((item, i) => {
-            const mark = selected.includes(item) ? chalk_1.default.green('[x]') : '[ ]';
-            const prefix = i === cursor ? chalk_1.default.cyan('> ') : '  ';
-            const extra = formatItem ? ` ${chalk_1.default.gray(formatItem(item))}` : '';
-            console.log(`${prefix}${mark} ${item.name}${extra}`);
-        });
-        console.log(chalk_1.default.gray('\nSpace to toggle, Enter to confirm'));
-    };
-    render();
-    return new Promise((resolve) => {
-        const handleKeypress = (str, key) => {
-            if (key.ctrl && key.name === 'c') {
-                if (process.stdin.isTTY)
-                    process.stdin.setRawMode(false);
-                process.exit(0);
-            }
-            if (key.name === 'up') {
-                cursor = Math.max(0, cursor - 1);
-                render();
-            }
-            else if (key.name === 'down') {
-                cursor = Math.min(items.length - 1, cursor + 1);
-                render();
-            }
-            else if (key.name === 'space' || str === ' ') {
-                const item = items[cursor];
-                if (selected.includes(item)) {
-                    selected.splice(selected.indexOf(item), 1);
-                }
-                else {
-                    selected.push(item);
-                }
-                render();
-            }
-            else if (key.name === 'return') {
-                process.stdin.removeListener('keypress', handleKeypress);
-                if (process.stdin.isTTY)
-                    process.stdin.setRawMode(false);
-                resolve(selected);
-            }
-        };
-        process.stdin.on('keypress', handleKeypress);
-    });
-}
-async function selectProvider() {
-    return selectOption(PROVIDERS, 'Select your provider');
-}
-async function selectProviderWithPrompt(message) {
-    console.log(chalk_1.default.cyan('\n' + message));
-    PROVIDERS.forEach((p, i) => { console.log(String(i + 1) + '. ' + p.name); });
-    const providerIds = ['openai', 'anthropic', 'custom'];
+const path_1 = __importDefault(require("path"));
+const runner_js_1 = require("./runner.js");
+const menu_js_1 = require("./menu.js");
+async function main() {
     while (true) {
-        const answer = await prompt('Enter a number (1-3) or name: ');
-        const trimmed = answer.trim().toLowerCase();
-        const num = parseInt(trimmed, 10);
-        if (num >= 1 && num <= PROVIDERS.length) {
-            return PROVIDERS[num - 1];
-        }
-        if (providerIds.includes(trimmed)) {
-            return PROVIDERS.find(p => p.id === trimmed);
-        }
-        console.log(chalk_1.default.yellow('Invalid selection. Please try again.'));
-    }
-}
-async function selectEndpoint(provider) {
-    if (provider === 'anthropic') {
-        return 'https://api.anthropic.com';
-    }
-    if (provider === 'custom' || provider === 'openai') {
-        const endpoints = COMMON_OPENAI_ENDPOINTS.map(e => ({ id: e.url || 'custom', name: e.name, url: e.url }));
-        const selection = await selectOption(endpoints, 'Select or enter endpoint');
-        if (selection.url === '') {
-            return (await prompt('Enter custom endpoint URL: ')).trim();
+        const choice = await (0, menu_js_1.showMainMenu)();
+        if (choice === 0) {
+            await (0, runner_js_1.runBenchmarks)();
         }
-        return selection.url;
-    }
-    return PROVIDERS.find(p => p.id === provider)?.defaultUrl || '';
-}
-async function selectEndpointWithPrompt(provider) {
-    if (provider === 'anthropic') {
-        return 'https://api.anthropic.com';
-    }
-    if (provider === 'custom') {
-        const answer = await prompt('Enter custom endpoint URL: ');
-        return answer.trim();
-    }
-    return PROVIDERS.find(p => p.id === provider)?.defaultUrl || '';
-}
-async function getConfig() {
-    const envProvider = process.env.LLM_PROVIDER || '';
-    const envApiKey = process.env.LLM_API_KEY || process.env.OPENAI_API_KEY || '';
-    const envBaseUrl = process.env.LLM_BASE_URL || process.env.OPENAI_BASE_URL || '';
-    const envModelName = process.env.LLM_MODEL || process.env.MODEL_NAME || '';
-    // Load saved config
-    const savedConfig = await loadConfig();
-    let provider;
-    let providerMode = savedConfig?.mode || 'openai';
-    let providerInfo;
-    if (envProvider && PROVIDERS.some(p => p.id === envProvider)) {
-        provider = envProvider;
-        providerInfo = PROVIDERS.find(p => p.id === provider);
-        console.log(chalk_1.default.green(`Provider: ${providerInfo.name} (from env)`));
-    }
-    else if (savedConfig?.provider) {
-        provider = savedConfig.provider;
-        providerInfo = PROVIDERS.find(p => p.id === provider);
-        console.log(chalk_1.default.green(`Provider: ${providerInfo.name} (from config)`));
-    }
-    else {
-        providerInfo = await selectProvider();
-        provider = providerInfo.id;
-    }
-    // Ask for mode when custom provider
-    if (provider === 'custom') {
-        const envMode = process.env.LLM_MODE || '';
-        if (envMode === 'anthropic') {
-            providerMode = 'anthropic';
+        else if (choice === 1) {
+            const tuiPath = path_1.default.join(__dirname, '..', 'bin', 'tui.js');
+            (0, child_process_1.execFileSync)('node', [tuiPath], { stdio: 'inherit' });
         }
-        else if (savedConfig?.mode) {
-            providerMode = savedConfig.mode;
-        }
-        else {
-            console.log(chalk_1.default.cyan(''));
-            const modeAnswer = (await prompt('API mode - openai (OpenAI-compatible) or anthropic (Anthropic-compatible)? (o/a): ')).trim().toLowerCase();
-            providerMode = modeAnswer === 'a' ? 'anthropic' : 'openai';
-        }
-        console.log(chalk_1.default.green('Mode: ' + providerMode));
-    }
-    let endpoint = envBaseUrl || savedConfig?.baseUrl || '';
-    if (!endpoint) {
-        endpoint = await selectEndpoint(provider);
-    }
-    else {
-        console.log(chalk_1.default.green(`Endpoint: ${endpoint}`));
-    }
-    let key = envApiKey || savedConfig?.apiKey || '';
-    if (!key) {
-        const keyPrompt = provider === 'anthropic'
-            ? 'Enter your Anthropic API key: '
-            : 'Enter your API key: ';
-        key = (await prompt(keyPrompt)).trim();
-    }
-    else {
-        const maskedKey = key.length > 8 ? `${key.slice(0, 4)}...${key.slice(-4)}` : '***';
-        console.log(chalk_1.default.green(`API key: ${maskedKey}`));
-    }
-    let model = envModelName || savedConfig?.modelName || '';
-    if (!model) {
-        model = (await prompt('Enter model name (e.g., gpt-4o, claude-3-opus): ')).trim();
-    }
-    else {
-        console.log(chalk_1.default.green(`Model: ${model}`));
-    }
-    const config = { provider, apiKey: key, baseUrl: endpoint, modelName: model };
-    if (provider === 'custom' && providerMode)
-        config.mode = providerMode;
-    // Preserve existing judge config when saving
-    if (savedConfig) {
-        if (savedConfig.judgeProvider)
-            config.judgeProvider = savedConfig.judgeProvider;
-        if (savedConfig.judgeApiKey)
-            config.judgeApiKey = savedConfig.judgeApiKey;
-        if (savedConfig.judgeBaseUrl)
-            config.judgeBaseUrl = savedConfig.judgeBaseUrl;
-        if (savedConfig.judgeModelName)
-            config.judgeModelName = savedConfig.judgeModelName;
-    }
-    // Save config if not from env
-    if (!envProvider && !envApiKey && !envModelName) {
-        await saveConfig(config);
-    }
-    return config;
-}
-async function selectBenchmarks() {
-    const available = Object.values(benchmarks_js_1.BENCHMARK_DEFINITIONS);
-    const selection = await selectMulti(available, 'Select benchmarks to run', (bench) => `(${bench.defaultSamples.toLocaleString()} tests) - ${bench.description}`);
-    if (selection.length === 0) {
-        console.log(chalk_1.default.yellow('No benchmarks selected, exiting'));
-        process.exit(0);
-    }
-    console.log(chalk_1.default.green(`\nSelected ${selection.length} benchmark(s)\n`));
-    const benchmarks = [];
-    for (const bench of selection) {
-        const pctStr = await prompt(`${bench.name}: Enter % to run (1-100, default 100): `);
-        const pct = parseInt(pctStr) || 100;
-        benchmarks.push({ ...bench, percentage: Math.min(100, Math.max(1, pct)) });
-    }
-    return benchmarks;
-}
-async function runBenchmarks() {
-    const config = await getConfig();
-    const client = (0, client_js_1.createLLMClient)(config.provider, config.apiKey, config.baseUrl, config.modelName, config.mode);
-    const benchmarks = await selectBenchmarks();
-    // Ask about shuffling
-    const shuffleAnswer = (await prompt('Shuffle samples for diverse distribution? (Y/n): ')).trim().toLowerCase();
-    const shouldShuffle = shuffleAnswer !== 'n';
-    benchmarks.forEach((b) => b.shuffle = shouldShuffle);
-    // Check if any selected benchmark supports judge
-    const judgeBenchmarks = benchmarks.filter(b => b.useJudge);
-    let useJudge = false;
-    let judgeClient;
-    let judgeProvider = config.judgeProvider || 'openai';
-    let judgeBaseUrl = config.judgeBaseUrl || '';
-    let judgeModel = config.judgeModelName || '';
-    if (judgeBenchmarks.length > 0) {
-        const hasCritical = judgeBenchmarks.some(b => b.id === 'truthfulqa' || b.id === 'spider' || b.id === 'math');
-        const label = hasCritical ? '(highly recommended)' : '(recommended)';
-        const judgeAnswer = (await prompt(`Use judge for evaluation? (y/N) - ${label}: `)).trim().toLowerCase();
-        useJudge = judgeAnswer === 'y';
-        if (useJudge) {
-            const useJudgeEnv = process.env.JUDGE_PROVIDER || config.judgeProvider || '';
-            if (useJudgeEnv && PROVIDERS.some(p => p.id === useJudgeEnv)) {
-                judgeProvider = useJudgeEnv;
-            }
-            else {
-                const providerInfo = await selectProviderWithPrompt('Select judge provider:');
-                judgeProvider = providerInfo.id;
-            }
-            judgeBaseUrl = process.env.JUDGE_BASE_URL || config.judgeBaseUrl || '';
-            if (!judgeBaseUrl) {
-                judgeBaseUrl = await selectEndpointWithPrompt(judgeProvider);
-            }
-            const judgeApiKey = process.env.JUDGE_API_KEY || config.judgeApiKey || '';
-            const finalJudgeApiKey = judgeApiKey || (await prompt('Enter judge API key: ')).trim();
-            judgeModel = process.env.JUDGE_MODEL || config.judgeModelName || '';
-            if (!judgeModel) {
-                const isOpenAI = judgeProvider === 'openai' || judgeBaseUrl.includes('openai.com');
-                const isOpenRouter = judgeBaseUrl.includes('openrouter.ai') || judgeBaseUrl.includes('openrouter');
-                const modelRecommendation = (isOpenAI || isOpenRouter) ? 'gpt-4o-mini' : '';
-                const promptText = modelRecommendation
-                    ? `Enter judge model name (recommended: ${modelRecommendation}): `
-                    : 'Enter judge model name: ';
-                judgeModel = (await prompt(promptText)).trim() || modelRecommendation;
-            }
-            judgeClient = (0, client_js_1.createLLMClient)(judgeProvider, finalJudgeApiKey, judgeBaseUrl, judgeModel);
-            // Save judge config if not from env
-            if (!process.env.JUDGE_PROVIDER && !process.env.JUDGE_API_KEY && !process.env.JUDGE_MODEL) {
-                const fullConfig = {
-                    ...config,
-                    judgeProvider,
-                    judgeApiKey: finalJudgeApiKey,
-                    judgeBaseUrl,
-                    judgeModelName: judgeModel
-                };
-                await saveConfig(fullConfig);
-            }
-        }
-    }
-    console.log(chalk_1.default.bold('\n=== Configuration ==='));
-    console.log(`  Model: ${config.modelName}`);
-    console.log(`  Provider: ${config.provider}`);
-    console.log(`  Endpoint: ${config.baseUrl}`);
-    if (useJudge) {
-        console.log(chalk_1.default.cyan('  Judge: enabled'));
-        console.log(`    Provider: ${judgeProvider}`);
-        console.log(`    Endpoint: ${judgeBaseUrl}`);
-        console.log(`    Model: ${judgeModel}`);
-    }
-    else {
-        console.log(chalk_1.default.gray('  Judge: disabled'));
-    }
-    if (shouldShuffle) {
-        console.log(chalk_1.default.green('  Shuffle: enabled'));
-    }
-    else {
-        console.log(chalk_1.default.gray('  Shuffle: disabled'));
-    }
-    const proceed = (await prompt('\nProceed with running benchmarks? (Y/n): ')).trim().toLowerCase();
-    if (proceed === 'n') {
-        console.log(chalk_1.default.yellow('Exiting...'));
-        process.exit(0);
-    }
-    const detailedLogsDir = (0, paths_js_1.getDetailedLogsDir)();
-    const progressDir = (0, paths_js_1.getProgressDir)();
-    await fs_extra_1.default.ensureDir(detailedLogsDir);
-    await fs_extra_1.default.ensureDir(progressDir);
-    console.log(chalk_1.default.gray(`\nResults: ${detailedLogsDir}`));
-    const logger = new logger_js_1.Logger(detailedLogsDir);
-    const progress = new progress_js_1.ProgressTracker(progressDir);
-    const evaluator = new evaluator_js_1.Evaluator(client, { timeout: 120, retries: 3, temperature: 0 }, judgeClient);
-    const results = [];
-    for (const benchmark of benchmarks) {
-        console.log(chalk_1.default.bold(`\n=== Running ${benchmark.name} ===`));
-        const safeModelName = config.modelName.replace(/[^a-zA-Z0-9]/g, '_');
-        const progressFile = `${safeModelName}_${benchmark.id}_progress.json`;
-        const previousProgress = progress.load(progressFile);
-        let startIdx = 0;
-        let runSeed = Date.now();
-        if (previousProgress && previousProgress.completed > 0) {
-            console.log(chalk_1.default.yellow(`Found existing progress: ${previousProgress.completed}/${previousProgress.total} completed`));
-            const resume = await prompt('Resume from where you left off? (y/N): ');
-            if (resume.trim().toLowerCase() === 'y') {
-                startIdx = previousProgress.completed;
-                runSeed = previousProgress.seed || runSeed;
-                console.log(chalk_1.default.cyan('Resuming...\n'));
-            }
-            else {
-                console.log(chalk_1.default.cyan('Starting fresh...\n'));
-                progress.clear(progressFile);
-            }
-        }
-        let data;
-        try {
-            data = await (0, benchmarks_js_1.fetchBenchmark)(benchmark.id, benchmark.percentage, benchmark.shuffle, runSeed);
-        }
-        catch (error) {
-            console.log(chalk_1.default.red(`Failed to fetch ${benchmark.name}: ${error}`));
-            continue;
-        }
-        let correct = 0;
-        const runTimestamp = Date.now();
-        const runLogEntries = [];
-        for (let i = startIdx; i < data.length; i++) {
-            const item = data[i];
-            try {
-                const pct = i === 0 ? '0.0' : ((correct / i) * 100).toFixed(1);
-                process.stdout.write(`\r${benchmark.name}: [solving... ${i + 1}/${data.length}] ${pct}% correct`);
-                const response = await evaluator.evaluate(benchmark, item);
-                let isCorrect;
-                let judgeResponse;
-                if (benchmark.useJudge && judgeClient) {
-                    process.stdout.write(`\r${benchmark.name}: [judging... ${i + 1}/${data.length}] ${pct}% correct`);
-                    const result = await evaluator.evaluateAndCheckWithJudge(benchmark, item, response);
-                    isCorrect = result.correct;
-                    judgeResponse = result.judgeResponse;
-                }
-                else {
-                    isCorrect = evaluator.checkAnswer(benchmark, item, response);
-                }
-                if (isCorrect)
-                    correct++;
-                runLogEntries.push({
-                    benchmark: benchmark.id,
-                    model: config.modelName,
-                    question: item,
-                    response,
-                    isCorrect,
-                    judgeResponse,
-                    timestamp: new Date().toISOString(),
-                    index: i,
-                });
-                progress.save(progressFile, { completed: i + 1, total: data.length, seed: runSeed });
-            }
-            catch (error) {
-                console.log(chalk_1.default.red(`\nError on item ${i}: ${error}`));
-            }
-        }
-        // Batch write all entries for this run
-        logger.logBatch(runLogEntries, `${benchmark.id}_${config.modelName}_${runTimestamp}`);
-        const result = {
-            benchmark: benchmark.id,
-            model: config.modelName,
-            total: data.length,
-            correct,
-            accuracy: (correct / data.length) * 100,
-            timestamp: new Date().toISOString(),
-            seed: runSeed,
-            judge: useJudge && judgeModel ? judgeModel : undefined,
-        };
-        results.push(result);
-        console.log(chalk_1.default.green(`\n${benchmark.name}: ${correct}/${data.length} (${result.accuracy.toFixed(2)}%)`));
-        progress.clear(progressFile);
-    }
-    const resultsFile = path_1.default.join((0, paths_js_1.getResultsDir)(), `eval_results_${Date.now()}.json`);
-    await fs_extra_1.default.ensureDir((0, paths_js_1.getResultsDir)());
-    await fs_extra_1.default.writeJson(resultsFile, { results, timestamp: new Date().toISOString() }, { spaces: 2 });
-    console.log(chalk_1.default.bold(`\n=== Results saved to ${resultsFile} ===`));
-    console.log(chalk_1.default.bold('\n=== Summary ==='));
-    for (const r of results) {
-        console.log(`${r.benchmark}: ${r.correct}/${r.total} (${r.accuracy.toFixed(2)}%)`);
-    }
-}
-async function showMenu() {
-    // Remove all existing keypress listeners first
-    process.stdin.removeAllListeners('keypress');
-    readline_1.default.emitKeypressEvents(process.stdin);
-    if (process.stdin.isTTY) {
-        process.stdin.setRawMode(true);
     }
-    let cursor = 0;
-    const items = ['Run benchmarks', 'Explore past results'];
-    const render = () => {
-        process.stdout.write('\x1b[H\x1b[2J');
-        console.log(chalk_1.default.bold.cyan('\n=== LLM Benchmark Runner ===\n'));
-        items.forEach((item, i) => {
-            const prefix = i === cursor ? chalk_1.default.cyan('> ') : '  ';
-            console.log(`${prefix}${item}`);
-        });
-        console.log(chalk_1.default.gray('\nArrow keys, Enter to select, Esc to quit'));
-    };
-    render();
-    return new Promise((resolve) => {
-        const handleKeypress = async (str, key) => {
-            if (key.ctrl && key.name === 'c') {
-                process.stdin.removeListener('keypress', handleKeypress);
-                if (process.stdin.isTTY)
-                    process.stdin.setRawMode(false);
-                console.log(chalk_1.default.green('\nGoodbye!'));
-                process.exit(0);
-            }
-            if (key.name === 'up') {
-                cursor = Math.max(0, cursor - 1);
-                render();
-            }
-            else if (key.name === 'down') {
-                cursor = Math.min(items.length - 1, cursor + 1);
-                render();
-            }
-            else if (key.name === 'escape') {
-                process.stdin.removeListener('keypress', handleKeypress);
-                if (process.stdin.isTTY)
-                    process.stdin.setRawMode(false);
-                console.log(chalk_1.default.green('\nGoodbye!'));
-                process.exit(0);
-            }
-            else if (key.name === 'return') {
-                process.stdin.removeListener('keypress', handleKeypress);
-                if (process.stdin.isTTY)
-                    process.stdin.setRawMode(false);
-                if (cursor === 0) {
-                    runBenchmarks().then(() => showMenu()).then(() => resolve());
-                }
-                else if (cursor === 1) {
-                    if (process.stdin.isTTY)
-                        process.stdin.setRawMode(false);
-                    process.stdin.removeAllListeners('keypress');
-                    process.stdout.write('\x1b[H\x1b[2J');
-                    const tuiPath = path_1.default.join(__dirname, '..', 'bin', 'tui.js');
-                    (0, child_process_1.execFileSync)('node', [tuiPath], { stdio: 'inherit' });
-                    showMenu().then(() => resolve());
-                }
-            }
-        };
-        process.stdin.on('keypress', handleKeypress);
-    });
 }
-showMenu().catch(console.error);
+main().catch(console.error);
 //# sourceMappingURL=index.js.map