llmtester 1.0.0

package/dist/client.d.ts

@@ -0,0 +1,69 @@
+export type ProviderType = 'openai' | 'anthropic' | 'custom';
+export interface LLMClient {
+    provider: ProviderType;
+    baseUrl: string;
+    model: string;
+    chat(messages: {
+        role: string;
+        content: string;
+    }[], options?: {
+        temperature?: number;
+        maxTokens?: number;
+    }): Promise<{
+        content: string;
+        finishReason: string;
+        usage?: {
+            promptTokens: number;
+            completionTokens: number;
+            totalTokens: number;
+        };
+    }>;
+}
+export declare class OpenAICompatibleClient implements LLMClient {
+    provider: ProviderType;
+    baseUrl: string;
+    model: string;
+    private apiKey;
+    private httpClient;
+    constructor(apiKey: string, baseUrl: string, model: string);
+    chat(messages: {
+        role: string;
+        content: string;
+    }[], options?: {
+        temperature?: number;
+        maxTokens?: number;
+    }): Promise<{
+        content: string;
+        finishReason: string;
+        usage?: {
+            promptTokens: number;
+            completionTokens: number;
+            totalTokens: number;
+        };
+    }>;
+}
+export declare class AnthropicClient implements LLMClient {
+    provider: ProviderType;
+    baseUrl: string;
+    model: string;
+    private apiKey;
+    private httpClient;
+    constructor(apiKey: string, baseUrl: string, model: string);
+    chat(messages: {
+        role: string;
+        content: string;
+    }[], options?: {
+        temperature?: number;
+        maxTokens?: number;
+    }): Promise<{
+        content: string;
+        finishReason: string;
+        usage?: {
+            promptTokens: number;
+            completionTokens: number;
+            totalTokens: number;
+        };
+    }>;
+}
+export declare function createLLMClient(provider: ProviderType, apiKey: string, baseUrl: string, model: string): LLMClient;
+//# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE7D,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,EAAE,OAAO,CAAC,EAAE;QAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC;QACV,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE;YAAE,YAAY,EAAE,MAAM,CAAC;YAAC,gBAAgB,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC;KACjF,CAAC,CAAC;CACJ;AAED,qBAAa,sBAAuB,YAAW,SAAS;IAC/C,QAAQ,EAAE,YAAY,CAAY;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,UAAU,CAAgB;gBAEtB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAcpD,IAAI,CAAC,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,EAAE,OAAO,CAAC,EAAE;QAClE,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC;QACV,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE;YAAE,YAAY,EAAE,MAAM,CAAC;YAAC,gBAAgB,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC;KACjF,CAAC;CAmBH;AAED,qBAAa,eAAgB,YAAW,SAAS;IACxC,QAAQ,EAAE,YAAY,CAAe;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,UAAU,CAAgB;gBAEtB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAgBpD,IAAI,CAAC,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,EAAE,OAAO,CAAC,EAAE;QAClE,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC;QACV,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE;YAAE,YAAY,EAAE,MAAM,CAAC;YAAC,gBAAgB,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC;KACjF,CAAC;CA8BH;AAED,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,GACZ,SAAS,CAKX"}

package/dist/client.js

@@ -0,0 +1,103 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AnthropicClient = exports.OpenAICompatibleClient = void 0;
+exports.createLLMClient = createLLMClient;
+const axios_1 = __importDefault(require("axios"));
+class OpenAICompatibleClient {
+    provider = 'openai';
+    baseUrl;
+    model;
+    apiKey;
+    httpClient;
+    constructor(apiKey, baseUrl, model) {
+        this.apiKey = apiKey;
+        this.baseUrl = baseUrl;
+        this.model = model;
+        this.httpClient = axios_1.default.create({
+            baseURL: baseUrl,
+            headers: {
+                'Authorization': `Bearer ${apiKey}`,
+                'Content-Type': 'application/json',
+            },
+            timeout: 120000,
+        });
+    }
+    async chat(messages, options) {
+        const response = await this.httpClient.post('/chat/completions', {
+            model: this.model,
+            messages,
+            temperature: options?.temperature ?? 0,
+            max_tokens: options?.maxTokens ?? 512,
+        });
+        const choice = response.data.choices?.[0];
+        return {
+            content: choice?.message?.content || '',
+            finishReason: choice?.finish_reason || '',
+            usage: response.data.usage ? {
+                promptTokens: response.data.usage.prompt_tokens || 0,
+                completionTokens: response.data.usage.completion_tokens || 0,
+                totalTokens: response.data.usage.total_tokens || 0,
+            } : undefined,
+        };
+    }
+}
+exports.OpenAICompatibleClient = OpenAICompatibleClient;
+class AnthropicClient {
+    provider = 'anthropic';
+    baseUrl;
+    model;
+    apiKey;
+    httpClient;
+    constructor(apiKey, baseUrl, model) {
+        this.apiKey = apiKey;
+        this.baseUrl = baseUrl;
+        this.model = model;
+        const baseURL = baseUrl || 'https://api.anthropic.com';
+        this.httpClient = axios_1.default.create({
+            baseURL,
+            headers: {
+                'x-api-key': apiKey,
+                'Content-Type': 'application/json',
+                'anthropic-version': '2023-06-01',
+            },
+            timeout: 120000,
+        });
+    }
+    async chat(messages, options) {
+        const systemMessage = messages.find(m => m.role === 'system');
+        const chatMessages = messages.filter(m => m.role !== 'system');
+        const requestBody = {
+            model: this.model,
+            messages: chatMessages.map(m => ({
+                role: m.role === 'assistant' ? 'assistant' : 'user',
+                content: m.content,
+            })),
+            temperature: options?.temperature ?? 0,
+            max_tokens: options?.maxTokens ?? 1024,
+        };
+        if (systemMessage) {
+            requestBody.system = systemMessage.content;
+        }
+        const response = await this.httpClient.post('/v1/messages', requestBody);
+        return {
+            content: response.data.content?.[0]?.text || '',
+            finishReason: response.data.stop_reason || '',
+            usage: response.data.usage ? {
+                promptTokens: response.data.usage.input_tokens || 0,
+                completionTokens: response.data.usage.output_tokens || 0,
+                totalTokens: (response.data.usage.input_tokens || 0) + (response.data.usage.output_tokens || 0),
+            } : undefined,
+        };
+    }
+}
+exports.AnthropicClient = AnthropicClient;
+function createLLMClient(provider, apiKey, baseUrl, model) {
+    if (provider === 'anthropic') {
+        return new AnthropicClient(apiKey, baseUrl, model);
+    }
+    return new OpenAICompatibleClient(apiKey, baseUrl, model);
+}
+//# sourceMappingURL=client.js.map

package/dist/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";;;;;;AAiIA,0CAUC;AA3ID,kDAA6C;AAkB7C,MAAa,sBAAsB;IAC1B,QAAQ,GAAiB,QAAQ,CAAC;IAClC,OAAO,CAAS;IAChB,KAAK,CAAS;IACb,MAAM,CAAS;IACf,UAAU,CAAgB;IAElC,YAAY,MAAc,EAAE,OAAe,EAAE,KAAa;QACxD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,eAAK,CAAC,MAAM,CAAC;YAC7B,OAAO,EAAE,OAAO;YAChB,OAAO,EAAE;gBACP,eAAe,EAAE,UAAU,MAAM,EAAE;gBACnC,cAAc,EAAE,kBAAkB;aACnC;YACD,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,QAA6C,EAAE,OAGzD;QAKC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC/D,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ;YACR,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,CAAC;YACtC,UAAU,EAAE,OAAO,EAAE,SAAS,IAAI,GAAG;SACtC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1C,OAAO;YACL,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE;YACvC,YAAY,EAAE,MAAM,EAAE,aAAa,IAAI,EAAE;YACzC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC3B,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC;gBACpD,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC;gBAC5D,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC;aACnD,CAAC,CAAC,CAAC,SAAS;SACd,CAAC;IACJ,CAAC;CACF;AA/CD,wDA+CC;AAED,MAAa,eAAe;IACnB,QAAQ,GAAiB,WAAW,CAAC;IACrC,OAAO,CAAS;IAChB,KAAK,CAAS;IACb,MAAM,CAAS;IACf,UAAU,CAAgB;IAElC,YAAY,MAAc,EAAE,OAAe,EAAE,KAAa;QACxD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,MAAM,OAAO,GAAG,OAAO,IAAI,2BAA2B,CAAC;QACvD,IAAI,CAAC,UAAU,GAAG,eAAK,CAAC,MAAM,CAAC;YAC7B,OAAO;YACP,OAAO,EAAE;gBACP,WAAW,EAAE,MAAM;gBACnB,cAAc,EAAE,kBAAkB;gBAClC,mBAAmB,EAAE,YAAY;aAClC;YACD,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,QAA6C,EAAE,OAGzD;QAKC,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QAC9D,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QAE/D,MAAM,WAAW,GAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC/B,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM;gBACnD,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,CAAC;YACtC,UAAU,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI;SACvC,CAAC;QAEF,IAAI,aAAa,EAAE,CAAC;YAClB,WAAW,CAAC,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC;QAC7C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAEzE,OAAO;YACL,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE;YAC/C,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE;YAC7C,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC3B,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC;gBACnD,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC;gBACxD,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC;aAChG,CAAC,CAAC,CAAC,SAAS;SACd,CAAC;IACJ,CAAC;CACF;AA5DD,0CA4DC;AAED,SAAgB,eAAe,CAC7B,QAAsB,EACtB,MAAc,EACd,OAAe,EACf,KAAa;IAEb,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;QAC7B,OAAO,IAAI,eAAe,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,IAAI,sBAAsB,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AAC5D,CAAC"}

package/dist/evaluator.d.ts

@@ -0,0 +1,57 @@
+import { LLMClient } from './client.js';
+import { Benchmark } from './benchmarks.js';
+export interface EvaluationResponse {
+    content: string;
+    finishReason: string;
+    usage?: {
+        promptTokens: number;
+        completionTokens: number;
+        totalTokens: number;
+    };
+}
+export interface EvaluationResult {
+    benchmark: string;
+    model: string;
+    total: number;
+    correct: number;
+    accuracy: number;
+    timestamp: string;
+    seed?: number;
+    judge?: string;
+}
+interface EvaluatorOptions {
+    timeout: number;
+    retries: number;
+    temperature: number;
+    judgeTemperature?: number;
+}
+export declare class Evaluator {
+    private client;
+    private judgeClient;
+    private model;
+    private options;
+    constructor(client: LLMClient, options?: Partial<EvaluatorOptions>, judgeClient?: LLMClient);
+    private buildPrompt;
+    private stripThinkingTags;
+    evaluate(benchmark: Benchmark, item: any): Promise<EvaluationResponse>;
+    checkAnswer(benchmark: Benchmark, item: any, response: EvaluationResponse): boolean;
+    evaluateAndCheckWithJudge(benchmark: Benchmark, item: any, modelResponse: EvaluationResponse): Promise<{
+        correct: boolean;
+        judgeResponse?: string;
+    }>;
+    private checkMathAnswer;
+    private extractFinalAnswer;
+    private extractNumber;
+    private checkCodeAnswer;
+    private checkTypeScriptAnswer;
+    private checkSqlAnswer;
+    private extractCode;
+    private checkMultipleChoice;
+    private checkTruthfulQA;
+    private checkBashAnswer;
+    private checkBBHAnswer;
+    private getMaxTokens;
+    private sleep;
+}
+export {};
+//# sourceMappingURL=evaluator.d.ts.map

package/dist/evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,gBAAgB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,WAAW,CAAmB;IACtC,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,OAAO,CAAmB;gBAEtB,MAAM,EAAE,SAAS,EAAE,OAAO,GAAE,OAAO,CAAC,gBAAgB,CAAM,EAAE,WAAW,CAAC,EAAE,SAAS;IAY/F,OAAO,CAAC,WAAW;IAsBnB,OAAO,CAAC,iBAAiB;IAOnB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA+B5E,WAAW,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,kBAAkB,GAAG,OAAO;IAgC7E,yBAAyB,CAC7B,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,GAAG,EACT,aAAa,EAAE,kBAAkB,GAChC,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAsFxD,OAAO,CAAC,eAAe;IAcvB,OAAO,CAAC,kBAAkB;IAW1B,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,qBAAqB;IAW7B,OAAO,CAAC,cAAc;IAiBtB,OAAO,CAAC,WAAW;IA2BnB,OAAO,CAAC,mBAAmB;IA2D3B,OAAO,CAAC,eAAe;IAuBvB,OAAO,CAAC,eAAe;IAmBvB,OAAO,CAAC,cAAc;IAkEtB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,KAAK;CAGd"}

package/dist/evaluator.js

@@ -0,0 +1,410 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Evaluator = void 0;
+const child_process_1 = require("child_process");
+class Evaluator {
+    client;
+    judgeClient;
+    model;
+    options;
+    constructor(client, options = {}, judgeClient) {
+        this.client = client;
+        this.judgeClient = judgeClient || null;
+        this.model = client.model;
+        this.options = {
+            timeout: options.timeout ?? 120,
+            retries: options.retries ?? 3,
+            temperature: options.temperature ?? 0,
+            judgeTemperature: options.judgeTemperature ?? 0,
+        };
+    }
+    buildPrompt(benchmark, item) {
+        const template = benchmark.promptTemplate || '{question}';
+        // Replace common placeholders
+        let prompt = template
+            .replace('{question}', item.question || item.problem || item.prompt || '')
+            .replace('{text}', item.text || item.question || item.problem || item.prompt || '')
+            .replace('{prompt}', item.prompt || item.question || item.problem || '');
+        if (item.choices) {
+            const choices = Array.isArray(item.choices)
+                ? item.choices
+                : Object.values(item.choices);
+            const choicesText = choices
+                .map((c, i) => `${String.fromCharCode(65 + i)}. ${c}`)
+                .join('\n');
+            prompt = prompt.replace('{choices}', choicesText);
+        }
+        return prompt;
+    }
+    stripThinkingTags(content) {
+        return content
+            .replace(/<think>[\s\S]*?<\/think>/gi, '')
+            .replace(/<think>[\s\S]*?<\/thinking>/gi, '')
+            .trim();
+    }
+    async evaluate(benchmark, item) {
+        const prompt = this.buildPrompt(benchmark, item);
+        let lastError = null;
+        for (let attempt = 0; attempt < this.options.retries; attempt++) {
+            try {
+                const response = await this.client.chat([{ role: 'user', content: prompt }], {
+                    temperature: this.options.temperature,
+                    maxTokens: this.getMaxTokens(benchmark),
+                });
+                return {
+                    content: response.content,
+                    finishReason: response.finishReason,
+                    usage: response.usage,
+                };
+            }
+            catch (error) {
+                lastError = error;
+                if (attempt < this.options.retries - 1) {
+                    const delay = Math.pow(2, attempt) * 1000;
+                    await this.sleep(delay);
+                }
+            }
+        }
+        throw lastError || new Error('Evaluation failed');
+    }
+    checkAnswer(benchmark, item, response) {
+        const answerField = benchmark.answerField || 'answer';
+        const correctAnswer = item[answerField];
+        if (correctAnswer === undefined || correctAnswer === null)
+            return false;
+        const responseText = response.content.trim();
+        switch (benchmark.type) {
+            case 'math_reasoning':
+                return this.checkMathAnswer(responseText, correctAnswer);
+            case 'code':
+                return this.checkCodeAnswer(responseText, correctAnswer);
+            case 'typescript':
+                return this.checkTypeScriptAnswer(responseText, correctAnswer);
+            case 'sql':
+                return this.checkSqlAnswer(responseText, correctAnswer);
+            case 'truthfulqa':
+                return this.checkTruthfulQA(responseText, correctAnswer, item);
+            case 'commonsense':
+            case 'reasoning':
+            case 'knowledge':
+                return this.checkMultipleChoice(responseText, correctAnswer, item);
+            case 'bbh':
+                return this.checkBBHAnswer(responseText, correctAnswer);
+            case 'terminal':
+                return this.checkBashAnswer(responseText, correctAnswer);
+            default:
+                return responseText.toLowerCase().includes(correctAnswer.toLowerCase());
+        }
+    }
+    async evaluateAndCheckWithJudge(benchmark, item, modelResponse) {
+        if (!benchmark.useJudge || !this.judgeClient) {
+            const correct = this.checkAnswer(benchmark, item, modelResponse);
+            return { correct, judgeResponse: undefined };
+        }
+        let judgePrompt = benchmark.judgePromptTemplate || '';
+        const cleanResponse = this.stripThinkingTags(modelResponse.content);
+        // Build prompt based on benchmark type
+        switch (benchmark.id) {
+            case 'truthfulqa': {
+                const labeledChoices = item.labeled_choices || [];
+                judgePrompt = judgePrompt
+                    .replace('{question}', item.question || '')
+                    .replace('{all_choices}', labeledChoices.join('\n'))
+                    .replace('{model_response}', cleanResponse);
+                break;
+            }
+            case 'humaneval':
+            case 'mbpp': {
+                const question = item.question || item.text || item.prompt || '';
+                const testCases = item.test_cases || item.test_list || [];
+                const testStr = Array.isArray(testCases)
+                    ? testCases.map((tc, i) => {
+                        const input = tc.split('===')[0]?.trim() || '';
+                        const expected = tc.split('===')[1]?.trim() || '';
+                        return `Test ${i + 1}: Input: ${input} Expected: ${expected}`;
+                    }).join('\n')
+                    : JSON.stringify(testCases);
+                judgePrompt = judgePrompt
+                    .replace('{question}', question)
+                    .replace('{test_cases}', testStr)
+                    .replace('{model_response}', cleanResponse);
+                break;
+            }
+            case 'nl2bash': {
+                const answerField = benchmark.answerField || 'cmd';
+                judgePrompt = judgePrompt
+                    .replace('{question}', item.question || '')
+                    .replace('{correct_answer}', item[answerField] || '')
+                    .replace('{model_response}', cleanResponse);
+                break;
+            }
+            case 'spider': {
+                const answerField = benchmark.answerField || 'query';
+                judgePrompt = judgePrompt
+                    .replace('{question}', item.question || '')
+                    .replace('{correct_answer}', item[answerField] || '')
+                    .replace('{model_response}', cleanResponse);
+                break;
+            }
+            case 'math': {
+                const answerField = benchmark.answerField || 'answer';
+                judgePrompt = judgePrompt
+                    .replace('{question}', item.question || '')
+                    .replace('{correct_answer}', item[answerField] || '')
+                    .replace('{model_response}', cleanResponse);
+                break;
+            }
+            default: {
+                judgePrompt = judgePrompt
+                    .replace('{question}', item.question || '')
+                    .replace('{model_response}', cleanResponse);
+            }
+        }
+        try {
+            const judgeResponse = await this.judgeClient.chat([{ role: 'user', content: judgePrompt }], { temperature: this.options.judgeTemperature ?? 0, maxTokens: 2000 });
+            const judgeAnswer = judgeResponse.content.trim().toUpperCase();
+            const isCorrect = judgeAnswer.includes('YES');
+            return {
+                correct: isCorrect,
+                judgeResponse: judgeResponse.content
+            };
+        }
+        catch (error) {
+            console.error('Judge evaluation failed:', error);
+            return { correct: false, judgeResponse: `Error: ${error}` };
+        }
+    }
+    checkMathAnswer(response, correctAnswer) {
+        const extractedResponse = this.extractFinalAnswer(response);
+        const extractedAnswer = this.extractFinalAnswer(correctAnswer);
+        const responseNum = this.extractNumber(extractedResponse);
+        const answerNum = this.extractNumber(extractedAnswer);
+        if (responseNum !== null && answerNum !== null) {
+            return Math.abs(responseNum - answerNum) < 0.01;
+        }
+        return extractedResponse.toLowerCase() === extractedAnswer.toLowerCase();
+    }
+    extractFinalAnswer(text) {
+        if (text.includes('####')) {
+            return text.split('####').pop()?.trim() || text;
+        }
+        const numbers = text.match(/-?\d+\.?\d*/g);
+        if (numbers && numbers.length > 0) {
+            return numbers[numbers.length - 1];
+        }
+        return text.trim();
+    }
+    extractNumber(text) {
+        const match = text.match(/-?\d+\.?\d*/);
+        return match ? parseFloat(match[0]) : null;
+    }
+    checkCodeAnswer(response, correctAnswer) {
+        const responseCode = this.extractCode(response);
+        if (!responseCode || responseCode.length === 0) {
+            return false;
+        }
+        try {
+            const encoded = Buffer.from(responseCode).toString('base64');
+            (0, child_process_1.execSync)(`python3 -c "import base64; code=base64.b64decode('${encoded}').decode(); compile(code,'<string>','exec')"`, { stdio: 'pipe' });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    checkTypeScriptAnswer(response, correctAnswer) {
+        // For TypeScript, check if response contains TypeScript-like code
+        // (function declaration with types)
+        const hasFunction = /function\s+\w+/.test(response);
+        const hasArrowFunction = /=>\s*{/.test(response);
+        const hasTypeScriptTypes = /:\s*(number|string|boolean|any|void|never)/.test(response);
+        const hasExport = /export\s+/.test(response);
+        return hasFunction || hasArrowFunction;
+    }
+    checkSqlAnswer(response, correctAnswer) {
+        // For SQL, check if response contains SQL keywords
+        const hasSelect = /SELECT/i.test(response);
+        const hasFrom = /FROM/i.test(response);
+        // Extract SQL query from response if it's in a code block
+        const sqlMatch = response.match(/```sql\n?([\s\S]*?)```/i) ||
+            response.match(/```\n?([\s\S]*?)```/i);
+        if (sqlMatch) {
+            const extractedSql = sqlMatch[1].trim().toUpperCase();
+            return /SELECT/.test(extractedSql) && /FROM/.test(extractedSql);
+        }
+        return hasSelect && hasFrom;
+    }
+    extractCode(text) {
+        // Try to find code blocks first
+        const codeBlockMatch = text.match(/```(?:\w+)?\n([\s\S]*?)```/);
+        if (codeBlockMatch)
+            return codeBlockMatch[1].trim();
+        // Try to find Python function definitions and extract from there
+        const funcMatch = text.match(/(?:^|\n)(def \w+.*?(?=\n(?:[^\s]|$)))/);
+        if (funcMatch) {
+            return text.slice(text.indexOf(funcMatch[1])).trim();
+        }
+        // If no code block or function found, try to find a line that looks like Python code
+        const lines = text.split('\n');
+        const codeStartIdx = lines.findIndex(line => line.trim().startsWith('def ') ||
+            line.trim().startsWith('class ') ||
+            line.trim().startsWith('import ') ||
+            line.trim().startsWith('from '));
+        if (codeStartIdx >= 0) {
+            return lines.slice(codeStartIdx).join('\n').trim();
+        }
+        return text.trim();
+    }
+    checkMultipleChoice(response, correctAnswer, item) {
+        const normalizedResponse = response.toUpperCase().trim();
+        // Handle numeric answer (e.g., MMLU returns 0, 1, 2, 3 as numbers)
+        let answerStr = String(correctAnswer);
+        let answerIndex;
+        if (/^[0-3]$/.test(answerStr)) {
+            // Numeric index (0-3)
+            answerIndex = parseInt(answerStr);
+            answerStr = String.fromCharCode(65 + answerIndex); // Convert to A, B, C, D
+        }
+        else {
+            // Letter answer (A, B, C, D)
+            answerStr = answerStr.toUpperCase().trim();
+            answerIndex = answerStr.charCodeAt(0) - 65;
+        }
+        const normalizedAnswer = answerStr;
+        // Direct match
+        if (normalizedResponse === normalizedAnswer)
+            return true;
+        // Single letter response (e.g., "A")
+        if (normalizedResponse.length === 1 && /^[A-D]$/.test(normalizedResponse)) {
+            return normalizedResponse === normalizedAnswer;
+        }
+        // Extract letter from various patterns
+        // "Choice A", "choice A", "Option B", "option B"
+        let letterMatch = normalizedResponse.match(/\b(CHOICE|OPTION)\s+([A-D])\b/);
+        if (letterMatch)
+            return letterMatch[2] === normalizedAnswer;
+        // "Answer is A", "answer is A", "is A"
+        letterMatch = normalizedResponse.match(/\b(ANSWER|IS)\s+([A-D])\b/);
+        if (letterMatch)
+            return letterMatch[2] === normalizedAnswer;
+        // "The correct answer is B"
+        letterMatch = normalizedResponse.match(/\bCORRECT\b.*?\b([A-D])\b/);
+        if (letterMatch)
+            return letterMatch[1] === normalizedAnswer;
+        // Single letter A, B, C, or D surrounded by word boundaries
+        letterMatch = normalizedResponse.match(/\b([A-D])\b/);
+        if (letterMatch)
+            return letterMatch[1] === normalizedAnswer;
+        const choices = Array.isArray(item.choices)
+            ? item.choices
+            : item.choices
+                ? Object.values(item.choices)
+                : [];
+        if (answerIndex >= 0 && answerIndex < choices.length) {
+            const expectedChoice = String(choices[answerIndex]).toUpperCase().trim();
+            // Check if response contains the expected choice or vice versa
+            return normalizedResponse.includes(expectedChoice) || expectedChoice.includes(normalizedResponse);
+        }
+        return false;
+    }
+    checkTruthfulQA(response, correctAnswer, item) {
+        const normalizedResponse = response.toUpperCase().trim();
+        // Get correct choice text
+        const answerIndex = typeof correctAnswer === 'number' ? correctAnswer : parseInt(correctAnswer);
+        const choices = Array.isArray(item.choices) ? item.choices : [];
+        if (answerIndex < 0 || answerIndex >= choices.length)
+            return false;
+        const correctChoice = choices[answerIndex].toUpperCase().trim();
+        // Check if response contains the correct answer or vice versa
+        if (normalizedResponse.includes(correctChoice) || correctChoice.includes(normalizedResponse)) {
+            return true;
+        }
+        // Check if it's the same first 20+ characters
+        if (normalizedResponse.slice(0, 20) === correctChoice.slice(0, 20)) {
+            return true;
+        }
+        return false;
+    }
+    checkBashAnswer(response, correctAnswer) {
+        const normalizedResponse = response.toLowerCase().trim();
+        const normalizedAnswer = correctAnswer.toLowerCase().trim();
+        // Exact match
+        if (normalizedResponse === normalizedAnswer)
+            return true;
+        // Check if main command is present (partial credit)
+        const answerParts = normalizedAnswer.split(/\s+/);
+        if (answerParts.length > 0) {
+            const mainCmd = answerParts[0];
+            if (mainCmd && normalizedResponse.includes(mainCmd)) {
+                return true; // Partial credit - main command matches
+            }
+        }
+        return false;
+    }
+    checkBBHAnswer(response, correctAnswer) {
+        // Strip thinking tags
+        const strippedResponse = response
+            .replace(/<think>[\s\S]*?<\/think>/gi, '')
+            .replace(/<think>[\s\S]*?<\/thinking>/gi, '')
+            .trim();
+        // Extract final answer after #### if present
+        let finalAnswer = strippedResponse;
+        if (strippedResponse.includes('####')) {
+            finalAnswer = strippedResponse.split('####').pop()?.trim() || strippedResponse;
+        }
+        const normalizedResponse = finalAnswer.toLowerCase().trim();
+        const normalizedAnswer = correctAnswer.toLowerCase().trim();
+        // Exact match
+        if (normalizedResponse === normalizedAnswer)
+            return true;
+        // Handle True/False style answers
+        if (normalizedAnswer === 'true' || normalizedAnswer === 'false') {
+            if (normalizedResponse === 'true' || normalizedResponse === 'false') {
+                return normalizedResponse === normalizedAnswer;
+            }
+        }
+        // Handle Yes/No style answers
+        if (normalizedAnswer === 'yes' || normalizedAnswer === 'no') {
+            if (normalizedResponse === 'yes' || normalizedResponse === 'no') {
+                return normalizedResponse === normalizedAnswer;
+            }
+        }
+        // Handle A/B/C/D style answers (multiple choice)
+        if (/^[a-d]$/i.test(normalizedAnswer)) {
+            const answerChar = normalizedAnswer.toUpperCase();
+            if (normalizedResponse === answerChar)
+                return true;
+            // Check if response contains the letter
+            if (new RegExp(`\\b${answerChar}\\b`).test(normalizedResponse))
+                return true;
+        }
+        // Handle numeric answers
+        const responseNum = normalizedResponse.match(/-?\d+\.?\d*/);
+        const answerNum = normalizedAnswer.match(/-?\d+\.?\d*/);
+        if (responseNum && answerNum) {
+            return parseFloat(responseNum[0]) === parseFloat(answerNum[0]);
+        }
+        // Handle comma-separated lists (like word sorting)
+        if (normalizedAnswer.includes(',') && normalizedResponse.includes(',')) {
+            const respSet = new Set(normalizedResponse.split(',').map(s => s.trim()));
+            const ansSet = new Set(normalizedAnswer.split(',').map(s => s.trim()));
+            // Check if sets are equal
+            if (respSet.size === ansSet.size && [...respSet].every(x => ansSet.has(x))) {
+                return true;
+            }
+        }
+        // Partial match - check if the answer appears in the response
+        if (normalizedResponse.includes(normalizedAnswer) || normalizedAnswer.includes(normalizedResponse)) {
+            return true;
+        }
+        return false;
+    }
+    getMaxTokens(benchmark) {
+        return 100000;
+    }
+    sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+}
+exports.Evaluator = Evaluator;
+//# sourceMappingURL=evaluator.js.map