judgeval 0.2.4 → 0.2.5

package/dist/esm/judges/index.js

@@ -0,0 +1,185 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import axios from 'axios';
+import { warn, error } from '../common/logger.js';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { execSync } from 'child_process';
+/**
+ * Default judge implementation using OpenAI API
+ */
+export class DefaultJudge {
+    constructor(modelName = 'gpt-3.5-turbo', apiKey, user) {
+        this.modelName = modelName;
+        this.apiKey = apiKey || process.env.OPENAI_API_KEY;
+        this.user = user;
+        if (!this.apiKey) {
+            warn('No API key provided for DefaultJudge. Set OPENAI_API_KEY environment variable or pass apiKey to constructor.');
+        }
+    }
+    generate(prompt) {
+        // For synchronous generation, we need to block until we get a response
+        // This is similar to how the Python SDK's fetch_litellm_api_response works
+        if (!this.apiKey) {
+            throw new Error('No API key provided for DefaultJudge');
+        }
+        try {
+            // In Node.js, we can use a synchronous HTTP request via child_process
+            // Create a temporary file for the request and response
+            const tempDir = os.tmpdir();
+            const requestFile = path.join(tempDir, `openai-request-${Date.now()}.json`);
+            const responseFile = path.join(tempDir, `openai-response-${Date.now()}.json`);
+            // Write request data to file
+            fs.writeFileSync(requestFile, JSON.stringify(Object.assign({ model: this.modelName, messages: [{ role: 'user', content: prompt }], temperature: 0.0 }, (this.user ? { user: this.user } : {}))));
+            // Make the request using curl
+            const curlCommand = `curl -s -X POST https://api.openai.com/v1/chat/completions \
+        -H "Content-Type: application/json" \
+        -H "Authorization: Bearer ${this.apiKey}" \
+        -d @${requestFile} > ${responseFile}`;
+            execSync(curlCommand);
+            // Read the response
+            const responseData = JSON.parse(fs.readFileSync(responseFile, 'utf8'));
+            // Clean up temporary files
+            fs.unlinkSync(requestFile);
+            fs.unlinkSync(responseFile);
+            // Return the content
+            return responseData.choices[0].message.content;
+        }
+        catch (e) {
+            error(`Error in synchronous generate: ${e.message}`);
+            throw new Error(`Failed to generate text: ${e.message}`);
+        }
+    }
+    aGenerate(prompt) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.apiKey) {
+                throw new Error('No API key provided for DefaultJudge');
+            }
+            try {
+                const response = yield axios.post('https://api.openai.com/v1/chat/completions', Object.assign({ model: this.modelName, messages: [{ role: 'user', content: prompt }], temperature: 0.0 }, (this.user ? { user: this.user } : {})), {
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'Authorization': `Bearer ${this.apiKey}`
+                    }
+                });
+                return response.data.choices[0].message.content;
+            }
+            catch (e) {
+                error(`Error generating text with OpenAI API: ${e.message}`);
+                throw new Error(`Failed to generate text: ${e.message}`);
+            }
+        });
+    }
+    getModelName() {
+        return this.modelName;
+    }
+}
+/**
+ * Together AI judge implementation
+ */
+export class TogetherJudge {
+    constructor(modelName = 'meta-llama/Meta-Llama-3-8B-Instruct-Turbo', apiKey) {
+        this.modelName = modelName;
+        this.apiKey = apiKey || process.env.TOGETHER_API_KEY;
+        if (!this.apiKey) {
+            warn('No API key provided for TogetherJudge. Set TOGETHER_API_KEY environment variable or pass apiKey to constructor.');
+        }
+    }
+    generate(prompt) {
+        // For synchronous generation, we need to block until we get a response
+        if (!this.apiKey) {
+            throw new Error('No API key provided for TogetherJudge');
+        }
+        try {
+            // In Node.js, we can use a synchronous HTTP request via child_process
+            const tempDir = os.tmpdir();
+            const requestFile = path.join(tempDir, `together-request-${Date.now()}.json`);
+            const responseFile = path.join(tempDir, `together-response-${Date.now()}.json`);
+            // Write request data to file
+            fs.writeFileSync(requestFile, JSON.stringify({
+                model: this.modelName,
+                prompt: prompt,
+                temperature: 0.0,
+                max_tokens: 1024
+            }));
+            // Make the request using curl
+            const curlCommand = `curl -s -X POST https://api.together.xyz/v1/completions \
+        -H "Content-Type: application/json" \
+        -H "Authorization: Bearer ${this.apiKey}" \
+        -d @${requestFile} > ${responseFile}`;
+            execSync(curlCommand);
+            // Read the response
+            const responseData = JSON.parse(fs.readFileSync(responseFile, 'utf8'));
+            // Clean up temporary files
+            fs.unlinkSync(requestFile);
+            fs.unlinkSync(responseFile);
+            // Return the content
+            return responseData.choices[0].text;
+        }
+        catch (e) {
+            error(`Error in synchronous generate: ${e.message}`);
+            throw new Error(`Failed to generate text: ${e.message}`);
+        }
+    }
+    aGenerate(prompt) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.apiKey) {
+                throw new Error('No API key provided for TogetherJudge');
+            }
+            try {
+                const response = yield axios.post('https://api.together.xyz/v1/completions', {
+                    model: this.modelName,
+                    prompt: prompt,
+                    temperature: 0.0,
+                    max_tokens: 1024
+                }, {
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'Authorization': `Bearer ${this.apiKey}`
+                    }
+                });
+                return response.data.choices[0].text;
+            }
+            catch (e) {
+                error(`Error generating text with Together API: ${e.message}`);
+                throw new Error(`Failed to generate text: ${e.message}`);
+            }
+        });
+    }
+    getModelName() {
+        return this.modelName;
+    }
+}
+/**
+ * Create a judge instance
+ * @param model Model name or Judge instance
+ * @param user Optional user identifier
+ * @returns Judge instance and whether it's a native model
+ */
+export function createJudge(model, user) {
+    if (!model) {
+        return { judge: new DefaultJudge(undefined, undefined, user), usingNativeModel: true };
+    }
+    if (typeof model === 'string') {
+        // Check if it's a Together AI model
+        if (model.startsWith('together/') ||
+            model.startsWith('meta-llama/') ||
+            model.startsWith('mistralai/') ||
+            model.includes('llama')) {
+            return { judge: new TogetherJudge(model), usingNativeModel: true };
+        }
+        // Default to OpenAI
+        return { judge: new DefaultJudge(model, undefined, user), usingNativeModel: true };
+    }
+    // It's already a Judge instance
+    return { judge: model, usingNativeModel: false };
+}
+//# sourceMappingURL=index.js.map

package/dist/esm/judges/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/judges/index.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAO,IAAI,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAsBzC;;GAEG;AACH,MAAM,OAAO,YAAY;IAKvB,YAAY,YAAoB,eAAe,EAAE,MAAe,EAAE,IAAa;QAC7E,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QACnD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,8GAA8G,CAAC,CAAC;QACvH,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,MAAc;QACrB,uEAAuE;QACvE,2EAA2E;QAC3E,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC;YACH,sEAAsE;YACtE,uDAAuD;YACvD,MAAM,OAAO,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC5E,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAE9E,6BAA6B;YAC7B,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,iBAC1C,KAAK,EAAE,IAAI,CAAC,SAAS,EACrB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAC7C,WAAW,EAAE,GAAG,IACb,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EACzC,CAAC,CAAC;YAEJ,8BAA8B;YAC9B,MAAM,WAAW,GAAG;;oCAEU,IAAI,CAAC,MAAM;cACjC,WAAW,MAAM,YAAY,EAAE,CAAC;YAExC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAEtB,oBAAoB;YACpB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;YAEvE,2BAA2B;YAC3B,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;YAC3B,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;YAE5B,qBAAqB;YACrB,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACjD,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,KAAK,CAAC,kCAAkC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAEK,SAAS,CAAC,MAAc;;YAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,4CAA4C,kBAE1C,KAAK,EAAE,IAAI,CAAC,SAAS,EACrB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAC7C,WAAW,EAAE,GAAG,IACb,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAE3C;oBACE,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,eAAe,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;qBACzC;iBACF,CACF,CAAC;gBAEF,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;YAClD,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBAChB,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC7D,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;KAAA;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,aAAa;IAIxB,YAAY,YAAoB,2CAA2C,EAAE,MAAe;QAC1F,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;QAErD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,iHAAiH,CAAC,CAAC;QAC1H,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,MAAc;QACrB,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC;YACH,sEAAsE;YACtE,MAAM,OAAO,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,oBAAoB,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9E,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,qBAAqB,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAEhF,6BAA6B;YAC7B,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC;gBAC3C,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,MAAM,EAAE,MAAM;gBACd,WAAW,EAAE,GAAG;gBAChB,UAAU,EAAE,IAAI;aACjB,CAAC,CAAC,CAAC;YAEJ,8BAA8B;YAC9B,MAAM,WAAW,GAAG;;oCAEU,IAAI,CAAC,MAAM;cACjC,WAAW,MAAM,YAAY,EAAE,CAAC;YAExC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAEtB,oBAAoB;YACpB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;YAEvE,2BAA2B;YAC3B,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;YAC3B,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;YAE5B,qBAAqB;YACrB,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACtC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,KAAK,CAAC,kCAAkC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAEK,SAAS,CAAC,MAAc;;YAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,yCAAyC,EACzC;oBACE,KAAK,EAAE,IAAI,CAAC,SAAS;oBACrB,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,GAAG;oBAChB,UAAU,EAAE,IAAI;iBACjB,EACD;oBACE,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,eAAe,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;qBACzC;iBACF,CACF,CAAC;gBAEF,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACvC,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBAChB,KAAK,CAAC,4CAA4C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/D,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;KAAA;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,KAAsB,EAAE,IAAa;IAC/D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,KAAK,EAAE,IAAI,YAAY,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;IACzF,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,oCAAoC;QACpC,IAAI,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC;YAC7B,KAAK,CAAC,UAAU,CAAC,aAAa,CAAC;YAC/B,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC;YAC9B,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,KAAK,EAAE,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;QACrE,CAAC;QAED,oBAAoB;QACpB,OAAO,EAAE,KAAK,EAAE,IAAI,YAAY,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;IACrF,CAAC;IAED,gCAAgC;IAChC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC;AACnD,CAAC"}