evals 1.0.7 → 2.0.0

package/src/index.js

@@ -1,190 +0,0 @@
-#!/usr/bin/env node
-import { program } from 'commander';
-import fs from 'fs';
-import OpenAI from 'openai';
-
-program
-    .command('fetch-evals')
-    .description('Fetch the latest evals for the project')
-    .action(async () => {
-        if (!process.env.UMBRAGE_EVALS_API_KEY) {
-            throw new Error('UMBRAGE_EVALS_API_KEY is not set in the environment variables.');
-        }
-
-        const UMBRAGE_EVALS_API_KEY = process.env.UMBRAGE_EVALS_API_KEY;
-
-        const fetchEvals = async () => {
-            const url = new URL('https://api-gateway.groff.workers.dev/evals');
-
-            url.searchParams.append('page', 0);
-            url.searchParams.append('pageSize', 100);
-            url.searchParams.append('eval_type', 'OpenAI-GPT-4');
-
-            try {
-                const response = await fetch(url, {
-                    method: 'GET',
-                    headers: {
-                        'X-API-KEY': UMBRAGE_EVALS_API_KEY,
-                    },
-                });
-
-                if (!response.ok) {
-                    throw new Error(`HTTP error! status: ${response.status}`);
-                }
-
-                const data = await response.json();
-                return data.evals;
-            } catch (error) {
-                console.error('Error fetching evals:', error);
-                return []; // Return an empty array to avoid breaking downstream code
-            }
-        };
-
-        const processPromptFile = async file => {
-            const promptFilename = file.split('.prompt.js')[0];
-            const evalsFolder = `prompts/${promptFilename}_evals`;
-
-            if (!fs.existsSync(evalsFolder)) {
-                fs.mkdirSync(evalsFolder, { recursive: true });
-            }
-
-            // Fetch evals from the cloudflare worker
-            const evalsForPrompt = await fetchEvals();
-
-            for (const evalObject of evalsForPrompt) {
-                const { name: evalName, eval_code } = evalObject;
-                const markdownFileName = `${evalsFolder}/${evalName.replace(/[^a-z0-9]/gi, '_')}.md`;
-                fs.writeFileSync(markdownFileName, eval_code);
-            }
-        };
-
-        try {
-            const currentWorkingDir = process.cwd(); // Get the current working directory
-            const promptsDir = `${currentWorkingDir}/prompts/`; // Use it as the base for your prompts directory
-            const promptFiles = fs.readdirSync(promptsDir).filter(file => file.endsWith('.prompt.js'));
-
-            const processingPromises = promptFiles.map(processPromptFile);
-            await Promise.all(processingPromises);
-
-            console.log('Done fetching evals!');
-        } catch (error) {
-            console.error('An error occurred:', error);
-        }
-    });
-
-program
-    .command('run-evals')
-    .description('Run evals in the current directory and log results')
-    .action(async () => {
-        if (!process.env.OPENAI_API_KEY) {
-            throw new Error('OPENAI_API_KEY is not set in the environment variables.');
-        }
-
-        const openai = new OpenAI(); // Assumes OPENAI_API_KEY is set in environment
-        const currentWorkingDir = process.cwd(); // Get the current working directory
-        const promptsDir = `${currentWorkingDir}/prompts/`; // Use it as the base for your prompts directory
-        const model = 'gpt-4-1106-preview';
-        const temperature = 0;
-
-        // Process a single markdown file
-        const processMarkdownFile = async (evalsFolder, evalFile, promptInstance) => {
-            console.log(`\nEvaluating: ${evalFile}`);
-            const evalName = evalFile.split('.md')[0];
-            const eval_code = fs.readFileSync(`${evalsFolder}/${evalFile}`, 'utf-8');
-
-            console.time('Model response time');
-            const { response: modelResponse, prompts: evalPrompts } = await promptInstance.callModel('Hi! What is your name?');
-            console.timeEnd('Model response time');
-
-            const messages = [
-                {
-                    role: 'system',
-                    content:
-                        'You are a prompt evaluation expert. \\nYou will respond in JSON format with an "explanation" of why you have given it a grade from 0-100, and "suggestions" for improving the response in order to get a higher grade, and lastly the "grade" from 0-100 in integer number format. \\nUse the rubric to accomplish this task.',
-                },
-                { role: 'function', name: 'grading_rubric', content: eval_code },
-                { role: 'user', content: `Grade the following response using the rubric: \\n ${modelResponse}` },
-            ];
-
-            console.time('Eval response time');
-            const evalResponse = await openai.chat.completions.create({
-                model,
-                messages,
-                temperature,
-                response_format: { type: 'json_object' },
-            });
-            console.timeEnd('Eval response time');
-
-            const evalResult = JSON.parse(evalResponse.choices[0].message.content);
-            console.log('evalResult', evalResult);
-
-            return {
-                evalName,
-                evalCode: eval_code,
-                evalResult,
-                isValid: evalResult.grade && evalResult.explanation && evalResult.suggestions,
-            };
-        };
-
-        // Process a single prompt file
-        const processPromptFile = async file => {
-            const promptFilename = file.split('.prompt.js')[0];
-            const evalsFolder = `${promptsDir}/${promptFilename}_evals`;
-
-            if (!fs.existsSync(evalsFolder)) {
-                console.error(`Evals folder not found for ${promptFilename}, please run fetch_latest_evals.js first.`);
-                return;
-            }
-
-            const promptInstance = await import(`${promptsDir}${file}`).then(mod => mod.default);
-            const evaluations = [];
-
-            const evalMarkdownFiles = fs.readdirSync(evalsFolder).filter(f => f.endsWith('.md'));
-
-            for (const evalFile of evalMarkdownFiles) {
-                const result = await processMarkdownFile(evalsFolder, evalFile, promptInstance);
-                if (result.isValid) {
-                    evaluations.push({
-                        promptName: promptInstance.promptName,
-                        modelName: promptInstance.modelName,
-                        modelSettings: promptInstance.modelSettings,
-                        modelResponse: result.modelResponse,
-                        evalPromptsJson: JSON.stringify(result.evalPrompts),
-                        evalName: result.evalName,
-                        evalCode: result.evalCode,
-                        grade: result.evalResult.grade,
-                        explanation: result.evalResult.explanation,
-                        suggestions: result.evalResult.suggestions,
-                    });
-                } else {
-                    // Handle invalid evaluation
-                    evaluations.push({
-                        promptName: promptInstance.promptName,
-                        modelName: promptInstance.modelName,
-                        modelSettings: promptInstance.modelSettings,
-                        modelResponse: result.modelResponse,
-                        evalPromptsJson: JSON.stringify(result.evalPrompts),
-                        evalName: result.evalName,
-                        evalCode: result.evalCode,
-                        grade: 'Evaluation failed.',
-                        explanation: 'Evaluation failed.',
-                        suggestions: 'Evaluation failed.',
-                    });
-                }
-            }
-
-            const jsonFilePath = `${evalsFolder}/${promptFilename}_evals_results_${new Date().toISOString()}.json`;
-            fs.writeFileSync(jsonFilePath, JSON.stringify(evaluations, null, 4));
-        };
-
-        try {
-            const promptFiles = fs.readdirSync(promptsDir).filter(file => file.endsWith('.prompt.js'));
-            const processingPromises = promptFiles.map(processPromptFile);
-            await Promise.all(processingPromises);
-            console.log('Done processing evals!');
-        } catch (error) {
-            console.error('An error occurred:', error);
-        }
-    });
-
-program.parse(process.argv);