evals 1.0.2 → 1.0.3

package/package.json

@@ -4,7 +4,7 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "Umbrage Evals CLI",
   "bin": {
     "evals": "./src/index.js"

package/src/index.js

@@ -1,16 +1,188 @@
 #!/usr/bin/env node
-const { program } = require('commander');
-const fetchEvals = require('./fetchEvals');
-const runEvals = require('./runEvals');
+import { program } from 'commander';
+import fs from 'fs';
+import OpenAI from 'openai';
 
 program
-  .command('fetch-evals')
-  .description('Fetch the latest evals for the project')
-  .action(fetchEvals);
+    .command('fetch-evals')
+    .description('Fetch the latest evals for the project')
+    .action(async () => {
+        if (!process.env.UMBRAGE_EVALS_API_KEY) {
+            throw new Error('UMBRAGE_EVALS_API_KEY is not set in the environment variables.');
+        }
+
+        const UMBRAGE_EVALS_API_KEY = process.env.UMBRAGE_EVALS_API_KEY;
+
+        const fetchEvals = async () => {
+            const url = new URL('https://api-gateway.groff.workers.dev/evals');
+
+            url.searchParams.append('page', 0);
+            url.searchParams.append('pageSize', 100);
+            url.searchParams.append('eval_type', 'OpenAI-GPT-4');
+
+            try {
+                const response = await fetch(url, {
+                    method: 'GET',
+                    headers: {
+                        Authorization: `Bearer ${UMBRAGE_EVALS_API_KEY}`,
+                    },
+                });
+
+                if (!response.ok) {
+                    throw new Error(`HTTP error! status: ${response.status}`);
+                }
+
+                const data = await response.json();
+                return data.evals;
+            } catch (error) {
+                console.error('Error fetching evals:', error);
+                return []; // Return an empty array to avoid breaking downstream code
+            }
+        };
+
+        const processPromptFile = async file => {
+            const promptFilename = file.split('.prompt.js')[0];
+            const evalsFolder = `${promptFilename}_evals`;
+
+            if (!fs.existsSync(evalsFolder)) {
+                fs.mkdirSync(evalsFolder, { recursive: true });
+            }
+
+            // Fetch evals from the cloudflare worker
+            const evalsForPrompt = await fetchEvals();
+
+            for (const evalObject of evalsForPrompt) {
+                const { name: evalName, eval_code } = evalObject;
+                const markdownFileName = `${evalsFolder}/${evalName.replace(/[^a-z0-9]/gi, '_')}.md`;
+                fs.writeFileSync(markdownFileName, eval_code);
+            }
+        };
+
+        try {
+            const promptsDir = './prompts/';
+            const promptFiles = fs.readdirSync(promptsDir).filter(file => file.endsWith('.prompt.js'));
+
+            const processingPromises = promptFiles.map(processPromptFile);
+            await Promise.all(processingPromises);
+
+            console.log('Done fetching evals!');
+        } catch (error) {
+            console.error('An error occurred:', error);
+        }
+    });
 
 program
-  .command('run-evals')
-  .description('Run evals in the current directory and log results')
-  .action(runEvals);
+    .command('run-evals')
+    .description('Run evals in the current directory and log results')
+    .action(async () => {
+        if (!process.env.OPENAI_API_KEY) {
+            throw new Error('OPENAI_API_KEY is not set in the environment variables.');
+        }
+
+        const openai = new OpenAI(); // Assumes OPENAI_API_KEY is set in environment
+        const promptsDir = './prompts/';
+        const model = 'gpt-4-1106-preview';
+        const temperature = 0;
+
+        // Process a single markdown file
+        const processMarkdownFile = async (evalsFolder, evalFile, promptInstance) => {
+            console.log(`\nEvaluating: ${evalFile}`);
+            const evalName = evalFile.split('.md')[0];
+            const eval_code = fs.readFileSync(`${evalsFolder}/${evalFile}`, 'utf-8');
+
+            console.time('Model response time');
+            const { response: modelResponse, prompts: evalPrompts } = await promptInstance.callModel('Hi! What is your name?');
+            console.timeEnd('Model response time');
+
+            const messages = [
+                {
+                    role: 'system',
+                    content:
+                        'You are a prompt evaluation expert. \\nYou will respond in JSON format with an "explanation" of why you have given it a grade from 0-100, and "suggestions" for improving the response in order to get a higher grade, and lastly the "grade" from 0-100 in integer number format. \\nUse the rubric to accomplish this task.',
+                },
+                { role: 'function', name: 'grading_rubric', content: eval_code },
+                { role: 'user', content: `Grade the following response using the rubric: \\n ${modelResponse}` },
+            ];
+
+            console.time('Eval response time');
+            const evalResponse = await openai.chat.completions.create({
+                model,
+                messages,
+                temperature,
+                response_format: { type: 'json_object' },
+            });
+            console.timeEnd('Eval response time');
+
+            const evalResult = JSON.parse(evalResponse.choices[0].message.content);
+            console.log('evalResult', evalResult);
+
+            return {
+                evalName,
+                evalCode: eval_code,
+                evalResult,
+                isValid: evalResult.grade && evalResult.explanation && evalResult.suggestions,
+            };
+        };
+
+        // Process a single prompt file
+        const processPromptFile = async file => {
+            const promptFilename = file.split('.prompt.js')[0];
+            const evalsFolder = `${promptsDir}/${promptFilename}_evals`;
+
+            if (!fs.existsSync(evalsFolder)) {
+                console.error(`Evals folder not found for ${promptFilename}, please run fetch_latest_evals.js first.`);
+                return;
+            }
+
+            const promptInstance = await import(`${promptsDir}${file}`).then(mod => mod.default);
+            const evaluations = [];
+
+            const evalMarkdownFiles = fs.readdirSync(evalsFolder).filter(f => f.endsWith('.md'));
+
+            for (const evalFile of evalMarkdownFiles) {
+                const result = await processMarkdownFile(evalsFolder, evalFile, promptInstance);
+                if (result.isValid) {
+                    evaluations.push({
+                        promptName: promptInstance.promptName,
+                        modelName: promptInstance.modelName,
+                        modelSettings: promptInstance.modelSettings,
+                        modelResponse: result.modelResponse,
+                        evalPromptsJson: JSON.stringify(result.evalPrompts),
+                        evalName: result.evalName,
+                        evalCode: result.evalCode,
+                        grade: result.evalResult.grade,
+                        explanation: result.evalResult.explanation,
+                        suggestions: result.evalResult.suggestions,
+                    });
+                } else {
+                    // Handle invalid evaluation
+                    evaluations.push({
+                        promptName: promptInstance.promptName,
+                        modelName: promptInstance.modelName,
+                        modelSettings: promptInstance.modelSettings,
+                        modelResponse: result.modelResponse,
+                        evalPromptsJson: JSON.stringify(result.evalPrompts),
+                        evalName: result.evalName,
+                        evalCode: result.evalCode,
+                        grade: 'Evaluation failed.',
+                        explanation: 'Evaluation failed.',
+                        suggestions: 'Evaluation failed.',
+                    });
+                }
+            }
+
+            const jsonFilePath = `${evalsFolder}/${promptFilename}_evals_results_${new Date().toISOString()}.json`;
+            fs.writeFileSync(jsonFilePath, JSON.stringify(evaluations, null, 4));
+        };
+
+        try {
+            const promptFiles = fs.readdirSync(promptsDir).filter(file => file.endsWith('.prompt.js'));
+            const processingPromises = promptFiles.map(processPromptFile);
+            await Promise.all(processingPromises);
+            console.log('Done processing evals!');
+        } catch (error) {
+            console.error('An error occurred:', error);
+        }
+    });
 
 program.parse(process.argv);