evals 0.0.6 → 1.0.1

package/README.md

@@ -1,8 +1,8 @@
-# Umbrage Evals JavaScript SDK
+# Umbrage Evals JavaScript CLI
 
 ## Overview
 
-This is the Javascript SDK for Umbrage Evals. It will have both TypeScript and JavaScript support.
+This is the Javascript CLI for Umbrage Evals. It also supports TypeScript.
 
 ## Installation
 
@@ -16,7 +16,137 @@ npm install evals
 
 -   Node.js (or another JavaScript runtime environment like Bun)
 
+## Environment
+
+You will need to set the following environment variables:
+
+```bash
+UMBRAGE_EVALS_API_KEY=<your api key for the umbrage evals project>
+OPENAI_API_KEY=<your openai api key>
+```
+
+These can be set in your `~/.bashrc` or `~/.zshrc` file on Mac or Linux, or in your `~/.bash_profile` file on Windows.
+
+It would look like:
+
+```bash
+export UMBRAGE_EVALS_API_KEY=<your api key for the umbrage evals project>
+export OPENAI_API_KEY=<your openai api key>
+```
+
+To save the file, press `Ctrl + X`, then `Y`, then `Enter`. Then, run `source ~/.bashrc` or `source ~/.zshrc` on Mac or Linux, or `source ~/.bash_profile` on Windows.
+
+## Example usage
+
+```bash
+umbrage-cli fetch-evals
+
+umbrage-cli run-evals
+```
+
+## Example BaseModel
+
+```typescript
+export interface ModelSettings {
+  temperature: number;
+  [key: string]: any;
+}
+
+export interface Prompt {
+  prompt_type: string;
+  prompt_text: string;
+}
+
+export interface Config {
+  promptName: string;
+  modelName: string;
+  modelSettings: ModelSettings;
+  environment: string;
+}
+
+class BaseModel {
+  promptName: string;
+  modelName: string;
+  modelSettings: ModelSettings;
+  environment: string;
+
+  constructor(config: Config) {
+    this.promptName = config.promptName;
+    this.modelName = config.modelName;
+    this.modelSettings = config.modelSettings;
+    this.environment = config.environment || 'development';
+}
+
+  /**
+   * This method should be overridden in subclasses.
+   * It should return an object containing the response and the prompts array.
+   * @param userMessage The user message to be processed by the model, optional.
+   * @returns Promise<{ response: string, prompts: Prompt[] }>
+   */
+  async callModel(userMessage: string = ''): Promise<{ response: string; prompts: Prompt[] }> {
+      throw new Error('callModel method needs to be implemented in subclasses');
+  }
+}
+
+export default BaseModel;
+```
+
+## Example Prompt (OpenAI API)
+
+```typescript
+import BaseModel from './BaseModel';
+import OpenAI from 'openai';
+
+class ChevySalesCopilotPrompt extends BaseModel {
+    constructor() {
+        const config = {
+            promptName: 'Chevy Sales Copilot',
+            modelName: 'gpt-4',
+            modelSettings: { temperature: 0.2 },
+            environment: 'development'
+        };
+        super(config);
+
+        this.openai = new OpenAI(); // Assumes OPENAI_API_KEY is set in environment
+        this.messages = [
+            { role: 'system', content: 'You are a Chevy Sales Copilot assistant. You can only help with topics related to sales at Chevrolet. If asked your name your name is "Sales Copilot".' }
+        ];
+    }
+
+    async callModel(userMessage) {
+        try {
+            // Update messages with user input
+            const updatedMessages = [...this.messages, { role: 'user', content: userMessage }];
+
+            const modelResponse = await this.openai.chat.completions.create({
+                model: this.modelName,
+                messages: updatedMessages,
+                temperature: this.modelSettings.temperature,
+            });
+
+            const response = modelResponse.choices[0].message.content;
+
+            const prompts = updatedMessages.map(message => ({
+                prompt_type: message.role,
+                prompt_text: message.content,
+            }));
+
+            return { response, prompts };
+        } catch (error) {
+            console.error('Error:', error);
+            throw error;
+        }
+    }
+}
+
+const prompt = new ChevySalesCopilotPrompt();
+
+export default prompt;
+
+```
+
 ## LICENSE
+
 Copyright (c) 2023 Umbrage Studios, LLC
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to use the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: