evals 1.0.0 → 1.0.1

package/README.md

@@ -1,8 +1,8 @@
-# Umbrage Evals JavaScript SDK
+# Umbrage Evals JavaScript CLI
 
 ## Overview
 
-This is the Javascript SDK for Umbrage Evals. It also supports TypeScript.
+This is the Javascript CLI for Umbrage Evals. It also supports TypeScript.
 
 ## Installation
 
@@ -16,53 +16,137 @@ npm install evals
 
 -   Node.js (or another JavaScript runtime environment like Bun)
 
+## Environment
+
+You will need to set the following environment variables:
+
+```bash
+UMBRAGE_EVALS_API_KEY=<your api key for the umbrage evals project>
+OPENAI_API_KEY=<your openai api key>
+```
+
+These can be set in your `~/.bashrc` or `~/.zshrc` file on Mac or Linux, or in your `~/.bash_profile` file on Windows.
+
+It would look like:
+
+```bash
+export UMBRAGE_EVALS_API_KEY=<your api key for the umbrage evals project>
+export OPENAI_API_KEY=<your openai api key>
+```
+
+To save the file, press `Ctrl + X`, then `Y`, then `Enter`. Then, run `source ~/.bashrc` or `source ~/.zshrc` on Mac or Linux, or `source ~/.bash_profile` on Windows.
+
 ## Example usage
 
+```bash
+umbrage-cli fetch-evals
+
+umbrage-cli run-evals
+```
+
+## Example BaseModel
+
 ```typescript
-import Evals from 'evals';
-
-const evals = new Evals('your-umbrage-evals-api-key-goes-here'); // Get your API key from the Umbrage Evals Dashboard
-
-// Get the timestamp for the request so we can log the response time from the model
-const request_timestamp = new Date();
-
-// Setting model in a variable since it's going to be passed in two places, one for the model and one for Umbrage Evals
-const model = 'gpt-3.5-turbo';
-
-// Call the model
-const chatResponse = await openai.createChatCompletion({
-  model,
-  messages,
-  max_tokens: reservedTokensForResponse,
-  temperature: 0.2,
-});
-
-// Get the timestamp for the response so we can log the response time from the model
-const response_timestamp = new Date();
-
-// Log the prompt and response data to Umbrage Evals
-const logResponse = await evals.logPrompt({
-  prompt_name: 'unique_prompt_name', // A unique name specific to this prompt. This is what will show up in the Dashboard under your Project.
-  environment: 'development', // Which environment you're using, e.g. development, test, staging, production
-  model,
-  model_settings: {   // Optional, but recommended if you want to track model settings
-    temperature: 0.2,
-    max_tokens: 512,
-  },
-  prompts: messages.map((message) => ({
-    prompt_type: message.role,    // Make sure you have an array with prompt_type and prompt_text
-    prompt_text: message.content,
-  })),
-  response: chatResponse.data.choices[0].message.content, // Map the final text response from the model to the response field
-  request_timestamp,
-  response_timestamp, // Pass the JavaScript Date objects for the request and response timestamps
-})
-
-console.log(logResponse) // { status: 200, body: 'Prompt and response logged successfully' }
+export interface ModelSettings {
+  temperature: number;
+  [key: string]: any;
+}
+
+export interface Prompt {
+  prompt_type: string;
+  prompt_text: string;
+}
+
+export interface Config {
+  promptName: string;
+  modelName: string;
+  modelSettings: ModelSettings;
+  environment: string;
+}
+
+class BaseModel {
+  promptName: string;
+  modelName: string;
+  modelSettings: ModelSettings;
+  environment: string;
+
+  constructor(config: Config) {
+    this.promptName = config.promptName;
+    this.modelName = config.modelName;
+    this.modelSettings = config.modelSettings;
+    this.environment = config.environment || 'development';
+}
+
+  /**
+   * This method should be overridden in subclasses.
+   * It should return an object containing the response and the prompts array.
+   * @param userMessage The user message to be processed by the model, optional.
+   * @returns Promise<{ response: string, prompts: Prompt[] }>
+   */
+  async callModel(userMessage: string = ''): Promise<{ response: string; prompts: Prompt[] }> {
+      throw new Error('callModel method needs to be implemented in subclasses');
+  }
+}
+
+export default BaseModel;
+```
+
+## Example Prompt (OpenAI API)
+
+```typescript
+import BaseModel from './BaseModel';
+import OpenAI from 'openai';
+
+class ChevySalesCopilotPrompt extends BaseModel {
+    constructor() {
+        const config = {
+            promptName: 'Chevy Sales Copilot',
+            modelName: 'gpt-4',
+            modelSettings: { temperature: 0.2 },
+            environment: 'development'
+        };
+        super(config);
+
+        this.openai = new OpenAI(); // Assumes OPENAI_API_KEY is set in environment
+        this.messages = [
+            { role: 'system', content: 'You are a Chevy Sales Copilot assistant. You can only help with topics related to sales at Chevrolet. If asked your name your name is "Sales Copilot".' }
+        ];
+    }
+
+    async callModel(userMessage) {
+        try {
+            // Update messages with user input
+            const updatedMessages = [...this.messages, { role: 'user', content: userMessage }];
+
+            const modelResponse = await this.openai.chat.completions.create({
+                model: this.modelName,
+                messages: updatedMessages,
+                temperature: this.modelSettings.temperature,
+            });
+
+            const response = modelResponse.choices[0].message.content;
+
+            const prompts = updatedMessages.map(message => ({
+                prompt_type: message.role,
+                prompt_text: message.content,
+            }));
+
+            return { response, prompts };
+        } catch (error) {
+            console.error('Error:', error);
+            throw error;
+        }
+    }
+}
+
+const prompt = new ChevySalesCopilotPrompt();
+
+export default prompt;
 
 ```
 
 ## LICENSE
+
 Copyright (c) 2023 Umbrage Studios, LLC
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to use the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: