evals 0.0.5 → 1.0.0

package/README.md

@@ -2,7 +2,7 @@
 
 ## Overview
 
-This is the Javascript SDK for Umbrage Evals. It will have both TypeScript and JavaScript support.
+This is the Javascript SDK for Umbrage Evals. It also supports TypeScript.
 
 ## Installation
 
@@ -16,6 +16,52 @@ npm install evals
 
 -   Node.js (or another JavaScript runtime environment like Bun)
 
+## Example usage
+
+```typescript
+import Evals from 'evals';
+
+const evals = new Evals('your-umbrage-evals-api-key-goes-here'); // Get your API key from the Umbrage Evals Dashboard
+
+// Get the timestamp for the request so we can log the response time from the model
+const request_timestamp = new Date();
+
+// Setting model in a variable since it's going to be passed in two places, one for the model and one for Umbrage Evals
+const model = 'gpt-3.5-turbo';
+
+// Call the model
+const chatResponse = await openai.createChatCompletion({
+  model,
+  messages,
+  max_tokens: reservedTokensForResponse,
+  temperature: 0.2,
+});
+
+// Get the timestamp for the response so we can log the response time from the model
+const response_timestamp = new Date();
+
+// Log the prompt and response data to Umbrage Evals
+const logResponse = await evals.logPrompt({
+  prompt_name: 'unique_prompt_name', // A unique name specific to this prompt. This is what will show up in the Dashboard under your Project.
+  environment: 'development', // Which environment you're using, e.g. development, test, staging, production
+  model,
+  model_settings: {   // Optional, but recommended if you want to track model settings
+    temperature: 0.2,
+    max_tokens: 512,
+  },
+  prompts: messages.map((message) => ({
+    prompt_type: message.role,    // Make sure you have an array with prompt_type and prompt_text
+    prompt_text: message.content,
+  })),
+  response: chatResponse.data.choices[0].message.content, // Map the final text response from the model to the response field
+  request_timestamp,
+  response_timestamp, // Pass the JavaScript Date objects for the request and response timestamps
+})
+
+console.log(logResponse) // { status: 200, body: 'Prompt and response logged successfully' }
+
+```
+
 ## LICENSE
 Copyright (c) 2023 Umbrage Studios, LLC
 

package/dist/evals.d.ts

@@ -1,9 +1,13 @@
-/// <reference types="bun-types" />
 import { RequestData } from './types';
 declare class Evals {
     private apiKey;
     private endpoint;
     constructor(apiKey: string, endpoint?: string);
-    logPrompt(requestData: RequestData): Promise<Response>;
+    logPrompt(requestData: RequestData): Promise<{
+        status: number;
+        body: string;
+    }>;
+    formatTimestamp(timestamp: Date): string;
+    validateRequestData(requestData: RequestData): void;
 }
 export default Evals;

package/dist/index.js

@@ -7,6 +7,7 @@ class Evals {
     this.endpoint = endpoint;
   }
   async logPrompt(requestData) {
+    this.validateRequestData(requestData);
     const response = await fetch(this.endpoint, {
       method: "POST",
       headers: {
@@ -14,10 +15,52 @@ class Evals {
       },
       body: JSON.stringify({
         api_key: this.apiKey,
-        ...requestData
+        ...requestData,
+        request_timestamp: this.formatTimestamp(new Date(requestData.request_timestamp)),
+        response_timestamp: this.formatTimestamp(new Date(requestData.response_timestamp))
       })
     });
-    return response;
+    const body = await response.text();
+    return { status: response.status, body };
+  }
+  formatTimestamp(timestamp) {
+    const pad = (num, size) => num.toString().padStart(size, "0");
+    const year = timestamp.getFullYear();
+    const month = pad(timestamp.getMonth() + 1, 2);
+    const day = pad(timestamp.getDate(), 2);
+    const hours = pad(timestamp.getHours(), 2);
+    const minutes = pad(timestamp.getMinutes(), 2);
+    const seconds = pad(timestamp.getSeconds(), 2);
+    const milliseconds = pad(timestamp.getMilliseconds(), 3);
+    return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${milliseconds}`;
+  }
+  validateRequestData(requestData) {
+    if (!requestData.request_timestamp || !requestData.response_timestamp || isNaN(Date.parse(requestData.request_timestamp)) || isNaN(Date.parse(requestData.response_timestamp))) {
+      throw new Error("request_timestamp and response_timestamp are required and must be valid MySQL DATETIME strings");
+    }
+    if (!requestData.prompts || !Array.isArray(requestData.prompts) || requestData.prompts.length === 0) {
+      throw new Error("prompts is required and must be an array with at least one item");
+    }
+    requestData.prompts.forEach((prompt) => {
+      if (typeof prompt.prompt_type !== "string" || typeof prompt.prompt_text !== "string") {
+        throw new Error("Each prompt must have a string prompt_type and prompt_text");
+      }
+    });
+    if (!requestData.model_settings && typeof requestData.model_settings !== "object") {
+      throw new Error("model_settings is required and must be an object");
+    }
+    if (!requestData.model || typeof requestData.model !== "string") {
+      throw new Error("model is required and must be a string");
+    }
+    if (!requestData.prompt_name || typeof requestData.prompt_name !== "string") {
+      throw new Error("prompt_name is required and must be a string");
+    }
+    if (!requestData.environment || typeof requestData.environment !== "string") {
+      throw new Error("environment is required and must be a string");
+    }
+    if (!requestData.response || typeof requestData.response !== "string") {
+      throw new Error("response is required and must be a string");
+    }
   }
 }
 var evals_default = Evals;

package/package.json

@@ -4,7 +4,7 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
-  "version": "0.0.5",
+  "version": "1.0.0",
   "description": "Umbrage Evals SDK",
   "scripts": {
     "build": "bun build --target=node ./src/index.ts --outfile=dist/index.js && bun run build:declaration",