deepeval 0.1.17

package/README.md

@@ -0,0 +1,162 @@
+# DeepEval.ts
+
+TypeScript client for Confident AI's DeepEval API - a framework for evaluating and testing Large Language Models (LLMs).
+
+## Installation
+
+```bash
+npm install deepeval-ts
+```
+
+## Authentication
+
+DeepEval.ts requires a Confident AI API key to authenticate with the service. You can set up your API key in one of the following ways:
+
+### Option 1: Environment Variables
+
+Set the `CONFIDENT_API_KEY` environment variable:
+
+```bash
+# In your terminal
+export CONFIDENT_API_KEY="your-api-key-here"
+
+# Or for Windows
+set CONFIDENT_API_KEY=your-api-key-here
+```
+
+### Option 2: .env File
+
+Create a `.env` file in your project root:
+
+```bash
+# .env file
+CONFIDENT_API_KEY="your-api-key-here"
+```
+
+Then use a package like `dotenv` to load it:
+
+```bash
+npm install dotenv
+```
+
+```javascript
+// At the top of your entry file
+import 'dotenv/config';
+```
+
+### Option 3: Pass API Key Directly
+
+You can also pass your API key directly when creating an API instance:
+
+```typescript
+import { Api } from 'deepeval-ts';
+
+const api = new Api("your-api-key-here");
+```
+
+## Usage Examples
+
+### Working with Datasets
+
+```typescript
+import { EvaluationDataset, LLMTestCase } from 'deepeval-ts';
+import * as path from 'path';
+
+// Load dataset from CSV
+const dataset = new EvaluationDataset();
+await dataset.addTestCasesFromCsvFile(
+  'path/to/dataset.csv',
+  'input_column',
+  'actual_output_column',
+  'expected_output_column'
+);
+
+// Create dataset programmatically
+const customDataset = new EvaluationDataset();
+customDataset.addTestCase(
+  new LLMTestCase({
+    input: "What is the capital of France?",
+    actualOutput: "Paris is the capital of France.",
+    expectedOutput: "Paris"
+  })
+);
+
+// Iterate through test cases
+for (const testCase of dataset.testCases) {
+  console.log(`Input: ${testCase.input}`);
+  console.log(`Output: ${testCase.actualOutput}`);
+}
+```
+
+## API Reference
+
+### EvaluationDataset
+
+The `EvaluationDataset` class manages collections of test cases for LLM evaluation.
+
+```typescript
+// Create a new dataset
+const dataset = new EvaluationDataset();
+
+// Add test cases from CSV
+await dataset.addTestCasesFromCsvFile(
+  filePath,           // Path to CSV file
+  inputColumn,        // Name of input column
+  actualOutputColumn, // Name of actual output column
+  expectedOutputColumn, // Name of expected output column (optional)
+  contextColumn,      // Name of context column (optional)
+  contextDelimiter,   // Delimiter for context values (optional)
+  retrievalContextColumn, // Name of retrieval context column (optional)
+  retrievalContextDelimiter // Delimiter for retrieval context values (optional)
+);
+
+// Add a test case programmatically
+dataset.addTestCase(
+  new LLMTestCase({
+    input: "What is the capital of France?",
+    actualOutput: "Paris is the capital of France.",
+    expectedOutput: "Paris",
+    context: ["France is a country in Europe.", "Paris is a city."],
+    retrievalContext: ["Paris is the capital and most populous city of France."]
+  })
+);
+```
+
+### LLMTestCase
+
+The `LLMTestCase` class represents individual test cases for LLM evaluation.
+
+```typescript
+const testCase = new LLMTestCase({
+  input: "What is the capital of France?",
+  actualOutput: "Paris is the capital of France.",
+  expectedOutput: "Paris",
+  context: ["France is a country in Europe.", "Paris is a city."],
+  retrievalContext: ["Paris is the capital and most populous city of France."],
+  toolCalls: [
+    {
+      name: "search",
+      input: { query: "capital of France" },
+      output: { result: "Paris is the capital of France" }
+    }
+  ]
+});
+```
+
+## Development
+
+To build the package locally:
+
+```bash
+npm run build
+```
+
+To run tests:
+
+```bash
+npm test
+```
+
+## License
+
+[MIT](LICENSE)

package/dist/annotation/api.d.ts

@@ -0,0 +1,15 @@
+export declare enum AnnotationType {
+    THUMBS_RATING = "THUMBS_RATING",
+    FIVE_STAR_RATING = "FIVE_STAR_RATING"
+}
+export interface APIAnnotation {
+    rating: number;
+    traceUuid?: string;
+    spanUuid?: string;
+    threadId?: string;
+    expectedOutput?: string;
+    expectedOutcome?: string;
+    explanation?: string;
+    type?: AnnotationType;
+    userId?: string;
+}

package/dist/annotation/api.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AnnotationType = void 0;
+var AnnotationType;
+(function (AnnotationType) {
+    AnnotationType["THUMBS_RATING"] = "THUMBS_RATING";
+    AnnotationType["FIVE_STAR_RATING"] = "FIVE_STAR_RATING";
+})(AnnotationType || (exports.AnnotationType = AnnotationType = {}));

package/dist/annotation/index.d.ts

@@ -0,0 +1,3 @@
+import { APIAnnotation } from "./api";
+export * from "./api";
+export declare const sendAnnotation: ({ rating, traceUuid, spanUuid, threadId, expectedOutput, expectedOutcome, explanation, type, userId, }: APIAnnotation) => Promise<void>;

package/dist/annotation/index.js

@@ -0,0 +1,36 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sendAnnotation = void 0;
+const utils_1 = require("./utils");
+const confident_1 = require("../confident");
+__exportStar(require("./api"), exports);
+const sendAnnotation = async ({ rating, traceUuid, spanUuid, threadId, expectedOutput, expectedOutcome, explanation, type, userId, }) => {
+    const apiAnnotation = (0, utils_1.validateAPIAnnotation)({
+        rating,
+        traceUuid,
+        spanUuid,
+        threadId,
+        expectedOutput,
+        expectedOutcome,
+        explanation,
+        type,
+        userId,
+    });
+    const api = new confident_1.Api();
+    await api.sendRequest(confident_1.HttpMethods.POST, confident_1.Endpoints.ANNOTATION_ENDPOINT, apiAnnotation);
+};
+exports.sendAnnotation = sendAnnotation;

package/dist/annotation/utils.d.ts

@@ -0,0 +1,2 @@
+import { APIAnnotation } from "./api";
+export declare const validateAPIAnnotation: (apiAnnotation: APIAnnotation) => APIAnnotation;

package/dist/annotation/utils.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateAPIAnnotation = void 0;
+const api_1 = require("./api");
+const validateAPIAnnotation = (apiAnnotation) => {
+    const hasTrace = !!apiAnnotation.traceUuid;
+    const hasSpan = !!apiAnnotation.spanUuid;
+    const hasThread = !!apiAnnotation.threadId;
+    const idCount = [hasTrace, hasSpan, hasThread].filter(Boolean).length;
+    if (idCount > 1) {
+        throw new Error("Only one of 'traceUuid', 'spanUuid', or 'threadId' should be provided.");
+    }
+    if (idCount === 0) {
+        throw new Error("One of 'traceUuid', 'spanUuid', or 'threadId' must be provided.");
+    }
+    if (apiAnnotation.type === api_1.AnnotationType.FIVE_STAR_RATING &&
+        apiAnnotation.rating !== undefined &&
+        (apiAnnotation.rating < 1 || apiAnnotation.rating > 5)) {
+        throw new Error("Five star rating must be between 1 and 5.");
+    }
+    if (apiAnnotation.type === api_1.AnnotationType.THUMBS_RATING &&
+        apiAnnotation.rating !== undefined &&
+        (apiAnnotation.rating < 0 || apiAnnotation.rating > 1)) {
+        throw new Error("Thumbs rating must be either 0 or 1.");
+    }
+    if (apiAnnotation.threadId && apiAnnotation.expectedOutput) {
+        throw new Error("Expected output cannot be provided for threads.");
+    }
+    if (!apiAnnotation.threadId && apiAnnotation.expectedOutcome) {
+        throw new Error("Expected outcome cannot be provided for traces or spans.");
+    }
+    return apiAnnotation;
+};
+exports.validateAPIAnnotation = validateAPIAnnotation;

package/dist/confident/api.d.ts

@@ -0,0 +1,40 @@
+export declare const DEEPEVAL_BASE_URL = "https://deepeval.confident-ai.com";
+export declare const API_BASE_URL: string;
+export declare enum HttpMethods {
+    GET = "GET",
+    POST = "POST",
+    DELETE = "DELETE",
+    PUT = "PUT"
+}
+export declare enum Endpoints {
+    DATASET_ENDPOINT = "/v1/datasets",
+    TEST_RUN_ENDPOINT = "/v1/test-run",
+    TRACING_ENDPOINT = "/v1/tracing",
+    TRACES_ENDPOINT = "/v1/traces",
+    EVENT_ENDPOINT = "/v1/event",
+    FEEDBACK_ENDPOINT = "/v1/feedback",
+    PROMPTS_VERSION_ID_ENDPOINT = "/v1/prompts/:alias/versions/:version",
+    PROMPTS_LABEL_ENDPOINT = "/v1/prompts/:alias/labels/:label",
+    PROMPTS_COMMITS_ENDPOINT = "/v1/prompts/:alias/commits",
+    PROMPTS_COMMIT_HASH_ENDPOINT = "/v1/prompts/:alias/commits/:hash",
+    PROMPTS_VERSIONS_ENDPOINT = "/v1/prompts/:alias/versions",
+    PROMPTS_ENDPOINT = "/v1/prompts",
+    RECOMMEND_ENDPOINT = "/v1/recommend-metrics",
+    EVALUATE_ENDPOINT = "/v1/evaluate",
+    EVALUATE_THREAD_ENDPOINT = "/v1/evaluate/threads/:threadId",
+    EVALUATE_TRACE_ENDPOINT = "/v1/evaluate/traces/:traceUuid",
+    EVALUATE_SPAN_ENDPOINT = "/v1/evaluate/spans/:spanUuid",
+    GUARD_ENDPOINT = "/guard",
+    GUARDRAILS_ENDPOINT = "/guardrails",
+    BASELINE_ATTACKS_ENDPOINT = "/generate-baseline-attacks",
+    SIMULATE_ENDPOINT = "/v1/simulate",
+    ANNOTATION_ENDPOINT = "/v1/annotations"
+}
+export declare class Api {
+    private apiKey;
+    private headers;
+    private baseApiUrl;
+    constructor(apiKey?: string, baseUrl?: string);
+    private static httpRequest;
+    sendRequest(method: HttpMethods, endpoint: Endpoints | string, body?: any, params?: Record<string, any>, endpointString?: string, urlParams?: Record<string, string>): Promise<any>;
+}

package/dist/confident/api.js

@@ -0,0 +1,206 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Api = exports.Endpoints = exports.HttpMethods = exports.API_BASE_URL = exports.DEEPEVAL_BASE_URL = void 0;
+const axios_1 = __importDefault(require("axios"));
+const utils_1 = require("../utils");
+const readline_1 = require("readline");
+var Regions;
+(function (Regions) {
+    Regions["EU"] = "EU";
+    Regions["US"] = "US";
+    Regions["AU"] = "AU";
+})(Regions || (Regions = {}));
+const _LOCAL_API_BASE_URL = "http://localhost:3001";
+const _LOCAL_DEEPEVAL_BASE_URL = "http://0.0.0.0:8000";
+const PROD_DEEPEVAL_BASE_URL = "https://deepeval.confident-ai.com";
+const CONFIDENT_BASE_URL = "https://api.confident-ai.com";
+const CONFIDENT_BASE_URL_EU = "https://eu.api.confident-ai.com";
+const CONFIDENT_BASE_URL_AU = "https://au.api.confident-ai.com";
+const region = process.env.CONFIDENT_REGION || Regions.US;
+exports.DEEPEVAL_BASE_URL = PROD_DEEPEVAL_BASE_URL;
+exports.API_BASE_URL = region === Regions.EU
+    ? CONFIDENT_BASE_URL_EU
+    : region === Regions.AU
+        ? CONFIDENT_BASE_URL_AU
+        : CONFIDENT_BASE_URL;
+const inferBaseUrlFromApiKey = (apiKey) => {
+    if (apiKey.startsWith("confident_eu_")) {
+        return CONFIDENT_BASE_URL_EU;
+    }
+    else if (apiKey.startsWith("confident_au_")) {
+        return CONFIDENT_BASE_URL_AU;
+    }
+    return CONFIDENT_BASE_URL;
+};
+const RETRYABLE_ERROR_CODES = [
+    "ECONNRESET",
+    "ETIMEDOUT",
+    "ECONNREFUSED",
+    "ENOTFOUND",
+    "ENETUNREACH",
+    "ESOCKETTIMEDOUT",
+    "CERT_HAS_EXPIRED",
+];
+function logRetryError(error, attempt) {
+    console.error(`Confident AI Error: ${error}. Retrying: ${attempt} time(s)...`);
+}
+var HttpMethods;
+(function (HttpMethods) {
+    HttpMethods["GET"] = "GET";
+    HttpMethods["POST"] = "POST";
+    HttpMethods["DELETE"] = "DELETE";
+    HttpMethods["PUT"] = "PUT";
+})(HttpMethods || (exports.HttpMethods = HttpMethods = {}));
+var Endpoints;
+(function (Endpoints) {
+    Endpoints["DATASET_ENDPOINT"] = "/v1/datasets";
+    Endpoints["TEST_RUN_ENDPOINT"] = "/v1/test-run";
+    Endpoints["TRACING_ENDPOINT"] = "/v1/tracing";
+    Endpoints["TRACES_ENDPOINT"] = "/v1/traces";
+    Endpoints["EVENT_ENDPOINT"] = "/v1/event";
+    Endpoints["FEEDBACK_ENDPOINT"] = "/v1/feedback";
+    Endpoints["PROMPTS_VERSION_ID_ENDPOINT"] = "/v1/prompts/:alias/versions/:version";
+    Endpoints["PROMPTS_LABEL_ENDPOINT"] = "/v1/prompts/:alias/labels/:label";
+    Endpoints["PROMPTS_COMMITS_ENDPOINT"] = "/v1/prompts/:alias/commits";
+    Endpoints["PROMPTS_COMMIT_HASH_ENDPOINT"] = "/v1/prompts/:alias/commits/:hash";
+    Endpoints["PROMPTS_VERSIONS_ENDPOINT"] = "/v1/prompts/:alias/versions";
+    Endpoints["PROMPTS_ENDPOINT"] = "/v1/prompts";
+    Endpoints["RECOMMEND_ENDPOINT"] = "/v1/recommend-metrics";
+    Endpoints["EVALUATE_ENDPOINT"] = "/v1/evaluate";
+    Endpoints["EVALUATE_THREAD_ENDPOINT"] = "/v1/evaluate/threads/:threadId";
+    Endpoints["EVALUATE_TRACE_ENDPOINT"] = "/v1/evaluate/traces/:traceUuid";
+    Endpoints["EVALUATE_SPAN_ENDPOINT"] = "/v1/evaluate/spans/:spanUuid";
+    Endpoints["GUARD_ENDPOINT"] = "/guard";
+    Endpoints["GUARDRAILS_ENDPOINT"] = "/guardrails";
+    Endpoints["BASELINE_ATTACKS_ENDPOINT"] = "/generate-baseline-attacks";
+    Endpoints["SIMULATE_ENDPOINT"] = "/v1/simulate";
+    Endpoints["ANNOTATION_ENDPOINT"] = "/v1/annotations";
+})(Endpoints || (exports.Endpoints = Endpoints = {}));
+const defaultRetryOptions = {
+    maxAttempts: 5,
+    initialDelay: 1000,
+    maxDelay: 10000,
+    factor: 2,
+    jitter: true,
+};
+class Api {
+    apiKey;
+    headers;
+    baseApiUrl;
+    constructor(apiKey, baseUrl) {
+        if (!apiKey) {
+            apiKey = process.env.CONFIDENT_API_KEY;
+        }
+        if (!apiKey) {
+            throw new Error("Please provide a valid Confident AI API Key.");
+        }
+        // if region is set or url is provided, respect that
+        if (!process.env.CONFIDENT_REGION && !baseUrl) {
+            this.baseApiUrl = inferBaseUrlFromApiKey(apiKey);
+        }
+        else {
+            this.baseApiUrl = baseUrl || exports.API_BASE_URL;
+        }
+        this.apiKey = apiKey;
+        this.headers = {
+            "Content-Type": "application/json",
+            CONFIDENT_API_KEY: apiKey,
+        };
+    }
+    static async httpRequest(method, url, headers, data, params, options = defaultRetryOptions) {
+        let attempt = 0;
+        let delay = options.initialDelay;
+        while (attempt < options.maxAttempts) {
+            try {
+                const response = await (0, axios_1.default)({
+                    method,
+                    url,
+                    headers,
+                    data,
+                    params,
+                });
+                return response;
+            }
+            catch (error) {
+                attempt++;
+                // Check if error is retryable
+                const isRetryable = error.code &&
+                    RETRYABLE_ERROR_CODES.some((code) => error.code.includes(code));
+                if (!isRetryable || attempt >= options.maxAttempts) {
+                    throw error;
+                }
+                logRetryError(error, attempt);
+                // Calculate delay with exponential backoff and jitter
+                if (options.jitter) {
+                    const jitterFactor = Math.random() + 0.5; // Random between 0.5 and 1.5
+                    delay = Math.min(delay * options.factor * jitterFactor, options.maxDelay);
+                }
+                else {
+                    delay = Math.min(delay * options.factor, options.maxDelay);
+                }
+                await (0, utils_1.wait)(delay);
+            }
+        }
+        throw new Error(`Request failed after ${options.maxAttempts} attempts`);
+    }
+    async sendRequest(method, endpoint, body, params, endpointString, urlParams) {
+        let endpointPath = endpointString || endpoint;
+        if (urlParams) {
+            for (const [key, value] of Object.entries(urlParams)) {
+                endpointPath = endpointPath.replace(`:${key}`, value);
+            }
+        }
+        const url = `${this.baseApiUrl}${endpointPath}`;
+        try {
+            const res = await Api.httpRequest(method, url, this.headers, body, params);
+            if (res.status === 200) {
+                return res.data;
+            }
+            else if (res.status === 409 && body) {
+                const message = res.data?.message || "Conflict occurred.";
+                // In Node.js environment
+                if (typeof process !== "undefined" && process.stdin && process.stdout) {
+                    const readline = (0, readline_1.createInterface)({
+                        input: process.stdin,
+                        output: process.stdout,
+                    });
+                    return new Promise((resolve) => {
+                        readline.question(`${message} Would you like to overwrite it? [y/N] or change the alias [c]: `, (answer) => {
+                            readline.close();
+                            const userInput = answer.trim().toLowerCase();
+                            if (userInput === "y") {
+                                body.overwrite = true;
+                                resolve(this.sendRequest(method, endpoint, body));
+                            }
+                            else if (userInput === "c") {
+                                readline.question("Enter a new alias: ", (newAlias) => {
+                                    readline.close();
+                                    body.alias = newAlias.trim();
+                                    resolve(this.sendRequest(method, endpoint, body));
+                                });
+                            }
+                            else {
+                                console.log("Aborted.");
+                                resolve(null);
+                            }
+                        });
+                    });
+                }
+                else {
+                    console.error("Conflict occurred. Please implement appropriate UI handling for this environment.");
+                    return null;
+                }
+            }
+            else {
+                throw new Error(res.data?.error || res.statusText);
+            }
+        }
+        catch (error) {
+            throw new Error(error.response?.data?.error || error.message);
+        }
+    }
+}
+exports.Api = Api;

package/dist/confident/evaluate.d.ts

@@ -0,0 +1,11 @@
+import { LLMTestCase, ConversationalTestCase } from "../test-case";
+import { Prompt } from "../prompt";
+export declare function evaluate(params: {
+    metricCollection: string;
+    llmTestCases?: LLMTestCase[];
+    conversationalTestCases?: ConversationalTestCase[];
+    hyperparameters?: {
+        [key: string]: string | number | boolean | Prompt;
+    };
+    identifier?: string;
+}): Promise<void>;