@operor/testing 0.1.0

package/src/TestCaseEvaluator.ts

@@ -0,0 +1,296 @@
+import type { LLMProvider } from '@operor/llm';
+import type { TestCase } from './types.js';
+
+export interface EvaluationResult {
+  passed: boolean;
+  score: number;
+  method: 'exact' | 'contains' | 'similarity' | 'llm_judge';
+  reasoning: string;
+  toolsCorrect: boolean;
+}
+
+export class TestCaseEvaluator {
+  constructor(private llm?: LLMProvider) {}
+
+  async evaluate(
+    testCase: TestCase,
+    agentResponse: string,
+    toolsCalled: Array<{ name: string; params: any; result: any }>,
+    strategy?: 'exact' | 'contains' | 'similarity' | 'semantic'
+  ): Promise<EvaluationResult> {
+    // Validate tools first
+    const toolsCorrect = this.validateTools(testCase.expectedTools, toolsCalled);
+
+    // Choose evaluation strategy based on explicit strategy parameter
+    if (strategy === 'exact') {
+      return this.evaluateByExact(testCase, agentResponse, toolsCorrect);
+    }
+
+    if (strategy === 'contains') {
+      return this.evaluateByContains(testCase, agentResponse, toolsCorrect);
+    }
+
+    if (strategy === 'similarity') {
+      return this.evaluateBySimilarity(testCase, agentResponse, toolsCorrect);
+    }
+
+    if (strategy === 'semantic' && this.llm) {
+      if (testCase.expectedAnswer) {
+        return await this.evaluateByLLMComparison(testCase, agentResponse, toolsCorrect);
+      }
+      return await this.evaluateByLLMJudge(testCase, agentResponse, toolsCorrect);
+    }
+
+    // Default behavior (no strategy specified)
+    if (!this.llm) {
+      // No LLM: use string similarity
+      return this.evaluateBySimilarity(testCase, agentResponse, toolsCorrect);
+    }
+
+    if (testCase.expectedAnswer) {
+      // LLM + expected answer: use LLM comparison
+      return await this.evaluateByLLMComparison(testCase, agentResponse, toolsCorrect);
+    }
+
+    // LLM + no expected answer: use LLM standalone judge
+    return await this.evaluateByLLMJudge(testCase, agentResponse, toolsCorrect);
+  }
+
+  private validateTools(
+    expectedTools: string[] | undefined,
+    toolsCalled: Array<{ name: string; params: any; result: any }>
+  ): boolean {
+    if (!expectedTools || expectedTools.length === 0) {
+      return true; // No tools expected, always pass
+    }
+
+    const calledNames = new Set(toolsCalled.map((t) => t.name));
+    return expectedTools.every((tool) => calledNames.has(tool));
+  }
+
+  private evaluateByExact(
+    testCase: TestCase,
+    agentResponse: string,
+    toolsCorrect: boolean
+  ): EvaluationResult {
+    if (!testCase.expectedAnswer) {
+      return {
+        passed: toolsCorrect,
+        score: toolsCorrect ? 1 : 0,
+        method: 'exact',
+        reasoning: 'No expected answer provided, evaluated tools only',
+        toolsCorrect,
+      };
+    }
+
+    const matches = agentResponse.trim().toLowerCase() === testCase.expectedAnswer.trim().toLowerCase();
+    const passed = matches && toolsCorrect;
+
+    return {
+      passed,
+      score: matches ? 1 : 0,
+      method: 'exact',
+      reasoning: matches ? 'Exact match' : 'Response does not exactly match expected answer',
+      toolsCorrect,
+    };
+  }
+
+  private evaluateByContains(
+    testCase: TestCase,
+    agentResponse: string,
+    toolsCorrect: boolean
+  ): EvaluationResult {
+    if (!testCase.expectedAnswer) {
+      return {
+        passed: toolsCorrect,
+        score: toolsCorrect ? 1 : 0,
+        method: 'contains',
+        reasoning: 'No expected answer provided, evaluated tools only',
+        toolsCorrect,
+      };
+    }
+
+    const normalizeDashes = (s: string) => s.replace(/[\u2013\u2011]/g, '-');
+    const contains = normalizeDashes(agentResponse.toLowerCase()).includes(normalizeDashes(testCase.expectedAnswer.toLowerCase()));
+    const passed = contains && toolsCorrect;
+
+    return {
+      passed,
+      score: contains ? 1 : 0,
+      method: 'contains',
+      reasoning: contains
+        ? `Response contains expected text: "${testCase.expectedAnswer}"`
+        : `Response does not contain expected text: "${testCase.expectedAnswer}"`,
+      toolsCorrect,
+    };
+  }
+
+  private evaluateBySimilarity(
+    testCase: TestCase,
+    agentResponse: string,
+    toolsCorrect: boolean
+  ): EvaluationResult {
+    if (!testCase.expectedAnswer) {
+      // No expected answer and no LLM: can't evaluate, pass by default
+      return {
+        passed: toolsCorrect,
+        score: toolsCorrect ? 1 : 0,
+        method: 'similarity',
+        reasoning: 'No expected answer provided, evaluated tools only',
+        toolsCorrect,
+      };
+    }
+
+    const similarity = this.normalizedLevenshtein(
+      testCase.expectedAnswer.toLowerCase(),
+      agentResponse.toLowerCase()
+    );
+
+    const passed = similarity > 0.7 && toolsCorrect;
+
+    return {
+      passed,
+      score: similarity,
+      method: 'similarity',
+      reasoning: `String similarity: ${(similarity * 100).toFixed(1)}% (threshold: 70%)`,
+      toolsCorrect,
+    };
+  }
+
+  private async evaluateByLLMComparison(
+    testCase: TestCase,
+    agentResponse: string,
+    toolsCorrect: boolean
+  ): Promise<EvaluationResult> {
+    const prompt = `You are evaluating an AI agent's response to a customer question.
+
+Question: ${testCase.question}
+Expected Answer: ${testCase.expectedAnswer}
+Actual Response: ${agentResponse}
+
+Rate the actual response on a scale of 1-5:
+1 = Completely wrong or irrelevant
+2 = Partially correct but missing key information
+3 = Mostly correct with minor issues
+4 = Correct with good quality
+5 = Excellent, matches or exceeds expected answer
+
+Respond with ONLY a JSON object in this format:
+{"score": <1-5>, "reasoning": "<brief explanation>"}`;
+
+    const result = await this.llm!.complete(
+      [{ role: 'user', content: prompt }],
+      { temperature: 0, maxTokens: 200 }
+    );
+
+    let score = 3;
+    let reasoning = 'LLM evaluation completed';
+
+    try {
+      const parsed = JSON.parse(result.text);
+      score = parsed.score;
+      reasoning = parsed.reasoning;
+    } catch {
+      // Fallback: try to extract score from text
+      const match = result.text.match(/score["\s:]+(\d)/i);
+      if (match) {
+        score = parseInt(match[1], 10);
+      }
+      reasoning = result.text.substring(0, 200);
+    }
+
+    const normalizedScore = score / 5;
+    const passed = normalizedScore >= 0.6 && toolsCorrect;
+
+    return {
+      passed,
+      score: normalizedScore,
+      method: 'llm_judge',
+      reasoning: `LLM comparison (${score}/5): ${reasoning}`,
+      toolsCorrect,
+    };
+  }
+
+  private async evaluateByLLMJudge(
+    testCase: TestCase,
+    agentResponse: string,
+    toolsCorrect: boolean
+  ): Promise<EvaluationResult> {
+    const prompt = `You are evaluating an AI agent's response to a customer question.
+
+Question: ${testCase.question}
+Agent Response: ${agentResponse}
+
+Rate the response quality on a scale of 1-5:
+1 = Unhelpful, incorrect, or inappropriate
+2 = Partially helpful but incomplete or unclear
+3 = Adequate, addresses the question reasonably
+4 = Good quality, helpful and accurate
+5 = Excellent, comprehensive and professional
+
+Respond with ONLY a JSON object in this format:
+{"score": <1-5>, "reasoning": "<brief explanation>"}`;
+
+    const result = await this.llm!.complete(
+      [{ role: 'user', content: prompt }],
+      { temperature: 0, maxTokens: 200 }
+    );
+
+    let score = 3;
+    let reasoning = 'LLM evaluation completed';
+
+    try {
+      const parsed = JSON.parse(result.text);
+      score = parsed.score;
+      reasoning = parsed.reasoning;
+    } catch {
+      // Fallback: try to extract score from text
+      const match = result.text.match(/score["\s:]+(\d)/i);
+      if (match) {
+        score = parseInt(match[1], 10);
+      }
+      reasoning = result.text.substring(0, 200);
+    }
+
+    const normalizedScore = score / 5;
+    const passed = normalizedScore >= 0.6 && toolsCorrect;
+
+    return {
+      passed,
+      score: normalizedScore,
+      method: 'llm_judge',
+      reasoning: `LLM standalone judge (${score}/5): ${reasoning}`,
+      toolsCorrect,
+    };
+  }
+
+  private normalizedLevenshtein(s1: string, s2: string): number {
+    const len1 = s1.length;
+    const len2 = s2.length;
+
+    if (len1 === 0) return len2 === 0 ? 1 : 0;
+    if (len2 === 0) return 0;
+
+    const matrix: number[][] = Array.from({ length: len1 + 1 }, () =>
+      Array(len2 + 1).fill(0)
+    );
+
+    for (let i = 0; i <= len1; i++) matrix[i][0] = i;
+    for (let j = 0; j <= len2; j++) matrix[0][j] = j;
+
+    for (let i = 1; i <= len1; i++) {
+      for (let j = 1; j <= len2; j++) {
+        const cost = s1[i - 1] === s2[j - 1] ? 0 : 1;
+        matrix[i][j] = Math.min(
+          matrix[i - 1][j] + 1,
+          matrix[i][j - 1] + 1,
+          matrix[i - 1][j - 1] + cost
+        );
+      }
+    }
+
+    const distance = matrix[len1][len2];
+    const maxLen = Math.max(len1, len2);
+    return 1 - distance / maxLen;
+  }
+}

package/src/TestSuiteRunner.ts

@@ -0,0 +1,151 @@
+import { Operor } from '@operor/core';
+import type { LLMProvider } from '@operor/llm';
+import { MockProvider } from '@operor/provider-mock';
+import type { TestCase, TestCaseResult, TestSuiteResult } from './types.js';
+import { TestCaseEvaluator } from './TestCaseEvaluator.js';
+
+export interface TestSuiteRunnerConfig {
+  agentOS: Operor;
+  llm?: LLMProvider;
+  timeout?: number;
+  strategy?: 'exact' | 'contains' | 'similarity' | 'semantic';
+}
+
+export class TestSuiteRunner {
+  private evaluator: TestCaseEvaluator;
+  private agentOS: Operor;
+  private timeout: number;
+  private strategy?: 'exact' | 'contains' | 'similarity' | 'semantic';
+
+  constructor(config: TestSuiteRunnerConfig) {
+    this.agentOS = config.agentOS;
+    this.evaluator = new TestCaseEvaluator(config.llm);
+    this.timeout = config.timeout || 30000;
+    this.strategy = config.strategy;
+  }
+
+  async runSuite(testCases: TestCase[]): Promise<TestSuiteResult> {
+    const results: TestCaseResult[] = [];
+    const startTime = Date.now();
+
+    for (const testCase of testCases) {
+      const result = await this.runTestCase(testCase);
+      results.push(result);
+    }
+
+    const totalDuration = Date.now() - startTime;
+    const passed = results.filter((r) => r.evaluation.passed).length;
+    const failed = results.length - passed;
+    const averageScore =
+      results.reduce((sum, r) => sum + r.evaluation.score, 0) / results.length;
+    const totalCost = results.reduce((sum, r) => sum + r.cost, 0);
+
+    // Group by tags
+    const byTag: Record<string, { total: number; passed: number; avgScore: number }> = {};
+    for (const result of results) {
+      const tags = result.testCase.tags || ['untagged'];
+      for (const tag of tags) {
+        if (!byTag[tag]) {
+          byTag[tag] = { total: 0, passed: 0, avgScore: 0 };
+        }
+        byTag[tag].total++;
+        if (result.evaluation.passed) {
+          byTag[tag].passed++;
+        }
+        byTag[tag].avgScore += result.evaluation.score;
+      }
+    }
+
+    // Calculate average scores per tag
+    for (const tag in byTag) {
+      byTag[tag].avgScore /= byTag[tag].total;
+    }
+
+    return {
+      total: results.length,
+      passed,
+      failed,
+      averageScore,
+      byTag,
+      results,
+      totalDuration,
+      totalCost,
+    };
+  }
+
+  private async runTestCase(testCase: TestCase): Promise<TestCaseResult> {
+    const startTime = Date.now();
+    let agentResponse = '';
+    let toolsCalled: Array<{ name: string; params: any; result: any }> = [];
+    let cost = 0;
+
+    try {
+      // Create a promise that resolves when we get a response
+      const responsePromise = new Promise<{
+        text: string;
+        toolCalls?: Array<{ name: string; params: any; result: any }>;
+        cost: number;
+      }>((resolve, reject) => {
+        const timeoutId = setTimeout(() => {
+          reject(new Error(`Test case ${testCase.id} timed out after ${this.timeout}ms`));
+        }, this.timeout);
+
+        // Listen for the response
+        this.agentOS.once('message:processed', (event: any) => {
+          clearTimeout(timeoutId);
+          resolve({
+            text: event.response.text,
+            toolCalls: event.response.toolCalls || [],
+            cost: event.cost || 0,
+          });
+        });
+
+        // Listen for errors
+        this.agentOS.once('error', (event: any) => {
+          clearTimeout(timeoutId);
+          reject(event.error);
+        });
+      });
+
+      // Get the mock provider
+      const mockProvider = Array.from((this.agentOS as any).providers.values()).find(
+        (p: any) => p.name === 'mock'
+      ) as MockProvider | undefined;
+
+      if (!mockProvider) {
+        throw new Error('MockProvider not found in Operor');
+      }
+
+      // Simulate incoming message
+      const testPhone = testCase.persona || 'test-user';
+      mockProvider.simulateIncomingMessage(testPhone, testCase.question);
+
+      // Wait for response
+      const response = await responsePromise;
+      agentResponse = response.text;
+      toolsCalled = response.toolCalls || [];
+      cost = response.cost;
+    } catch (error) {
+      agentResponse = `Error: ${error instanceof Error ? error.message : String(error)}`;
+    }
+
+    const duration = Date.now() - startTime;
+
+    // Evaluate the response
+    const evaluation = await this.evaluator.evaluate(
+      testCase,
+      agentResponse,
+      toolsCalled,
+      this.strategy
+    );
+
+    return {
+      testCase,
+      agentResponse,
+      toolsCalled,
+      evaluation,
+      duration,
+      cost,
+    };
+  }
+}

package/src/__tests__/CSVLoader.test.ts

@@ -0,0 +1,122 @@
+import { describe, it, expect } from 'vitest';
+import { CSVLoader } from '../CSVLoader.js';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { dirname } from 'node:path';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const fixturesDir = resolve(__dirname, '../../fixtures');
+
+describe('CSVLoader', () => {
+  describe('fromCSVString', () => {
+    it('parses basic CSV', () => {
+      const csv = `id,question,expected_answer,expected_tools,persona,tags
+t1,Hello,Hi there,,friendly,greeting`;
+      const cases = CSVLoader.fromCSVString(csv);
+      expect(cases).toHaveLength(1);
+      expect(cases[0]).toEqual({
+        id: 't1',
+        question: 'Hello',
+        expectedAnswer: 'Hi there',
+        persona: 'friendly',
+        tags: ['greeting'],
+      });
+    });
+
+    it('handles quoted fields with commas', () => {
+      const csv = `id,question,expected_answer,expected_tools,persona,tags
+t1,"Hello, world",response,"toolA,toolB",,`;
+      const cases = CSVLoader.fromCSVString(csv);
+      expect(cases[0].question).toBe('Hello, world');
+      expect(cases[0].expectedTools).toEqual(['toolA', 'toolB']);
+    });
+
+    it('strips UTF-8 BOM', () => {
+      const csv = `\uFEFFid,question,expected_answer,expected_tools,persona,tags
+t1,Hello,Hi,,friendly,`;
+      const cases = CSVLoader.fromCSVString(csv);
+      expect(cases[0].id).toBe('t1');
+    });
+
+    it('skips empty optional fields', () => {
+      const csv = `id,question,expected_answer,expected_tools,persona,tags
+t1,Hello,,,, `;
+      const cases = CSVLoader.fromCSVString(csv);
+      expect(cases[0]).toEqual({ id: 't1', question: 'Hello' });
+    });
+
+    it('throws on missing id', () => {
+      const csv = `id,question
+,Hello`;
+      expect(() => CSVLoader.fromCSVString(csv)).toThrow('missing required field');
+    });
+
+    it('throws on missing question', () => {
+      const csv = `id,question
+t1,`;
+      expect(() => CSVLoader.fromCSVString(csv)).toThrow('missing required field');
+    });
+
+    it('handles multiple tags', () => {
+      const csv = `id,question,expected_answer,expected_tools,persona,tags
+t1,Hello,,,,greeting,edge-case`;
+      const cases = CSVLoader.fromCSVString(csv);
+      expect(cases[0].tags).toContain('greeting');
+    });
+  });
+
+  describe('fromJSON', () => {
+    it('parses JSON array', () => {
+      const json = JSON.stringify([
+        { id: 't1', question: 'Hello', expectedAnswer: 'Hi' },
+      ]);
+      const cases = CSVLoader.fromJSON(json);
+      expect(cases).toHaveLength(1);
+      expect(cases[0].expectedAnswer).toBe('Hi');
+    });
+
+    it('parses object with testCases key', () => {
+      const json = JSON.stringify({
+        testCases: [{ id: 't1', question: 'Hello' }],
+      });
+      const cases = CSVLoader.fromJSON(json);
+      expect(cases).toHaveLength(1);
+    });
+
+    it('throws on missing required fields', () => {
+      const json = JSON.stringify([{ id: 't1' }]);
+      expect(() => CSVLoader.fromJSON(json)).toThrow('missing required field');
+    });
+
+    it('throws on invalid structure', () => {
+      expect(() => CSVLoader.fromJSON('{"foo": "bar"}')).toThrow(
+        'JSON must be an array'
+      );
+    });
+  });
+
+  describe('fromFile', () => {
+    it('loads sample CSV fixture', async () => {
+      const cases = await CSVLoader.fromFile(
+        resolve(fixturesDir, 'sample-tests.csv')
+      );
+      expect(cases.length).toBeGreaterThanOrEqual(5);
+      expect(cases[0].id).toBe('greeting-1');
+      expect(cases[0].question).toBe('Hello');
+    });
+
+    it('loads JSON file', async () => {
+      const tmpPath = resolve(fixturesDir, '_test-tmp.json');
+      const { writeFile, unlink } = await import('node:fs/promises');
+      const data = [{ id: 'j1', question: 'Hi from JSON' }];
+      await writeFile(tmpPath, JSON.stringify(data));
+      try {
+        const cases = await CSVLoader.fromFile(tmpPath);
+        expect(cases).toHaveLength(1);
+        expect(cases[0].question).toBe('Hi from JSON');
+      } finally {
+        await unlink(tmpPath);
+      }
+    });
+  });
+});