@operor/testing 0.1.0

package/src/__tests__/ConversationEvaluator.test.ts

@@ -0,0 +1,221 @@
+import { describe, it, expect } from 'vitest';
+import { ConversationEvaluator } from '../ConversationEvaluator.js';
+import type { ConversationTurn, ConversationSuccessCriteria } from '../types.js';
+
+const agentTurns: ConversationTurn[] = [
+  { role: 'customer', message: 'Where is my order #1234?' },
+  {
+    role: 'agent',
+    message: 'Let me look that up for you. Your order #1234 is currently in transit and should arrive by Friday.',
+    toolCalls: [{ name: 'get_order', params: { orderId: '1234' }, result: { status: 'in_transit' } }],
+  },
+  { role: 'customer', message: 'Great, thanks!' },
+  { role: 'agent', message: 'You\'re welcome! Is there anything else I can help with?' },
+];
+
+describe('ConversationEvaluator', () => {
+  describe('criteria-based evaluation (no LLM)', () => {
+    const evaluator = new ConversationEvaluator();
+
+    it('passes tool_called criteria when tool was called', async () => {
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [{ name: 'get_order', params: { orderId: '1234' }, result: { status: 'in_transit' } }],
+        successCriteria: [{ type: 'tool_called', value: 'get_order' }],
+      });
+      expect(result.overall).toBe('pass');
+      expect(result.criteriaResults![0].passed).toBe(true);
+    });
+
+    it('fails tool_called criteria when tool was not called', async () => {
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+        successCriteria: [{ type: 'tool_called', value: 'create_discount' }],
+      });
+      expect(result.overall).toBe('fail');
+      expect(result.criteriaResults![0].passed).toBe(false);
+      expect(result.criteriaResults![0].details).toContain('create_discount');
+    });
+
+    it('passes response_contains criteria (case-insensitive)', async () => {
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+        successCriteria: [{ type: 'response_contains', value: 'in transit' }],
+      });
+      expect(result.overall).toBe('pass');
+      expect(result.criteriaResults![0].passed).toBe(true);
+    });
+
+    it('fails response_contains when text not found', async () => {
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+        successCriteria: [{ type: 'response_contains', value: 'refund issued' }],
+      });
+      expect(result.overall).toBe('fail');
+    });
+
+    it('passes turns_under criteria when under limit', async () => {
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+        successCriteria: [{ type: 'turns_under', value: 10 }],
+      });
+      expect(result.overall).toBe('pass');
+      expect(result.criteriaResults![0].details).toContain('4 turns');
+    });
+
+    it('fails turns_under criteria when over limit', async () => {
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+        successCriteria: [{ type: 'turns_under', value: 2 }],
+      });
+      expect(result.overall).toBe('fail');
+    });
+
+    it('evaluates custom criteria with function', async () => {
+      const customFn = (turns: ConversationTurn[]) =>
+        turns.some((t) => t.role === 'agent' && t.message.includes('welcome'));
+
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+        successCriteria: [{ type: 'custom', value: customFn }],
+      });
+      expect(result.overall).toBe('pass');
+    });
+
+    it('returns partial when some criteria pass and some fail', async () => {
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+        successCriteria: [
+          { type: 'tool_called', value: 'get_order' },
+          { type: 'tool_called', value: 'create_discount' },
+        ],
+      });
+      expect(result.overall).toBe('partial');
+      expect(result.criteriaResults![0].passed).toBe(true);
+      expect(result.criteriaResults![1].passed).toBe(false);
+    });
+
+    it('returns pass with no criteria and no LLM', async () => {
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+      });
+      expect(result.overall).toBe('fail');
+      expect(result.feedback).toBe('No criteria specified');
+    });
+
+    it('handles multiple criteria all passing', async () => {
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+        successCriteria: [
+          { type: 'tool_called', value: 'get_order' },
+          { type: 'response_contains', value: 'transit' },
+          { type: 'turns_under', value: 10 },
+        ],
+      });
+      expect(result.overall).toBe('pass');
+      expect(result.criteriaResults!.every((cr) => cr.passed)).toBe(true);
+    });
+  });
+
+  describe('LLM-based evaluation', () => {
+    it('calls LLM with temperature 0 and parses response', async () => {
+      const mockLLM = {
+        complete: async (messages: any[], options: any) => {
+          expect(options.temperature).toBe(0);
+          expect(messages[0].content).toContain('Order tracking');
+          return {
+            text: JSON.stringify({
+              overall: 'pass',
+              scores: { accuracy: 5, toolUsage: 4, tone: 5, resolution: 4 },
+              feedback: 'Agent handled the order inquiry well.',
+            }),
+          };
+        },
+      };
+
+      const evaluator = new ConversationEvaluator({ llmProvider: mockLLM as any });
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [{ name: 'get_order', params: { orderId: '1234' }, result: { status: 'in_transit' } }],
+      });
+
+      expect(result.overall).toBe('pass');
+      expect(result.scores.accuracy).toBe(5);
+      expect(result.scores.toolUsage).toBe(4);
+      expect(result.feedback).toContain('order inquiry');
+    });
+
+    it('downgrades LLM pass to partial when criteria fail', async () => {
+      const mockLLM = {
+        complete: async () => ({
+          text: JSON.stringify({
+            overall: 'pass',
+            scores: { accuracy: 5, toolUsage: 5, tone: 5, resolution: 5 },
+            feedback: 'Perfect.',
+          }),
+        }),
+      };
+
+      const evaluator = new ConversationEvaluator({ llmProvider: mockLLM as any });
+      const result = await evaluator.evaluate({
+        scenario: 'Order tracking',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+        successCriteria: [{ type: 'tool_called', value: 'nonexistent_tool' }],
+      });
+
+      expect(result.overall).toBe('partial');
+      expect(result.criteriaResults![0].passed).toBe(false);
+    });
+
+    it('handles malformed LLM response gracefully', async () => {
+      const mockLLM = {
+        complete: async () => ({ text: 'not valid json at all' }),
+      };
+
+      const evaluator = new ConversationEvaluator({ llmProvider: mockLLM as any });
+      const result = await evaluator.evaluate({
+        scenario: 'Test',
+        persona: 'polite',
+        turns: agentTurns,
+        toolsCalled: [],
+      });
+
+      expect(result.overall).toBe('fail');
+      expect(result.scores.accuracy).toBe(1);
+      expect(result.feedback).toContain('Failed to parse');
+    });
+  });
+});

package/src/__tests__/ConversationRunner.test.ts

@@ -0,0 +1,270 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { ConversationRunner } from '../ConversationRunner.js';
+import { CustomerSimulator } from '../CustomerSimulator.js';
+import { ConversationEvaluator } from '../ConversationEvaluator.js';
+import type { ConversationScenario } from '../types.js';
+import { EventEmitter } from 'node:events';
+
+// Minimal mock Operor that emits events
+function createMockOperor(responses: string[], toolCalls?: any[][]) {
+  const emitter = new EventEmitter();
+  let callIndex = 0;
+
+  const mockProvider = {
+    name: 'mock',
+    simulateIncomingMessage: (_from: string, _text: string) => {
+      // Simulate async agent processing
+      setTimeout(() => {
+        const idx = callIndex++;
+        emitter.emit('message:processed', {
+          response: {
+            text: responses[idx] || 'Default response',
+            toolCalls: toolCalls?.[idx] || [],
+          },
+          cost: 0.001,
+        });
+      }, 5);
+    },
+  };
+
+  const providers = new Map();
+  providers.set('mock', mockProvider);
+
+  return Object.assign(emitter, { providers });
+}
+
+describe('ConversationRunner', () => {
+  const scriptedScenario: ConversationScenario = {
+    id: 'order-tracking',
+    name: 'Order Tracking',
+    description: 'Customer asks about a delayed order',
+    persona: 'polite',
+    maxTurns: 3,
+    scriptedResponses: [
+      'Where is my order #1234?',
+      'When will it arrive?',
+    ],
+    successCriteria: [
+      { type: 'response_contains', value: 'order' },
+    ],
+  };
+
+  it('runs a scripted scenario end-to-end', async () => {
+    const agentOS = createMockOperor([
+      'Let me look up order #1234 for you. It is currently in transit.',
+      'It should arrive by Friday.',
+    ]);
+
+    const runner = new ConversationRunner({
+      agentOS: agentOS as any,
+      customerSimulator: new CustomerSimulator(),
+      conversationEvaluator: new ConversationEvaluator(),
+    });
+
+    const result = await runner.runScenario(scriptedScenario);
+
+    expect(result.scenario.id).toBe('order-tracking');
+    expect(result.turns).toHaveLength(4); // 2 customer + 2 agent
+    expect(result.turns[0].role).toBe('customer');
+    expect(result.turns[0].message).toBe('Where is my order #1234?');
+    expect(result.turns[1].role).toBe('agent');
+    expect(result.turns[1].message).toContain('order #1234');
+    expect(result.turns[2].role).toBe('customer');
+    expect(result.turns[2].message).toBe('When will it arrive?');
+    expect(result.turns[3].role).toBe('agent');
+    expect(result.duration).toBeGreaterThan(0);
+    expect(result.cost).toBeGreaterThan(0);
+  });
+
+  it('evaluates success criteria from scenario', async () => {
+    const agentOS = createMockOperor([
+      'Your order is on its way!',
+      'It arrives Friday.',
+    ]);
+
+    const runner = new ConversationRunner({
+      agentOS: agentOS as any,
+      customerSimulator: new CustomerSimulator(),
+      conversationEvaluator: new ConversationEvaluator(),
+    });
+
+    const result = await runner.runScenario(scriptedScenario);
+    expect(result.passed).toBe(true);
+    expect(result.evaluation.overall).toBe('pass');
+  });
+
+  it('fails when success criteria not met', async () => {
+    const scenario: ConversationScenario = {
+      ...scriptedScenario,
+      successCriteria: [
+        { type: 'tool_called', value: 'get_order' },
+      ],
+    };
+
+    const agentOS = createMockOperor([
+      'I can help with that.',
+      'Let me check.',
+    ]);
+
+    const runner = new ConversationRunner({
+      agentOS: agentOS as any,
+      customerSimulator: new CustomerSimulator(),
+      conversationEvaluator: new ConversationEvaluator(),
+    });
+
+    const result = await runner.runScenario(scenario);
+    expect(result.passed).toBe(false);
+  });
+
+  it('tracks tool calls across turns', async () => {
+    const agentOS = createMockOperor(
+      ['Found your order.', 'Discount applied.'],
+      [
+        [{ name: 'get_order', params: { id: '1234' }, result: { status: 'delayed' } }],
+        [{ name: 'create_discount', params: { amount: 10 }, result: { code: 'SORRY10' } }],
+      ]
+    );
+
+    const runner = new ConversationRunner({
+      agentOS: agentOS as any,
+      customerSimulator: new CustomerSimulator(),
+      conversationEvaluator: new ConversationEvaluator(),
+    });
+
+    const result = await runner.runScenario(scriptedScenario);
+    expect(result.turns[1].toolCalls).toHaveLength(1);
+    expect(result.turns[1].toolCalls![0].name).toBe('get_order');
+    expect(result.turns[3].toolCalls).toHaveLength(1);
+    expect(result.turns[3].toolCalls![0].name).toBe('create_discount');
+  });
+
+  it('respects maxTurns limit', async () => {
+    const scenario: ConversationScenario = {
+      id: 'limited',
+      name: 'Limited Turns',
+      description: 'Test max turns',
+      persona: 'verbose',
+      maxTurns: 1,
+      scriptedResponses: ['Hello', 'More', 'Even more'],
+    };
+
+    const agentOS = createMockOperor(['Hi there!']);
+
+    const runner = new ConversationRunner({
+      agentOS: agentOS as any,
+      customerSimulator: new CustomerSimulator(),
+      conversationEvaluator: new ConversationEvaluator(),
+    });
+
+    const result = await runner.runScenario(scenario);
+    // maxTurns=1 means only 1 exchange (1 customer + 1 agent = 2 turns)
+    expect(result.turns).toHaveLength(2);
+  });
+
+  it('handles agent timeout gracefully', async () => {
+    const emitter = new EventEmitter();
+    const mockProvider = {
+      name: 'mock',
+      simulateIncomingMessage: () => {
+        // Never emits response — simulates timeout
+      },
+    };
+    const providers = new Map();
+    providers.set('mock', mockProvider);
+    const agentOS = Object.assign(emitter, { providers });
+
+    const runner = new ConversationRunner({
+      agentOS: agentOS as any,
+      customerSimulator: new CustomerSimulator(),
+      conversationEvaluator: new ConversationEvaluator(),
+      timeout: 50, // Very short timeout
+    });
+
+    const result = await runner.runScenario(scriptedScenario);
+    expect(result.passed).toBe(false);
+    expect(result.evaluation.feedback).toContain('timed out');
+  });
+
+  it('runs multiple scenarios sequentially', async () => {
+    const scenario2: ConversationScenario = {
+      id: 'greeting',
+      name: 'Simple Greeting',
+      description: 'Customer says hello',
+      persona: 'terse',
+      maxTurns: 1,
+      scriptedResponses: ['Hi'],
+      successCriteria: [{ type: 'turns_under', value: 10 }],
+    };
+
+    // Need separate agentOS instances since event listeners are consumed
+    let callCount = 0;
+    const emitter = new EventEmitter();
+    const mockProvider = {
+      name: 'mock',
+      simulateIncomingMessage: () => {
+        setTimeout(() => {
+          callCount++;
+          emitter.emit('message:processed', {
+            response: { text: `Response ${callCount}`, toolCalls: [] },
+            cost: 0,
+          });
+        }, 5);
+      },
+    };
+    const providers = new Map();
+    providers.set('mock', mockProvider);
+    const agentOS = Object.assign(emitter, { providers });
+
+    const runner = new ConversationRunner({
+      agentOS: agentOS as any,
+      customerSimulator: new CustomerSimulator(),
+      conversationEvaluator: new ConversationEvaluator(),
+    });
+
+    const results = await runner.runScenarios([scriptedScenario, scenario2]);
+    expect(results).toHaveLength(2);
+    expect(results[0].scenario.id).toBe('order-tracking');
+    expect(results[1].scenario.id).toBe('greeting');
+  });
+
+  it('handles error events from Operor', async () => {
+    const emitter = new EventEmitter();
+    const mockProvider = {
+      name: 'mock',
+      simulateIncomingMessage: () => {
+        setTimeout(() => {
+          emitter.emit('error', { error: new Error('Agent processing failed') });
+        }, 5);
+      },
+    };
+    const providers = new Map();
+    providers.set('mock', mockProvider);
+    const agentOS = Object.assign(emitter, { providers });
+
+    const runner = new ConversationRunner({
+      agentOS: agentOS as any,
+      customerSimulator: new CustomerSimulator(),
+      conversationEvaluator: new ConversationEvaluator(),
+    });
+
+    const result = await runner.runScenario(scriptedScenario);
+    expect(result.passed).toBe(false);
+    expect(result.evaluation.feedback).toContain('Agent processing failed');
+  });
+
+  it('throws when MockProvider not found', async () => {
+    const emitter = new EventEmitter();
+    const providers = new Map(); // Empty — no mock provider
+    const agentOS = Object.assign(emitter, { providers });
+
+    const runner = new ConversationRunner({
+      agentOS: agentOS as any,
+      customerSimulator: new CustomerSimulator(),
+      conversationEvaluator: new ConversationEvaluator(),
+    });
+
+    const result = await runner.runScenario(scriptedScenario);
+    expect(result.passed).toBe(false);
+    expect(result.evaluation.feedback).toContain('MockProvider not found');
+  });
+});

package/src/__tests__/CustomerSimulator.test.ts

@@ -0,0 +1,160 @@
+import { describe, it, expect } from 'vitest';
+import { CustomerSimulator } from '../CustomerSimulator.js';
+import type { ConversationTurn } from '../types.js';
+
+describe('CustomerSimulator', () => {
+  describe('script mode', () => {
+    const sim = new CustomerSimulator();
+
+    it('returns first scripted response on turn 0', async () => {
+      const result = await sim.generateMessage('polite', [], {
+        scriptedResponses: ['Hi, where is my order?', 'Order #1234', 'Thanks!'],
+        currentTurn: 0,
+      });
+      expect(result.message).toBe('Hi, where is my order?');
+      expect(result.shouldContinue).toBe(true);
+    });
+
+    it('returns subsequent responses based on currentTurn', async () => {
+      const history: ConversationTurn[] = [
+        { role: 'customer', message: 'Hi, where is my order?' },
+        { role: 'agent', message: 'Can you give me your order number?' },
+      ];
+      const result = await sim.generateMessage('polite', history, {
+        scriptedResponses: ['Hi, where is my order?', 'Order #1234', 'Thanks!'],
+        currentTurn: 1,
+      });
+      expect(result.message).toBe('Order #1234');
+      expect(result.shouldContinue).toBe(true);
+    });
+
+    it('sets shouldContinue=false on last scripted response', async () => {
+      const result = await sim.generateMessage('polite', [], {
+        scriptedResponses: ['Hi, where is my order?', 'Order #1234', 'Thanks!'],
+        currentTurn: 2,
+      });
+      expect(result.message).toBe('Thanks!');
+      expect(result.shouldContinue).toBe(false);
+    });
+
+    it('returns last response with shouldContinue=false when turns exceed scripts', async () => {
+      const result = await sim.generateMessage('polite', [], {
+        scriptedResponses: ['Hello', 'Goodbye'],
+        currentTurn: 5,
+      });
+      expect(result.message).toBe('Goodbye');
+      expect(result.shouldContinue).toBe(false);
+    });
+
+    it('infers turn from history when currentTurn not provided', async () => {
+      const history: ConversationTurn[] = [
+        { role: 'customer', message: 'Hi' },
+        { role: 'agent', message: 'Hello!' },
+      ];
+      const result = await sim.generateMessage('polite', history, {
+        scriptedResponses: ['Hi', 'My order is late', 'Thanks'],
+      });
+      // 1 customer message in history → turn 1
+      expect(result.message).toBe('My order is late');
+      expect(result.shouldContinue).toBe(true);
+    });
+
+    it('works with a single scripted response', async () => {
+      const result = await sim.generateMessage('terse', [], {
+        scriptedResponses: ['Just checking'],
+        currentTurn: 0,
+      });
+      expect(result.message).toBe('Just checking');
+      expect(result.shouldContinue).toBe(false);
+    });
+  });
+
+  describe('LLM mode', () => {
+    it('throws when no LLM provider and no scripted responses', async () => {
+      const sim = new CustomerSimulator();
+      await expect(
+        sim.generateMessage('polite', [])
+      ).rejects.toThrow('CustomerSimulator requires an LLM provider');
+    });
+
+    it('calls LLM with correct messages and parses JSON response', async () => {
+      const mockLLM = {
+        complete: async (messages: any[], options: any) => {
+          // Verify system prompt is first
+          expect(messages[0].role).toBe('system');
+          expect(messages[0].content).toContain('simulating a customer');
+          // Verify temperature
+          expect(options.temperature).toBe(0.7);
+          return {
+            text: '{"message": "I need help with my order", "shouldContinue": true}',
+          };
+        },
+      };
+
+      const sim = new CustomerSimulator({ llmProvider: mockLLM as any });
+      const result = await sim.generateMessage('frustrated', [
+        { role: 'agent', message: 'How can I help you today?' },
+      ]);
+      expect(result.message).toBe('I need help with my order');
+      expect(result.shouldContinue).toBe(true);
+    });
+
+    it('handles non-JSON LLM response gracefully', async () => {
+      const mockLLM = {
+        complete: async () => ({
+          text: 'I just want to know where my package is!',
+        }),
+      };
+
+      const sim = new CustomerSimulator({ llmProvider: mockLLM as any });
+      const result = await sim.generateMessage('frustrated', []);
+      expect(result.message).toBe('I just want to know where my package is!');
+      expect(result.shouldContinue).toBe(true);
+    });
+
+    it('includes scenario context in system prompt', async () => {
+      let capturedMessages: any[] = [];
+      const mockLLM = {
+        complete: async (messages: any[]) => {
+          capturedMessages = messages;
+          return { text: '{"message": "test", "shouldContinue": false}' };
+        },
+      };
+
+      const sim = new CustomerSimulator({ llmProvider: mockLLM as any });
+      await sim.generateMessage('polite', [], {
+        scenario: 'Customer ordered headphones 5 days ago',
+        maxTurns: 5,
+        currentTurn: 2,
+      });
+
+      const systemMsg = capturedMessages[0].content;
+      expect(systemMsg).toContain('Customer ordered headphones 5 days ago');
+      expect(systemMsg).toContain('maximum of 5 turns');
+      expect(systemMsg).toContain('turn 2');
+    });
+
+    it('converts history turns to LLM messages', async () => {
+      let capturedMessages: any[] = [];
+      const mockLLM = {
+        complete: async (messages: any[]) => {
+          capturedMessages = messages;
+          return { text: '{"message": "ok", "shouldContinue": false}' };
+        },
+      };
+
+      const sim = new CustomerSimulator({ llmProvider: mockLLM as any });
+      await sim.generateMessage('polite', [
+        { role: 'customer', message: 'Hi there' },
+        { role: 'agent', message: 'Hello! How can I help?' },
+        { role: 'customer', message: 'I have a question' },
+      ]);
+
+      // system + 3 history messages
+      expect(capturedMessages).toHaveLength(4);
+      expect(capturedMessages[1]).toEqual({ role: 'user', content: 'Hi there' });
+      expect(capturedMessages[2]).toEqual({ role: 'assistant', content: 'Hello! How can I help?' });
+      expect(capturedMessages[3]).toEqual({ role: 'user', content: 'I have a question' });
+    });
+  });
+});