@operor/testing 0.1.0

package/src/scenarios/order-tracking.ts

@@ -0,0 +1,56 @@
+import type { ConversationScenario } from '../types.js';
+
+export const delayedOrderScenario: ConversationScenario = {
+  id: 'order-tracking-delayed',
+  name: 'Delayed Order with Compensation',
+  description: 'Customer asks about a delayed order and expects compensation offer',
+  persona: 'polite',
+  maxTurns: 3,
+  expectedTools: ['get_order', 'create_discount'],
+  expectedOutcome: 'Agent looks up order, identifies delay, and offers discount code',
+  successCriteria: [
+    { type: 'tool_called', value: 'get_order' },
+    { type: 'tool_called', value: 'create_discount' },
+  ],
+  scriptedResponses: [
+    'Where is my order #12345? It was supposed to arrive two days ago.',
+    'That is really frustrating. Is there anything you can do to make up for this?',
+    'Thank you, I appreciate the discount code.',
+  ],
+};
+
+export const onTimeOrderScenario: ConversationScenario = {
+  id: 'order-tracking-ontime',
+  name: 'On-time Order Status Check',
+  description: 'Customer asks about an order that is on time or already delivered',
+  persona: 'polite',
+  maxTurns: 2,
+  expectedTools: ['get_order'],
+  expectedOutcome: 'Agent looks up order and confirms status without offering compensation',
+  successCriteria: [
+    { type: 'tool_called', value: 'get_order' },
+    { type: 'turns_under', value: 3 },
+  ],
+  scriptedResponses: [
+    'Can you check the status of order #67890?',
+    'Great, thanks for the update!',
+  ],
+};
+
+export const orderNotFoundScenario: ConversationScenario = {
+  id: 'order-tracking-not-found',
+  name: 'Order Not Found',
+  description: 'Customer asks about an order that does not exist',
+  persona: 'confused',
+  maxTurns: 2,
+  expectedTools: ['get_order'],
+  expectedOutcome: 'Agent attempts lookup and explains order was not found',
+  successCriteria: [
+    { type: 'tool_called', value: 'get_order' },
+    { type: 'response_contains', value: 'not found' },
+  ],
+  scriptedResponses: [
+    'Where is order #99999?',
+    'Hmm, let me double check the order number and get back to you.',
+  ],
+};

package/src/scenarios.ts

@@ -0,0 +1,142 @@
+import type { ConversationScenario } from './types.js';
+
+export const ECOMMERCE_SCENARIOS: ConversationScenario[] = [
+  {
+    id: 'delayed-order-compensation',
+    name: 'Delayed order with compensation',
+    description: 'Customer asks about a delayed order and expects compensation',
+    persona: 'Frustrated customer whose order #12345 was supposed to arrive 3 days ago',
+    maxTurns: 6,
+    expectedTools: ['get_order', 'create_discount'],
+    expectedOutcome: 'Agent finds order, acknowledges delay, offers discount',
+    scriptedResponses: [
+      'Where is my order #12345? It was supposed to arrive 3 days ago!',
+      'This is really frustrating. Can you do anything to make up for this?',
+      'Okay, I appreciate the discount. Thank you.',
+    ],
+  },
+  {
+    id: 'order-not-found',
+    name: 'Order not found',
+    description: 'Customer asks about an order that does not exist in the system',
+    persona: 'Confused customer who may have the wrong order number',
+    maxTurns: 4,
+    expectedTools: ['get_order'],
+    expectedOutcome: 'Agent attempts lookup, explains order not found, asks customer to verify',
+    scriptedResponses: [
+      'Can you check on order #99999 for me?',
+      'Hmm, let me double check the number and get back to you.',
+    ],
+  },
+  {
+    id: 'product-inquiry',
+    name: 'Product inquiry',
+    description: 'Customer searches for a product and asks about availability',
+    persona: 'Curious shopper looking for electronics',
+    maxTurns: 4,
+    expectedTools: ['search_products'],
+    expectedOutcome: 'Agent searches products and provides relevant results',
+    scriptedResponses: [
+      'Do you have any wireless headphones in stock?',
+      'What about the price range? Anything under $200?',
+      'Great, thanks for the info!',
+    ],
+  },
+  {
+    id: 'return-request',
+    name: 'Return request',
+    description: 'Customer wants to return a recently delivered item',
+    persona: 'Polite customer who received a defective product from order #67890',
+    maxTurns: 5,
+    expectedTools: ['get_order'],
+    expectedOutcome: 'Agent looks up order, acknowledges issue, explains return process',
+    scriptedResponses: [
+      'I received order #67890 but one of the items is defective. I would like to return it.',
+      'Yes, the wireless mouse stopped working after one day.',
+      'Okay, how do I send it back?',
+    ],
+  },
+  {
+    id: 'greeting',
+    name: 'Simple greeting',
+    description: 'Customer says hello and expects a friendly welcome',
+    persona: 'Friendly first-time visitor',
+    maxTurns: 2,
+    expectedTools: [],
+    expectedOutcome: 'Agent responds with a friendly greeting and offers help',
+    scriptedResponses: [
+      'Hello!',
+    ],
+  },
+  {
+    id: 'billing-dispute',
+    name: 'Billing dispute',
+    description: 'Customer believes they were charged incorrectly',
+    persona: 'Concerned customer who noticed a double charge on order #12345',
+    maxTurns: 5,
+    expectedTools: ['get_order'],
+    expectedOutcome: 'Agent looks up order, reviews charges, and addresses the billing concern',
+    scriptedResponses: [
+      'I think I was charged twice for order #12345. Can you check?',
+      'My credit card shows two charges of $299.99 on the same day.',
+      'Can you escalate this to someone who can issue a correction?',
+    ],
+  },
+  {
+    id: 'multi-issue',
+    name: 'Multi-issue conversation',
+    description: 'Customer has multiple problems: a delayed order and a product question',
+    persona: 'Busy customer who wants to resolve everything in one conversation',
+    maxTurns: 6,
+    expectedTools: ['get_order', 'search_products'],
+    expectedOutcome: 'Agent handles both issues sequentially without losing context',
+    scriptedResponses: [
+      'Two things: first, where is my order #12345?',
+      'Okay thanks. Also, do you carry mechanical keyboards?',
+      'Nice, I might order one. Can you also give me a discount for the late delivery?',
+      'Sounds good, thanks for handling both issues.',
+    ],
+  },
+  {
+    id: 'lead-qualification',
+    name: 'Lead qualification',
+    description: 'Potential customer asking pre-purchase questions about products and shipping',
+    persona: 'Prospective buyer evaluating whether to make a purchase',
+    maxTurns: 4,
+    expectedTools: ['search_products'],
+    expectedOutcome: 'Agent answers product questions and encourages purchase',
+    scriptedResponses: [
+      'I am thinking about buying some electronics. What do you have?',
+      'How fast is shipping usually?',
+      'Do you offer any discounts for first-time buyers?',
+    ],
+  },
+  {
+    id: 'frustrated-escalation',
+    name: 'Frustrated customer escalation',
+    description: 'Angry customer escalates through multiple complaints',
+    persona: 'Very frustrated customer who has contacted support multiple times about order #12345',
+    maxTurns: 6,
+    expectedTools: ['get_order', 'create_discount'],
+    expectedOutcome: 'Agent remains professional, empathizes, and offers concrete resolution',
+    scriptedResponses: [
+      'This is the THIRD time I am contacting you about order #12345. Still not here!',
+      'I have been waiting over a week. This is completely unacceptable.',
+      'I want a refund or serious compensation. A 5% coupon is insulting.',
+      'Fine, that is more reasonable. But I expect the order to arrive this week.',
+    ],
+  },
+  {
+    id: 'on-time-order-check',
+    name: 'On-time order status check',
+    description: 'Customer checks on an order that is on time or already delivered',
+    persona: 'Polite customer just checking in on order #67890',
+    maxTurns: 2,
+    expectedTools: ['get_order'],
+    expectedOutcome: 'Agent confirms order status, no compensation needed',
+    scriptedResponses: [
+      'Hi, can I get an update on order #67890?',
+      'Perfect, thanks!',
+    ],
+  },
+];

package/src/types.ts

@@ -0,0 +1,133 @@
+export interface TestCase {
+  id: string;
+  question: string;
+  expectedAnswer?: string;
+  expectedTools?: string[];
+  persona?: string;
+  tags?: string[];
+  metadata?: Record<string, any>;
+}
+
+export interface TestCaseResult {
+  testCase: TestCase;
+  agentResponse: string;
+  toolsCalled: Array<{ name: string; params: any; result: any }>;
+  evaluation: {
+    passed: boolean;
+    score: number;
+    method: 'exact' | 'contains' | 'similarity' | 'llm_judge';
+    reasoning: string;
+    toolsCorrect: boolean;
+  };
+  duration: number;
+  cost: number;
+}
+
+export interface TestSuiteResult {
+  total: number;
+  passed: number;
+  failed: number;
+  averageScore: number;
+  byTag: Record<string, { total: number; passed: number; avgScore: number }>;
+  results: TestCaseResult[];
+  totalDuration: number;
+  totalCost: number;
+}
+
+// --- Multi-turn conversation testing types ---
+
+export interface CustomerPersona {
+  name: string;
+  style: 'polite' | 'frustrated' | 'confused' | 'terse' | 'verbose';
+  context?: string;
+}
+
+export interface ConversationSuccessCriteria {
+  type: 'tool_called' | 'response_contains' | 'intent_matched' | 'turns_under' | 'custom';
+  value: string | number | ((history: ConversationTurn[]) => boolean);
+}
+
+export interface ConversationScenario {
+  id: string;
+  name: string;
+  description: string;
+  persona: string;
+  maxTurns: number;
+  expectedTools?: string[];
+  expectedOutcome?: string;
+  successCriteria?: ConversationSuccessCriteria[];
+  scriptedResponses?: string[];
+}
+
+export interface ConversationTurn {
+  role: 'customer' | 'agent';
+  message: string;
+  toolCalls?: Array<{ name: string; params: any; result: any }>;
+}
+
+export interface CustomerSimulatorResponse {
+  message: string;
+  shouldContinue: boolean;
+}
+
+export interface CriteriaResult {
+  criteria: ConversationSuccessCriteria;
+  passed: boolean;
+  details: string;
+}
+
+export interface ConversationEvaluation {
+  overall: 'pass' | 'fail' | 'partial';
+  scores: { accuracy: number; toolUsage: number; tone: number; resolution: number };
+  feedback: string;
+  criteriaResults?: CriteriaResult[];
+}
+
+export interface ConversationTestResult {
+  scenario: ConversationScenario;
+  passed: boolean;
+  turns: ConversationTurn[];
+  evaluation: ConversationEvaluation;
+  duration: number;
+  cost: number;
+}
+
+// --- Simulation testing types ---
+
+export interface SimulationConfig {
+  testSuiteFiles?: string[];
+  conversationScenarios?: ConversationScenario[] | 'builtin';
+  totalConversations?: number;
+  pauseBetweenMs?: number;
+  evaluationStrategy?: 'exact' | 'similarity' | 'llm_judge';
+  integrationMode?: 'mock' | 'real' | 'dry-run';
+  allowWrites?: boolean;
+  timeout?: number;
+  parallel?: boolean;
+}
+
+export interface SimulationReport {
+  timestamp: Date;
+  duration: number;
+  totalConversations: number;
+  passed: number;
+  failed: number;
+  averageScores: { accuracy: number; toolUsage: number; tone: number; resolution: number };
+  scenarioBreakdown: Array<{ scenario: string; runs: number; passRate: number; avgScore: number }>;
+  toolUsageStats: Record<string, number>;
+  commonFailurePatterns: string[];
+  recommendations: string[];
+  testSuiteResults: TestSuiteResult[];
+  conversationResults: ConversationTestResult[];
+  overallPassed: boolean;
+  totalCost: number;
+  summary: {
+    totalTests: number;
+    passedTests: number;
+    failedTests: number;
+    totalConversations: number;
+    passedConversations: number;
+    failedConversations: number;
+    overallPassRate: number;
+  };
+}

package/src/utils.ts

@@ -0,0 +1,6 @@
+/**
+ * Format current time as HH:MM:SS timestamp
+ */
+export function formatTimestamp(): string {
+  return new Date().toLocaleTimeString('en-US', { hour12: false });
+}

package/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/tsdown.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'tsdown';
+
+export default defineConfig({
+  entry: ['src/index.ts'],
+  format: ['esm'],
+  dts: true,
+  clean: true,
+  sourcemap: true,
+  outExtensions: () => ({ js: '.js', dts: '.d.ts' }),
+});