@operor/testing 0.1.0

package/src/SimulationRunner.ts

@@ -0,0 +1,425 @@
+import type { Operor } from '@operor/core';
+import type { LLMProvider } from '@operor/llm';
+import type {
+  SimulationConfig,
+  SimulationReport,
+  TestSuiteResult,
+  ConversationTestResult,
+  ConversationScenario,
+} from './types.js';
+import { TestSuiteRunner } from './TestSuiteRunner.js';
+import { ConversationRunner } from './ConversationRunner.js';
+import { CustomerSimulator } from './CustomerSimulator.js';
+import { ConversationEvaluator } from './ConversationEvaluator.js';
+import { CSVLoader } from './CSVLoader.js';
+import { ECOMMERCE_SCENARIOS } from './scenarios.js';
+
+export interface SimulationRunnerOptions {
+  agentOS: Operor;
+  config: SimulationConfig;
+  llm?: LLMProvider;
+}
+
+export class SimulationRunner {
+  private agentOS: Operor;
+  private config: SimulationConfig;
+  private llm?: LLMProvider;
+
+  constructor(options: SimulationRunnerOptions) {
+    this.agentOS = options.agentOS;
+    this.config = options.config;
+    this.llm = options.llm;
+  }
+
+  async run(
+    onProgress?: (completed: number, total: number, result: ConversationTestResult) => void
+  ): Promise<SimulationReport> {
+    const startTime = Date.now();
+    const testSuiteResults: TestSuiteResult[] = [];
+    const conversationResults: ConversationTestResult[] = [];
+    let totalCost = 0;
+
+    // 1. Run test suites from CSV/JSON files
+    if (this.config.testSuiteFiles?.length) {
+      const suiteRunner = new TestSuiteRunner({
+        agentOS: this.agentOS,
+        llm: this.llm,
+        timeout: this.config.timeout,
+      });
+
+      for (const file of this.config.testSuiteFiles) {
+        const testCases = await CSVLoader.fromFile(file);
+        const result = await suiteRunner.runSuite(testCases);
+        testSuiteResults.push(result);
+        totalCost += result.totalCost;
+      }
+    }
+
+    // 2. Run conversation scenarios
+    const scenarios = this.resolveScenarios();
+    if (scenarios.length) {
+      const conversationRunner = new ConversationRunner({
+        agentOS: this.agentOS,
+        customerSimulator: new CustomerSimulator({ llmProvider: this.llm }),
+        conversationEvaluator: new ConversationEvaluator({ llmProvider: this.llm }),
+        timeout: this.config.timeout,
+      });
+
+      // Distribute totalConversations across scenarios round-robin
+      const schedule = this.buildSchedule(scenarios);
+      const pauseMs = this.config.pauseBetweenMs ?? 500;
+
+      for (let i = 0; i < schedule.length; i++) {
+        const scenario = schedule[i];
+
+        // Add timeout protection around conversation execution
+        const timeoutMs = this.config.timeout || 60000;
+        const result = await Promise.race([
+          conversationRunner.runScenario(scenario),
+          new Promise<ConversationTestResult>((_, reject) =>
+            setTimeout(() => reject(new Error(`Conversation timed out after ${timeoutMs}ms`)), timeoutMs)
+          ),
+        ]).catch((error): ConversationTestResult => {
+          // If timeout or error, return a failed result
+          return {
+            scenario,
+            passed: false,
+            turns: [],
+            evaluation: {
+              overall: 'fail',
+              scores: { accuracy: 1, toolUsage: 1, tone: 1, resolution: 1 },
+              feedback: `Timeout or error: ${error instanceof Error ? error.message : String(error)}`,
+            },
+            duration: timeoutMs,
+            cost: 0,
+          };
+        });
+
+        conversationResults.push(result);
+        totalCost += result.cost;
+
+        if (onProgress) {
+          onProgress(i + 1, schedule.length, result);
+        }
+
+        // Pause between conversations (skip after last)
+        if (i < schedule.length - 1 && pauseMs > 0) {
+          await new Promise((resolve) => setTimeout(resolve, pauseMs));
+        }
+      }
+    }
+
+    const duration = Date.now() - startTime;
+
+    // 3. Aggregate results
+    const totalTests = testSuiteResults.reduce((sum, r) => sum + r.total, 0);
+    const passedTests = testSuiteResults.reduce((sum, r) => sum + r.passed, 0);
+    const failedTests = testSuiteResults.reduce((sum, r) => sum + r.failed, 0);
+    const totalConversations = conversationResults.length;
+    const passedConversations = conversationResults.filter((r) => r.passed).length;
+    const failedConversations = totalConversations - passedConversations;
+
+    const totalItems = totalTests + totalConversations;
+    const passedItems = passedTests + passedConversations;
+    const overallPassRate = totalItems > 0 ? passedItems / totalItems : 0;
+
+    const averageScores = this.computeAverageScores(conversationResults);
+    const scenarioBreakdown = this.computeScenarioBreakdown(conversationResults);
+    const toolUsageStats = this.computeToolUsageStats(conversationResults);
+
+    // 4. Failure analysis
+    const failedResults = conversationResults.filter((r) => !r.passed);
+    let commonFailurePatterns: string[] = [];
+    let recommendations: string[] = [];
+
+    if (failedResults.length > 0) {
+      if (this.llm) {
+        const analysis = await this.analyzeFailuresWithLLM(failedResults);
+        commonFailurePatterns = analysis.patterns;
+        recommendations = analysis.recommendations;
+      } else {
+        commonFailurePatterns = this.heuristicFailurePatterns(failedResults);
+        recommendations = this.heuristicRecommendations(failedResults);
+      }
+    }
+
+    return {
+      timestamp: new Date(),
+      duration,
+      totalConversations,
+      passed: passedConversations,
+      failed: failedConversations,
+      averageScores,
+      scenarioBreakdown,
+      toolUsageStats,
+      commonFailurePatterns,
+      recommendations,
+      testSuiteResults,
+      conversationResults,
+      overallPassed: failedTests === 0 && failedConversations === 0 && totalItems > 0,
+      totalCost,
+      summary: {
+        totalTests,
+        passedTests,
+        failedTests,
+        totalConversations,
+        passedConversations,
+        failedConversations,
+        overallPassRate,
+      },
+    };
+  }
+
+  static formatReport(report: SimulationReport): string {
+    const lines: string[] = [];
+
+    lines.push('=== Simulation Report ===');
+    lines.push(`Date: ${report.timestamp.toISOString()}`);
+    lines.push(`Duration: ${(report.duration / 1000).toFixed(1)}s`);
+    lines.push(`Cost: $${report.totalCost.toFixed(4)}`);
+    lines.push('');
+
+    // Test suite results
+    if (report.testSuiteResults.length) {
+      lines.push('--- Test Suites ---');
+      for (const suite of report.testSuiteResults) {
+        lines.push(`  ${suite.passed}/${suite.total} passed (avg score: ${suite.averageScore.toFixed(2)})`);
+        for (const result of suite.results) {
+          const status = result.evaluation.passed ? 'PASS' : 'FAIL';
+          lines.push(`    [${status}] ${result.testCase.id}: ${result.testCase.question}`);
+        }
+      }
+      lines.push('');
+    }
+
+    // Scenario breakdown
+    if (report.scenarioBreakdown.length) {
+      lines.push('--- Scenario Breakdown ---');
+      for (const s of report.scenarioBreakdown) {
+        const pct = (s.passRate * 100).toFixed(0);
+        lines.push(`  ${s.scenario}: ${s.runs} run(s), ${pct}% pass rate, avg score ${s.avgScore.toFixed(2)}`);
+      }
+      lines.push('');
+    }
+
+    // Tool usage
+    const toolEntries = Object.entries(report.toolUsageStats);
+    if (toolEntries.length) {
+      lines.push('--- Tool Usage ---');
+      for (const [tool, count] of toolEntries.sort((a, b) => b[1] - a[1])) {
+        lines.push(`  ${tool}: ${count} call(s)`);
+      }
+      lines.push('');
+    }
+
+    // Average scores
+    const { averageScores } = report;
+    if (report.totalConversations > 0) {
+      lines.push('--- Average Scores ---');
+      lines.push(`  Accuracy:   ${averageScores.accuracy.toFixed(2)}`);
+      lines.push(`  Tool Usage: ${averageScores.toolUsage.toFixed(2)}`);
+      lines.push(`  Tone:       ${averageScores.tone.toFixed(2)}`);
+      lines.push(`  Resolution: ${averageScores.resolution.toFixed(2)}`);
+      lines.push('');
+    }
+
+    // Failure patterns
+    if (report.commonFailurePatterns.length) {
+      lines.push('--- Common Failure Patterns ---');
+      for (const pattern of report.commonFailurePatterns) {
+        lines.push(`  - ${pattern}`);
+      }
+      lines.push('');
+    }
+
+    // Recommendations
+    if (report.recommendations.length) {
+      lines.push('--- Recommendations ---');
+      for (const rec of report.recommendations) {
+        lines.push(`  - ${rec}`);
+      }
+      lines.push('');
+    }
+
+    // Summary
+    const { summary } = report;
+    lines.push('--- Summary ---');
+    if (summary.totalTests > 0) {
+      lines.push(`Tests: ${summary.passedTests}/${summary.totalTests} passed`);
+    }
+    lines.push(`Conversations: ${summary.passedConversations}/${summary.totalConversations} passed`);
+    lines.push(`Overall pass rate: ${(summary.overallPassRate * 100).toFixed(1)}%`);
+    lines.push(`Result: ${report.overallPassed ? 'PASSED' : 'FAILED'}`);
+
+    return lines.join('\n');
+  }
+
+  private resolveScenarios(): ConversationScenario[] {
+    if (!this.config.conversationScenarios) return [];
+    if (this.config.conversationScenarios === 'builtin') return ECOMMERCE_SCENARIOS;
+    return this.config.conversationScenarios;
+  }
+
+  private buildSchedule(scenarios: ConversationScenario[]): ConversationScenario[] {
+    const total = this.config.totalConversations ?? scenarios.length;
+    const schedule: ConversationScenario[] = [];
+    for (let i = 0; i < total; i++) {
+      schedule.push(scenarios[i % scenarios.length]);
+    }
+    return schedule;
+  }
+
+  private computeAverageScores(
+    results: ConversationTestResult[]
+  ): { accuracy: number; toolUsage: number; tone: number; resolution: number } {
+    if (results.length === 0) {
+      return { accuracy: 0, toolUsage: 0, tone: 0, resolution: 0 };
+    }
+    const totals = results.reduce(
+      (acc, r) => ({
+        accuracy: acc.accuracy + r.evaluation.scores.accuracy,
+        toolUsage: acc.toolUsage + r.evaluation.scores.toolUsage,
+        tone: acc.tone + r.evaluation.scores.tone,
+        resolution: acc.resolution + r.evaluation.scores.resolution,
+      }),
+      { accuracy: 0, toolUsage: 0, tone: 0, resolution: 0 }
+    );
+    const n = results.length;
+    return {
+      accuracy: totals.accuracy / n,
+      toolUsage: totals.toolUsage / n,
+      tone: totals.tone / n,
+      resolution: totals.resolution / n,
+    };
+  }
+
+  private computeScenarioBreakdown(
+    results: ConversationTestResult[]
+  ): Array<{ scenario: string; runs: number; passRate: number; avgScore: number }> {
+    const byScenario = new Map<string, ConversationTestResult[]>();
+    for (const r of results) {
+      const name = r.scenario.name;
+      if (!byScenario.has(name)) byScenario.set(name, []);
+      byScenario.get(name)!.push(r);
+    }
+    return Array.from(byScenario.entries()).map(([scenario, runs]) => {
+      const passed = runs.filter((r) => r.passed).length;
+      const scores = runs.map((r) => {
+        const s = r.evaluation.scores;
+        return (s.accuracy + s.toolUsage + s.tone + s.resolution) / 4;
+      });
+      const avgScore = scores.reduce((a, b) => a + b, 0) / scores.length;
+      return { scenario, runs: runs.length, passRate: passed / runs.length, avgScore };
+    });
+  }
+
+  private computeToolUsageStats(results: ConversationTestResult[]): Record<string, number> {
+    const stats: Record<string, number> = {};
+    for (const r of results) {
+      for (const turn of r.turns) {
+        if (turn.toolCalls) {
+          for (const tc of turn.toolCalls) {
+            stats[tc.name] = (stats[tc.name] || 0) + 1;
+          }
+        }
+      }
+    }
+    return stats;
+  }
+
+  private async analyzeFailuresWithLLM(
+    failedResults: ConversationTestResult[]
+  ): Promise<{ patterns: string[]; recommendations: string[] }> {
+    if (!this.llm) {
+      return { patterns: [], recommendations: [] };
+    }
+
+    const summaries = failedResults.slice(0, 10).map((r) => {
+      const turns = r.turns.map((t) => `[${t.role}]: ${t.message}`).join('\n');
+      return `Scenario: ${r.scenario.name}\nFeedback: ${r.evaluation.feedback}\nConversation:\n${turns}`;
+    });
+
+    const prompt = `Analyze these failed customer support conversation tests and identify patterns.
+
+${summaries.join('\n\n---\n\n')}
+
+Respond with ONLY valid JSON (no markdown, no code fences):
+{
+  "patterns": ["pattern 1", "pattern 2"],
+  "recommendations": ["recommendation 1", "recommendation 2"]
+}`;
+
+    try {
+      const result = await this.llm.complete(
+        [{ role: 'user', content: prompt }],
+        { temperature: 0, maxTokens: 1000 }
+      );
+      const cleaned = result.text.replace(/```(?:json)?\s*/g, '').replace(/```/g, '').trim();
+      const parsed = JSON.parse(cleaned);
+      return {
+        patterns: Array.isArray(parsed.patterns) ? parsed.patterns.map(String) : [],
+        recommendations: Array.isArray(parsed.recommendations) ? parsed.recommendations.map(String) : [],
+      };
+    } catch {
+      return {
+        patterns: this.heuristicFailurePatterns(failedResults),
+        recommendations: this.heuristicRecommendations(failedResults),
+      };
+    }
+  }
+
+  private heuristicFailurePatterns(failedResults: ConversationTestResult[]): string[] {
+    const patterns: string[] = [];
+
+    const noToolCalls = failedResults.filter((r) =>
+      r.turns.every((t) => !t.toolCalls?.length)
+    );
+    if (noToolCalls.length > 0) {
+      patterns.push(`${noToolCalls.length} conversation(s) failed with no tool calls`);
+    }
+
+    const lowResolution = failedResults.filter((r) => r.evaluation.scores.resolution <= 2);
+    if (lowResolution.length > 0) {
+      patterns.push(`${lowResolution.length} conversation(s) had low resolution scores`);
+    }
+
+    const lowTone = failedResults.filter((r) => r.evaluation.scores.tone <= 2);
+    if (lowTone.length > 0) {
+      patterns.push(`${lowTone.length} conversation(s) had low tone scores`);
+    }
+
+    if (patterns.length === 0) {
+      patterns.push(`${failedResults.length} conversation(s) failed evaluation criteria`);
+    }
+
+    return patterns;
+  }
+
+  private heuristicRecommendations(failedResults: ConversationTestResult[]): string[] {
+    const recs: string[] = [];
+
+    const noTools = failedResults.filter((r) =>
+      r.turns.every((t) => !t.toolCalls?.length)
+    );
+    if (noTools.length > 0) {
+      recs.push('Ensure agent is configured to use available tools for customer queries');
+    }
+
+    const expectedButMissing = new Set<string>();
+    for (const r of failedResults) {
+      for (const tool of r.scenario.expectedTools || []) {
+        const used = r.turns.some((t) => t.toolCalls?.some((tc) => tc.name === tool));
+        if (!used) expectedButMissing.add(tool);
+      }
+    }
+    if (expectedButMissing.size > 0) {
+      recs.push(`Tools expected but not called: ${Array.from(expectedButMissing).join(', ')}`);
+    }
+
+    if (recs.length === 0) {
+      recs.push('Review failed scenarios and adjust agent rules or prompts');
+    }
+
+    return recs;
+  }
+}

package/src/SkillTestHarness.ts

@@ -0,0 +1,220 @@
+import type { Skill, Tool } from '@operor/core';
+
+export interface SkillTestHarnessConfig {
+  allowWrites?: boolean;
+  allowDestructive?: boolean;
+  maxOperations?: number;
+  timeoutMs?: number;
+  dryRun?: boolean;
+}
+
+export interface AuditLogEntry {
+  name: string;
+  params: any;
+  result: any;
+  timestamp: number;
+  duration: number;
+  classification: 'read' | 'write' | 'destructive';
+}
+
+/**
+ * Safety wrapper for Integration instances during testing.
+ * Provides operation limits, dry-run mode, and audit logging.
+ */
+export class SkillTestHarness implements Skill {
+  public readonly name: string;
+  private inner: Skill;
+  private config: Required<SkillTestHarnessConfig>;
+  private auditLog: AuditLogEntry[] = [];
+  private operationCount = 0;
+
+  // Tool classification rules
+  private static readonly READ_TOOLS = new Set([
+    'get_order',
+    'search_products',
+    'salesforce_get_contact',
+    'salesforce_get_cases',
+    'stripe_get_customer',
+  ]);
+
+  private static readonly WRITE_TOOLS = new Set([
+    'create_discount',
+    'salesforce_update_contact',
+    'salesforce_create_case',
+    'salesforce_add_note',
+  ]);
+
+  private static readonly DESTRUCTIVE_TOOLS = new Set([
+    'stripe_create_refund',
+  ]);
+
+  constructor(inner: Skill, config: SkillTestHarnessConfig = {}) {
+    this.inner = inner;
+    this.name = inner.name;
+    this.config = {
+      allowWrites: config.allowWrites ?? false,
+      allowDestructive: config.allowDestructive ?? false,
+      maxOperations: config.maxOperations ?? 10,
+      timeoutMs: config.timeoutMs ?? 30000,
+      dryRun: config.dryRun ?? false,
+    };
+
+    // Wrap all tools with safety checks
+    this.tools = this.wrapTools(inner.tools);
+  }
+
+  async initialize(): Promise<void> {
+    return this.inner.initialize();
+  }
+
+  /** @deprecated Use initialize() instead. */
+  async authenticate(): Promise<void> {
+    return this.inner.initialize();
+  }
+
+  isReady(): boolean {
+    return this.inner.isReady();
+  }
+
+  /** @deprecated Use isReady() instead. */
+  isAuthenticated(): boolean {
+    return this.inner.isReady();
+  }
+
+  public tools: Record<string, Tool>;
+
+  private wrapTools(innerTools: Record<string, Tool>): Record<string, Tool> {
+    const wrapped: Record<string, Tool> = {};
+
+    for (const [toolName, tool] of Object.entries(innerTools)) {
+      wrapped[toolName] = {
+        ...tool,
+        execute: async (params: any) => {
+          const startTime = Date.now();
+          const classification = this.classifyTool(toolName);
+
+          // Check operation limit
+          if (this.operationCount >= this.config.maxOperations) {
+            throw new Error(
+              `SkillTestHarness: Max operations limit reached (${this.config.maxOperations})`
+            );
+          }
+
+          // Check write permissions
+          if (classification === 'write' && !this.config.allowWrites) {
+            throw new Error(
+              `SkillTestHarness: Write operation '${toolName}' blocked (allowWrites=false)`
+            );
+          }
+
+          // Check destructive permissions
+          if (classification === 'destructive' && !this.config.allowDestructive) {
+            throw new Error(
+              `SkillTestHarness: Destructive operation '${toolName}' blocked (allowDestructive=false)`
+            );
+          }
+
+          this.operationCount++;
+
+          // Dry-run mode: return mock result without executing
+          if (this.config.dryRun) {
+            const result = {
+              dryRun: true,
+              wouldExecute: toolName,
+              params,
+            };
+
+            this.auditLog.push({
+              name: toolName,
+              params,
+              result,
+              timestamp: startTime,
+              duration: Date.now() - startTime,
+              classification,
+            });
+
+            return result;
+          }
+
+          // Execute with timeout
+          const timeoutPromise = new Promise((_, reject) => {
+            setTimeout(() => {
+              reject(new Error(`SkillTestHarness: Operation '${toolName}' timed out after ${this.config.timeoutMs}ms`));
+            }, this.config.timeoutMs);
+          });
+
+          try {
+            const result = await Promise.race([
+              tool.execute(params),
+              timeoutPromise,
+            ]);
+
+            const duration = Date.now() - startTime;
+
+            this.auditLog.push({
+              name: toolName,
+              params,
+              result,
+              timestamp: startTime,
+              duration,
+              classification,
+            });
+
+            return result;
+          } catch (error) {
+            const duration = Date.now() - startTime;
+
+            this.auditLog.push({
+              name: toolName,
+              params,
+              result: { error: error instanceof Error ? error.message : 'Unknown error' },
+              timestamp: startTime,
+              duration,
+              classification,
+            });
+
+            throw error;
+          }
+        },
+      };
+    }
+
+    return wrapped;
+  }
+
+  private classifyTool(toolName: string): 'read' | 'write' | 'destructive' {
+    if (SkillTestHarness.READ_TOOLS.has(toolName)) {
+      return 'read';
+    }
+    if (SkillTestHarness.DESTRUCTIVE_TOOLS.has(toolName)) {
+      return 'destructive';
+    }
+    if (SkillTestHarness.WRITE_TOOLS.has(toolName)) {
+      return 'write';
+    }
+    // Unknown tools default to 'write' (safe by default)
+    return 'write';
+  }
+
+  /**
+   * Get the audit log of all operations performed
+   */
+  getAuditLog(): AuditLogEntry[] {
+    return [...this.auditLog];
+  }
+
+  /**
+   * Reset the audit log and operation counter
+   */
+  resetAuditLog(): void {
+    this.auditLog = [];
+    this.operationCount = 0;
+  }
+
+  /**
+   * Get the current operation count
+   */
+  getOperationCount(): number {
+    return this.operationCount;
+  }
+}