@operor/testing 0.1.0

package/src/ConversationRunner.ts

@@ -0,0 +1,267 @@
+import type { Operor } from '@operor/core';
+import type { MockProvider } from '@operor/provider-mock';
+import type {
+  ConversationScenario,
+  ConversationTurn,
+  ConversationTestResult,
+} from './types.js';
+import { CustomerSimulator } from './CustomerSimulator.js';
+import { ConversationEvaluator } from './ConversationEvaluator.js';
+import { formatTimestamp } from './utils.js';
+
+export interface ConversationRunnerConfig {
+  agentOS: Operor;
+  customerSimulator: CustomerSimulator;
+  conversationEvaluator: ConversationEvaluator;
+  timeout?: number;
+  verbose?: boolean;
+}
+
+export class ConversationRunner {
+  private agentOS: Operor;
+  private customerSimulator: CustomerSimulator;
+  private conversationEvaluator: ConversationEvaluator;
+  private timeout: number;
+  private verbose: boolean;
+
+  constructor(config: ConversationRunnerConfig) {
+    this.agentOS = config.agentOS;
+    this.customerSimulator = config.customerSimulator;
+    this.conversationEvaluator = config.conversationEvaluator;
+    this.timeout = config.timeout || 30000;
+    this.verbose = config.verbose ?? false;
+  }
+
+  async runScenario(scenario: ConversationScenario): Promise<ConversationTestResult> {
+    const startTime = Date.now();
+    const turns: ConversationTurn[] = [];
+    const toolsCalled: Array<{ name: string; params: any; result: any }> = [];
+    let totalCost = 0;
+
+    try {
+      // Get MockProvider
+      const mockProvider = this.getMockProvider();
+      const customerId = `test-customer-${scenario.id}`;
+
+      // Start conversation with initial message (from scenario or first scripted response)
+      const initialMessage = scenario.scriptedResponses?.[0] || 'Hello, I need help';
+
+      if (this.verbose) {
+        console.log(`\n=== Starting Scenario: ${scenario.name} ===`);
+        console.log(`Persona: ${scenario.persona}`);
+        console.log(`Max Turns: ${scenario.maxTurns}\n`);
+      }
+
+      let shouldContinue = true;
+      let currentTurn = 0;
+
+      while (shouldContinue && currentTurn < scenario.maxTurns) {
+        // Determine customer message
+        const customerMessage =
+          currentTurn === 0
+            ? initialMessage
+            : (
+                await this.customerSimulator.generateMessage(scenario.persona, turns, {
+                  scenario: scenario.description,
+                  maxTurns: scenario.maxTurns,
+                  currentTurn,
+                  scriptedResponses: scenario.scriptedResponses,
+                })
+              ).message;
+
+        if (this.verbose) {
+          console.log(`[${formatTimestamp()}] Turn ${currentTurn + 1} [customer]: ${customerMessage}`);
+        }
+
+        // Wait for agent response
+        const agentResponse = await this.waitForAgentResponse(
+          mockProvider,
+          customerId,
+          customerMessage
+        );
+
+        if (this.verbose) {
+          console.log(`[${formatTimestamp()}] Turn ${currentTurn + 1} [agent]: ${agentResponse.text}`);
+          if (agentResponse.toolCalls?.length) {
+            console.log(
+              `  Tools called: ${agentResponse.toolCalls.map((tc) => tc.name).join(', ')}`
+            );
+          }
+        }
+
+        // Record turn
+        turns.push({
+          role: 'customer',
+          message: customerMessage,
+        });
+        turns.push({
+          role: 'agent',
+          message: agentResponse.text,
+          toolCalls: agentResponse.toolCalls,
+        });
+
+        // Track tools and cost
+        if (agentResponse.toolCalls) {
+          toolsCalled.push(...agentResponse.toolCalls);
+        }
+        totalCost += agentResponse.cost || 0;
+
+        currentTurn++;
+
+        // Check if conversation should continue
+        if (scenario.scriptedResponses) {
+          // Script mode: continue until scripts exhausted
+          shouldContinue = currentTurn < scenario.scriptedResponses.length;
+        } else {
+          // LLM mode: ask CustomerSimulator
+          if (currentTurn < scenario.maxTurns) {
+            const nextResponse = await this.customerSimulator.generateMessage(
+              scenario.persona,
+              turns,
+              {
+                scenario: scenario.description,
+                maxTurns: scenario.maxTurns,
+                currentTurn,
+              }
+            );
+            shouldContinue = nextResponse.shouldContinue;
+            if (!shouldContinue && this.verbose) {
+              console.log('Customer satisfied, ending conversation.');
+            }
+          }
+        }
+      }
+
+      if (this.verbose && currentTurn >= scenario.maxTurns) {
+        console.log(`\nReached max turns (${scenario.maxTurns})`);
+      }
+
+      // Evaluate conversation
+      const evaluation = await this.conversationEvaluator.evaluate({
+        scenario: scenario.description,
+        persona: scenario.persona,
+        turns,
+        toolsCalled,
+        expectedTools: scenario.expectedTools,
+        expectedOutcome: scenario.expectedOutcome,
+        successCriteria: scenario.successCriteria,
+      });
+
+      if (this.verbose) {
+        console.log(`\n=== Evaluation ===`);
+        console.log(`Overall: ${evaluation.overall}`);
+        console.log(`Scores: ${JSON.stringify(evaluation.scores)}`);
+        console.log(`Feedback: ${evaluation.feedback}\n`);
+      }
+
+      const duration = Date.now() - startTime;
+      const passed = evaluation.overall === 'pass';
+
+      return {
+        scenario,
+        passed,
+        turns,
+        evaluation,
+        duration,
+        cost: totalCost,
+      };
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      const errorMessage = error instanceof Error ? error.message : String(error);
+
+      if (this.verbose) {
+        console.error(`\nError in scenario ${scenario.name}: ${errorMessage}\n`);
+      }
+
+      return {
+        scenario,
+        passed: false,
+        turns,
+        evaluation: {
+          overall: 'fail',
+          scores: { accuracy: 1, toolUsage: 1, tone: 1, resolution: 1 },
+          feedback: `Error: ${errorMessage}`,
+        },
+        duration,
+        cost: totalCost,
+      };
+    }
+  }
+
+  async runScenarios(scenarios: ConversationScenario[]): Promise<ConversationTestResult[]> {
+    const results: ConversationTestResult[] = [];
+
+    for (const scenario of scenarios) {
+      const result = await this.runScenario(scenario);
+      results.push(result);
+    }
+
+    return results;
+  }
+
+  private getMockProvider(): MockProvider {
+    const mockProvider = Array.from((this.agentOS as any).providers.values()).find(
+      (p: any) => p.name === 'mock'
+    ) as MockProvider | undefined;
+
+    if (!mockProvider) {
+      throw new Error('MockProvider not found in Operor. Add it with agentOS.addProvider()');
+    }
+
+    return mockProvider;
+  }
+
+  private async waitForAgentResponse(
+    mockProvider: MockProvider,
+    customerId: string,
+    message: string
+  ): Promise<{ text: string; toolCalls?: Array<{ name: string; params: any; result: any }>; cost: number }> {
+    return new Promise((resolve, reject) => {
+      let settled = false;
+
+      const cleanup = () => {
+        this.agentOS.removeListener('message:processed', onProcessed);
+        this.agentOS.removeListener('error', onError);
+      };
+
+      const timeoutId = setTimeout(() => {
+        if (!settled) {
+          settled = true;
+          cleanup();
+          reject(new Error(`Agent response timed out after ${this.timeout}ms`));
+        }
+      }, this.timeout);
+
+      const onProcessed = (event: any) => {
+        if (!settled) {
+          settled = true;
+          clearTimeout(timeoutId);
+          cleanup();
+          resolve({
+            text: event.response.text,
+            toolCalls: event.response.toolCalls || [],
+            cost: event.cost || 0,
+          });
+        }
+      };
+
+      const onError = (event: any) => {
+        if (!settled) {
+          settled = true;
+          clearTimeout(timeoutId);
+          cleanup();
+          reject(event.error);
+        }
+      };
+
+      // Listen for the response
+      this.agentOS.once('message:processed', onProcessed);
+
+      // Listen for errors
+      this.agentOS.once('error', onError);
+
+      // Simulate incoming message
+      mockProvider.simulateIncomingMessage(customerId, message);
+    });
+  }
+}

package/src/CustomerSimulator.ts

@@ -0,0 +1,106 @@
+import type { LLMProvider, LLMMessage } from '@operor/llm';
+import type { ConversationTurn, CustomerSimulatorResponse } from './types.js';
+
+const PERSONA_PROMPTS: Record<string, string> = {
+  polite: 'You are polite, patient, and use courteous language.',
+  frustrated: 'You are frustrated and impatient. You express dissatisfaction but remain civil.',
+  confused: 'You are confused and unsure. You ask clarifying questions and sometimes misunderstand.',
+  terse: 'You give very short, minimal responses. One sentence max.',
+  verbose: 'You are detailed and talkative. You provide lots of context and background.',
+};
+
+function buildSystemPrompt(
+  persona: string,
+  context?: { scenario?: string; maxTurns?: number; currentTurn?: number }
+): string {
+  const style = PERSONA_PROMPTS[persona] || `You are a customer with a ${persona} communication style.`;
+  const scenarioLine = context?.scenario ? `\nScenario: ${context.scenario}` : '';
+  const turnInfo = context?.maxTurns
+    ? `\nThis conversation has a maximum of ${context.maxTurns} turns. You are on turn ${context.currentTurn ?? 1}.`
+    : '';
+
+  return `You are simulating a customer in a support conversation for testing purposes.
+${style}${scenarioLine}${turnInfo}
+
+Rules:
+- Stay in character throughout the conversation.
+- Escalate naturally if your issue isn't being resolved.
+- Set shouldContinue to false when your issue is resolved or you have no more questions.
+- If the agent asks a question, answer it in character.
+
+Respond with ONLY valid JSON (no markdown, no code fences):
+{"message": "your response as the customer", "shouldContinue": true}`;
+}
+
+function formatHistory(history: ConversationTurn[]): LLMMessage[] {
+  return history.map((turn) => ({
+    role: turn.role === 'customer' ? 'user' as const : 'assistant' as const,
+    content: turn.message,
+  }));
+}
+
+function parseResponse(text: string): CustomerSimulatorResponse {
+  // Try to extract JSON from the response, handling markdown fences
+  const cleaned = text.replace(/```(?:json)?\s*/g, '').replace(/```/g, '').trim();
+  try {
+    const parsed = JSON.parse(cleaned);
+    return {
+      message: String(parsed.message ?? ''),
+      shouldContinue: Boolean(parsed.shouldContinue),
+    };
+  } catch {
+    // If JSON parsing fails, treat the whole text as the message
+    return { message: text.trim(), shouldContinue: true };
+  }
+}
+
+export class CustomerSimulator {
+  private llm?: LLMProvider;
+
+  constructor(options?: { llmProvider?: LLMProvider }) {
+    this.llm = options?.llmProvider;
+  }
+
+  async generateMessage(
+    persona: string,
+    history: ConversationTurn[],
+    context?: {
+      scenario?: string;
+      maxTurns?: number;
+      currentTurn?: number;
+      scriptedResponses?: string[];
+    }
+  ): Promise<CustomerSimulatorResponse> {
+    // Script mode: use pre-defined responses
+    if (context?.scriptedResponses?.length) {
+      const turn = context.currentTurn ?? history.filter((t) => t.role === 'customer').length;
+      const responses = context.scriptedResponses;
+      if (turn < responses.length) {
+        return {
+          message: responses[turn],
+          shouldContinue: turn < responses.length - 1,
+        };
+      }
+      // Exhausted scripted responses
+      return { message: responses[responses.length - 1], shouldContinue: false };
+    }
+
+    // LLM mode
+    if (!this.llm) {
+      throw new Error('CustomerSimulator requires an LLM provider for non-scripted mode');
+    }
+
+    const systemPrompt = buildSystemPrompt(persona, context);
+    const messages: LLMMessage[] = [
+      { role: 'system', content: systemPrompt },
+      ...formatHistory(history),
+    ];
+
+    const result = await this.llm.complete(messages, {
+      temperature: 0.7,
+      maxTokens: 500,
+    });
+
+    return parseResponse(result.text);
+  }
+}

package/src/MockShopifySkill.ts

@@ -0,0 +1,336 @@
+import type { Skill, Tool } from '@operor/core';
+
+interface MockOrder {
+  id: string;
+  name: string;
+  status: string;
+  financialStatus: string;
+  createdAt: string;
+  expectedDelivery: Date;
+  actualDelivery?: Date;
+  tracking?: string;
+  trackingUrl?: string;
+  items: Array<{ name: string; quantity: number; price: string }>;
+  total: string;
+}
+
+interface MockProduct {
+  id: number;
+  title: string;
+  vendor: string;
+  type: string;
+  price: string;
+  available: boolean;
+}
+
+interface MockDiscount {
+  code: string;
+  percent: number;
+  validDays: number;
+  startsAt: string;
+  expiresAt: string;
+  priceRuleId: number;
+  createdAt: Date;
+}
+
+export class MockShopifySkill implements Skill {
+  public readonly name = 'shopify';
+  private ready = false;
+  private mockOrders: Map<string, MockOrder> = new Map();
+  private mockProducts: MockProduct[] = [];
+  private mockDiscounts: MockDiscount[] = [];
+  private nextPriceRuleId = 1000;
+
+  constructor() {
+    // Seed with mock data
+    this.seedMockData();
+  }
+
+  async initialize(): Promise<void> {
+    this.ready = true;
+    console.log('✅ Mock Shopify initialized');
+  }
+
+  /** @deprecated Use initialize() instead. */
+  async authenticate(): Promise<void> {
+    return this.initialize();
+  }
+
+  isReady(): boolean {
+    return this.ready;
+  }
+
+  /** @deprecated Use isReady() instead. */
+  isAuthenticated(): boolean {
+    return this.ready;
+  }
+
+  /**
+   * Reset all mock data to initial state (for testing)
+   */
+  reset(): void {
+    this.mockOrders.clear();
+    this.mockProducts = [];
+    this.mockDiscounts = [];
+    this.nextPriceRuleId = 1000;
+    this.seedMockData();
+  }
+
+  /**
+   * Seed custom test data (for testing)
+   */
+  seedData(config: {
+    orders?: Array<Partial<MockOrder>>;
+    products?: Array<Partial<MockProduct>>;
+    discounts?: Array<Partial<MockDiscount>>;
+  }): void {
+    if (config.orders) {
+      for (const order of config.orders) {
+        const fullOrder: MockOrder = {
+          id: order.id || String(Date.now()),
+          name: order.name || `#${order.id || '1001'}`,
+          status: order.status || 'unfulfilled',
+          financialStatus: order.financialStatus || 'paid',
+          createdAt: order.createdAt || new Date().toISOString(),
+          expectedDelivery: order.expectedDelivery || new Date(),
+          actualDelivery: order.actualDelivery,
+          tracking: order.tracking,
+          trackingUrl: order.trackingUrl,
+          items: order.items || [],
+          total: order.total || '0.00',
+        };
+        this.mockOrders.set(fullOrder.id, fullOrder);
+      }
+    }
+
+    if (config.products) {
+      for (const product of config.products) {
+        const fullProduct: MockProduct = {
+          id: product.id || Date.now(),
+          title: product.title || 'Product',
+          vendor: product.vendor || 'Mock Vendor',
+          type: product.type || 'General',
+          price: product.price || '0.00',
+          available: product.available !== undefined ? product.available : true,
+        };
+        this.mockProducts.push(fullProduct);
+      }
+    }
+
+    if (config.discounts) {
+      for (const discount of config.discounts) {
+        const fullDiscount: MockDiscount = {
+          code: discount.code || 'DISCOUNT',
+          percent: discount.percent || 10,
+          validDays: discount.validDays || 30,
+          startsAt: discount.startsAt || new Date().toISOString(),
+          expiresAt: discount.expiresAt || new Date(Date.now() + 30 * 24 * 60 * 60 * 1000).toISOString(),
+          priceRuleId: discount.priceRuleId || this.nextPriceRuleId++,
+          createdAt: discount.createdAt || new Date(),
+        };
+        this.mockDiscounts.push(fullDiscount);
+      }
+    }
+  }
+
+  private seedMockData(): void {
+    // Create a delayed order for testing
+    const twoDaysAgo = new Date();
+    twoDaysAgo.setDate(twoDaysAgo.getDate() - 2);
+
+    this.mockOrders.set('12345', {
+      id: '12345',
+      name: '#1001',
+      status: 'in_transit',
+      financialStatus: 'paid',
+      createdAt: new Date(Date.now() - 5 * 24 * 60 * 60 * 1000).toISOString(),
+      expectedDelivery: twoDaysAgo,
+      tracking: 'TRACK123456789',
+      trackingUrl: 'https://track.example.com/TRACK123456789',
+      items: [
+        { name: 'Premium Headphones', quantity: 1, price: '299.99' },
+      ],
+      total: '299.99',
+    });
+
+    this.mockOrders.set('67890', {
+      id: '67890',
+      name: '#1002',
+      status: 'delivered',
+      financialStatus: 'paid',
+      createdAt: new Date(Date.now() - 10 * 24 * 60 * 60 * 1000).toISOString(),
+      expectedDelivery: new Date(Date.now() - 5 * 24 * 60 * 60 * 1000),
+      actualDelivery: new Date(Date.now() - 3 * 24 * 60 * 60 * 1000),
+      tracking: 'TRACK987654321',
+      trackingUrl: 'https://track.example.com/TRACK987654321',
+      items: [
+        { name: 'Wireless Mouse', quantity: 2, price: '49.99' },
+      ],
+      total: '99.98',
+    });
+
+    // Seed default products
+    this.mockProducts = [
+      {
+        id: 1001,
+        title: 'Premium Headphones',
+        vendor: 'AudioTech',
+        type: 'Electronics',
+        price: '299.99',
+        available: true,
+      },
+      {
+        id: 1002,
+        title: 'Wireless Mouse',
+        vendor: 'TechGear',
+        type: 'Electronics',
+        price: '49.99',
+        available: true,
+      },
+      {
+        id: 1003,
+        title: 'Mechanical Keyboard',
+        vendor: 'KeyMaster',
+        type: 'Electronics',
+        price: '149.99',
+        available: true,
+      },
+      {
+        id: 1004,
+        title: 'USB-C Cable',
+        vendor: 'CableCo',
+        type: 'Accessories',
+        price: '19.99',
+        available: false,
+      },
+    ];
+  }
+
+  public tools: Record<string, Tool> = {
+    get_order: {
+      name: 'get_order',
+      description: 'Get order details by order ID or order name (e.g., #1001)',
+      parameters: {
+        orderId: { type: 'string', required: true },
+      },
+      execute: async (params: { orderId: string }) => {
+        const orderIdentifier = params.orderId.replace('#', '');
+        const order = this.mockOrders.get(orderIdentifier);
+
+        if (!order) {
+          return {
+            found: false,
+            error: `Order ${params.orderId} not found`,
+          };
+        }
+
+        const delayMs = new Date().getTime() - order.expectedDelivery.getTime();
+        const delayDays = Math.floor(delayMs / (1000 * 60 * 60 * 24));
+
+        return {
+          found: true,
+          id: order.id,
+          name: order.name,
+          status: order.status,
+          financialStatus: order.financialStatus,
+          createdAt: order.createdAt,
+          total: order.total,
+          items: order.items,
+          tracking: order.tracking || null,
+          trackingUrl: order.trackingUrl || null,
+          // Mock-specific extra fields
+          isDelayed: delayDays > 0,
+          delayDays: Math.max(0, delayDays),
+        };
+      },
+    },
+
+    create_discount: {
+      name: 'create_discount',
+      description: 'Create a percentage discount code',
+      parameters: {
+        percent: { type: 'number', required: true },
+        validDays: { type: 'number', required: true },
+      },
+      execute: async (params: { percent: number; validDays: number }) => {
+        const code = `SORRY${params.percent}`;
+        const startsAt = new Date().toISOString();
+        const expiresAt = new Date(Date.now() + params.validDays * 24 * 60 * 60 * 1000).toISOString();
+        const priceRuleId = this.nextPriceRuleId++;
+
+        const discount: MockDiscount = {
+          code,
+          percent: params.percent,
+          validDays: params.validDays,
+          startsAt,
+          expiresAt,
+          priceRuleId,
+          createdAt: new Date(),
+        };
+
+        this.mockDiscounts.push(discount);
+
+        console.log(`\n💰 Created discount code: ${code}`);
+        console.log(`   ${params.percent}% off, valid for ${params.validDays} days\n`);
+
+        return {
+          code,
+          percent: params.percent,
+          validDays: params.validDays,
+          startsAt,
+          expiresAt,
+          priceRuleId,
+        };
+      },
+    },
+
+    search_products: {
+      name: 'search_products',
+      description: 'Search for products in the store',
+      parameters: {
+        query: { type: 'string', required: true },
+        limit: { type: 'number', required: false },
+      },
+      execute: async (params: { query: string; limit?: number }) => {
+        const limit = params.limit || 10;
+        const queryLower = params.query.toLowerCase();
+
+        const matched = this.mockProducts
+          .filter((p) =>
+            p.title.toLowerCase().includes(queryLower) ||
+            p.vendor.toLowerCase().includes(queryLower) ||
+            p.type.toLowerCase().includes(queryLower)
+          )
+          .slice(0, limit);
+
+        return {
+          found: matched.length,
+          products: matched.map((p) => ({
+            id: p.id,
+            title: p.title,
+            vendor: p.vendor,
+            type: p.type,
+            price: p.price,
+            available: p.available,
+          })),
+        };
+      },
+    },
+  };
+
+  /**
+   * Get all created discounts (for testing)
+   */
+  getDiscounts(): MockDiscount[] {
+    return [...this.mockDiscounts];
+  }
+
+  /**
+   * Get all orders (for testing)
+   */
+  getOrders(): MockOrder[] {
+    return Array.from(this.mockOrders.values());
+  }
+}
+
+export type { MockOrder, MockProduct, MockDiscount };