@operor/testing 0.1.0

package/dist/index.js

@@ -0,0 +1,1608 @@
+import { parse } from "csv-parse/sync";
+import { readFile } from "node:fs/promises";
+
+//#region src/CSVLoader.ts
+var CSVLoader = class CSVLoader {
+	static async fromFile(path) {
+		const content = await readFile(path, "utf-8");
+		if (path.endsWith(".json")) return CSVLoader.fromJSON(content);
+		return CSVLoader.fromCSVString(content);
+	}
+	static fromCSVString(csv) {
+		return parse(csv.replace(/^\uFEFF/, ""), {
+			columns: true,
+			skip_empty_lines: true,
+			trim: true,
+			relax_column_count: true
+		}).map((row, i) => {
+			const id = row.id?.trim();
+			const question = row.question?.trim();
+			if (!id || !question) throw new Error(`Row ${i + 1}: missing required field(s) — id and question are required`);
+			const testCase = {
+				id,
+				question
+			};
+			if (row.expected_answer?.trim()) testCase.expectedAnswer = row.expected_answer.trim();
+			if (row.expected_tools?.trim()) testCase.expectedTools = row.expected_tools.split(",").map((t) => t.trim()).filter(Boolean);
+			if (row.persona?.trim()) testCase.persona = row.persona.trim();
+			if (row.tags?.trim()) testCase.tags = row.tags.split(",").map((t) => t.trim()).filter(Boolean);
+			return testCase;
+		});
+	}
+	static fromJSON(json) {
+		const data = JSON.parse(json);
+		const arr = Array.isArray(data) ? data : data.testCases ?? data.tests;
+		if (!Array.isArray(arr)) throw new Error("JSON must be an array or contain a testCases/tests array");
+		return arr.map((item, i) => {
+			if (!item.id || !item.question) throw new Error(`Item ${i}: missing required field(s) — id and question are required`);
+			const testCase = {
+				id: item.id,
+				question: item.question
+			};
+			if (item.expectedAnswer) testCase.expectedAnswer = item.expectedAnswer;
+			if (item.expectedTools) testCase.expectedTools = item.expectedTools;
+			if (item.persona) testCase.persona = item.persona;
+			if (item.tags) testCase.tags = item.tags;
+			if (item.metadata) testCase.metadata = item.metadata;
+			return testCase;
+		});
+	}
+};
+
+//#endregion
+//#region src/TestCaseEvaluator.ts
+var TestCaseEvaluator = class {
+	constructor(llm) {
+		this.llm = llm;
+	}
+	async evaluate(testCase, agentResponse, toolsCalled, strategy) {
+		const toolsCorrect = this.validateTools(testCase.expectedTools, toolsCalled);
+		if (strategy === "exact") return this.evaluateByExact(testCase, agentResponse, toolsCorrect);
+		if (strategy === "contains") return this.evaluateByContains(testCase, agentResponse, toolsCorrect);
+		if (strategy === "similarity") return this.evaluateBySimilarity(testCase, agentResponse, toolsCorrect);
+		if (strategy === "semantic" && this.llm) {
+			if (testCase.expectedAnswer) return await this.evaluateByLLMComparison(testCase, agentResponse, toolsCorrect);
+			return await this.evaluateByLLMJudge(testCase, agentResponse, toolsCorrect);
+		}
+		if (!this.llm) return this.evaluateBySimilarity(testCase, agentResponse, toolsCorrect);
+		if (testCase.expectedAnswer) return await this.evaluateByLLMComparison(testCase, agentResponse, toolsCorrect);
+		return await this.evaluateByLLMJudge(testCase, agentResponse, toolsCorrect);
+	}
+	validateTools(expectedTools, toolsCalled) {
+		if (!expectedTools || expectedTools.length === 0) return true;
+		const calledNames = new Set(toolsCalled.map((t) => t.name));
+		return expectedTools.every((tool) => calledNames.has(tool));
+	}
+	evaluateByExact(testCase, agentResponse, toolsCorrect) {
+		if (!testCase.expectedAnswer) return {
+			passed: toolsCorrect,
+			score: toolsCorrect ? 1 : 0,
+			method: "exact",
+			reasoning: "No expected answer provided, evaluated tools only",
+			toolsCorrect
+		};
+		const matches = agentResponse.trim().toLowerCase() === testCase.expectedAnswer.trim().toLowerCase();
+		return {
+			passed: matches && toolsCorrect,
+			score: matches ? 1 : 0,
+			method: "exact",
+			reasoning: matches ? "Exact match" : "Response does not exactly match expected answer",
+			toolsCorrect
+		};
+	}
+	evaluateByContains(testCase, agentResponse, toolsCorrect) {
+		if (!testCase.expectedAnswer) return {
+			passed: toolsCorrect,
+			score: toolsCorrect ? 1 : 0,
+			method: "contains",
+			reasoning: "No expected answer provided, evaluated tools only",
+			toolsCorrect
+		};
+		const normalizeDashes = (s) => s.replace(/[\u2013\u2011]/g, "-");
+		const contains = normalizeDashes(agentResponse.toLowerCase()).includes(normalizeDashes(testCase.expectedAnswer.toLowerCase()));
+		return {
+			passed: contains && toolsCorrect,
+			score: contains ? 1 : 0,
+			method: "contains",
+			reasoning: contains ? `Response contains expected text: "${testCase.expectedAnswer}"` : `Response does not contain expected text: "${testCase.expectedAnswer}"`,
+			toolsCorrect
+		};
+	}
+	evaluateBySimilarity(testCase, agentResponse, toolsCorrect) {
+		if (!testCase.expectedAnswer) return {
+			passed: toolsCorrect,
+			score: toolsCorrect ? 1 : 0,
+			method: "similarity",
+			reasoning: "No expected answer provided, evaluated tools only",
+			toolsCorrect
+		};
+		const similarity = this.normalizedLevenshtein(testCase.expectedAnswer.toLowerCase(), agentResponse.toLowerCase());
+		return {
+			passed: similarity > .7 && toolsCorrect,
+			score: similarity,
+			method: "similarity",
+			reasoning: `String similarity: ${(similarity * 100).toFixed(1)}% (threshold: 70%)`,
+			toolsCorrect
+		};
+	}
+	async evaluateByLLMComparison(testCase, agentResponse, toolsCorrect) {
+		const prompt = `You are evaluating an AI agent's response to a customer question.
+
+Question: ${testCase.question}
+Expected Answer: ${testCase.expectedAnswer}
+Actual Response: ${agentResponse}
+
+Rate the actual response on a scale of 1-5:
+1 = Completely wrong or irrelevant
+2 = Partially correct but missing key information
+3 = Mostly correct with minor issues
+4 = Correct with good quality
+5 = Excellent, matches or exceeds expected answer
+
+Respond with ONLY a JSON object in this format:
+{"score": <1-5>, "reasoning": "<brief explanation>"}`;
+		const result = await this.llm.complete([{
+			role: "user",
+			content: prompt
+		}], {
+			temperature: 0,
+			maxTokens: 200
+		});
+		let score = 3;
+		let reasoning = "LLM evaluation completed";
+		try {
+			const parsed = JSON.parse(result.text);
+			score = parsed.score;
+			reasoning = parsed.reasoning;
+		} catch {
+			const match = result.text.match(/score["\s:]+(\d)/i);
+			if (match) score = parseInt(match[1], 10);
+			reasoning = result.text.substring(0, 200);
+		}
+		const normalizedScore = score / 5;
+		return {
+			passed: normalizedScore >= .6 && toolsCorrect,
+			score: normalizedScore,
+			method: "llm_judge",
+			reasoning: `LLM comparison (${score}/5): ${reasoning}`,
+			toolsCorrect
+		};
+	}
+	async evaluateByLLMJudge(testCase, agentResponse, toolsCorrect) {
+		const prompt = `You are evaluating an AI agent's response to a customer question.
+
+Question: ${testCase.question}
+Agent Response: ${agentResponse}
+
+Rate the response quality on a scale of 1-5:
+1 = Unhelpful, incorrect, or inappropriate
+2 = Partially helpful but incomplete or unclear
+3 = Adequate, addresses the question reasonably
+4 = Good quality, helpful and accurate
+5 = Excellent, comprehensive and professional
+
+Respond with ONLY a JSON object in this format:
+{"score": <1-5>, "reasoning": "<brief explanation>"}`;
+		const result = await this.llm.complete([{
+			role: "user",
+			content: prompt
+		}], {
+			temperature: 0,
+			maxTokens: 200
+		});
+		let score = 3;
+		let reasoning = "LLM evaluation completed";
+		try {
+			const parsed = JSON.parse(result.text);
+			score = parsed.score;
+			reasoning = parsed.reasoning;
+		} catch {
+			const match = result.text.match(/score["\s:]+(\d)/i);
+			if (match) score = parseInt(match[1], 10);
+			reasoning = result.text.substring(0, 200);
+		}
+		const normalizedScore = score / 5;
+		return {
+			passed: normalizedScore >= .6 && toolsCorrect,
+			score: normalizedScore,
+			method: "llm_judge",
+			reasoning: `LLM standalone judge (${score}/5): ${reasoning}`,
+			toolsCorrect
+		};
+	}
+	normalizedLevenshtein(s1, s2) {
+		const len1 = s1.length;
+		const len2 = s2.length;
+		if (len1 === 0) return len2 === 0 ? 1 : 0;
+		if (len2 === 0) return 0;
+		const matrix = Array.from({ length: len1 + 1 }, () => Array(len2 + 1).fill(0));
+		for (let i = 0; i <= len1; i++) matrix[i][0] = i;
+		for (let j = 0; j <= len2; j++) matrix[0][j] = j;
+		for (let i = 1; i <= len1; i++) for (let j = 1; j <= len2; j++) {
+			const cost = s1[i - 1] === s2[j - 1] ? 0 : 1;
+			matrix[i][j] = Math.min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost);
+		}
+		return 1 - matrix[len1][len2] / Math.max(len1, len2);
+	}
+};
+
+//#endregion
+//#region src/TestSuiteRunner.ts
+var TestSuiteRunner = class {
+	evaluator;
+	agentOS;
+	timeout;
+	strategy;
+	constructor(config) {
+		this.agentOS = config.agentOS;
+		this.evaluator = new TestCaseEvaluator(config.llm);
+		this.timeout = config.timeout || 3e4;
+		this.strategy = config.strategy;
+	}
+	async runSuite(testCases) {
+		const results = [];
+		const startTime = Date.now();
+		for (const testCase of testCases) {
+			const result = await this.runTestCase(testCase);
+			results.push(result);
+		}
+		const totalDuration = Date.now() - startTime;
+		const passed = results.filter((r) => r.evaluation.passed).length;
+		const failed = results.length - passed;
+		const averageScore = results.reduce((sum, r) => sum + r.evaluation.score, 0) / results.length;
+		const totalCost = results.reduce((sum, r) => sum + r.cost, 0);
+		const byTag = {};
+		for (const result of results) {
+			const tags = result.testCase.tags || ["untagged"];
+			for (const tag of tags) {
+				if (!byTag[tag]) byTag[tag] = {
+					total: 0,
+					passed: 0,
+					avgScore: 0
+				};
+				byTag[tag].total++;
+				if (result.evaluation.passed) byTag[tag].passed++;
+				byTag[tag].avgScore += result.evaluation.score;
+			}
+		}
+		for (const tag in byTag) byTag[tag].avgScore /= byTag[tag].total;
+		return {
+			total: results.length,
+			passed,
+			failed,
+			averageScore,
+			byTag,
+			results,
+			totalDuration,
+			totalCost
+		};
+	}
+	async runTestCase(testCase) {
+		const startTime = Date.now();
+		let agentResponse = "";
+		let toolsCalled = [];
+		let cost = 0;
+		try {
+			const responsePromise = new Promise((resolve, reject) => {
+				const timeoutId = setTimeout(() => {
+					reject(/* @__PURE__ */ new Error(`Test case ${testCase.id} timed out after ${this.timeout}ms`));
+				}, this.timeout);
+				this.agentOS.once("message:processed", (event) => {
+					clearTimeout(timeoutId);
+					resolve({
+						text: event.response.text,
+						toolCalls: event.response.toolCalls || [],
+						cost: event.cost || 0
+					});
+				});
+				this.agentOS.once("error", (event) => {
+					clearTimeout(timeoutId);
+					reject(event.error);
+				});
+			});
+			const mockProvider = Array.from(this.agentOS.providers.values()).find((p) => p.name === "mock");
+			if (!mockProvider) throw new Error("MockProvider not found in Operor");
+			const testPhone = testCase.persona || "test-user";
+			mockProvider.simulateIncomingMessage(testPhone, testCase.question);
+			const response = await responsePromise;
+			agentResponse = response.text;
+			toolsCalled = response.toolCalls || [];
+			cost = response.cost;
+		} catch (error) {
+			agentResponse = `Error: ${error instanceof Error ? error.message : String(error)}`;
+		}
+		const duration = Date.now() - startTime;
+		const evaluation = await this.evaluator.evaluate(testCase, agentResponse, toolsCalled, this.strategy);
+		return {
+			testCase,
+			agentResponse,
+			toolsCalled,
+			evaluation,
+			duration,
+			cost
+		};
+	}
+};
+
+//#endregion
+//#region src/SkillTestHarness.ts
+/**
+* Safety wrapper for Integration instances during testing.
+* Provides operation limits, dry-run mode, and audit logging.
+*/
+var SkillTestHarness = class SkillTestHarness {
+	name;
+	inner;
+	config;
+	auditLog = [];
+	operationCount = 0;
+	static READ_TOOLS = new Set([
+		"get_order",
+		"search_products",
+		"salesforce_get_contact",
+		"salesforce_get_cases",
+		"stripe_get_customer"
+	]);
+	static WRITE_TOOLS = new Set([
+		"create_discount",
+		"salesforce_update_contact",
+		"salesforce_create_case",
+		"salesforce_add_note"
+	]);
+	static DESTRUCTIVE_TOOLS = new Set(["stripe_create_refund"]);
+	constructor(inner, config = {}) {
+		this.inner = inner;
+		this.name = inner.name;
+		this.config = {
+			allowWrites: config.allowWrites ?? false,
+			allowDestructive: config.allowDestructive ?? false,
+			maxOperations: config.maxOperations ?? 10,
+			timeoutMs: config.timeoutMs ?? 3e4,
+			dryRun: config.dryRun ?? false
+		};
+		this.tools = this.wrapTools(inner.tools);
+	}
+	async initialize() {
+		return this.inner.initialize();
+	}
+	/** @deprecated Use initialize() instead. */
+	async authenticate() {
+		return this.inner.initialize();
+	}
+	isReady() {
+		return this.inner.isReady();
+	}
+	/** @deprecated Use isReady() instead. */
+	isAuthenticated() {
+		return this.inner.isReady();
+	}
+	tools;
+	wrapTools(innerTools) {
+		const wrapped = {};
+		for (const [toolName, tool] of Object.entries(innerTools)) wrapped[toolName] = {
+			...tool,
+			execute: async (params) => {
+				const startTime = Date.now();
+				const classification = this.classifyTool(toolName);
+				if (this.operationCount >= this.config.maxOperations) throw new Error(`SkillTestHarness: Max operations limit reached (${this.config.maxOperations})`);
+				if (classification === "write" && !this.config.allowWrites) throw new Error(`SkillTestHarness: Write operation '${toolName}' blocked (allowWrites=false)`);
+				if (classification === "destructive" && !this.config.allowDestructive) throw new Error(`SkillTestHarness: Destructive operation '${toolName}' blocked (allowDestructive=false)`);
+				this.operationCount++;
+				if (this.config.dryRun) {
+					const result = {
+						dryRun: true,
+						wouldExecute: toolName,
+						params
+					};
+					this.auditLog.push({
+						name: toolName,
+						params,
+						result,
+						timestamp: startTime,
+						duration: Date.now() - startTime,
+						classification
+					});
+					return result;
+				}
+				const timeoutPromise = new Promise((_, reject) => {
+					setTimeout(() => {
+						reject(/* @__PURE__ */ new Error(`SkillTestHarness: Operation '${toolName}' timed out after ${this.config.timeoutMs}ms`));
+					}, this.config.timeoutMs);
+				});
+				try {
+					const result = await Promise.race([tool.execute(params), timeoutPromise]);
+					const duration = Date.now() - startTime;
+					this.auditLog.push({
+						name: toolName,
+						params,
+						result,
+						timestamp: startTime,
+						duration,
+						classification
+					});
+					return result;
+				} catch (error) {
+					const duration = Date.now() - startTime;
+					this.auditLog.push({
+						name: toolName,
+						params,
+						result: { error: error instanceof Error ? error.message : "Unknown error" },
+						timestamp: startTime,
+						duration,
+						classification
+					});
+					throw error;
+				}
+			}
+		};
+		return wrapped;
+	}
+	classifyTool(toolName) {
+		if (SkillTestHarness.READ_TOOLS.has(toolName)) return "read";
+		if (SkillTestHarness.DESTRUCTIVE_TOOLS.has(toolName)) return "destructive";
+		if (SkillTestHarness.WRITE_TOOLS.has(toolName)) return "write";
+		return "write";
+	}
+	/**
+	* Get the audit log of all operations performed
+	*/
+	getAuditLog() {
+		return [...this.auditLog];
+	}
+	/**
+	* Reset the audit log and operation counter
+	*/
+	resetAuditLog() {
+		this.auditLog = [];
+		this.operationCount = 0;
+	}
+	/**
+	* Get the current operation count
+	*/
+	getOperationCount() {
+		return this.operationCount;
+	}
+};
+
+//#endregion
+//#region src/CustomerSimulator.ts
+const PERSONA_PROMPTS = {
+	polite: "You are polite, patient, and use courteous language.",
+	frustrated: "You are frustrated and impatient. You express dissatisfaction but remain civil.",
+	confused: "You are confused and unsure. You ask clarifying questions and sometimes misunderstand.",
+	terse: "You give very short, minimal responses. One sentence max.",
+	verbose: "You are detailed and talkative. You provide lots of context and background."
+};
+function buildSystemPrompt(persona, context) {
+	return `You are simulating a customer in a support conversation for testing purposes.
+${PERSONA_PROMPTS[persona] || `You are a customer with a ${persona} communication style.`}${context?.scenario ? `\nScenario: ${context.scenario}` : ""}${context?.maxTurns ? `\nThis conversation has a maximum of ${context.maxTurns} turns. You are on turn ${context.currentTurn ?? 1}.` : ""}
+
+Rules:
+- Stay in character throughout the conversation.
+- Escalate naturally if your issue isn't being resolved.
+- Set shouldContinue to false when your issue is resolved or you have no more questions.
+- If the agent asks a question, answer it in character.
+
+Respond with ONLY valid JSON (no markdown, no code fences):
+{"message": "your response as the customer", "shouldContinue": true}`;
+}
+function formatHistory(history) {
+	return history.map((turn) => ({
+		role: turn.role === "customer" ? "user" : "assistant",
+		content: turn.message
+	}));
+}
+function parseResponse(text) {
+	const cleaned = text.replace(/```(?:json)?\s*/g, "").replace(/```/g, "").trim();
+	try {
+		const parsed = JSON.parse(cleaned);
+		return {
+			message: String(parsed.message ?? ""),
+			shouldContinue: Boolean(parsed.shouldContinue)
+		};
+	} catch {
+		return {
+			message: text.trim(),
+			shouldContinue: true
+		};
+	}
+}
+var CustomerSimulator = class {
+	llm;
+	constructor(options) {
+		this.llm = options?.llmProvider;
+	}
+	async generateMessage(persona, history, context) {
+		if (context?.scriptedResponses?.length) {
+			const turn = context.currentTurn ?? history.filter((t) => t.role === "customer").length;
+			const responses = context.scriptedResponses;
+			if (turn < responses.length) return {
+				message: responses[turn],
+				shouldContinue: turn < responses.length - 1
+			};
+			return {
+				message: responses[responses.length - 1],
+				shouldContinue: false
+			};
+		}
+		if (!this.llm) throw new Error("CustomerSimulator requires an LLM provider for non-scripted mode");
+		const messages = [{
+			role: "system",
+			content: buildSystemPrompt(persona, context)
+		}, ...formatHistory(history)];
+		return parseResponse((await this.llm.complete(messages, {
+			temperature: .7,
+			maxTokens: 500
+		})).text);
+	}
+};
+
+//#endregion
+//#region src/ConversationEvaluator.ts
+function buildEvaluationPrompt(config) {
+	const { scenario, persona, turns, toolsCalled, expectedTools, expectedOutcome } = config;
+	const conversationText = turns.map((t, i) => `Turn ${i + 1} [${t.role}]: ${t.message}`).join("\n");
+	const toolsText = toolsCalled.length ? toolsCalled.map((tc) => `- ${tc.name}(${JSON.stringify(tc.params)}) → ${JSON.stringify(tc.result)}`).join("\n") : "None";
+	return `You are evaluating a customer support conversation for testing purposes.
+
+Scenario: ${scenario}
+Customer Persona: ${persona}
+Expected Tools: ${expectedTools?.length ? expectedTools.join(", ") : "Not specified"}
+Expected Outcome: ${expectedOutcome || "Not specified"}
+
+Conversation:
+${conversationText}
+
+Tools Called:
+${toolsText}
+
+Evaluate the conversation on these dimensions (score 1-5 for each):
+
+1. Accuracy (1-5): Did the agent provide factually correct information based on tool results?
+2. Tool Usage (1-5): Did the agent call the right tools at the right time?
+3. Tone (1-5): Was the agent's tone appropriate, professional, and empathetic?
+4. Resolution (1-5): Was the customer's issue resolved or properly addressed?
+
+Overall assessment:
+- "pass": All criteria met, customer satisfied (scores mostly 4-5)
+- "partial": Some issues but acceptable (scores mostly 3-4)
+- "fail": Significant problems (any score 1-2, or multiple scores below 3)
+
+Respond with ONLY valid JSON (no markdown, no code fences):
+{
+  "overall": "pass" | "fail" | "partial",
+  "scores": {
+    "accuracy": <integer 1-5>,
+    "toolUsage": <integer 1-5>,
+    "tone": <integer 1-5>,
+    "resolution": <integer 1-5>
+  },
+  "feedback": "<brief explanation of the evaluation>"
+}`;
+}
+function parseEvaluationResponse(text) {
+	const cleaned = text.replace(/```(?:json)?\s*/g, "").replace(/```/g, "").trim();
+	try {
+		const parsed = JSON.parse(cleaned);
+		return {
+			overall: parsed.overall || "fail",
+			scores: {
+				accuracy: Math.round(parsed.scores?.accuracy ?? 1),
+				toolUsage: Math.round(parsed.scores?.toolUsage ?? 1),
+				tone: Math.round(parsed.scores?.tone ?? 1),
+				resolution: Math.round(parsed.scores?.resolution ?? 1)
+			},
+			feedback: String(parsed.feedback ?? "No feedback provided")
+		};
+	} catch {
+		return {
+			overall: "fail",
+			scores: {
+				accuracy: 1,
+				toolUsage: 1,
+				tone: 1,
+				resolution: 1
+			},
+			feedback: "Failed to parse evaluation response"
+		};
+	}
+}
+function evaluateCriteria(criteria, turns) {
+	const { type, value } = criteria;
+	switch (type) {
+		case "tool_called": {
+			const toolName = String(value);
+			const called = turns.some((turn) => turn.toolCalls?.some((tc) => tc.name === toolName));
+			return {
+				criteria,
+				passed: called,
+				details: called ? `Tool "${toolName}" was called` : `Tool "${toolName}" was not called`
+			};
+		}
+		case "response_contains": {
+			const searchText = String(value).toLowerCase();
+			const found = turns.some((turn) => turn.role === "agent" && turn.message.toLowerCase().includes(searchText));
+			return {
+				criteria,
+				passed: found,
+				details: found ? `Agent response contains "${value}"` : `Agent response does not contain "${value}"`
+			};
+		}
+		case "intent_matched": {
+			const intentText = String(value).toLowerCase();
+			const matched = turns.some((turn) => turn.message.toLowerCase().includes(intentText));
+			return {
+				criteria,
+				passed: matched,
+				details: matched ? `Intent "${value}" was matched` : `Intent "${value}" was not matched`
+			};
+		}
+		case "turns_under": {
+			const maxTurns = Number(value);
+			const actualTurns = turns.length;
+			const passed = actualTurns < maxTurns;
+			return {
+				criteria,
+				passed,
+				details: passed ? `Conversation completed in ${actualTurns} turns (under ${maxTurns})` : `Conversation took ${actualTurns} turns (expected under ${maxTurns})`
+			};
+		}
+		case "custom":
+			if (typeof value === "function") try {
+				const passed = value(turns);
+				return {
+					criteria,
+					passed,
+					details: passed ? "Custom criteria passed" : "Custom criteria failed"
+				};
+			} catch (error) {
+				return {
+					criteria,
+					passed: false,
+					details: `Custom criteria error: ${error}`
+				};
+			}
+			return {
+				criteria,
+				passed: false,
+				details: "Custom criteria value must be a function"
+			};
+		default: return {
+			criteria,
+			passed: false,
+			details: `Unknown criteria type: ${type}`
+		};
+	}
+}
+var ConversationEvaluator = class {
+	llm;
+	constructor(options) {
+		this.llm = options?.llmProvider;
+	}
+	async evaluate(config) {
+		const { turns, successCriteria } = config;
+		const criteriaResults = successCriteria ? successCriteria.map((criteria) => evaluateCriteria(criteria, turns)) : [];
+		if (this.llm) {
+			const messages = [{
+				role: "user",
+				content: buildEvaluationPrompt(config)
+			}];
+			const evaluation = parseEvaluationResponse((await this.llm.complete(messages, {
+				temperature: 0,
+				maxTokens: 1e3
+			})).text);
+			evaluation.criteriaResults = criteriaResults;
+			if (!criteriaResults.every((cr) => cr.passed) && evaluation.overall === "pass") evaluation.overall = "partial";
+			return evaluation;
+		}
+		if (!criteriaResults.length) return {
+			overall: "fail",
+			scores: {
+				accuracy: 1,
+				toolUsage: 1,
+				tone: 1,
+				resolution: 1
+			},
+			feedback: "No criteria specified",
+			criteriaResults
+		};
+		const allPassed = criteriaResults.every((cr) => cr.passed);
+		const somePassed = criteriaResults.some((cr) => cr.passed);
+		return {
+			overall: allPassed ? "pass" : somePassed ? "partial" : "fail",
+			scores: {
+				accuracy: 3,
+				toolUsage: 3,
+				tone: 3,
+				resolution: 3
+			},
+			feedback: criteriaResults.map((cr) => cr.details).join("; "),
+			criteriaResults
+		};
+	}
+};
+
+//#endregion
+//#region src/utils.ts
+/**
+* Format current time as HH:MM:SS timestamp
+*/
+function formatTimestamp() {
+	return (/* @__PURE__ */ new Date()).toLocaleTimeString("en-US", { hour12: false });
+}
+
+//#endregion
+//#region src/ConversationRunner.ts
+var ConversationRunner = class {
+	agentOS;
+	customerSimulator;
+	conversationEvaluator;
+	timeout;
+	verbose;
+	constructor(config) {
+		this.agentOS = config.agentOS;
+		this.customerSimulator = config.customerSimulator;
+		this.conversationEvaluator = config.conversationEvaluator;
+		this.timeout = config.timeout || 3e4;
+		this.verbose = config.verbose ?? false;
+	}
+	async runScenario(scenario) {
+		const startTime = Date.now();
+		const turns = [];
+		const toolsCalled = [];
+		let totalCost = 0;
+		try {
+			const mockProvider = this.getMockProvider();
+			const customerId = `test-customer-${scenario.id}`;
+			const initialMessage = scenario.scriptedResponses?.[0] || "Hello, I need help";
+			if (this.verbose) {
+				console.log(`\n=== Starting Scenario: ${scenario.name} ===`);
+				console.log(`Persona: ${scenario.persona}`);
+				console.log(`Max Turns: ${scenario.maxTurns}\n`);
+			}
+			let shouldContinue = true;
+			let currentTurn = 0;
+			while (shouldContinue && currentTurn < scenario.maxTurns) {
+				const customerMessage = currentTurn === 0 ? initialMessage : (await this.customerSimulator.generateMessage(scenario.persona, turns, {
+					scenario: scenario.description,
+					maxTurns: scenario.maxTurns,
+					currentTurn,
+					scriptedResponses: scenario.scriptedResponses
+				})).message;
+				if (this.verbose) console.log(`[${formatTimestamp()}] Turn ${currentTurn + 1} [customer]: ${customerMessage}`);
+				const agentResponse = await this.waitForAgentResponse(mockProvider, customerId, customerMessage);
+				if (this.verbose) {
+					console.log(`[${formatTimestamp()}] Turn ${currentTurn + 1} [agent]: ${agentResponse.text}`);
+					if (agentResponse.toolCalls?.length) console.log(`  Tools called: ${agentResponse.toolCalls.map((tc) => tc.name).join(", ")}`);
+				}
+				turns.push({
+					role: "customer",
+					message: customerMessage
+				});
+				turns.push({
+					role: "agent",
+					message: agentResponse.text,
+					toolCalls: agentResponse.toolCalls
+				});
+				if (agentResponse.toolCalls) toolsCalled.push(...agentResponse.toolCalls);
+				totalCost += agentResponse.cost || 0;
+				currentTurn++;
+				if (scenario.scriptedResponses) shouldContinue = currentTurn < scenario.scriptedResponses.length;
+				else if (currentTurn < scenario.maxTurns) {
+					shouldContinue = (await this.customerSimulator.generateMessage(scenario.persona, turns, {
+						scenario: scenario.description,
+						maxTurns: scenario.maxTurns,
+						currentTurn
+					})).shouldContinue;
+					if (!shouldContinue && this.verbose) console.log("Customer satisfied, ending conversation.");
+				}
+			}
+			if (this.verbose && currentTurn >= scenario.maxTurns) console.log(`\nReached max turns (${scenario.maxTurns})`);
+			const evaluation = await this.conversationEvaluator.evaluate({
+				scenario: scenario.description,
+				persona: scenario.persona,
+				turns,
+				toolsCalled,
+				expectedTools: scenario.expectedTools,
+				expectedOutcome: scenario.expectedOutcome,
+				successCriteria: scenario.successCriteria
+			});
+			if (this.verbose) {
+				console.log(`\n=== Evaluation ===`);
+				console.log(`Overall: ${evaluation.overall}`);
+				console.log(`Scores: ${JSON.stringify(evaluation.scores)}`);
+				console.log(`Feedback: ${evaluation.feedback}\n`);
+			}
+			const duration = Date.now() - startTime;
+			return {
+				scenario,
+				passed: evaluation.overall === "pass",
+				turns,
+				evaluation,
+				duration,
+				cost: totalCost
+			};
+		} catch (error) {
+			const duration = Date.now() - startTime;
+			const errorMessage = error instanceof Error ? error.message : String(error);
+			if (this.verbose) console.error(`\nError in scenario ${scenario.name}: ${errorMessage}\n`);
+			return {
+				scenario,
+				passed: false,
+				turns,
+				evaluation: {
+					overall: "fail",
+					scores: {
+						accuracy: 1,
+						toolUsage: 1,
+						tone: 1,
+						resolution: 1
+					},
+					feedback: `Error: ${errorMessage}`
+				},
+				duration,
+				cost: totalCost
+			};
+		}
+	}
+	async runScenarios(scenarios) {
+		const results = [];
+		for (const scenario of scenarios) {
+			const result = await this.runScenario(scenario);
+			results.push(result);
+		}
+		return results;
+	}
+	getMockProvider() {
+		const mockProvider = Array.from(this.agentOS.providers.values()).find((p) => p.name === "mock");
+		if (!mockProvider) throw new Error("MockProvider not found in Operor. Add it with agentOS.addProvider()");
+		return mockProvider;
+	}
+	async waitForAgentResponse(mockProvider, customerId, message) {
+		return new Promise((resolve, reject) => {
+			let settled = false;
+			const cleanup = () => {
+				this.agentOS.removeListener("message:processed", onProcessed);
+				this.agentOS.removeListener("error", onError);
+			};
+			const timeoutId = setTimeout(() => {
+				if (!settled) {
+					settled = true;
+					cleanup();
+					reject(/* @__PURE__ */ new Error(`Agent response timed out after ${this.timeout}ms`));
+				}
+			}, this.timeout);
+			const onProcessed = (event) => {
+				if (!settled) {
+					settled = true;
+					clearTimeout(timeoutId);
+					cleanup();
+					resolve({
+						text: event.response.text,
+						toolCalls: event.response.toolCalls || [],
+						cost: event.cost || 0
+					});
+				}
+			};
+			const onError = (event) => {
+				if (!settled) {
+					settled = true;
+					clearTimeout(timeoutId);
+					cleanup();
+					reject(event.error);
+				}
+			};
+			this.agentOS.once("message:processed", onProcessed);
+			this.agentOS.once("error", onError);
+			mockProvider.simulateIncomingMessage(customerId, message);
+		});
+	}
+};
+
+//#endregion
+//#region src/scenarios.ts
+const ECOMMERCE_SCENARIOS = [
+	{
+		id: "delayed-order-compensation",
+		name: "Delayed order with compensation",
+		description: "Customer asks about a delayed order and expects compensation",
+		persona: "Frustrated customer whose order #12345 was supposed to arrive 3 days ago",
+		maxTurns: 6,
+		expectedTools: ["get_order", "create_discount"],
+		expectedOutcome: "Agent finds order, acknowledges delay, offers discount",
+		scriptedResponses: [
+			"Where is my order #12345? It was supposed to arrive 3 days ago!",
+			"This is really frustrating. Can you do anything to make up for this?",
+			"Okay, I appreciate the discount. Thank you."
+		]
+	},
+	{
+		id: "order-not-found",
+		name: "Order not found",
+		description: "Customer asks about an order that does not exist in the system",
+		persona: "Confused customer who may have the wrong order number",
+		maxTurns: 4,
+		expectedTools: ["get_order"],
+		expectedOutcome: "Agent attempts lookup, explains order not found, asks customer to verify",
+		scriptedResponses: ["Can you check on order #99999 for me?", "Hmm, let me double check the number and get back to you."]
+	},
+	{
+		id: "product-inquiry",
+		name: "Product inquiry",
+		description: "Customer searches for a product and asks about availability",
+		persona: "Curious shopper looking for electronics",
+		maxTurns: 4,
+		expectedTools: ["search_products"],
+		expectedOutcome: "Agent searches products and provides relevant results",
+		scriptedResponses: [
+			"Do you have any wireless headphones in stock?",
+			"What about the price range? Anything under $200?",
+			"Great, thanks for the info!"
+		]
+	},
+	{
+		id: "return-request",
+		name: "Return request",
+		description: "Customer wants to return a recently delivered item",
+		persona: "Polite customer who received a defective product from order #67890",
+		maxTurns: 5,
+		expectedTools: ["get_order"],
+		expectedOutcome: "Agent looks up order, acknowledges issue, explains return process",
+		scriptedResponses: [
+			"I received order #67890 but one of the items is defective. I would like to return it.",
+			"Yes, the wireless mouse stopped working after one day.",
+			"Okay, how do I send it back?"
+		]
+	},
+	{
+		id: "greeting",
+		name: "Simple greeting",
+		description: "Customer says hello and expects a friendly welcome",
+		persona: "Friendly first-time visitor",
+		maxTurns: 2,
+		expectedTools: [],
+		expectedOutcome: "Agent responds with a friendly greeting and offers help",
+		scriptedResponses: ["Hello!"]
+	},
+	{
+		id: "billing-dispute",
+		name: "Billing dispute",
+		description: "Customer believes they were charged incorrectly",
+		persona: "Concerned customer who noticed a double charge on order #12345",
+		maxTurns: 5,
+		expectedTools: ["get_order"],
+		expectedOutcome: "Agent looks up order, reviews charges, and addresses the billing concern",
+		scriptedResponses: [
+			"I think I was charged twice for order #12345. Can you check?",
+			"My credit card shows two charges of $299.99 on the same day.",
+			"Can you escalate this to someone who can issue a correction?"
+		]
+	},
+	{
+		id: "multi-issue",
+		name: "Multi-issue conversation",
+		description: "Customer has multiple problems: a delayed order and a product question",
+		persona: "Busy customer who wants to resolve everything in one conversation",
+		maxTurns: 6,
+		expectedTools: ["get_order", "search_products"],
+		expectedOutcome: "Agent handles both issues sequentially without losing context",
+		scriptedResponses: [
+			"Two things: first, where is my order #12345?",
+			"Okay thanks. Also, do you carry mechanical keyboards?",
+			"Nice, I might order one. Can you also give me a discount for the late delivery?",
+			"Sounds good, thanks for handling both issues."
+		]
+	},
+	{
+		id: "lead-qualification",
+		name: "Lead qualification",
+		description: "Potential customer asking pre-purchase questions about products and shipping",
+		persona: "Prospective buyer evaluating whether to make a purchase",
+		maxTurns: 4,
+		expectedTools: ["search_products"],
+		expectedOutcome: "Agent answers product questions and encourages purchase",
+		scriptedResponses: [
+			"I am thinking about buying some electronics. What do you have?",
+			"How fast is shipping usually?",
+			"Do you offer any discounts for first-time buyers?"
+		]
+	},
+	{
+		id: "frustrated-escalation",
+		name: "Frustrated customer escalation",
+		description: "Angry customer escalates through multiple complaints",
+		persona: "Very frustrated customer who has contacted support multiple times about order #12345",
+		maxTurns: 6,
+		expectedTools: ["get_order", "create_discount"],
+		expectedOutcome: "Agent remains professional, empathizes, and offers concrete resolution",
+		scriptedResponses: [
+			"This is the THIRD time I am contacting you about order #12345. Still not here!",
+			"I have been waiting over a week. This is completely unacceptable.",
+			"I want a refund or serious compensation. A 5% coupon is insulting.",
+			"Fine, that is more reasonable. But I expect the order to arrive this week."
+		]
+	},
+	{
+		id: "on-time-order-check",
+		name: "On-time order status check",
+		description: "Customer checks on an order that is on time or already delivered",
+		persona: "Polite customer just checking in on order #67890",
+		maxTurns: 2,
+		expectedTools: ["get_order"],
+		expectedOutcome: "Agent confirms order status, no compensation needed",
+		scriptedResponses: ["Hi, can I get an update on order #67890?", "Perfect, thanks!"]
+	}
+];
+
+//#endregion
+//#region src/SimulationRunner.ts
+var SimulationRunner = class {
+	agentOS;
+	config;
+	llm;
+	constructor(options) {
+		this.agentOS = options.agentOS;
+		this.config = options.config;
+		this.llm = options.llm;
+	}
+	async run(onProgress) {
+		const startTime = Date.now();
+		const testSuiteResults = [];
+		const conversationResults = [];
+		let totalCost = 0;
+		if (this.config.testSuiteFiles?.length) {
+			const suiteRunner = new TestSuiteRunner({
+				agentOS: this.agentOS,
+				llm: this.llm,
+				timeout: this.config.timeout
+			});
+			for (const file of this.config.testSuiteFiles) {
+				const testCases = await CSVLoader.fromFile(file);
+				const result = await suiteRunner.runSuite(testCases);
+				testSuiteResults.push(result);
+				totalCost += result.totalCost;
+			}
+		}
+		const scenarios = this.resolveScenarios();
+		if (scenarios.length) {
+			const conversationRunner = new ConversationRunner({
+				agentOS: this.agentOS,
+				customerSimulator: new CustomerSimulator({ llmProvider: this.llm }),
+				conversationEvaluator: new ConversationEvaluator({ llmProvider: this.llm }),
+				timeout: this.config.timeout
+			});
+			const schedule = this.buildSchedule(scenarios);
+			const pauseMs = this.config.pauseBetweenMs ?? 500;
+			for (let i = 0; i < schedule.length; i++) {
+				const scenario = schedule[i];
+				const timeoutMs = this.config.timeout || 6e4;
+				const result = await Promise.race([conversationRunner.runScenario(scenario), new Promise((_, reject) => setTimeout(() => reject(/* @__PURE__ */ new Error(`Conversation timed out after ${timeoutMs}ms`)), timeoutMs))]).catch((error) => {
+					return {
+						scenario,
+						passed: false,
+						turns: [],
+						evaluation: {
+							overall: "fail",
+							scores: {
+								accuracy: 1,
+								toolUsage: 1,
+								tone: 1,
+								resolution: 1
+							},
+							feedback: `Timeout or error: ${error instanceof Error ? error.message : String(error)}`
+						},
+						duration: timeoutMs,
+						cost: 0
+					};
+				});
+				conversationResults.push(result);
+				totalCost += result.cost;
+				if (onProgress) onProgress(i + 1, schedule.length, result);
+				if (i < schedule.length - 1 && pauseMs > 0) await new Promise((resolve) => setTimeout(resolve, pauseMs));
+			}
+		}
+		const duration = Date.now() - startTime;
+		const totalTests = testSuiteResults.reduce((sum, r) => sum + r.total, 0);
+		const passedTests = testSuiteResults.reduce((sum, r) => sum + r.passed, 0);
+		const failedTests = testSuiteResults.reduce((sum, r) => sum + r.failed, 0);
+		const totalConversations = conversationResults.length;
+		const passedConversations = conversationResults.filter((r) => r.passed).length;
+		const failedConversations = totalConversations - passedConversations;
+		const totalItems = totalTests + totalConversations;
+		const passedItems = passedTests + passedConversations;
+		const overallPassRate = totalItems > 0 ? passedItems / totalItems : 0;
+		const averageScores = this.computeAverageScores(conversationResults);
+		const scenarioBreakdown = this.computeScenarioBreakdown(conversationResults);
+		const toolUsageStats = this.computeToolUsageStats(conversationResults);
+		const failedResults = conversationResults.filter((r) => !r.passed);
+		let commonFailurePatterns = [];
+		let recommendations = [];
+		if (failedResults.length > 0) if (this.llm) {
+			const analysis = await this.analyzeFailuresWithLLM(failedResults);
+			commonFailurePatterns = analysis.patterns;
+			recommendations = analysis.recommendations;
+		} else {
+			commonFailurePatterns = this.heuristicFailurePatterns(failedResults);
+			recommendations = this.heuristicRecommendations(failedResults);
+		}
+		return {
+			timestamp: /* @__PURE__ */ new Date(),
+			duration,
+			totalConversations,
+			passed: passedConversations,
+			failed: failedConversations,
+			averageScores,
+			scenarioBreakdown,
+			toolUsageStats,
+			commonFailurePatterns,
+			recommendations,
+			testSuiteResults,
+			conversationResults,
+			overallPassed: failedTests === 0 && failedConversations === 0 && totalItems > 0,
+			totalCost,
+			summary: {
+				totalTests,
+				passedTests,
+				failedTests,
+				totalConversations,
+				passedConversations,
+				failedConversations,
+				overallPassRate
+			}
+		};
+	}
+	static formatReport(report) {
+		const lines = [];
+		lines.push("=== Simulation Report ===");
+		lines.push(`Date: ${report.timestamp.toISOString()}`);
+		lines.push(`Duration: ${(report.duration / 1e3).toFixed(1)}s`);
+		lines.push(`Cost: $${report.totalCost.toFixed(4)}`);
+		lines.push("");
+		if (report.testSuiteResults.length) {
+			lines.push("--- Test Suites ---");
+			for (const suite of report.testSuiteResults) {
+				lines.push(`  ${suite.passed}/${suite.total} passed (avg score: ${suite.averageScore.toFixed(2)})`);
+				for (const result of suite.results) {
+					const status = result.evaluation.passed ? "PASS" : "FAIL";
+					lines.push(`    [${status}] ${result.testCase.id}: ${result.testCase.question}`);
+				}
+			}
+			lines.push("");
+		}
+		if (report.scenarioBreakdown.length) {
+			lines.push("--- Scenario Breakdown ---");
+			for (const s of report.scenarioBreakdown) {
+				const pct = (s.passRate * 100).toFixed(0);
+				lines.push(`  ${s.scenario}: ${s.runs} run(s), ${pct}% pass rate, avg score ${s.avgScore.toFixed(2)}`);
+			}
+			lines.push("");
+		}
+		const toolEntries = Object.entries(report.toolUsageStats);
+		if (toolEntries.length) {
+			lines.push("--- Tool Usage ---");
+			for (const [tool, count] of toolEntries.sort((a, b) => b[1] - a[1])) lines.push(`  ${tool}: ${count} call(s)`);
+			lines.push("");
+		}
+		const { averageScores } = report;
+		if (report.totalConversations > 0) {
+			lines.push("--- Average Scores ---");
+			lines.push(`  Accuracy:   ${averageScores.accuracy.toFixed(2)}`);
+			lines.push(`  Tool Usage: ${averageScores.toolUsage.toFixed(2)}`);
+			lines.push(`  Tone:       ${averageScores.tone.toFixed(2)}`);
+			lines.push(`  Resolution: ${averageScores.resolution.toFixed(2)}`);
+			lines.push("");
+		}
+		if (report.commonFailurePatterns.length) {
+			lines.push("--- Common Failure Patterns ---");
+			for (const pattern of report.commonFailurePatterns) lines.push(`  - ${pattern}`);
+			lines.push("");
+		}
+		if (report.recommendations.length) {
+			lines.push("--- Recommendations ---");
+			for (const rec of report.recommendations) lines.push(`  - ${rec}`);
+			lines.push("");
+		}
+		const { summary } = report;
+		lines.push("--- Summary ---");
+		if (summary.totalTests > 0) lines.push(`Tests: ${summary.passedTests}/${summary.totalTests} passed`);
+		lines.push(`Conversations: ${summary.passedConversations}/${summary.totalConversations} passed`);
+		lines.push(`Overall pass rate: ${(summary.overallPassRate * 100).toFixed(1)}%`);
+		lines.push(`Result: ${report.overallPassed ? "PASSED" : "FAILED"}`);
+		return lines.join("\n");
+	}
+	resolveScenarios() {
+		if (!this.config.conversationScenarios) return [];
+		if (this.config.conversationScenarios === "builtin") return ECOMMERCE_SCENARIOS;
+		return this.config.conversationScenarios;
+	}
+	buildSchedule(scenarios) {
+		const total = this.config.totalConversations ?? scenarios.length;
+		const schedule = [];
+		for (let i = 0; i < total; i++) schedule.push(scenarios[i % scenarios.length]);
+		return schedule;
+	}
+	computeAverageScores(results) {
+		if (results.length === 0) return {
+			accuracy: 0,
+			toolUsage: 0,
+			tone: 0,
+			resolution: 0
+		};
+		const totals = results.reduce((acc, r) => ({
+			accuracy: acc.accuracy + r.evaluation.scores.accuracy,
+			toolUsage: acc.toolUsage + r.evaluation.scores.toolUsage,
+			tone: acc.tone + r.evaluation.scores.tone,
+			resolution: acc.resolution + r.evaluation.scores.resolution
+		}), {
+			accuracy: 0,
+			toolUsage: 0,
+			tone: 0,
+			resolution: 0
+		});
+		const n = results.length;
+		return {
+			accuracy: totals.accuracy / n,
+			toolUsage: totals.toolUsage / n,
+			tone: totals.tone / n,
+			resolution: totals.resolution / n
+		};
+	}
+	computeScenarioBreakdown(results) {
+		const byScenario = /* @__PURE__ */ new Map();
+		for (const r of results) {
+			const name = r.scenario.name;
+			if (!byScenario.has(name)) byScenario.set(name, []);
+			byScenario.get(name).push(r);
+		}
+		return Array.from(byScenario.entries()).map(([scenario, runs]) => {
+			const passed = runs.filter((r) => r.passed).length;
+			const scores = runs.map((r) => {
+				const s = r.evaluation.scores;
+				return (s.accuracy + s.toolUsage + s.tone + s.resolution) / 4;
+			});
+			const avgScore = scores.reduce((a, b) => a + b, 0) / scores.length;
+			return {
+				scenario,
+				runs: runs.length,
+				passRate: passed / runs.length,
+				avgScore
+			};
+		});
+	}
+	computeToolUsageStats(results) {
+		const stats = {};
+		for (const r of results) for (const turn of r.turns) if (turn.toolCalls) for (const tc of turn.toolCalls) stats[tc.name] = (stats[tc.name] || 0) + 1;
+		return stats;
+	}
+	async analyzeFailuresWithLLM(failedResults) {
+		if (!this.llm) return {
+			patterns: [],
+			recommendations: []
+		};
+		const prompt = `Analyze these failed customer support conversation tests and identify patterns.
+
+${failedResults.slice(0, 10).map((r) => {
+			const turns = r.turns.map((t) => `[${t.role}]: ${t.message}`).join("\n");
+			return `Scenario: ${r.scenario.name}\nFeedback: ${r.evaluation.feedback}\nConversation:\n${turns}`;
+		}).join("\n\n---\n\n")}
+
+Respond with ONLY valid JSON (no markdown, no code fences):
+{
+  "patterns": ["pattern 1", "pattern 2"],
+  "recommendations": ["recommendation 1", "recommendation 2"]
+}`;
+		try {
+			const cleaned = (await this.llm.complete([{
+				role: "user",
+				content: prompt
+			}], {
+				temperature: 0,
+				maxTokens: 1e3
+			})).text.replace(/```(?:json)?\s*/g, "").replace(/```/g, "").trim();
+			const parsed = JSON.parse(cleaned);
+			return {
+				patterns: Array.isArray(parsed.patterns) ? parsed.patterns.map(String) : [],
+				recommendations: Array.isArray(parsed.recommendations) ? parsed.recommendations.map(String) : []
+			};
+		} catch {
+			return {
+				patterns: this.heuristicFailurePatterns(failedResults),
+				recommendations: this.heuristicRecommendations(failedResults)
+			};
+		}
+	}
+	heuristicFailurePatterns(failedResults) {
+		const patterns = [];
+		const noToolCalls = failedResults.filter((r) => r.turns.every((t) => !t.toolCalls?.length));
+		if (noToolCalls.length > 0) patterns.push(`${noToolCalls.length} conversation(s) failed with no tool calls`);
+		const lowResolution = failedResults.filter((r) => r.evaluation.scores.resolution <= 2);
+		if (lowResolution.length > 0) patterns.push(`${lowResolution.length} conversation(s) had low resolution scores`);
+		const lowTone = failedResults.filter((r) => r.evaluation.scores.tone <= 2);
+		if (lowTone.length > 0) patterns.push(`${lowTone.length} conversation(s) had low tone scores`);
+		if (patterns.length === 0) patterns.push(`${failedResults.length} conversation(s) failed evaluation criteria`);
+		return patterns;
+	}
+	heuristicRecommendations(failedResults) {
+		const recs = [];
+		if (failedResults.filter((r) => r.turns.every((t) => !t.toolCalls?.length)).length > 0) recs.push("Ensure agent is configured to use available tools for customer queries");
+		const expectedButMissing = /* @__PURE__ */ new Set();
+		for (const r of failedResults) for (const tool of r.scenario.expectedTools || []) if (!r.turns.some((t) => t.toolCalls?.some((tc) => tc.name === tool))) expectedButMissing.add(tool);
+		if (expectedButMissing.size > 0) recs.push(`Tools expected but not called: ${Array.from(expectedButMissing).join(", ")}`);
+		if (recs.length === 0) recs.push("Review failed scenarios and adjust agent rules or prompts");
+		return recs;
+	}
+};
+
+//#endregion
+//#region src/MockShopifySkill.ts
+var MockShopifySkill = class {
+	name = "shopify";
+	ready = false;
+	mockOrders = /* @__PURE__ */ new Map();
+	mockProducts = [];
+	mockDiscounts = [];
+	nextPriceRuleId = 1e3;
+	constructor() {
+		this.seedMockData();
+	}
+	async initialize() {
+		this.ready = true;
+		console.log("✅ Mock Shopify initialized");
+	}
+	/** @deprecated Use initialize() instead. */
+	async authenticate() {
+		return this.initialize();
+	}
+	isReady() {
+		return this.ready;
+	}
+	/** @deprecated Use isReady() instead. */
+	isAuthenticated() {
+		return this.ready;
+	}
+	/**
+	* Reset all mock data to initial state (for testing)
+	*/
+	reset() {
+		this.mockOrders.clear();
+		this.mockProducts = [];
+		this.mockDiscounts = [];
+		this.nextPriceRuleId = 1e3;
+		this.seedMockData();
+	}
+	/**
+	* Seed custom test data (for testing)
+	*/
+	seedData(config) {
+		if (config.orders) for (const order of config.orders) {
+			const fullOrder = {
+				id: order.id || String(Date.now()),
+				name: order.name || `#${order.id || "1001"}`,
+				status: order.status || "unfulfilled",
+				financialStatus: order.financialStatus || "paid",
+				createdAt: order.createdAt || (/* @__PURE__ */ new Date()).toISOString(),
+				expectedDelivery: order.expectedDelivery || /* @__PURE__ */ new Date(),
+				actualDelivery: order.actualDelivery,
+				tracking: order.tracking,
+				trackingUrl: order.trackingUrl,
+				items: order.items || [],
+				total: order.total || "0.00"
+			};
+			this.mockOrders.set(fullOrder.id, fullOrder);
+		}
+		if (config.products) for (const product of config.products) {
+			const fullProduct = {
+				id: product.id || Date.now(),
+				title: product.title || "Product",
+				vendor: product.vendor || "Mock Vendor",
+				type: product.type || "General",
+				price: product.price || "0.00",
+				available: product.available !== void 0 ? product.available : true
+			};
+			this.mockProducts.push(fullProduct);
+		}
+		if (config.discounts) for (const discount of config.discounts) {
+			const fullDiscount = {
+				code: discount.code || "DISCOUNT",
+				percent: discount.percent || 10,
+				validDays: discount.validDays || 30,
+				startsAt: discount.startsAt || (/* @__PURE__ */ new Date()).toISOString(),
+				expiresAt: discount.expiresAt || new Date(Date.now() + 720 * 60 * 60 * 1e3).toISOString(),
+				priceRuleId: discount.priceRuleId || this.nextPriceRuleId++,
+				createdAt: discount.createdAt || /* @__PURE__ */ new Date()
+			};
+			this.mockDiscounts.push(fullDiscount);
+		}
+	}
+	seedMockData() {
+		const twoDaysAgo = /* @__PURE__ */ new Date();
+		twoDaysAgo.setDate(twoDaysAgo.getDate() - 2);
+		this.mockOrders.set("12345", {
+			id: "12345",
+			name: "#1001",
+			status: "in_transit",
+			financialStatus: "paid",
+			createdAt: (/* @__PURE__ */ new Date(Date.now() - 7200 * 60 * 1e3)).toISOString(),
+			expectedDelivery: twoDaysAgo,
+			tracking: "TRACK123456789",
+			trackingUrl: "https://track.example.com/TRACK123456789",
+			items: [{
+				name: "Premium Headphones",
+				quantity: 1,
+				price: "299.99"
+			}],
+			total: "299.99"
+		});
+		this.mockOrders.set("67890", {
+			id: "67890",
+			name: "#1002",
+			status: "delivered",
+			financialStatus: "paid",
+			createdAt: (/* @__PURE__ */ new Date(Date.now() - 14400 * 60 * 1e3)).toISOString(),
+			expectedDelivery: /* @__PURE__ */ new Date(Date.now() - 7200 * 60 * 1e3),
+			actualDelivery: /* @__PURE__ */ new Date(Date.now() - 4320 * 60 * 1e3),
+			tracking: "TRACK987654321",
+			trackingUrl: "https://track.example.com/TRACK987654321",
+			items: [{
+				name: "Wireless Mouse",
+				quantity: 2,
+				price: "49.99"
+			}],
+			total: "99.98"
+		});
+		this.mockProducts = [
+			{
+				id: 1001,
+				title: "Premium Headphones",
+				vendor: "AudioTech",
+				type: "Electronics",
+				price: "299.99",
+				available: true
+			},
+			{
+				id: 1002,
+				title: "Wireless Mouse",
+				vendor: "TechGear",
+				type: "Electronics",
+				price: "49.99",
+				available: true
+			},
+			{
+				id: 1003,
+				title: "Mechanical Keyboard",
+				vendor: "KeyMaster",
+				type: "Electronics",
+				price: "149.99",
+				available: true
+			},
+			{
+				id: 1004,
+				title: "USB-C Cable",
+				vendor: "CableCo",
+				type: "Accessories",
+				price: "19.99",
+				available: false
+			}
+		];
+	}
+	tools = {
+		get_order: {
+			name: "get_order",
+			description: "Get order details by order ID or order name (e.g., #1001)",
+			parameters: { orderId: {
+				type: "string",
+				required: true
+			} },
+			execute: async (params) => {
+				const orderIdentifier = params.orderId.replace("#", "");
+				const order = this.mockOrders.get(orderIdentifier);
+				if (!order) return {
+					found: false,
+					error: `Order ${params.orderId} not found`
+				};
+				const delayMs = (/* @__PURE__ */ new Date()).getTime() - order.expectedDelivery.getTime();
+				const delayDays = Math.floor(delayMs / (1e3 * 60 * 60 * 24));
+				return {
+					found: true,
+					id: order.id,
+					name: order.name,
+					status: order.status,
+					financialStatus: order.financialStatus,
+					createdAt: order.createdAt,
+					total: order.total,
+					items: order.items,
+					tracking: order.tracking || null,
+					trackingUrl: order.trackingUrl || null,
+					isDelayed: delayDays > 0,
+					delayDays: Math.max(0, delayDays)
+				};
+			}
+		},
+		create_discount: {
+			name: "create_discount",
+			description: "Create a percentage discount code",
+			parameters: {
+				percent: {
+					type: "number",
+					required: true
+				},
+				validDays: {
+					type: "number",
+					required: true
+				}
+			},
+			execute: async (params) => {
+				const code = `SORRY${params.percent}`;
+				const startsAt = (/* @__PURE__ */ new Date()).toISOString();
+				const expiresAt = new Date(Date.now() + params.validDays * 24 * 60 * 60 * 1e3).toISOString();
+				const priceRuleId = this.nextPriceRuleId++;
+				const discount = {
+					code,
+					percent: params.percent,
+					validDays: params.validDays,
+					startsAt,
+					expiresAt,
+					priceRuleId,
+					createdAt: /* @__PURE__ */ new Date()
+				};
+				this.mockDiscounts.push(discount);
+				console.log(`\n💰 Created discount code: ${code}`);
+				console.log(`   ${params.percent}% off, valid for ${params.validDays} days\n`);
+				return {
+					code,
+					percent: params.percent,
+					validDays: params.validDays,
+					startsAt,
+					expiresAt,
+					priceRuleId
+				};
+			}
+		},
+		search_products: {
+			name: "search_products",
+			description: "Search for products in the store",
+			parameters: {
+				query: {
+					type: "string",
+					required: true
+				},
+				limit: {
+					type: "number",
+					required: false
+				}
+			},
+			execute: async (params) => {
+				const limit = params.limit || 10;
+				const queryLower = params.query.toLowerCase();
+				const matched = this.mockProducts.filter((p) => p.title.toLowerCase().includes(queryLower) || p.vendor.toLowerCase().includes(queryLower) || p.type.toLowerCase().includes(queryLower)).slice(0, limit);
+				return {
+					found: matched.length,
+					products: matched.map((p) => ({
+						id: p.id,
+						title: p.title,
+						vendor: p.vendor,
+						type: p.type,
+						price: p.price,
+						available: p.available
+					}))
+				};
+			}
+		}
+	};
+	/**
+	* Get all created discounts (for testing)
+	*/
+	getDiscounts() {
+		return [...this.mockDiscounts];
+	}
+	/**
+	* Get all orders (for testing)
+	*/
+	getOrders() {
+		return Array.from(this.mockOrders.values());
+	}
+};
+
+//#endregion
+export { CSVLoader, ConversationEvaluator, ConversationRunner, CustomerSimulator, ECOMMERCE_SCENARIOS, MockShopifySkill, SimulationRunner, SkillTestHarness, TestCaseEvaluator, TestSuiteRunner, formatTimestamp };
+//# sourceMappingURL=index.js.map