@operor/testing 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,281 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import { SimulationRunner } from '../SimulationRunner.js';
3
+ import { ECOMMERCE_SCENARIOS } from '../scenarios.js';
4
+ import type { SimulationReport, ConversationScenario } from '../types.js';
5
+ import { EventEmitter } from 'node:events';
6
+
7
+ // Minimal mock Operor that emits events for each simulated message
8
+ function createMockOperor(responseText = 'I can help with that.') {
9
+ const emitter = new EventEmitter();
10
+
11
+ const mockProvider = {
12
+ name: 'mock',
13
+ simulateIncomingMessage: (_from: string, _text: string) => {
14
+ setTimeout(() => {
15
+ emitter.emit('message:processed', {
16
+ response: {
17
+ text: responseText,
18
+ toolCalls: [],
19
+ },
20
+ cost: 0.001,
21
+ });
22
+ }, 5);
23
+ },
24
+ };
25
+
26
+ const providers = new Map();
27
+ providers.set('mock', mockProvider);
28
+
29
+ return Object.assign(emitter, { providers });
30
+ }
31
+
32
+ describe('SimulationRunner', () => {
33
+ it('runs conversation scenarios and produces a report', async () => {
34
+ const agentOS = createMockOperor('Your order is on its way!');
35
+
36
+ const scenario: ConversationScenario = {
37
+ id: 'test-scenario',
38
+ name: 'Test Scenario',
39
+ description: 'Simple test',
40
+ persona: 'polite',
41
+ maxTurns: 1,
42
+ scriptedResponses: ['Hello'],
43
+ successCriteria: [{ type: 'turns_under', value: 10 }],
44
+ };
45
+
46
+ const runner = new SimulationRunner({
47
+ agentOS: agentOS as any,
48
+ config: {
49
+ conversationScenarios: [scenario],
50
+ },
51
+ });
52
+
53
+ const report = await runner.run();
54
+
55
+ expect(report.timestamp).toBeInstanceOf(Date);
56
+ expect(report.duration).toBeGreaterThanOrEqual(0);
57
+ expect(report.conversationResults).toHaveLength(1);
58
+ expect(report.summary.totalConversations).toBe(1);
59
+ expect(report.summary.passedConversations).toBe(1);
60
+ expect(report.summary.overallPassRate).toBe(1);
61
+ expect(report.overallPassed).toBe(true);
62
+ });
63
+
64
+ it('resolves "builtin" to ECOMMERCE_SCENARIOS', async () => {
65
+ const agentOS = createMockOperor('Here is your order info.');
66
+
67
+ const runner = new SimulationRunner({
68
+ agentOS: agentOS as any,
69
+ config: {
70
+ conversationScenarios: 'builtin',
71
+ },
72
+ });
73
+
74
+ const report = await runner.run();
75
+
76
+ expect(report.conversationResults).toHaveLength(ECOMMERCE_SCENARIOS.length);
77
+ expect(report.summary.totalConversations).toBe(ECOMMERCE_SCENARIOS.length);
78
+ });
79
+
80
+ it('returns empty report when no scenarios or test files configured', async () => {
81
+ const agentOS = createMockOperor();
82
+
83
+ const runner = new SimulationRunner({
84
+ agentOS: agentOS as any,
85
+ config: {},
86
+ });
87
+
88
+ const report = await runner.run();
89
+
90
+ expect(report.testSuiteResults).toHaveLength(0);
91
+ expect(report.conversationResults).toHaveLength(0);
92
+ expect(report.summary.totalTests).toBe(0);
93
+ expect(report.summary.totalConversations).toBe(0);
94
+ expect(report.overallPassed).toBe(false); // No items = not passed
95
+ });
96
+
97
+ it('handles failed scenarios in the report', async () => {
98
+ const agentOS = createMockOperor('Sorry, I cannot help.');
99
+
100
+ const scenario: ConversationScenario = {
101
+ id: 'fail-scenario',
102
+ name: 'Failing Scenario',
103
+ description: 'Should fail criteria',
104
+ persona: 'frustrated',
105
+ maxTurns: 1,
106
+ scriptedResponses: ['Fix my order!'],
107
+ successCriteria: [{ type: 'tool_called', value: 'get_order' }],
108
+ };
109
+
110
+ const runner = new SimulationRunner({
111
+ agentOS: agentOS as any,
112
+ config: {
113
+ conversationScenarios: [scenario],
114
+ },
115
+ });
116
+
117
+ const report = await runner.run();
118
+
119
+ expect(report.overallPassed).toBe(false);
120
+ expect(report.summary.failedConversations).toBe(1);
121
+ expect(report.summary.overallPassRate).toBe(0);
122
+ });
123
+
124
+ it('calculates mixed pass rate correctly', async () => {
125
+ const agentOS = createMockOperor('Your order is being processed.');
126
+
127
+ const passingScenario: ConversationScenario = {
128
+ id: 'pass',
129
+ name: 'Passing',
130
+ description: 'Will pass',
131
+ persona: 'polite',
132
+ maxTurns: 1,
133
+ scriptedResponses: ['Hi'],
134
+ successCriteria: [{ type: 'turns_under', value: 10 }],
135
+ };
136
+
137
+ const failingScenario: ConversationScenario = {
138
+ id: 'fail',
139
+ name: 'Failing',
140
+ description: 'Will fail',
141
+ persona: 'frustrated',
142
+ maxTurns: 1,
143
+ scriptedResponses: ['Help!'],
144
+ successCriteria: [{ type: 'tool_called', value: 'nonexistent' }],
145
+ };
146
+
147
+ const runner = new SimulationRunner({
148
+ agentOS: agentOS as any,
149
+ config: {
150
+ conversationScenarios: [passingScenario, failingScenario],
151
+ },
152
+ });
153
+
154
+ const report = await runner.run();
155
+
156
+ expect(report.summary.passedConversations).toBe(1);
157
+ expect(report.summary.failedConversations).toBe(1);
158
+ expect(report.summary.overallPassRate).toBe(0.5);
159
+ expect(report.overallPassed).toBe(false);
160
+ });
161
+
162
+ describe('formatReport', () => {
163
+ it('formats a report with conversations', () => {
164
+ const report: SimulationReport = {
165
+ timestamp: new Date('2026-02-15T00:00:00Z'),
166
+ duration: 1500,
167
+ totalConversations: 1,
168
+ passed: 1,
169
+ failed: 0,
170
+ averageScores: { accuracy: 4, toolUsage: 3, tone: 5, resolution: 4 },
171
+ scenarioBreakdown: [{ scenario: 'Order Check', runs: 1, passRate: 1, avgScore: 4 }],
172
+ toolUsageStats: {},
173
+ commonFailurePatterns: [],
174
+ recommendations: [],
175
+ totalCost: 0.01,
176
+ testSuiteResults: [],
177
+ conversationResults: [
178
+ {
179
+ scenario: {
180
+ id: 'test',
181
+ name: 'Order Check',
182
+ description: 'Test',
183
+ persona: 'polite',
184
+ maxTurns: 3,
185
+ },
186
+ passed: true,
187
+ turns: [
188
+ { role: 'customer', message: 'Hi' },
189
+ { role: 'agent', message: 'Hello!' },
190
+ ],
191
+ evaluation: {
192
+ overall: 'pass',
193
+ scores: { accuracy: 4, toolUsage: 3, tone: 5, resolution: 4 },
194
+ feedback: 'Good interaction',
195
+ },
196
+ duration: 500,
197
+ cost: 0.01,
198
+ },
199
+ ],
200
+ overallPassed: true,
201
+ summary: {
202
+ totalTests: 0,
203
+ passedTests: 0,
204
+ failedTests: 0,
205
+ totalConversations: 1,
206
+ passedConversations: 1,
207
+ failedConversations: 0,
208
+ overallPassRate: 1,
209
+ },
210
+ };
211
+
212
+ const output = SimulationRunner.formatReport(report);
213
+
214
+ expect(output).toContain('Simulation Report');
215
+ expect(output).toContain('1.5s');
216
+ expect(output).toContain('Order Check');
217
+ expect(output).toContain('100% pass rate');
218
+ expect(output).toContain('pass rate: 100.0%');
219
+ expect(output).toContain('PASSED');
220
+ });
221
+
222
+ it('formats a failed report', () => {
223
+ const report: SimulationReport = {
224
+ timestamp: new Date('2026-02-15T00:00:00Z'),
225
+ duration: 3000,
226
+ totalConversations: 1,
227
+ passed: 0,
228
+ failed: 1,
229
+ averageScores: { accuracy: 1, toolUsage: 1, tone: 2, resolution: 1 },
230
+ scenarioBreakdown: [{ scenario: 'Bad Scenario', runs: 1, passRate: 0, avgScore: 1.25 }],
231
+ toolUsageStats: {},
232
+ commonFailurePatterns: ['Did not resolve the issue'],
233
+ recommendations: [],
234
+ totalCost: 0.02,
235
+ testSuiteResults: [],
236
+ conversationResults: [
237
+ {
238
+ scenario: {
239
+ id: 'fail',
240
+ name: 'Bad Scenario',
241
+ description: 'Fails',
242
+ persona: 'frustrated',
243
+ maxTurns: 2,
244
+ },
245
+ passed: false,
246
+ turns: [
247
+ { role: 'customer', message: 'Help!' },
248
+ { role: 'agent', message: 'Sorry.' },
249
+ { role: 'customer', message: 'Useless!' },
250
+ { role: 'agent', message: 'I apologize.' },
251
+ ],
252
+ evaluation: {
253
+ overall: 'fail',
254
+ scores: { accuracy: 1, toolUsage: 1, tone: 2, resolution: 1 },
255
+ feedback: 'Did not resolve the issue',
256
+ },
257
+ duration: 2000,
258
+ cost: 0.02,
259
+ },
260
+ ],
261
+ overallPassed: false,
262
+ summary: {
263
+ totalTests: 0,
264
+ passedTests: 0,
265
+ failedTests: 0,
266
+ totalConversations: 1,
267
+ passedConversations: 0,
268
+ failedConversations: 1,
269
+ overallPassRate: 0,
270
+ },
271
+ };
272
+
273
+ const output = SimulationRunner.formatReport(report);
274
+
275
+ expect(output).toContain('Bad Scenario');
276
+ expect(output).toContain('0% pass rate');
277
+ expect(output).toContain('FAILED');
278
+ expect(output).toContain('pass rate: 0.0%');
279
+ });
280
+ });
281
+ });
@@ -0,0 +1,181 @@
1
+ import { describe, it, expect, beforeEach } from 'vitest';
2
+ import { SkillTestHarness } from '../SkillTestHarness.js';
3
+ import type { Skill, Tool } from '@operor/core';
4
+
5
+ function createMockSkill(): Skill {
6
+ return {
7
+ name: 'test-integration',
8
+ initialize: async () => {},
9
+ isReady: () => true,
10
+ tools: {
11
+ get_order: {
12
+ name: 'get_order',
13
+ description: 'Get order details',
14
+ parameters: { orderId: { type: 'string', required: true } },
15
+ execute: async (params: any) => ({ found: true, id: params.orderId, status: 'delivered' }),
16
+ },
17
+ search_products: {
18
+ name: 'search_products',
19
+ description: 'Search products',
20
+ parameters: { query: { type: 'string', required: true } },
21
+ execute: async (params: any) => ({ found: 1, products: [{ title: params.query }] }),
22
+ },
23
+ create_discount: {
24
+ name: 'create_discount',
25
+ description: 'Create discount',
26
+ parameters: { percent: { type: 'number', required: true } },
27
+ execute: async (params: any) => ({ code: `DISC${params.percent}`, percent: params.percent }),
28
+ },
29
+ stripe_create_refund: {
30
+ name: 'stripe_create_refund',
31
+ description: 'Create refund',
32
+ parameters: { amount: { type: 'number', required: true } },
33
+ execute: async (params: any) => ({ refunded: true, amount: params.amount }),
34
+ },
35
+ unknown_tool: {
36
+ name: 'unknown_tool',
37
+ description: 'Some unknown tool',
38
+ parameters: {},
39
+ execute: async () => ({ ok: true }),
40
+ },
41
+ },
42
+ };
43
+ }
44
+
45
+ describe('SkillTestHarness', () => {
46
+ let inner: Skill;
47
+
48
+ beforeEach(() => {
49
+ inner = createMockSkill();
50
+ });
51
+
52
+ it('should preserve the inner integration name', () => {
53
+ const harness = new SkillTestHarness(inner);
54
+ expect(harness.name).toBe('test-integration');
55
+ });
56
+
57
+ it('should delegate initialize and isReady', async () => {
58
+ const harness = new SkillTestHarness(inner);
59
+ await harness.initialize();
60
+ expect(harness.isReady()).toBe(true);
61
+ });
62
+
63
+ it('should expose all inner tools', () => {
64
+ const harness = new SkillTestHarness(inner);
65
+ expect(Object.keys(harness.tools)).toEqual(Object.keys(inner.tools));
66
+ });
67
+
68
+ describe('read operations', () => {
69
+ it('should always allow read tools', async () => {
70
+ const harness = new SkillTestHarness(inner, { allowWrites: false });
71
+ const result = await harness.tools.get_order.execute({ orderId: '123' });
72
+ expect(result).toEqual({ found: true, id: '123', status: 'delivered' });
73
+ });
74
+
75
+ it('should allow search_products as read', async () => {
76
+ const harness = new SkillTestHarness(inner, { allowWrites: false });
77
+ const result = await harness.tools.search_products.execute({ query: 'test' });
78
+ expect(result.found).toBe(1);
79
+ });
80
+ });
81
+
82
+ describe('write operations', () => {
83
+ it('should block write tools when allowWrites=false', async () => {
84
+ const harness = new SkillTestHarness(inner, { allowWrites: false });
85
+ await expect(harness.tools.create_discount.execute({ percent: 10 }))
86
+ .rejects.toThrow('Write operation');
87
+ });
88
+
89
+ it('should allow write tools when allowWrites=true', async () => {
90
+ const harness = new SkillTestHarness(inner, { allowWrites: true });
91
+ const result = await harness.tools.create_discount.execute({ percent: 10 });
92
+ expect(result).toEqual({ code: 'DISC10', percent: 10 });
93
+ });
94
+ });
95
+
96
+ describe('destructive operations', () => {
97
+ it('should block destructive tools by default', async () => {
98
+ const harness = new SkillTestHarness(inner, { allowWrites: true });
99
+ await expect(harness.tools.stripe_create_refund.execute({ amount: 50 }))
100
+ .rejects.toThrow('Destructive operation');
101
+ });
102
+
103
+ it('should allow destructive tools when allowDestructive=true', async () => {
104
+ const harness = new SkillTestHarness(inner, { allowWrites: true, allowDestructive: true });
105
+ const result = await harness.tools.stripe_create_refund.execute({ amount: 50 });
106
+ expect(result).toEqual({ refunded: true, amount: 50 });
107
+ });
108
+ });
109
+
110
+ describe('unknown tools', () => {
111
+ it('should classify unknown tools as write (safe by default)', async () => {
112
+ const harness = new SkillTestHarness(inner, { allowWrites: false });
113
+ await expect(harness.tools.unknown_tool.execute({}))
114
+ .rejects.toThrow('Write operation');
115
+ });
116
+ });
117
+
118
+ describe('maxOperations', () => {
119
+ it('should enforce operation limit', async () => {
120
+ const harness = new SkillTestHarness(inner, { maxOperations: 2 });
121
+ await harness.tools.get_order.execute({ orderId: '1' });
122
+ await harness.tools.get_order.execute({ orderId: '2' });
123
+ await expect(harness.tools.get_order.execute({ orderId: '3' }))
124
+ .rejects.toThrow('Max operations limit reached');
125
+ });
126
+ });
127
+
128
+ describe('dryRun', () => {
129
+ it('should return dry-run result without executing', async () => {
130
+ const harness = new SkillTestHarness(inner, { dryRun: true, allowWrites: true });
131
+ const result = await harness.tools.create_discount.execute({ percent: 15 });
132
+ expect(result).toEqual({
133
+ dryRun: true,
134
+ wouldExecute: 'create_discount',
135
+ params: { percent: 15 },
136
+ });
137
+ });
138
+
139
+ it('should still enforce permission checks in dry-run mode', async () => {
140
+ const harness = new SkillTestHarness(inner, { dryRun: true, allowWrites: false });
141
+ await expect(harness.tools.create_discount.execute({ percent: 10 }))
142
+ .rejects.toThrow('Write operation');
143
+ });
144
+ });
145
+
146
+ describe('audit log', () => {
147
+ it('should record all operations', async () => {
148
+ const harness = new SkillTestHarness(inner, { allowWrites: true });
149
+ await harness.tools.get_order.execute({ orderId: '123' });
150
+ await harness.tools.create_discount.execute({ percent: 10 });
151
+
152
+ const log = harness.getAuditLog();
153
+ expect(log).toHaveLength(2);
154
+ expect(log[0].name).toBe('get_order');
155
+ expect(log[0].classification).toBe('read');
156
+ expect(log[1].name).toBe('create_discount');
157
+ expect(log[1].classification).toBe('write');
158
+ });
159
+
160
+ it('should record failed operations in audit log', async () => {
161
+ const harness = new SkillTestHarness(inner, { allowWrites: false });
162
+ try {
163
+ await harness.tools.create_discount.execute({ percent: 10 });
164
+ } catch {}
165
+
166
+ // Blocked operations are not logged (they throw before incrementing)
167
+ const log = harness.getAuditLog();
168
+ expect(log).toHaveLength(0);
169
+ });
170
+
171
+ it('should reset audit log and operation count', async () => {
172
+ const harness = new SkillTestHarness(inner);
173
+ await harness.tools.get_order.execute({ orderId: '1' });
174
+ expect(harness.getOperationCount()).toBe(1);
175
+
176
+ harness.resetAuditLog();
177
+ expect(harness.getAuditLog()).toHaveLength(0);
178
+ expect(harness.getOperationCount()).toBe(0);
179
+ });
180
+ });
181
+ });
@@ -0,0 +1,71 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import { ECOMMERCE_SCENARIOS } from '../scenarios.js';
3
+
4
+ describe('ECOMMERCE_SCENARIOS', () => {
5
+ it('should export 10 scenarios', () => {
6
+ expect(ECOMMERCE_SCENARIOS).toHaveLength(10);
7
+ });
8
+
9
+ it('should have unique IDs', () => {
10
+ const ids = ECOMMERCE_SCENARIOS.map((s) => s.id);
11
+ expect(new Set(ids).size).toBe(ids.length);
12
+ });
13
+
14
+ it('should have unique names', () => {
15
+ const names = ECOMMERCE_SCENARIOS.map((s) => s.name);
16
+ expect(new Set(names).size).toBe(names.length);
17
+ });
18
+
19
+ it('every scenario should have required fields', () => {
20
+ for (const s of ECOMMERCE_SCENARIOS) {
21
+ expect(s.id).toBeTruthy();
22
+ expect(s.name).toBeTruthy();
23
+ expect(s.description).toBeTruthy();
24
+ expect(s.persona).toBeTruthy();
25
+ expect(s.maxTurns).toBeGreaterThan(0);
26
+ expect(s.scriptedResponses).toBeDefined();
27
+ expect(s.scriptedResponses!.length).toBeGreaterThan(0);
28
+ expect(s.scriptedResponses!.length).toBeLessThanOrEqual(s.maxTurns);
29
+ }
30
+ });
31
+
32
+ it('should include a delayed order scenario with get_order and create_discount', () => {
33
+ const delayed = ECOMMERCE_SCENARIOS.find((s) => s.id === 'delayed-order-compensation');
34
+ expect(delayed).toBeDefined();
35
+ expect(delayed!.expectedTools).toContain('get_order');
36
+ expect(delayed!.expectedTools).toContain('create_discount');
37
+ });
38
+
39
+ it('should include an order-not-found scenario', () => {
40
+ const notFound = ECOMMERCE_SCENARIOS.find((s) => s.id === 'order-not-found');
41
+ expect(notFound).toBeDefined();
42
+ expect(notFound!.expectedTools).toContain('get_order');
43
+ });
44
+
45
+ it('should include a product inquiry scenario with search_products', () => {
46
+ const inquiry = ECOMMERCE_SCENARIOS.find((s) => s.id === 'product-inquiry');
47
+ expect(inquiry).toBeDefined();
48
+ expect(inquiry!.expectedTools).toContain('search_products');
49
+ });
50
+
51
+ it('should include a greeting scenario with no tools', () => {
52
+ const greeting = ECOMMERCE_SCENARIOS.find((s) => s.id === 'greeting');
53
+ expect(greeting).toBeDefined();
54
+ expect(greeting!.expectedTools).toEqual([]);
55
+ });
56
+
57
+ it('should include a multi-issue scenario', () => {
58
+ const multi = ECOMMERCE_SCENARIOS.find((s) => s.id === 'multi-issue');
59
+ expect(multi).toBeDefined();
60
+ expect(multi!.expectedTools).toContain('get_order');
61
+ expect(multi!.expectedTools).toContain('search_products');
62
+ });
63
+
64
+ it('should include return, billing, lead-qualification, and escalation scenarios', () => {
65
+ const ids = ECOMMERCE_SCENARIOS.map((s) => s.id);
66
+ expect(ids).toContain('return-request');
67
+ expect(ids).toContain('billing-dispute');
68
+ expect(ids).toContain('lead-qualification');
69
+ expect(ids).toContain('frustrated-escalation');
70
+ });
71
+ });
package/src/index.ts ADDED
@@ -0,0 +1,32 @@
1
+ export type {
2
+ TestCase,
3
+ TestCaseResult,
4
+ TestSuiteResult,
5
+ CustomerPersona,
6
+ ConversationSuccessCriteria,
7
+ ConversationScenario,
8
+ ConversationTurn,
9
+ CustomerSimulatorResponse,
10
+ ConversationEvaluation,
11
+ ConversationTestResult,
12
+ CriteriaResult,
13
+ SimulationConfig,
14
+ SimulationReport,
15
+ } from './types.js';
16
+ export { CSVLoader } from './CSVLoader.js';
17
+ export { TestCaseEvaluator } from './TestCaseEvaluator.js';
18
+ export type { EvaluationResult } from './TestCaseEvaluator.js';
19
+ export { TestSuiteRunner } from './TestSuiteRunner.js';
20
+ export type { TestSuiteRunnerConfig } from './TestSuiteRunner.js';
21
+ export { SkillTestHarness } from './SkillTestHarness.js';
22
+ export type { SkillTestHarnessConfig, AuditLogEntry } from './SkillTestHarness.js';
23
+ export { CustomerSimulator } from './CustomerSimulator.js';
24
+ export { ConversationEvaluator } from './ConversationEvaluator.js';
25
+ export { ConversationRunner } from './ConversationRunner.js';
26
+ export type { ConversationRunnerConfig } from './ConversationRunner.js';
27
+ export { SimulationRunner } from './SimulationRunner.js';
28
+ export type { SimulationRunnerOptions } from './SimulationRunner.js';
29
+ export { ECOMMERCE_SCENARIOS } from './scenarios.js';
30
+ export { MockShopifySkill } from './MockShopifySkill.js';
31
+ export type { MockOrder, MockProduct, MockDiscount } from './MockShopifySkill.js';
32
+ export { formatTimestamp } from './utils.js';
@@ -0,0 +1,52 @@
1
+ import type { ConversationScenario } from '../types.js';
2
+
3
+ export const emptyMessageScenario: ConversationScenario = {
4
+ id: 'edge-empty-message',
5
+ name: 'Empty Message',
6
+ description: 'Customer sends an empty or whitespace-only message',
7
+ persona: 'terse',
8
+ maxTurns: 2,
9
+ expectedTools: [],
10
+ expectedOutcome: 'Agent handles gracefully and asks how it can help',
11
+ successCriteria: [
12
+ { type: 'turns_under', value: 3 },
13
+ ],
14
+ scriptedResponses: [
15
+ ' ',
16
+ 'Oh sorry, I meant to ask about my order.',
17
+ ],
18
+ };
19
+
20
+ export const longMessageScenario: ConversationScenario = {
21
+ id: 'edge-long-message',
22
+ name: 'Very Long Message',
23
+ description: 'Customer sends an extremely long message with embedded order question',
24
+ persona: 'verbose',
25
+ maxTurns: 2,
26
+ expectedTools: ['get_order'],
27
+ expectedOutcome: 'Agent extracts the key question and responds appropriately',
28
+ successCriteria: [
29
+ { type: 'tool_called', value: 'get_order' },
30
+ ],
31
+ scriptedResponses: [
32
+ 'So I ordered something a while back and I have been checking every day and it still has not arrived. I remember when I placed the order I was really excited because it was exactly what I was looking for and the price was great too. Anyway the order number is #12345 and I just want to know where it is because I really need it for an event coming up soon. Can you please help me track it down? I would really appreciate it. Thanks so much in advance for your help with this matter.',
33
+ 'Got it, thanks for looking into that for me.',
34
+ ],
35
+ };
36
+
37
+ export const multiLanguageScenario: ConversationScenario = {
38
+ id: 'edge-multi-language',
39
+ name: 'Multi-language Message',
40
+ description: 'Customer writes in a different language (Spanish)',
41
+ persona: 'polite',
42
+ maxTurns: 2,
43
+ expectedTools: [],
44
+ expectedOutcome: 'Agent responds helpfully despite language difference',
45
+ successCriteria: [
46
+ { type: 'turns_under', value: 3 },
47
+ ],
48
+ scriptedResponses: [
49
+ 'Hola, donde esta mi pedido numero 12345?',
50
+ 'Gracias por la ayuda.',
51
+ ],
52
+ };
@@ -0,0 +1,37 @@
1
+ import type { ConversationScenario } from '../types.js';
2
+
3
+ export const greetingScenario: ConversationScenario = {
4
+ id: 'general-greeting',
5
+ name: 'Simple Greeting',
6
+ description: 'Customer says hello and expects a friendly response',
7
+ persona: 'polite',
8
+ maxTurns: 1,
9
+ expectedTools: [],
10
+ expectedOutcome: 'Agent responds with a friendly greeting',
11
+ successCriteria: [
12
+ { type: 'turns_under', value: 2 },
13
+ ],
14
+ scriptedResponses: [
15
+ 'Hello!',
16
+ ],
17
+ };
18
+
19
+ export const frustratedCustomerScenario: ConversationScenario = {
20
+ id: 'general-frustrated',
21
+ name: 'Frustrated Customer with Multiple Complaints',
22
+ description: 'Angry customer escalates through multiple complaints; agent should stay professional',
23
+ persona: 'frustrated',
24
+ maxTurns: 4,
25
+ expectedTools: ['get_order'],
26
+ expectedOutcome: 'Agent remains professional and offers resolution',
27
+ successCriteria: [
28
+ { type: 'response_contains', value: 'sorry' },
29
+ { type: 'turns_under', value: 5 },
30
+ ],
31
+ scriptedResponses: [
32
+ 'This is ridiculous! My order #12345 is STILL not here!',
33
+ 'I have been waiting forever. This is the worst service I have ever experienced.',
34
+ 'I want a refund or at least some kind of compensation for this mess.',
35
+ 'Fine. I guess that will have to do.',
36
+ ],
37
+ };
@@ -0,0 +1,32 @@
1
+ export {
2
+ delayedOrderScenario,
3
+ onTimeOrderScenario,
4
+ orderNotFoundScenario,
5
+ } from './order-tracking.js';
6
+
7
+ export {
8
+ greetingScenario,
9
+ frustratedCustomerScenario,
10
+ } from './general.js';
11
+
12
+ export {
13
+ emptyMessageScenario,
14
+ longMessageScenario,
15
+ multiLanguageScenario,
16
+ } from './edge-cases.js';
17
+
18
+ import { delayedOrderScenario, onTimeOrderScenario, orderNotFoundScenario } from './order-tracking.js';
19
+ import { greetingScenario, frustratedCustomerScenario } from './general.js';
20
+ import { emptyMessageScenario, longMessageScenario, multiLanguageScenario } from './edge-cases.js';
21
+ import type { ConversationScenario } from '../types.js';
22
+
23
+ export const allScenarios: ConversationScenario[] = [
24
+ delayedOrderScenario,
25
+ onTimeOrderScenario,
26
+ orderNotFoundScenario,
27
+ greetingScenario,
28
+ frustratedCustomerScenario,
29
+ emptyMessageScenario,
30
+ longMessageScenario,
31
+ multiLanguageScenario,
32
+ ];