@artemiskit/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +229 -0
  2. package/README.md +44 -1
  3. package/dist/artemiskit.d.ts +149 -0
  4. package/dist/artemiskit.d.ts.map +1 -0
  5. package/dist/builders/index.d.ts +352 -0
  6. package/dist/builders/index.d.ts.map +1 -0
  7. package/dist/builders/index.js +354 -0
  8. package/dist/contracts/index.d.ts +253 -0
  9. package/dist/contracts/index.d.ts.map +1 -0
  10. package/dist/contracts/index.js +53 -0
  11. package/dist/guardian/action-validator.d.ts +92 -0
  12. package/dist/guardian/action-validator.d.ts.map +1 -0
  13. package/dist/guardian/circuit-breaker.d.ts +211 -0
  14. package/dist/guardian/circuit-breaker.d.ts.map +1 -0
  15. package/dist/guardian/guardian.d.ts +213 -0
  16. package/dist/guardian/guardian.d.ts.map +1 -0
  17. package/dist/guardian/guardrails.d.ts +107 -0
  18. package/dist/guardian/guardrails.d.ts.map +1 -0
  19. package/dist/guardian/index.d.ts +35 -0
  20. package/dist/guardian/index.d.ts.map +1 -0
  21. package/dist/guardian/intent-classifier.d.ts +81 -0
  22. package/dist/guardian/intent-classifier.d.ts.map +1 -0
  23. package/dist/guardian/interceptor.d.ts +141 -0
  24. package/dist/guardian/interceptor.d.ts.map +1 -0
  25. package/dist/guardian/policy.d.ts +52 -0
  26. package/dist/guardian/policy.d.ts.map +1 -0
  27. package/dist/guardian/semantic-validator.d.ts +61 -0
  28. package/dist/guardian/semantic-validator.d.ts.map +1 -0
  29. package/dist/guardian/types.d.ts +653 -0
  30. package/dist/guardian/types.d.ts.map +1 -0
  31. package/dist/index.d.ts +23 -0
  32. package/dist/index.d.ts.map +1 -0
  33. package/dist/index.js +44918 -28644
  34. package/dist/matchers/core.d.ts +81 -0
  35. package/dist/matchers/core.d.ts.map +1 -0
  36. package/dist/matchers/index.d.ts +6 -0
  37. package/dist/matchers/index.d.ts.map +1 -0
  38. package/dist/matchers/index.js +13 -59
  39. package/dist/matchers/jest.d.ts +46 -0
  40. package/dist/matchers/jest.d.ts.map +1 -0
  41. package/dist/matchers/jest.js +0 -46
  42. package/dist/matchers/vitest.d.ts +24 -0
  43. package/dist/matchers/vitest.d.ts.map +1 -0
  44. package/dist/matchers/vitest.js +0 -46
  45. package/dist/types-only.d.ts +31 -0
  46. package/dist/types-only.d.ts.map +1 -0
  47. package/dist/types-only.js +1 -0
  48. package/dist/types.d.ts +379 -0
  49. package/dist/types.d.ts.map +1 -0
  50. package/dist/utils/index.d.ts +112 -0
  51. package/dist/utils/index.d.ts.map +1 -0
  52. package/dist/utils/index.js +58 -0
  53. package/package.json +19 -4
  54. package/src/__tests__/builders.test.ts +450 -0
  55. package/src/__tests__/circuit-breaker.test.ts +704 -0
  56. package/src/__tests__/compare.test.ts +469 -0
  57. package/src/__tests__/contracts.test.ts +354 -0
  58. package/src/__tests__/guardian-modes.test.ts +215 -0
  59. package/src/__tests__/guardrails.test.ts +404 -0
  60. package/src/__tests__/semantic-validator.test.ts +421 -0
  61. package/src/__tests__/utils.test.ts +324 -0
  62. package/src/__tests__/validate.test.ts +310 -0
  63. package/src/artemiskit.ts +479 -5
  64. package/src/builders/index.ts +700 -0
  65. package/src/contracts/index.ts +357 -0
  66. package/src/guardian/circuit-breaker.ts +74 -28
  67. package/src/guardian/guardian.ts +307 -20
  68. package/src/guardian/guardrails.ts +143 -1
  69. package/src/guardian/index.ts +35 -1
  70. package/src/guardian/interceptor.ts +87 -11
  71. package/src/guardian/semantic-validator.ts +399 -0
  72. package/src/guardian/types.ts +413 -2
  73. package/src/index.ts +258 -14
  74. package/src/types-only.ts +276 -0
  75. package/src/types.ts +186 -0
  76. package/src/utils/index.ts +225 -0
  77. package/tsconfig.json +2 -1
  78. package/adapters/openai/dist/index.js +0 -5625
@@ -0,0 +1,324 @@
1
+ /**
2
+ * Tests for SDK utility types and functions
3
+ */
4
+
5
+ import { describe, expect, it } from 'bun:test';
6
+ import type { CaseResult, RedTeamManifest, RunManifest, StressManifest } from '@artemiskit/core';
7
+ import type { RedTeamResult, RunResult, StressResult } from '../types';
8
+ import {
9
+ assert,
10
+ assertDefined,
11
+ calculateSuccessRate,
12
+ getCasesByTag,
13
+ getFailedCases,
14
+ getPassedCases,
15
+ isRedTeamManifestType,
16
+ isRedTeamResult,
17
+ isRunManifestType,
18
+ isRunResult,
19
+ isStressManifestType,
20
+ isStressResult,
21
+ } from '../utils';
22
+
23
+ // Test fixtures
24
+ const createCaseResult = (ok: boolean, tags: string[] = []): CaseResult => ({
25
+ id: `case-${Math.random().toString(36).slice(2)}`,
26
+ ok,
27
+ score: ok ? 1 : 0,
28
+ matcherType: 'contains',
29
+ latencyMs: 100,
30
+ tokens: { prompt: 10, completion: 20, total: 30 },
31
+ prompt: 'test prompt',
32
+ response: 'test response',
33
+ expected: { type: 'contains', values: ['test'] },
34
+ tags,
35
+ });
36
+
37
+ const createRunResult = (cases: CaseResult[]): RunResult => ({
38
+ success: cases.every((c) => c.ok),
39
+ cases,
40
+ manifest: {
41
+ version: '1.0',
42
+ run_id: 'test-run',
43
+ project: 'test',
44
+ start_time: new Date().toISOString(),
45
+ end_time: new Date().toISOString(),
46
+ duration_ms: 1000,
47
+ config: { scenario: 'test', provider: 'openai' },
48
+ metrics: {
49
+ success_rate: cases.filter((c) => c.ok).length / cases.length || 0,
50
+ total_cases: cases.length,
51
+ passed_cases: cases.filter((c) => c.ok).length,
52
+ failed_cases: cases.filter((c) => !c.ok).length,
53
+ median_latency_ms: 100,
54
+ p95_latency_ms: 150,
55
+ total_tokens: cases.length * 30,
56
+ total_prompt_tokens: cases.length * 10,
57
+ total_completion_tokens: cases.length * 20,
58
+ },
59
+ git: { commit: 'abc123', branch: 'main', dirty: false },
60
+ provenance: { run_by: 'test' },
61
+ cases,
62
+ environment: { node_version: '20.0.0', platform: 'darwin', arch: 'arm64' },
63
+ },
64
+ });
65
+
66
+ const createRedTeamResult = (): RedTeamResult => ({
67
+ success: true,
68
+ defenseRate: 0.95,
69
+ unsafeCount: 1,
70
+ manifest: {
71
+ version: '1.0',
72
+ type: 'redteam',
73
+ run_id: 'test-redteam',
74
+ project: 'test',
75
+ start_time: new Date().toISOString(),
76
+ end_time: new Date().toISOString(),
77
+ duration_ms: 1000,
78
+ config: { scenario: 'test', provider: 'openai', mutations: [], count_per_case: 1 },
79
+ metrics: {
80
+ total_tests: 20,
81
+ safe_responses: 18,
82
+ blocked_responses: 1,
83
+ unsafe_responses: 1,
84
+ error_responses: 0,
85
+ defended: 19,
86
+ defense_rate: 0.95,
87
+ by_severity: { low: 0, medium: 1, high: 0, critical: 0 },
88
+ },
89
+ git: { commit: 'abc123', branch: 'main', dirty: false },
90
+ provenance: { run_by: 'test' },
91
+ results: [],
92
+ environment: { node_version: '20.0.0', platform: 'darwin', arch: 'arm64' },
93
+ },
94
+ });
95
+
96
+ const createStressResult = (): StressResult => ({
97
+ success: true,
98
+ successRate: 0.98,
99
+ rps: 50,
100
+ p95LatencyMs: 250,
101
+ manifest: {
102
+ version: '1.0',
103
+ type: 'stress',
104
+ run_id: 'test-stress',
105
+ project: 'test',
106
+ start_time: new Date().toISOString(),
107
+ end_time: new Date().toISOString(),
108
+ duration_ms: 10000,
109
+ config: {
110
+ scenario: 'test',
111
+ provider: 'openai',
112
+ concurrency: 10,
113
+ duration_seconds: 10,
114
+ ramp_up_seconds: 2,
115
+ },
116
+ metrics: {
117
+ total_requests: 500,
118
+ successful_requests: 490,
119
+ failed_requests: 10,
120
+ success_rate: 0.98,
121
+ requests_per_second: 50,
122
+ min_latency_ms: 50,
123
+ max_latency_ms: 500,
124
+ avg_latency_ms: 150,
125
+ p50_latency_ms: 140,
126
+ p90_latency_ms: 200,
127
+ p95_latency_ms: 250,
128
+ p99_latency_ms: 400,
129
+ },
130
+ git: { commit: 'abc123', branch: 'main', dirty: false },
131
+ provenance: { run_by: 'test' },
132
+ sample_results: [],
133
+ environment: { node_version: '20.0.0', platform: 'darwin', arch: 'arm64' },
134
+ },
135
+ });
136
+
137
+ describe('Utils - Type Guards', () => {
138
+ describe('isRunResult', () => {
139
+ it('should return true for RunResult', () => {
140
+ const result = createRunResult([createCaseResult(true)]);
141
+ expect(isRunResult(result)).toBe(true);
142
+ });
143
+
144
+ it('should return false for RedTeamResult', () => {
145
+ const result = createRedTeamResult();
146
+ expect(isRunResult(result)).toBe(false);
147
+ });
148
+
149
+ it('should return false for StressResult', () => {
150
+ const result = createStressResult();
151
+ expect(isRunResult(result)).toBe(false);
152
+ });
153
+ });
154
+
155
+ describe('isRedTeamResult', () => {
156
+ it('should return true for RedTeamResult', () => {
157
+ const result = createRedTeamResult();
158
+ expect(isRedTeamResult(result)).toBe(true);
159
+ });
160
+
161
+ it('should return false for RunResult', () => {
162
+ const result = createRunResult([createCaseResult(true)]);
163
+ expect(isRedTeamResult(result)).toBe(false);
164
+ });
165
+
166
+ it('should return false for StressResult', () => {
167
+ const result = createStressResult();
168
+ expect(isRedTeamResult(result)).toBe(false);
169
+ });
170
+ });
171
+
172
+ describe('isStressResult', () => {
173
+ it('should return true for StressResult', () => {
174
+ const result = createStressResult();
175
+ expect(isStressResult(result)).toBe(true);
176
+ });
177
+
178
+ it('should return false for RunResult', () => {
179
+ const result = createRunResult([createCaseResult(true)]);
180
+ expect(isStressResult(result)).toBe(false);
181
+ });
182
+
183
+ it('should return false for RedTeamResult', () => {
184
+ const result = createRedTeamResult();
185
+ expect(isStressResult(result)).toBe(false);
186
+ });
187
+ });
188
+
189
+ describe('Manifest type guards', () => {
190
+ it('isRunManifestType should identify run manifests', () => {
191
+ const runManifest: RunManifest = createRunResult([]).manifest;
192
+ expect(isRunManifestType(runManifest)).toBe(true);
193
+ });
194
+
195
+ it('isRedTeamManifestType should identify redteam manifests', () => {
196
+ const redteamManifest: RedTeamManifest = createRedTeamResult().manifest;
197
+ expect(isRedTeamManifestType(redteamManifest)).toBe(true);
198
+ });
199
+
200
+ it('isStressManifestType should identify stress manifests', () => {
201
+ const stressManifest: StressManifest = createStressResult().manifest;
202
+ expect(isStressManifestType(stressManifest)).toBe(true);
203
+ });
204
+ });
205
+ });
206
+
207
+ describe('Utils - Assertion Helpers', () => {
208
+ describe('assertDefined', () => {
209
+ it('should not throw for defined values', () => {
210
+ expect(() => assertDefined('value')).not.toThrow();
211
+ expect(() => assertDefined(0)).not.toThrow();
212
+ expect(() => assertDefined(false)).not.toThrow();
213
+ expect(() => assertDefined({})).not.toThrow();
214
+ });
215
+
216
+ it('should throw for null', () => {
217
+ expect(() => assertDefined(null)).toThrow();
218
+ });
219
+
220
+ it('should throw for undefined', () => {
221
+ expect(() => assertDefined(undefined)).toThrow();
222
+ });
223
+
224
+ it('should use custom message', () => {
225
+ expect(() => assertDefined(null, 'Custom error')).toThrow('Custom error');
226
+ });
227
+ });
228
+
229
+ describe('assert', () => {
230
+ it('should not throw for true condition', () => {
231
+ expect(() => assert(true)).not.toThrow();
232
+ const truthyValue = 1;
233
+ expect(() => assert(truthyValue === 1)).not.toThrow();
234
+ });
235
+
236
+ it('should throw for false condition', () => {
237
+ expect(() => assert(false)).toThrow();
238
+ });
239
+
240
+ it('should use custom message', () => {
241
+ expect(() => assert(false, 'Custom assertion')).toThrow('Custom assertion');
242
+ });
243
+ });
244
+ });
245
+
246
+ describe('Utils - Result Analysis Helpers', () => {
247
+ describe('getFailedCases', () => {
248
+ it('should return empty array when all cases pass', () => {
249
+ const result = createRunResult([createCaseResult(true), createCaseResult(true)]);
250
+ expect(getFailedCases(result)).toHaveLength(0);
251
+ });
252
+
253
+ it('should return failed cases only', () => {
254
+ const passedCase = createCaseResult(true);
255
+ const failedCase = createCaseResult(false);
256
+ const result = createRunResult([passedCase, failedCase]);
257
+
258
+ const failed = getFailedCases(result);
259
+ expect(failed).toHaveLength(1);
260
+ expect(failed[0].ok).toBe(false);
261
+ });
262
+ });
263
+
264
+ describe('getPassedCases', () => {
265
+ it('should return empty array when all cases fail', () => {
266
+ const result = createRunResult([createCaseResult(false), createCaseResult(false)]);
267
+ expect(getPassedCases(result)).toHaveLength(0);
268
+ });
269
+
270
+ it('should return passed cases only', () => {
271
+ const passedCase = createCaseResult(true);
272
+ const failedCase = createCaseResult(false);
273
+ const result = createRunResult([passedCase, failedCase]);
274
+
275
+ const passed = getPassedCases(result);
276
+ expect(passed).toHaveLength(1);
277
+ expect(passed[0].ok).toBe(true);
278
+ });
279
+ });
280
+
281
+ describe('getCasesByTag', () => {
282
+ it('should return cases with matching tag', () => {
283
+ const taggedCase = createCaseResult(true, ['important', 'smoke']);
284
+ const untaggedCase = createCaseResult(true, ['other']);
285
+ const result = createRunResult([taggedCase, untaggedCase]);
286
+
287
+ const tagged = getCasesByTag(result, 'important');
288
+ expect(tagged).toHaveLength(1);
289
+ expect(tagged[0].tags).toContain('important');
290
+ });
291
+
292
+ it('should return empty array when no cases match', () => {
293
+ const result = createRunResult([createCaseResult(true, ['other'])]);
294
+ expect(getCasesByTag(result, 'nonexistent')).toHaveLength(0);
295
+ });
296
+ });
297
+
298
+ describe('calculateSuccessRate', () => {
299
+ it('should return 1 when all cases pass', () => {
300
+ const result = createRunResult([createCaseResult(true), createCaseResult(true)]);
301
+ expect(calculateSuccessRate(result)).toBe(1);
302
+ });
303
+
304
+ it('should return 0 when all cases fail', () => {
305
+ const result = createRunResult([createCaseResult(false), createCaseResult(false)]);
306
+ expect(calculateSuccessRate(result)).toBe(0);
307
+ });
308
+
309
+ it('should return correct ratio', () => {
310
+ const result = createRunResult([
311
+ createCaseResult(true),
312
+ createCaseResult(true),
313
+ createCaseResult(false),
314
+ createCaseResult(false),
315
+ ]);
316
+ expect(calculateSuccessRate(result)).toBe(0.5);
317
+ });
318
+
319
+ it('should return 0 for empty cases', () => {
320
+ const result = createRunResult([]);
321
+ expect(calculateSuccessRate(result)).toBe(0);
322
+ });
323
+ });
324
+ });
@@ -0,0 +1,310 @@
1
+ /**
2
+ * Tests for ArtemisKit.validate() method (v0.3.2+)
3
+ */
4
+
5
+ import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test';
6
+ import { createStorageAdapter as createRealStorageAdapter } from '../../../core/src/storage/factory';
7
+
8
+ // Track parseScenarioFile calls and responses
9
+ let parseScenarioFileResponse: unknown = {};
10
+ let parseScenarioFileError: Error | null = null;
11
+
12
+ // Track resolveScenarioPaths behavior
13
+ let resolveScenarioPathsResponse: string[] = [];
14
+
15
+ // Mock the core module
16
+ mock.module('@artemiskit/core', () => ({
17
+ parseScenarioFile: async () => {
18
+ if (parseScenarioFileError) {
19
+ throw parseScenarioFileError;
20
+ }
21
+ return parseScenarioFileResponse;
22
+ },
23
+ resolveScenarioPaths: async (inputPath: string) => {
24
+ // If custom response is set, use it; otherwise return the input path as-is
25
+ if (resolveScenarioPathsResponse.length > 0) {
26
+ return resolveScenarioPathsResponse;
27
+ }
28
+ return [inputPath];
29
+ },
30
+ createAdapter: async () => ({
31
+ provider: 'mock',
32
+ generate: async () => ({}),
33
+ capabilities: async () => ({}),
34
+ close: () => {},
35
+ }),
36
+ runScenario: async () => ({}),
37
+ getGitInfo: async () => ({ commit: 'abc123', branch: 'main', dirty: false }),
38
+ // Bun's module mocks are shared by test files in a monorepo run. Preserve the
39
+ // real storage factory so unrelated CLI integration tests can create storage.
40
+ createStorageAdapter: createRealStorageAdapter,
41
+ }));
42
+
43
+ mock.module('@artemiskit/redteam', () => ({
44
+ TypoMutation: class {
45
+ name = 'typo';
46
+ },
47
+ RoleSpoofMutation: class {
48
+ name = 'role-spoof';
49
+ },
50
+ InstructionFlipMutation: class {
51
+ name = 'instruction-flip';
52
+ },
53
+ CotInjectionMutation: class {
54
+ name = 'cot-injection';
55
+ },
56
+ EncodingMutation: class {
57
+ name = 'encoding';
58
+ },
59
+ MultiTurnMutation: class {
60
+ name = 'multi-turn';
61
+ },
62
+ RedTeamGenerator: class {
63
+ generate() {
64
+ return [];
65
+ }
66
+ listMutations() {
67
+ return [];
68
+ }
69
+ },
70
+ UnsafeResponseDetector: class {
71
+ detect() {
72
+ return { unsafe: false, blocked: false, reasons: [], severity: 'none' };
73
+ }
74
+ },
75
+ }));
76
+
77
+ mock.module('nanoid', () => ({
78
+ nanoid: () => 'test-id-12345',
79
+ }));
80
+
81
+ import { ArtemisKit } from '../artemiskit';
82
+
83
+ // Helper to set mock response
84
+ function setParseResponse(response: unknown) {
85
+ parseScenarioFileResponse = response;
86
+ parseScenarioFileError = null;
87
+ }
88
+
89
+ function setParseError(error: Error) {
90
+ parseScenarioFileError = error;
91
+ }
92
+
93
+ function setResolvePaths(paths: string[]) {
94
+ resolveScenarioPathsResponse = paths;
95
+ }
96
+
97
+ function resetMocks() {
98
+ parseScenarioFileResponse = {};
99
+ parseScenarioFileError = null;
100
+ resolveScenarioPathsResponse = [];
101
+ }
102
+
103
+ describe('ArtemisKit.validate()', () => {
104
+ let kit: ArtemisKit;
105
+
106
+ beforeEach(() => {
107
+ kit = new ArtemisKit({
108
+ project: 'test-project',
109
+ provider: 'openai',
110
+ model: 'gpt-4',
111
+ });
112
+ resetMocks();
113
+ });
114
+
115
+ afterEach(() => {
116
+ resetMocks();
117
+ });
118
+
119
+ describe('single scenario validation', () => {
120
+ it('should validate a valid scenario file', async () => {
121
+ setParseResponse({
122
+ name: 'Valid Scenario',
123
+ version: '1.0',
124
+ model: 'gpt-4',
125
+ cases: [
126
+ {
127
+ id: 'case-1',
128
+ name: 'Test Case',
129
+ prompt: 'Hello',
130
+ expected: { type: 'contains', values: ['Hello'], mode: 'any' },
131
+ tags: [],
132
+ metadata: {},
133
+ retries: 0,
134
+ },
135
+ ],
136
+ tags: [],
137
+ });
138
+
139
+ const result = await kit.validate({ scenario: './valid-scenario.yaml' });
140
+
141
+ expect(result.valid).toBe(true);
142
+ expect(result.errors).toHaveLength(0);
143
+ expect(result.scenarios).toHaveLength(1);
144
+ expect(result.scenarios[0].valid).toBe(true);
145
+ expect(result.scenarios[0].file).toBe('./valid-scenario.yaml');
146
+ });
147
+
148
+ it('should detect missing required fields', async () => {
149
+ setParseResponse({
150
+ name: 'Incomplete Scenario',
151
+ version: '1.0',
152
+ model: 'gpt-4',
153
+ cases: [
154
+ {
155
+ id: 'case-1',
156
+ name: 'Test Case',
157
+ prompt: '', // Empty prompt
158
+ expected: { type: 'contains', values: ['Hello'], mode: 'any' },
159
+ tags: [],
160
+ metadata: {},
161
+ retries: 0,
162
+ },
163
+ ],
164
+ tags: [],
165
+ });
166
+
167
+ const result = await kit.validate({ scenario: './incomplete.yaml' });
168
+
169
+ expect(result.valid).toBe(false);
170
+ expect(result.errors.length).toBeGreaterThan(0);
171
+ expect(result.errors[0].message).toContain('prompt');
172
+ });
173
+
174
+ it('should detect invalid expectation types', async () => {
175
+ setParseResponse({
176
+ name: 'Invalid Expectation',
177
+ version: '1.0',
178
+ model: 'gpt-4',
179
+ cases: [
180
+ {
181
+ id: 'case-1',
182
+ name: 'Test Case',
183
+ prompt: 'Hello',
184
+ expected: { type: 'invalid_type', values: ['Hello'], mode: 'any' },
185
+ tags: [],
186
+ metadata: {},
187
+ retries: 0,
188
+ },
189
+ ],
190
+ tags: [],
191
+ });
192
+
193
+ const result = await kit.validate({ scenario: './invalid-expectation.yaml' });
194
+
195
+ expect(result.valid).toBe(false);
196
+ expect(
197
+ result.errors.some((e) => e.message.includes('type') || e.message.includes('expectation'))
198
+ ).toBe(true);
199
+ });
200
+
201
+ it('should detect duplicate case IDs', async () => {
202
+ setParseResponse({
203
+ name: 'Duplicate IDs',
204
+ version: '1.0',
205
+ model: 'gpt-4',
206
+ cases: [
207
+ {
208
+ id: 'same-id',
209
+ name: 'Test Case 1',
210
+ prompt: 'Hello',
211
+ expected: { type: 'contains', values: ['Hello'], mode: 'any' },
212
+ tags: [],
213
+ metadata: {},
214
+ retries: 0,
215
+ },
216
+ {
217
+ id: 'same-id',
218
+ name: 'Test Case 2',
219
+ prompt: 'World',
220
+ expected: { type: 'contains', values: ['World'], mode: 'any' },
221
+ tags: [],
222
+ metadata: {},
223
+ retries: 0,
224
+ },
225
+ ],
226
+ tags: [],
227
+ });
228
+
229
+ const result = await kit.validate({ scenario: './duplicate-ids.yaml' });
230
+
231
+ expect(result.valid).toBe(false);
232
+ expect(
233
+ result.errors.some((e) => e.message.includes('duplicate') || e.message.includes('same-id'))
234
+ ).toBe(true);
235
+ });
236
+
237
+ it('should handle parse errors gracefully', async () => {
238
+ setParseError(new Error('YAML parse error: invalid syntax'));
239
+
240
+ const result = await kit.validate({ scenario: './broken.yaml' });
241
+
242
+ expect(result.valid).toBe(false);
243
+ expect(result.errors.length).toBeGreaterThan(0);
244
+ expect(result.errors[0].message).toContain('parse');
245
+ });
246
+ });
247
+
248
+ describe('strict mode', () => {
249
+ it('should pass with warnings in non-strict mode', async () => {
250
+ setParseResponse({
251
+ name: 'Scenario with Warnings',
252
+ version: '1.0',
253
+ model: 'gpt-4',
254
+ cases: [
255
+ {
256
+ id: 'case-1',
257
+ name: 'Test',
258
+ prompt: 'Hello',
259
+ expected: { type: 'contains', values: ['Hello'], mode: 'any' },
260
+ tags: [],
261
+ metadata: {},
262
+ retries: 0,
263
+ },
264
+ ],
265
+ tags: [],
266
+ });
267
+
268
+ const result = await kit.validate({
269
+ scenario: './scenario.yaml',
270
+ strict: false,
271
+ });
272
+
273
+ // If there are warnings but no errors, valid should be true in non-strict mode
274
+ if (result.errors.length === 0) {
275
+ expect(result.valid).toBe(true);
276
+ }
277
+ });
278
+ });
279
+
280
+ describe('progress events', () => {
281
+ it('should emit progress events during validation', async () => {
282
+ setParseResponse({
283
+ name: 'Test Scenario',
284
+ version: '1.0',
285
+ model: 'gpt-4',
286
+ cases: [
287
+ {
288
+ id: 'case-1',
289
+ name: 'Test',
290
+ prompt: 'Hello',
291
+ expected: { type: 'contains', values: ['Hello'], mode: 'any' },
292
+ tags: [],
293
+ metadata: {},
294
+ retries: 0,
295
+ },
296
+ ],
297
+ tags: [],
298
+ });
299
+
300
+ let progressCalled = false;
301
+ kit.onProgress(() => {
302
+ progressCalled = true;
303
+ });
304
+
305
+ await kit.validate({ scenario: './test.yaml' });
306
+
307
+ expect(progressCalled).toBe(true);
308
+ });
309
+ });
310
+ });