@artemiskit/sdk 0.3.0 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CHANGELOG.md +225 -0
  2. package/README.md +44 -1
  3. package/dist/artemiskit.d.ts +149 -0
  4. package/dist/artemiskit.d.ts.map +1 -0
  5. package/dist/builders/index.d.ts +350 -0
  6. package/dist/builders/index.d.ts.map +1 -0
  7. package/dist/builders/index.js +398 -0
  8. package/dist/contracts/index.d.ts +253 -0
  9. package/dist/contracts/index.d.ts.map +1 -0
  10. package/dist/contracts/index.js +99 -0
  11. package/dist/guardian/action-validator.d.ts +92 -0
  12. package/dist/guardian/action-validator.d.ts.map +1 -0
  13. package/dist/guardian/circuit-breaker.d.ts +211 -0
  14. package/dist/guardian/circuit-breaker.d.ts.map +1 -0
  15. package/dist/guardian/guardian.d.ts +213 -0
  16. package/dist/guardian/guardian.d.ts.map +1 -0
  17. package/dist/guardian/guardrails.d.ts +107 -0
  18. package/dist/guardian/guardrails.d.ts.map +1 -0
  19. package/dist/guardian/index.d.ts +35 -0
  20. package/dist/guardian/index.d.ts.map +1 -0
  21. package/dist/guardian/intent-classifier.d.ts +81 -0
  22. package/dist/guardian/intent-classifier.d.ts.map +1 -0
  23. package/dist/guardian/interceptor.d.ts +141 -0
  24. package/dist/guardian/interceptor.d.ts.map +1 -0
  25. package/dist/guardian/policy.d.ts +52 -0
  26. package/dist/guardian/policy.d.ts.map +1 -0
  27. package/dist/guardian/semantic-validator.d.ts +61 -0
  28. package/dist/guardian/semantic-validator.d.ts.map +1 -0
  29. package/dist/guardian/types.d.ts +653 -0
  30. package/dist/guardian/types.d.ts.map +1 -0
  31. package/dist/index.d.ts +23 -0
  32. package/dist/index.d.ts.map +1 -0
  33. package/dist/index.js +24847 -13653
  34. package/dist/matchers/core.d.ts +81 -0
  35. package/dist/matchers/core.d.ts.map +1 -0
  36. package/dist/matchers/index.d.ts +6 -0
  37. package/dist/matchers/index.d.ts.map +1 -0
  38. package/dist/matchers/jest.d.ts +46 -0
  39. package/dist/matchers/jest.d.ts.map +1 -0
  40. package/dist/matchers/vitest.d.ts +24 -0
  41. package/dist/matchers/vitest.d.ts.map +1 -0
  42. package/dist/types-only.d.ts +31 -0
  43. package/dist/types-only.d.ts.map +1 -0
  44. package/dist/types-only.js +1 -0
  45. package/dist/types.d.ts +377 -0
  46. package/dist/types.d.ts.map +1 -0
  47. package/dist/utils/index.d.ts +112 -0
  48. package/dist/utils/index.d.ts.map +1 -0
  49. package/dist/utils/index.js +104 -0
  50. package/package.json +19 -4
  51. package/src/__tests__/builders.test.ts +445 -0
  52. package/src/__tests__/circuit-breaker.test.ts +704 -0
  53. package/src/__tests__/compare.test.ts +469 -0
  54. package/src/__tests__/contracts.test.ts +354 -0
  55. package/src/__tests__/guardian-modes.test.ts +215 -0
  56. package/src/__tests__/guardrails.test.ts +404 -0
  57. package/src/__tests__/semantic-validator.test.ts +421 -0
  58. package/src/__tests__/utils.test.ts +324 -0
  59. package/src/__tests__/validate.test.ts +307 -0
  60. package/src/artemiskit.ts +478 -5
  61. package/src/builders/index.ts +698 -0
  62. package/src/contracts/index.ts +357 -0
  63. package/src/guardian/circuit-breaker.ts +74 -28
  64. package/src/guardian/guardian.ts +307 -20
  65. package/src/guardian/guardrails.ts +143 -1
  66. package/src/guardian/index.ts +35 -1
  67. package/src/guardian/interceptor.ts +87 -11
  68. package/src/guardian/semantic-validator.ts +399 -0
  69. package/src/guardian/types.ts +413 -2
  70. package/src/index.ts +258 -14
  71. package/src/types-only.ts +276 -0
  72. package/src/types.ts +183 -0
  73. package/src/utils/index.ts +225 -0
  74. package/tsconfig.json +2 -1
@@ -0,0 +1,324 @@
1
+ /**
2
+ * Tests for SDK utility types and functions
3
+ */
4
+
5
+ import { describe, expect, it } from 'bun:test';
6
+ import type { CaseResult, RedTeamManifest, RunManifest, StressManifest } from '@artemiskit/core';
7
+ import type { RedTeamResult, RunResult, StressResult } from '../types';
8
+ import {
9
+ assert,
10
+ assertDefined,
11
+ calculateSuccessRate,
12
+ getCasesByTag,
13
+ getFailedCases,
14
+ getPassedCases,
15
+ isRedTeamManifestType,
16
+ isRedTeamResult,
17
+ isRunManifestType,
18
+ isRunResult,
19
+ isStressManifestType,
20
+ isStressResult,
21
+ } from '../utils';
22
+
23
+ // Test fixtures
24
+ const createCaseResult = (ok: boolean, tags: string[] = []): CaseResult => ({
25
+ id: `case-${Math.random().toString(36).slice(2)}`,
26
+ ok,
27
+ score: ok ? 1 : 0,
28
+ matcherType: 'contains',
29
+ latencyMs: 100,
30
+ tokens: { prompt: 10, completion: 20, total: 30 },
31
+ prompt: 'test prompt',
32
+ response: 'test response',
33
+ expected: { type: 'contains', values: ['test'] },
34
+ tags,
35
+ });
36
+
37
+ const createRunResult = (cases: CaseResult[]): RunResult => ({
38
+ success: cases.every((c) => c.ok),
39
+ cases,
40
+ manifest: {
41
+ version: '1.0',
42
+ run_id: 'test-run',
43
+ project: 'test',
44
+ start_time: new Date().toISOString(),
45
+ end_time: new Date().toISOString(),
46
+ duration_ms: 1000,
47
+ config: { scenario: 'test', provider: 'openai' },
48
+ metrics: {
49
+ success_rate: cases.filter((c) => c.ok).length / cases.length || 0,
50
+ total_cases: cases.length,
51
+ passed_cases: cases.filter((c) => c.ok).length,
52
+ failed_cases: cases.filter((c) => !c.ok).length,
53
+ median_latency_ms: 100,
54
+ p95_latency_ms: 150,
55
+ total_tokens: cases.length * 30,
56
+ total_prompt_tokens: cases.length * 10,
57
+ total_completion_tokens: cases.length * 20,
58
+ },
59
+ git: { commit: 'abc123', branch: 'main', dirty: false },
60
+ provenance: { run_by: 'test' },
61
+ cases,
62
+ environment: { node_version: '20.0.0', platform: 'darwin', arch: 'arm64' },
63
+ },
64
+ });
65
+
66
+ const createRedTeamResult = (): RedTeamResult => ({
67
+ success: true,
68
+ defenseRate: 0.95,
69
+ unsafeCount: 1,
70
+ manifest: {
71
+ version: '1.0',
72
+ type: 'redteam',
73
+ run_id: 'test-redteam',
74
+ project: 'test',
75
+ start_time: new Date().toISOString(),
76
+ end_time: new Date().toISOString(),
77
+ duration_ms: 1000,
78
+ config: { scenario: 'test', provider: 'openai', mutations: [], count_per_case: 1 },
79
+ metrics: {
80
+ total_tests: 20,
81
+ safe_responses: 18,
82
+ blocked_responses: 1,
83
+ unsafe_responses: 1,
84
+ error_responses: 0,
85
+ defended: 19,
86
+ defense_rate: 0.95,
87
+ by_severity: { low: 0, medium: 1, high: 0, critical: 0 },
88
+ },
89
+ git: { commit: 'abc123', branch: 'main', dirty: false },
90
+ provenance: { run_by: 'test' },
91
+ results: [],
92
+ environment: { node_version: '20.0.0', platform: 'darwin', arch: 'arm64' },
93
+ },
94
+ });
95
+
96
+ const createStressResult = (): StressResult => ({
97
+ success: true,
98
+ successRate: 0.98,
99
+ rps: 50,
100
+ p95LatencyMs: 250,
101
+ manifest: {
102
+ version: '1.0',
103
+ type: 'stress',
104
+ run_id: 'test-stress',
105
+ project: 'test',
106
+ start_time: new Date().toISOString(),
107
+ end_time: new Date().toISOString(),
108
+ duration_ms: 10000,
109
+ config: {
110
+ scenario: 'test',
111
+ provider: 'openai',
112
+ concurrency: 10,
113
+ duration_seconds: 10,
114
+ ramp_up_seconds: 2,
115
+ },
116
+ metrics: {
117
+ total_requests: 500,
118
+ successful_requests: 490,
119
+ failed_requests: 10,
120
+ success_rate: 0.98,
121
+ requests_per_second: 50,
122
+ min_latency_ms: 50,
123
+ max_latency_ms: 500,
124
+ avg_latency_ms: 150,
125
+ p50_latency_ms: 140,
126
+ p90_latency_ms: 200,
127
+ p95_latency_ms: 250,
128
+ p99_latency_ms: 400,
129
+ },
130
+ git: { commit: 'abc123', branch: 'main', dirty: false },
131
+ provenance: { run_by: 'test' },
132
+ sample_results: [],
133
+ environment: { node_version: '20.0.0', platform: 'darwin', arch: 'arm64' },
134
+ },
135
+ });
136
+
137
+ describe('Utils - Type Guards', () => {
138
+ describe('isRunResult', () => {
139
+ it('should return true for RunResult', () => {
140
+ const result = createRunResult([createCaseResult(true)]);
141
+ expect(isRunResult(result)).toBe(true);
142
+ });
143
+
144
+ it('should return false for RedTeamResult', () => {
145
+ const result = createRedTeamResult();
146
+ expect(isRunResult(result)).toBe(false);
147
+ });
148
+
149
+ it('should return false for StressResult', () => {
150
+ const result = createStressResult();
151
+ expect(isRunResult(result)).toBe(false);
152
+ });
153
+ });
154
+
155
+ describe('isRedTeamResult', () => {
156
+ it('should return true for RedTeamResult', () => {
157
+ const result = createRedTeamResult();
158
+ expect(isRedTeamResult(result)).toBe(true);
159
+ });
160
+
161
+ it('should return false for RunResult', () => {
162
+ const result = createRunResult([createCaseResult(true)]);
163
+ expect(isRedTeamResult(result)).toBe(false);
164
+ });
165
+
166
+ it('should return false for StressResult', () => {
167
+ const result = createStressResult();
168
+ expect(isRedTeamResult(result)).toBe(false);
169
+ });
170
+ });
171
+
172
+ describe('isStressResult', () => {
173
+ it('should return true for StressResult', () => {
174
+ const result = createStressResult();
175
+ expect(isStressResult(result)).toBe(true);
176
+ });
177
+
178
+ it('should return false for RunResult', () => {
179
+ const result = createRunResult([createCaseResult(true)]);
180
+ expect(isStressResult(result)).toBe(false);
181
+ });
182
+
183
+ it('should return false for RedTeamResult', () => {
184
+ const result = createRedTeamResult();
185
+ expect(isStressResult(result)).toBe(false);
186
+ });
187
+ });
188
+
189
+ describe('Manifest type guards', () => {
190
+ it('isRunManifestType should identify run manifests', () => {
191
+ const runManifest: RunManifest = createRunResult([]).manifest;
192
+ expect(isRunManifestType(runManifest)).toBe(true);
193
+ });
194
+
195
+ it('isRedTeamManifestType should identify redteam manifests', () => {
196
+ const redteamManifest: RedTeamManifest = createRedTeamResult().manifest;
197
+ expect(isRedTeamManifestType(redteamManifest)).toBe(true);
198
+ });
199
+
200
+ it('isStressManifestType should identify stress manifests', () => {
201
+ const stressManifest: StressManifest = createStressResult().manifest;
202
+ expect(isStressManifestType(stressManifest)).toBe(true);
203
+ });
204
+ });
205
+ });
206
+
207
+ describe('Utils - Assertion Helpers', () => {
208
+ describe('assertDefined', () => {
209
+ it('should not throw for defined values', () => {
210
+ expect(() => assertDefined('value')).not.toThrow();
211
+ expect(() => assertDefined(0)).not.toThrow();
212
+ expect(() => assertDefined(false)).not.toThrow();
213
+ expect(() => assertDefined({})).not.toThrow();
214
+ });
215
+
216
+ it('should throw for null', () => {
217
+ expect(() => assertDefined(null)).toThrow();
218
+ });
219
+
220
+ it('should throw for undefined', () => {
221
+ expect(() => assertDefined(undefined)).toThrow();
222
+ });
223
+
224
+ it('should use custom message', () => {
225
+ expect(() => assertDefined(null, 'Custom error')).toThrow('Custom error');
226
+ });
227
+ });
228
+
229
+ describe('assert', () => {
230
+ it('should not throw for true condition', () => {
231
+ expect(() => assert(true)).not.toThrow();
232
+ const truthyValue = 1;
233
+ expect(() => assert(truthyValue === 1)).not.toThrow();
234
+ });
235
+
236
+ it('should throw for false condition', () => {
237
+ expect(() => assert(false)).toThrow();
238
+ });
239
+
240
+ it('should use custom message', () => {
241
+ expect(() => assert(false, 'Custom assertion')).toThrow('Custom assertion');
242
+ });
243
+ });
244
+ });
245
+
246
+ describe('Utils - Result Analysis Helpers', () => {
247
+ describe('getFailedCases', () => {
248
+ it('should return empty array when all cases pass', () => {
249
+ const result = createRunResult([createCaseResult(true), createCaseResult(true)]);
250
+ expect(getFailedCases(result)).toHaveLength(0);
251
+ });
252
+
253
+ it('should return failed cases only', () => {
254
+ const passedCase = createCaseResult(true);
255
+ const failedCase = createCaseResult(false);
256
+ const result = createRunResult([passedCase, failedCase]);
257
+
258
+ const failed = getFailedCases(result);
259
+ expect(failed).toHaveLength(1);
260
+ expect(failed[0].ok).toBe(false);
261
+ });
262
+ });
263
+
264
+ describe('getPassedCases', () => {
265
+ it('should return empty array when all cases fail', () => {
266
+ const result = createRunResult([createCaseResult(false), createCaseResult(false)]);
267
+ expect(getPassedCases(result)).toHaveLength(0);
268
+ });
269
+
270
+ it('should return passed cases only', () => {
271
+ const passedCase = createCaseResult(true);
272
+ const failedCase = createCaseResult(false);
273
+ const result = createRunResult([passedCase, failedCase]);
274
+
275
+ const passed = getPassedCases(result);
276
+ expect(passed).toHaveLength(1);
277
+ expect(passed[0].ok).toBe(true);
278
+ });
279
+ });
280
+
281
+ describe('getCasesByTag', () => {
282
+ it('should return cases with matching tag', () => {
283
+ const taggedCase = createCaseResult(true, ['important', 'smoke']);
284
+ const untaggedCase = createCaseResult(true, ['other']);
285
+ const result = createRunResult([taggedCase, untaggedCase]);
286
+
287
+ const tagged = getCasesByTag(result, 'important');
288
+ expect(tagged).toHaveLength(1);
289
+ expect(tagged[0].tags).toContain('important');
290
+ });
291
+
292
+ it('should return empty array when no cases match', () => {
293
+ const result = createRunResult([createCaseResult(true, ['other'])]);
294
+ expect(getCasesByTag(result, 'nonexistent')).toHaveLength(0);
295
+ });
296
+ });
297
+
298
+ describe('calculateSuccessRate', () => {
299
+ it('should return 1 when all cases pass', () => {
300
+ const result = createRunResult([createCaseResult(true), createCaseResult(true)]);
301
+ expect(calculateSuccessRate(result)).toBe(1);
302
+ });
303
+
304
+ it('should return 0 when all cases fail', () => {
305
+ const result = createRunResult([createCaseResult(false), createCaseResult(false)]);
306
+ expect(calculateSuccessRate(result)).toBe(0);
307
+ });
308
+
309
+ it('should return correct ratio', () => {
310
+ const result = createRunResult([
311
+ createCaseResult(true),
312
+ createCaseResult(true),
313
+ createCaseResult(false),
314
+ createCaseResult(false),
315
+ ]);
316
+ expect(calculateSuccessRate(result)).toBe(0.5);
317
+ });
318
+
319
+ it('should return 0 for empty cases', () => {
320
+ const result = createRunResult([]);
321
+ expect(calculateSuccessRate(result)).toBe(0);
322
+ });
323
+ });
324
+ });
@@ -0,0 +1,307 @@
1
+ /**
2
+ * Tests for ArtemisKit.validate() method (v0.3.2+)
3
+ */
4
+
5
+ import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test';
6
+
7
+ // Track parseScenarioFile calls and responses
8
+ let parseScenarioFileResponse: unknown = {};
9
+ let parseScenarioFileError: Error | null = null;
10
+
11
+ // Track resolveScenarioPaths behavior
12
+ let resolveScenarioPathsResponse: string[] = [];
13
+
14
+ // Mock the core module
15
+ mock.module('@artemiskit/core', () => ({
16
+ parseScenarioFile: async () => {
17
+ if (parseScenarioFileError) {
18
+ throw parseScenarioFileError;
19
+ }
20
+ return parseScenarioFileResponse;
21
+ },
22
+ resolveScenarioPaths: async (inputPath: string) => {
23
+ // If custom response is set, use it; otherwise return the input path as-is
24
+ if (resolveScenarioPathsResponse.length > 0) {
25
+ return resolveScenarioPathsResponse;
26
+ }
27
+ return [inputPath];
28
+ },
29
+ createAdapter: async () => ({
30
+ provider: 'mock',
31
+ generate: async () => ({}),
32
+ capabilities: async () => ({}),
33
+ close: () => {},
34
+ }),
35
+ runScenario: async () => ({}),
36
+ getGitInfo: async () => ({ commit: 'abc123', branch: 'main', dirty: false }),
37
+ createStorageAdapter: () => null,
38
+ }));
39
+
40
+ mock.module('@artemiskit/redteam', () => ({
41
+ TypoMutation: class {
42
+ name = 'typo';
43
+ },
44
+ RoleSpoofMutation: class {
45
+ name = 'role-spoof';
46
+ },
47
+ InstructionFlipMutation: class {
48
+ name = 'instruction-flip';
49
+ },
50
+ CotInjectionMutation: class {
51
+ name = 'cot-injection';
52
+ },
53
+ EncodingMutation: class {
54
+ name = 'encoding';
55
+ },
56
+ MultiTurnMutation: class {
57
+ name = 'multi-turn';
58
+ },
59
+ RedTeamGenerator: class {
60
+ generate() {
61
+ return [];
62
+ }
63
+ listMutations() {
64
+ return [];
65
+ }
66
+ },
67
+ UnsafeResponseDetector: class {
68
+ detect() {
69
+ return { unsafe: false, blocked: false, reasons: [], severity: 'none' };
70
+ }
71
+ },
72
+ }));
73
+
74
+ mock.module('nanoid', () => ({
75
+ nanoid: () => 'test-id-12345',
76
+ }));
77
+
78
+ import { ArtemisKit } from '../artemiskit';
79
+
80
+ // Helper to set mock response
81
+ function setParseResponse(response: unknown) {
82
+ parseScenarioFileResponse = response;
83
+ parseScenarioFileError = null;
84
+ }
85
+
86
+ function setParseError(error: Error) {
87
+ parseScenarioFileError = error;
88
+ }
89
+
90
+ function setResolvePaths(paths: string[]) {
91
+ resolveScenarioPathsResponse = paths;
92
+ }
93
+
94
+ function resetMocks() {
95
+ parseScenarioFileResponse = {};
96
+ parseScenarioFileError = null;
97
+ resolveScenarioPathsResponse = [];
98
+ }
99
+
100
+ describe('ArtemisKit.validate()', () => {
101
+ let kit: ArtemisKit;
102
+
103
+ beforeEach(() => {
104
+ kit = new ArtemisKit({
105
+ project: 'test-project',
106
+ provider: 'openai',
107
+ model: 'gpt-4',
108
+ });
109
+ resetMocks();
110
+ });
111
+
112
+ afterEach(() => {
113
+ resetMocks();
114
+ });
115
+
116
+ describe('single scenario validation', () => {
117
+ it('should validate a valid scenario file', async () => {
118
+ setParseResponse({
119
+ name: 'Valid Scenario',
120
+ version: '1.0',
121
+ model: 'gpt-4',
122
+ cases: [
123
+ {
124
+ id: 'case-1',
125
+ name: 'Test Case',
126
+ prompt: 'Hello',
127
+ expected: { type: 'contains', values: ['Hello'], mode: 'any' },
128
+ tags: [],
129
+ metadata: {},
130
+ retries: 0,
131
+ },
132
+ ],
133
+ tags: [],
134
+ });
135
+
136
+ const result = await kit.validate({ scenario: './valid-scenario.yaml' });
137
+
138
+ expect(result.valid).toBe(true);
139
+ expect(result.errors).toHaveLength(0);
140
+ expect(result.scenarios).toHaveLength(1);
141
+ expect(result.scenarios[0].valid).toBe(true);
142
+ expect(result.scenarios[0].file).toBe('./valid-scenario.yaml');
143
+ });
144
+
145
+ it('should detect missing required fields', async () => {
146
+ setParseResponse({
147
+ name: 'Incomplete Scenario',
148
+ version: '1.0',
149
+ model: 'gpt-4',
150
+ cases: [
151
+ {
152
+ id: 'case-1',
153
+ name: 'Test Case',
154
+ prompt: '', // Empty prompt
155
+ expected: { type: 'contains', values: ['Hello'], mode: 'any' },
156
+ tags: [],
157
+ metadata: {},
158
+ retries: 0,
159
+ },
160
+ ],
161
+ tags: [],
162
+ });
163
+
164
+ const result = await kit.validate({ scenario: './incomplete.yaml' });
165
+
166
+ expect(result.valid).toBe(false);
167
+ expect(result.errors.length).toBeGreaterThan(0);
168
+ expect(result.errors[0].message).toContain('prompt');
169
+ });
170
+
171
+ it('should detect invalid expectation types', async () => {
172
+ setParseResponse({
173
+ name: 'Invalid Expectation',
174
+ version: '1.0',
175
+ model: 'gpt-4',
176
+ cases: [
177
+ {
178
+ id: 'case-1',
179
+ name: 'Test Case',
180
+ prompt: 'Hello',
181
+ expected: { type: 'invalid_type', values: ['Hello'], mode: 'any' },
182
+ tags: [],
183
+ metadata: {},
184
+ retries: 0,
185
+ },
186
+ ],
187
+ tags: [],
188
+ });
189
+
190
+ const result = await kit.validate({ scenario: './invalid-expectation.yaml' });
191
+
192
+ expect(result.valid).toBe(false);
193
+ expect(
194
+ result.errors.some((e) => e.message.includes('type') || e.message.includes('expectation'))
195
+ ).toBe(true);
196
+ });
197
+
198
+ it('should detect duplicate case IDs', async () => {
199
+ setParseResponse({
200
+ name: 'Duplicate IDs',
201
+ version: '1.0',
202
+ model: 'gpt-4',
203
+ cases: [
204
+ {
205
+ id: 'same-id',
206
+ name: 'Test Case 1',
207
+ prompt: 'Hello',
208
+ expected: { type: 'contains', values: ['Hello'], mode: 'any' },
209
+ tags: [],
210
+ metadata: {},
211
+ retries: 0,
212
+ },
213
+ {
214
+ id: 'same-id',
215
+ name: 'Test Case 2',
216
+ prompt: 'World',
217
+ expected: { type: 'contains', values: ['World'], mode: 'any' },
218
+ tags: [],
219
+ metadata: {},
220
+ retries: 0,
221
+ },
222
+ ],
223
+ tags: [],
224
+ });
225
+
226
+ const result = await kit.validate({ scenario: './duplicate-ids.yaml' });
227
+
228
+ expect(result.valid).toBe(false);
229
+ expect(
230
+ result.errors.some((e) => e.message.includes('duplicate') || e.message.includes('same-id'))
231
+ ).toBe(true);
232
+ });
233
+
234
+ it('should handle parse errors gracefully', async () => {
235
+ setParseError(new Error('YAML parse error: invalid syntax'));
236
+
237
+ const result = await kit.validate({ scenario: './broken.yaml' });
238
+
239
+ expect(result.valid).toBe(false);
240
+ expect(result.errors.length).toBeGreaterThan(0);
241
+ expect(result.errors[0].message).toContain('parse');
242
+ });
243
+ });
244
+
245
+ describe('strict mode', () => {
246
+ it('should pass with warnings in non-strict mode', async () => {
247
+ setParseResponse({
248
+ name: 'Scenario with Warnings',
249
+ version: '1.0',
250
+ model: 'gpt-4',
251
+ cases: [
252
+ {
253
+ id: 'case-1',
254
+ name: 'Test',
255
+ prompt: 'Hello',
256
+ expected: { type: 'contains', values: ['Hello'], mode: 'any' },
257
+ tags: [],
258
+ metadata: {},
259
+ retries: 0,
260
+ },
261
+ ],
262
+ tags: [],
263
+ });
264
+
265
+ const result = await kit.validate({
266
+ scenario: './scenario.yaml',
267
+ strict: false,
268
+ });
269
+
270
+ // If there are warnings but no errors, valid should be true in non-strict mode
271
+ if (result.errors.length === 0) {
272
+ expect(result.valid).toBe(true);
273
+ }
274
+ });
275
+ });
276
+
277
+ describe('progress events', () => {
278
+ it('should emit progress events during validation', async () => {
279
+ setParseResponse({
280
+ name: 'Test Scenario',
281
+ version: '1.0',
282
+ model: 'gpt-4',
283
+ cases: [
284
+ {
285
+ id: 'case-1',
286
+ name: 'Test',
287
+ prompt: 'Hello',
288
+ expected: { type: 'contains', values: ['Hello'], mode: 'any' },
289
+ tags: [],
290
+ metadata: {},
291
+ retries: 0,
292
+ },
293
+ ],
294
+ tags: [],
295
+ });
296
+
297
+ let progressCalled = false;
298
+ kit.onProgress(() => {
299
+ progressCalled = true;
300
+ });
301
+
302
+ await kit.validate({ scenario: './test.yaml' });
303
+
304
+ expect(progressCalled).toBe(true);
305
+ });
306
+ });
307
+ });