@artemiskit/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CHANGELOG.md +213 -0
  2. package/README.md +44 -1
  3. package/dist/artemiskit.d.ts +149 -0
  4. package/dist/artemiskit.d.ts.map +1 -0
  5. package/dist/builders/index.d.ts +350 -0
  6. package/dist/builders/index.d.ts.map +1 -0
  7. package/dist/builders/index.js +398 -0
  8. package/dist/contracts/index.d.ts +253 -0
  9. package/dist/contracts/index.d.ts.map +1 -0
  10. package/dist/contracts/index.js +99 -0
  11. package/dist/guardian/action-validator.d.ts +92 -0
  12. package/dist/guardian/action-validator.d.ts.map +1 -0
  13. package/dist/guardian/circuit-breaker.d.ts +211 -0
  14. package/dist/guardian/circuit-breaker.d.ts.map +1 -0
  15. package/dist/guardian/guardian.d.ts +213 -0
  16. package/dist/guardian/guardian.d.ts.map +1 -0
  17. package/dist/guardian/guardrails.d.ts +107 -0
  18. package/dist/guardian/guardrails.d.ts.map +1 -0
  19. package/dist/guardian/index.d.ts +35 -0
  20. package/dist/guardian/index.d.ts.map +1 -0
  21. package/dist/guardian/intent-classifier.d.ts +81 -0
  22. package/dist/guardian/intent-classifier.d.ts.map +1 -0
  23. package/dist/guardian/interceptor.d.ts +141 -0
  24. package/dist/guardian/interceptor.d.ts.map +1 -0
  25. package/dist/guardian/policy.d.ts +52 -0
  26. package/dist/guardian/policy.d.ts.map +1 -0
  27. package/dist/guardian/semantic-validator.d.ts +61 -0
  28. package/dist/guardian/semantic-validator.d.ts.map +1 -0
  29. package/dist/guardian/types.d.ts +653 -0
  30. package/dist/guardian/types.d.ts.map +1 -0
  31. package/dist/index.d.ts +23 -0
  32. package/dist/index.d.ts.map +1 -0
  33. package/dist/index.js +24847 -13653
  34. package/dist/matchers/core.d.ts +81 -0
  35. package/dist/matchers/core.d.ts.map +1 -0
  36. package/dist/matchers/index.d.ts +6 -0
  37. package/dist/matchers/index.d.ts.map +1 -0
  38. package/dist/matchers/jest.d.ts +46 -0
  39. package/dist/matchers/jest.d.ts.map +1 -0
  40. package/dist/matchers/vitest.d.ts +24 -0
  41. package/dist/matchers/vitest.d.ts.map +1 -0
  42. package/dist/types-only.d.ts +31 -0
  43. package/dist/types-only.d.ts.map +1 -0
  44. package/dist/types-only.js +1 -0
  45. package/dist/types.d.ts +377 -0
  46. package/dist/types.d.ts.map +1 -0
  47. package/dist/utils/index.d.ts +112 -0
  48. package/dist/utils/index.d.ts.map +1 -0
  49. package/dist/utils/index.js +104 -0
  50. package/package.json +17 -2
  51. package/src/__tests__/builders.test.ts +445 -0
  52. package/src/__tests__/circuit-breaker.test.ts +704 -0
  53. package/src/__tests__/compare.test.ts +469 -0
  54. package/src/__tests__/contracts.test.ts +354 -0
  55. package/src/__tests__/guardian-modes.test.ts +215 -0
  56. package/src/__tests__/guardrails.test.ts +404 -0
  57. package/src/__tests__/semantic-validator.test.ts +421 -0
  58. package/src/__tests__/utils.test.ts +324 -0
  59. package/src/__tests__/validate.test.ts +307 -0
  60. package/src/artemiskit.ts +478 -5
  61. package/src/builders/index.ts +698 -0
  62. package/src/contracts/index.ts +357 -0
  63. package/src/guardian/circuit-breaker.ts +74 -28
  64. package/src/guardian/guardian.ts +307 -20
  65. package/src/guardian/guardrails.ts +143 -1
  66. package/src/guardian/index.ts +35 -1
  67. package/src/guardian/interceptor.ts +87 -11
  68. package/src/guardian/semantic-validator.ts +399 -0
  69. package/src/guardian/types.ts +413 -2
  70. package/src/index.ts +258 -14
  71. package/src/types-only.ts +276 -0
  72. package/src/types.ts +183 -0
  73. package/src/utils/index.ts +225 -0
  74. package/tsconfig.json +2 -1
@@ -0,0 +1,469 @@
1
+ /**
2
+ * Tests for ArtemisKit.compare() method (v0.3.2+)
3
+ */
4
+
5
+ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
6
+
7
+ // Mock the core module
8
+ const mockStorageAdapter = {
9
+ save: vi.fn(),
10
+ load: vi.fn(),
11
+ loadRun: vi.fn(),
12
+ list: vi.fn(),
13
+ delete: vi.fn(),
14
+ compare: vi.fn(),
15
+ setBaseline: vi.fn(),
16
+ getBaseline: vi.fn(),
17
+ getBaselineByRunId: vi.fn(),
18
+ listBaselines: vi.fn(),
19
+ removeBaseline: vi.fn(),
20
+ removeBaselineByRunId: vi.fn(),
21
+ compareToBaseline: vi.fn(),
22
+ };
23
+
24
+ vi.mock('@artemiskit/core', () => ({
25
+ parseScenarioFile: vi.fn(),
26
+ createAdapter: vi.fn().mockResolvedValue({
27
+ provider: 'mock',
28
+ generate: vi.fn(),
29
+ capabilities: vi.fn(),
30
+ close: vi.fn(),
31
+ }),
32
+ runScenario: vi.fn(),
33
+ getGitInfo: vi.fn().mockResolvedValue({ commit: 'abc123', branch: 'main', dirty: false }),
34
+ createStorageAdapter: vi.fn().mockReturnValue(mockStorageAdapter),
35
+ LocalStorageAdapter: vi.fn().mockImplementation(() => mockStorageAdapter),
36
+ }));
37
+
38
+ vi.mock('@artemiskit/redteam', () => ({
39
+ TypoMutation: class {
40
+ name = 'typo';
41
+ },
42
+ RoleSpoofMutation: class {
43
+ name = 'role-spoof';
44
+ },
45
+ InstructionFlipMutation: class {
46
+ name = 'instruction-flip';
47
+ },
48
+ CotInjectionMutation: class {
49
+ name = 'cot-injection';
50
+ },
51
+ EncodingMutation: class {
52
+ name = 'encoding';
53
+ },
54
+ MultiTurnMutation: class {
55
+ name = 'multi-turn';
56
+ },
57
+ RedTeamGenerator: class {
58
+ generate() {
59
+ return [];
60
+ }
61
+ listMutations() {
62
+ return [];
63
+ }
64
+ },
65
+ UnsafeResponseDetector: class {
66
+ detect() {
67
+ return { unsafe: false, blocked: false, reasons: [], severity: 'none' };
68
+ }
69
+ },
70
+ }));
71
+
72
+ vi.mock('nanoid', () => ({
73
+ nanoid: () => 'test-id-12345',
74
+ }));
75
+
76
+ import { ArtemisKit } from '../artemiskit';
77
+
78
+ // Sample manifests for testing
79
+ const createMockManifest = (
80
+ runId: string,
81
+ successRate: number,
82
+ cases: { id: string; ok: boolean }[]
83
+ ) => ({
84
+ version: '1.0',
85
+ run_id: runId,
86
+ project: 'test-project',
87
+ start_time: '2026-03-17T10:00:00Z',
88
+ end_time: '2026-03-17T10:01:00Z',
89
+ duration_ms: 60000,
90
+ config: {
91
+ scenario: 'test-scenario',
92
+ provider: 'openai',
93
+ model: 'gpt-4',
94
+ },
95
+ metrics: {
96
+ success_rate: successRate,
97
+ total_cases: cases.length,
98
+ passed_cases: cases.filter((c) => c.ok).length,
99
+ failed_cases: cases.filter((c) => !c.ok).length,
100
+ median_latency_ms: 100,
101
+ p95_latency_ms: 150,
102
+ total_tokens: 1000,
103
+ total_prompt_tokens: 600,
104
+ total_completion_tokens: 400,
105
+ },
106
+ cases: cases.map((c) => ({
107
+ id: c.id,
108
+ name: `Case ${c.id}`,
109
+ ok: c.ok,
110
+ score: c.ok ? 1 : 0,
111
+ matcherType: 'contains',
112
+ reason: c.ok ? 'Passed' : 'Failed',
113
+ latencyMs: 100,
114
+ tokens: { prompt: 10, completion: 5, total: 15 },
115
+ prompt: 'Test prompt',
116
+ response: 'Test response',
117
+ expected: { type: 'contains', values: ['test'], mode: 'any' },
118
+ tags: [],
119
+ })),
120
+ });
121
+
122
+ describe('ArtemisKit.compare()', () => {
123
+ let kit: ArtemisKit;
124
+
125
+ beforeEach(() => {
126
+ vi.clearAllMocks();
127
+ kit = new ArtemisKit({
128
+ project: 'test-project',
129
+ provider: 'openai',
130
+ model: 'gpt-4',
131
+ storage: {
132
+ type: 'local',
133
+ basePath: './test-runs',
134
+ },
135
+ });
136
+ });
137
+
138
+ afterEach(() => {
139
+ vi.clearAllMocks();
140
+ });
141
+
142
+ describe('basic comparison', () => {
143
+ it('should compare two runs successfully', async () => {
144
+ const baselineManifest = createMockManifest('baseline-run', 0.9, [
145
+ { id: 'case-1', ok: true },
146
+ { id: 'case-2', ok: true },
147
+ { id: 'case-3', ok: true },
148
+ { id: 'case-4', ok: false },
149
+ ]);
150
+
151
+ const currentManifest = createMockManifest('current-run', 0.8, [
152
+ { id: 'case-1', ok: true },
153
+ { id: 'case-2', ok: false },
154
+ { id: 'case-3', ok: true },
155
+ { id: 'case-4', ok: true },
156
+ ]);
157
+
158
+ mockStorageAdapter.load
159
+ .mockResolvedValueOnce(baselineManifest)
160
+ .mockResolvedValueOnce(currentManifest);
161
+
162
+ const result = await kit.compare({
163
+ baseline: 'baseline-run',
164
+ current: 'current-run',
165
+ threshold: 0.05,
166
+ });
167
+
168
+ expect(result.baseline.runId).toBe('baseline-run');
169
+ expect(result.current.runId).toBe('current-run');
170
+ expect(result.baseline.successRate).toBeCloseTo(0.9, 5);
171
+ expect(result.current.successRate).toBeCloseTo(0.8, 5);
172
+ expect(result.comparison.successRateDelta).toBeCloseTo(-0.1, 5);
173
+ expect(result.hasRegression).toBe(true);
174
+ });
175
+
176
+ it('should detect no regression when improvement occurs', async () => {
177
+ const baselineManifest = createMockManifest('baseline-run', 0.8, [
178
+ { id: 'case-1', ok: true },
179
+ { id: 'case-2', ok: true },
180
+ { id: 'case-3', ok: false },
181
+ { id: 'case-4', ok: false },
182
+ ]);
183
+
184
+ const currentManifest = createMockManifest('current-run', 0.9, [
185
+ { id: 'case-1', ok: true },
186
+ { id: 'case-2', ok: true },
187
+ { id: 'case-3', ok: true },
188
+ { id: 'case-4', ok: false },
189
+ ]);
190
+
191
+ mockStorageAdapter.load
192
+ .mockResolvedValueOnce(baselineManifest)
193
+ .mockResolvedValueOnce(currentManifest);
194
+
195
+ const result = await kit.compare({
196
+ baseline: 'baseline-run',
197
+ current: 'current-run',
198
+ threshold: 0.05,
199
+ });
200
+
201
+ expect(result.comparison.successRateDelta).toBeCloseTo(0.1, 5);
202
+ expect(result.hasRegression).toBe(false);
203
+ });
204
+
205
+ it('should detect no regression within threshold', async () => {
206
+ const baselineManifest = createMockManifest('baseline-run', 0.9, [
207
+ { id: 'case-1', ok: true },
208
+ { id: 'case-2', ok: true },
209
+ { id: 'case-3', ok: true },
210
+ { id: 'case-4', ok: false },
211
+ ]);
212
+
213
+ const currentManifest = createMockManifest('current-run', 0.87, [
214
+ { id: 'case-1', ok: true },
215
+ { id: 'case-2', ok: true },
216
+ { id: 'case-3', ok: false },
217
+ { id: 'case-4', ok: true },
218
+ ]);
219
+
220
+ mockStorageAdapter.load
221
+ .mockResolvedValueOnce(baselineManifest)
222
+ .mockResolvedValueOnce(currentManifest);
223
+
224
+ const result = await kit.compare({
225
+ baseline: 'baseline-run',
226
+ current: 'current-run',
227
+ threshold: 0.05, // 5% threshold
228
+ });
229
+
230
+ // 3% drop is within 5% threshold
231
+ expect(result.comparison.successRateDelta).toBeCloseTo(-0.03, 1);
232
+ expect(result.hasRegression).toBe(false);
233
+ });
234
+ });
235
+
236
+ describe('case-level comparison', () => {
237
+ it('should identify new failures', async () => {
238
+ const baselineManifest = createMockManifest('baseline-run', 1.0, [
239
+ { id: 'case-1', ok: true },
240
+ { id: 'case-2', ok: true },
241
+ ]);
242
+
243
+ const currentManifest = createMockManifest('current-run', 0.5, [
244
+ { id: 'case-1', ok: true },
245
+ { id: 'case-2', ok: false },
246
+ ]);
247
+
248
+ mockStorageAdapter.load
249
+ .mockResolvedValueOnce(baselineManifest)
250
+ .mockResolvedValueOnce(currentManifest);
251
+
252
+ const result = await kit.compare({
253
+ baseline: 'baseline-run',
254
+ current: 'current-run',
255
+ });
256
+
257
+ expect(result.comparison.newFailures).toHaveLength(1);
258
+ expect(result.comparison.newFailures[0].caseId).toBe('case-2');
259
+ expect(result.comparison.newFailures[0].baselineStatus).toBe('passed');
260
+ expect(result.comparison.newFailures[0].currentStatus).toBe('failed');
261
+ });
262
+
263
+ it('should identify new passes', async () => {
264
+ const baselineManifest = createMockManifest('baseline-run', 0.5, [
265
+ { id: 'case-1', ok: true },
266
+ { id: 'case-2', ok: false },
267
+ ]);
268
+
269
+ const currentManifest = createMockManifest('current-run', 1.0, [
270
+ { id: 'case-1', ok: true },
271
+ { id: 'case-2', ok: true },
272
+ ]);
273
+
274
+ mockStorageAdapter.load
275
+ .mockResolvedValueOnce(baselineManifest)
276
+ .mockResolvedValueOnce(currentManifest);
277
+
278
+ const result = await kit.compare({
279
+ baseline: 'baseline-run',
280
+ current: 'current-run',
281
+ });
282
+
283
+ expect(result.comparison.newPasses).toHaveLength(1);
284
+ expect(result.comparison.newPasses[0].caseId).toBe('case-2');
285
+ expect(result.comparison.newPasses[0].baselineStatus).toBe('failed');
286
+ expect(result.comparison.newPasses[0].currentStatus).toBe('passed');
287
+ });
288
+
289
+ it('should identify unchanged cases', async () => {
290
+ const baselineManifest = createMockManifest('baseline-run', 0.5, [
291
+ { id: 'case-1', ok: true },
292
+ { id: 'case-2', ok: false },
293
+ ]);
294
+
295
+ const currentManifest = createMockManifest('current-run', 0.5, [
296
+ { id: 'case-1', ok: true },
297
+ { id: 'case-2', ok: false },
298
+ ]);
299
+
300
+ mockStorageAdapter.load
301
+ .mockResolvedValueOnce(baselineManifest)
302
+ .mockResolvedValueOnce(currentManifest);
303
+
304
+ const result = await kit.compare({
305
+ baseline: 'baseline-run',
306
+ current: 'current-run',
307
+ });
308
+
309
+ expect(result.comparison.unchanged).toHaveLength(2);
310
+ expect(result.comparison.newFailures).toHaveLength(0);
311
+ expect(result.comparison.newPasses).toHaveLength(0);
312
+ });
313
+ });
314
+
315
+ describe('threshold configuration', () => {
316
+ it('should use default threshold of 0.05', async () => {
317
+ const baselineManifest = createMockManifest('baseline-run', 0.9, [
318
+ { id: 'case-1', ok: true },
319
+ ]);
320
+ // 4% drop (0.86 - 0.9 = -0.04) should be within 5% threshold
321
+ const currentManifest = createMockManifest('current-run', 0.86, [
322
+ { id: 'case-1', ok: false },
323
+ ]);
324
+
325
+ mockStorageAdapter.load
326
+ .mockResolvedValueOnce(baselineManifest)
327
+ .mockResolvedValueOnce(currentManifest);
328
+
329
+ const result = await kit.compare({
330
+ baseline: 'baseline-run',
331
+ current: 'current-run',
332
+ // No threshold specified, should default to 0.05
333
+ });
334
+
335
+ expect(result.threshold).toBe(0.05);
336
+ // 4% drop is within 5% threshold - not a regression
337
+ expect(result.hasRegression).toBe(false);
338
+ });
339
+
340
+ it('should respect custom threshold', async () => {
341
+ const baselineManifest = createMockManifest('baseline-run', 0.9, [
342
+ { id: 'case-1', ok: true },
343
+ ]);
344
+ const currentManifest = createMockManifest('current-run', 0.88, [
345
+ { id: 'case-1', ok: false },
346
+ ]);
347
+
348
+ mockStorageAdapter.load
349
+ .mockResolvedValueOnce(baselineManifest)
350
+ .mockResolvedValueOnce(currentManifest);
351
+
352
+ const result = await kit.compare({
353
+ baseline: 'baseline-run',
354
+ current: 'current-run',
355
+ threshold: 0.01, // 1% threshold - more strict
356
+ });
357
+
358
+ expect(result.threshold).toBe(0.01);
359
+ // 2% drop exceeds 1% threshold
360
+ expect(result.hasRegression).toBe(true);
361
+ });
362
+ });
363
+
364
+ describe('special baseline values', () => {
365
+ it('should handle "latest" as baseline', async () => {
366
+ const latestRun = {
367
+ runId: 'latest-run-123',
368
+ project: 'test-project',
369
+ scenario: 'test-scenario',
370
+ type: 'run' as const,
371
+ startTime: '2026-03-17T10:00:00Z',
372
+ endTime: '2026-03-17T10:01:00Z',
373
+ durationMs: 60000,
374
+ successRate: 0.9,
375
+ totalCases: 4,
376
+ };
377
+
378
+ mockStorageAdapter.list.mockResolvedValue([latestRun]);
379
+
380
+ const baselineManifest = createMockManifest('latest-run-123', 0.9, [
381
+ { id: 'case-1', ok: true },
382
+ ]);
383
+ const currentManifest = createMockManifest('current-run', 0.8, [{ id: 'case-1', ok: false }]);
384
+
385
+ mockStorageAdapter.load
386
+ .mockResolvedValueOnce(baselineManifest)
387
+ .mockResolvedValueOnce(currentManifest);
388
+
389
+ const result = await kit.compare({
390
+ baseline: 'latest',
391
+ current: 'current-run',
392
+ });
393
+
394
+ expect(result.baseline.runId).toBe('latest-run-123');
395
+ expect(mockStorageAdapter.list).toHaveBeenCalled();
396
+ });
397
+ });
398
+
399
+ describe('error handling', () => {
400
+ it('should throw error when baseline run not found', async () => {
401
+ mockStorageAdapter.load.mockRejectedValue(new Error('Run not found: invalid-id'));
402
+
403
+ await expect(
404
+ kit.compare({
405
+ baseline: 'invalid-id',
406
+ current: 'current-run',
407
+ })
408
+ ).rejects.toThrow('Run not found');
409
+ });
410
+
411
+ it('should throw error when current run not found', async () => {
412
+ const baselineManifest = createMockManifest('baseline-run', 0.9, []);
413
+ mockStorageAdapter.load
414
+ .mockResolvedValueOnce(baselineManifest)
415
+ .mockRejectedValueOnce(new Error('Run not found: invalid-id'));
416
+
417
+ await expect(
418
+ kit.compare({
419
+ baseline: 'baseline-run',
420
+ current: 'invalid-id',
421
+ })
422
+ ).rejects.toThrow('Run not found');
423
+ });
424
+
425
+ it('should throw error when storage is not configured', async () => {
426
+ const kitWithoutStorage = new ArtemisKit({
427
+ project: 'test-project',
428
+ provider: 'openai',
429
+ model: 'gpt-4',
430
+ // No storage configuration
431
+ });
432
+
433
+ await expect(
434
+ kitWithoutStorage.compare({
435
+ baseline: 'baseline-run',
436
+ current: 'current-run',
437
+ })
438
+ ).rejects.toThrow(/storage/i);
439
+ });
440
+ });
441
+
442
+ describe('progress events', () => {
443
+ it('should emit progress events during comparison', async () => {
444
+ const baselineManifest = createMockManifest('baseline-run', 0.9, [
445
+ { id: 'case-1', ok: true },
446
+ ]);
447
+ const currentManifest = createMockManifest('current-run', 0.9, [{ id: 'case-1', ok: true }]);
448
+
449
+ mockStorageAdapter.load
450
+ .mockResolvedValueOnce(baselineManifest)
451
+ .mockResolvedValueOnce(currentManifest);
452
+
453
+ const progressHandler = vi.fn();
454
+ kit.onProgress(progressHandler);
455
+
456
+ await kit.compare({
457
+ baseline: 'baseline-run',
458
+ current: 'current-run',
459
+ });
460
+
461
+ expect(progressHandler).toHaveBeenCalled();
462
+ expect(progressHandler).toHaveBeenCalledWith(
463
+ expect.objectContaining({
464
+ phase: expect.any(String),
465
+ })
466
+ );
467
+ });
468
+ });
469
+ });