@artemiskit/sdk 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +213 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +350 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +398 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +99 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24847 -13653
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +377 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +104 -0
- package/package.json +17 -2
- package/src/__tests__/builders.test.ts +445 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +307 -0
- package/src/artemiskit.ts +478 -5
- package/src/builders/index.ts +698 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +183 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for ArtemisKit.compare() method (v0.3.2+)
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
6
|
+
|
|
7
|
+
// Mock the core module
|
|
8
|
+
const mockStorageAdapter = {
|
|
9
|
+
save: vi.fn(),
|
|
10
|
+
load: vi.fn(),
|
|
11
|
+
loadRun: vi.fn(),
|
|
12
|
+
list: vi.fn(),
|
|
13
|
+
delete: vi.fn(),
|
|
14
|
+
compare: vi.fn(),
|
|
15
|
+
setBaseline: vi.fn(),
|
|
16
|
+
getBaseline: vi.fn(),
|
|
17
|
+
getBaselineByRunId: vi.fn(),
|
|
18
|
+
listBaselines: vi.fn(),
|
|
19
|
+
removeBaseline: vi.fn(),
|
|
20
|
+
removeBaselineByRunId: vi.fn(),
|
|
21
|
+
compareToBaseline: vi.fn(),
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
vi.mock('@artemiskit/core', () => ({
|
|
25
|
+
parseScenarioFile: vi.fn(),
|
|
26
|
+
createAdapter: vi.fn().mockResolvedValue({
|
|
27
|
+
provider: 'mock',
|
|
28
|
+
generate: vi.fn(),
|
|
29
|
+
capabilities: vi.fn(),
|
|
30
|
+
close: vi.fn(),
|
|
31
|
+
}),
|
|
32
|
+
runScenario: vi.fn(),
|
|
33
|
+
getGitInfo: vi.fn().mockResolvedValue({ commit: 'abc123', branch: 'main', dirty: false }),
|
|
34
|
+
createStorageAdapter: vi.fn().mockReturnValue(mockStorageAdapter),
|
|
35
|
+
LocalStorageAdapter: vi.fn().mockImplementation(() => mockStorageAdapter),
|
|
36
|
+
}));
|
|
37
|
+
|
|
38
|
+
vi.mock('@artemiskit/redteam', () => ({
|
|
39
|
+
TypoMutation: class {
|
|
40
|
+
name = 'typo';
|
|
41
|
+
},
|
|
42
|
+
RoleSpoofMutation: class {
|
|
43
|
+
name = 'role-spoof';
|
|
44
|
+
},
|
|
45
|
+
InstructionFlipMutation: class {
|
|
46
|
+
name = 'instruction-flip';
|
|
47
|
+
},
|
|
48
|
+
CotInjectionMutation: class {
|
|
49
|
+
name = 'cot-injection';
|
|
50
|
+
},
|
|
51
|
+
EncodingMutation: class {
|
|
52
|
+
name = 'encoding';
|
|
53
|
+
},
|
|
54
|
+
MultiTurnMutation: class {
|
|
55
|
+
name = 'multi-turn';
|
|
56
|
+
},
|
|
57
|
+
RedTeamGenerator: class {
|
|
58
|
+
generate() {
|
|
59
|
+
return [];
|
|
60
|
+
}
|
|
61
|
+
listMutations() {
|
|
62
|
+
return [];
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
UnsafeResponseDetector: class {
|
|
66
|
+
detect() {
|
|
67
|
+
return { unsafe: false, blocked: false, reasons: [], severity: 'none' };
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
}));
|
|
71
|
+
|
|
72
|
+
vi.mock('nanoid', () => ({
|
|
73
|
+
nanoid: () => 'test-id-12345',
|
|
74
|
+
}));
|
|
75
|
+
|
|
76
|
+
import { ArtemisKit } from '../artemiskit';
|
|
77
|
+
|
|
78
|
+
// Sample manifests for testing
|
|
79
|
+
const createMockManifest = (
|
|
80
|
+
runId: string,
|
|
81
|
+
successRate: number,
|
|
82
|
+
cases: { id: string; ok: boolean }[]
|
|
83
|
+
) => ({
|
|
84
|
+
version: '1.0',
|
|
85
|
+
run_id: runId,
|
|
86
|
+
project: 'test-project',
|
|
87
|
+
start_time: '2026-03-17T10:00:00Z',
|
|
88
|
+
end_time: '2026-03-17T10:01:00Z',
|
|
89
|
+
duration_ms: 60000,
|
|
90
|
+
config: {
|
|
91
|
+
scenario: 'test-scenario',
|
|
92
|
+
provider: 'openai',
|
|
93
|
+
model: 'gpt-4',
|
|
94
|
+
},
|
|
95
|
+
metrics: {
|
|
96
|
+
success_rate: successRate,
|
|
97
|
+
total_cases: cases.length,
|
|
98
|
+
passed_cases: cases.filter((c) => c.ok).length,
|
|
99
|
+
failed_cases: cases.filter((c) => !c.ok).length,
|
|
100
|
+
median_latency_ms: 100,
|
|
101
|
+
p95_latency_ms: 150,
|
|
102
|
+
total_tokens: 1000,
|
|
103
|
+
total_prompt_tokens: 600,
|
|
104
|
+
total_completion_tokens: 400,
|
|
105
|
+
},
|
|
106
|
+
cases: cases.map((c) => ({
|
|
107
|
+
id: c.id,
|
|
108
|
+
name: `Case ${c.id}`,
|
|
109
|
+
ok: c.ok,
|
|
110
|
+
score: c.ok ? 1 : 0,
|
|
111
|
+
matcherType: 'contains',
|
|
112
|
+
reason: c.ok ? 'Passed' : 'Failed',
|
|
113
|
+
latencyMs: 100,
|
|
114
|
+
tokens: { prompt: 10, completion: 5, total: 15 },
|
|
115
|
+
prompt: 'Test prompt',
|
|
116
|
+
response: 'Test response',
|
|
117
|
+
expected: { type: 'contains', values: ['test'], mode: 'any' },
|
|
118
|
+
tags: [],
|
|
119
|
+
})),
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
describe('ArtemisKit.compare()', () => {
|
|
123
|
+
let kit: ArtemisKit;
|
|
124
|
+
|
|
125
|
+
beforeEach(() => {
|
|
126
|
+
vi.clearAllMocks();
|
|
127
|
+
kit = new ArtemisKit({
|
|
128
|
+
project: 'test-project',
|
|
129
|
+
provider: 'openai',
|
|
130
|
+
model: 'gpt-4',
|
|
131
|
+
storage: {
|
|
132
|
+
type: 'local',
|
|
133
|
+
basePath: './test-runs',
|
|
134
|
+
},
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
afterEach(() => {
|
|
139
|
+
vi.clearAllMocks();
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
describe('basic comparison', () => {
|
|
143
|
+
it('should compare two runs successfully', async () => {
|
|
144
|
+
const baselineManifest = createMockManifest('baseline-run', 0.9, [
|
|
145
|
+
{ id: 'case-1', ok: true },
|
|
146
|
+
{ id: 'case-2', ok: true },
|
|
147
|
+
{ id: 'case-3', ok: true },
|
|
148
|
+
{ id: 'case-4', ok: false },
|
|
149
|
+
]);
|
|
150
|
+
|
|
151
|
+
const currentManifest = createMockManifest('current-run', 0.8, [
|
|
152
|
+
{ id: 'case-1', ok: true },
|
|
153
|
+
{ id: 'case-2', ok: false },
|
|
154
|
+
{ id: 'case-3', ok: true },
|
|
155
|
+
{ id: 'case-4', ok: true },
|
|
156
|
+
]);
|
|
157
|
+
|
|
158
|
+
mockStorageAdapter.load
|
|
159
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
160
|
+
.mockResolvedValueOnce(currentManifest);
|
|
161
|
+
|
|
162
|
+
const result = await kit.compare({
|
|
163
|
+
baseline: 'baseline-run',
|
|
164
|
+
current: 'current-run',
|
|
165
|
+
threshold: 0.05,
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
expect(result.baseline.runId).toBe('baseline-run');
|
|
169
|
+
expect(result.current.runId).toBe('current-run');
|
|
170
|
+
expect(result.baseline.successRate).toBeCloseTo(0.9, 5);
|
|
171
|
+
expect(result.current.successRate).toBeCloseTo(0.8, 5);
|
|
172
|
+
expect(result.comparison.successRateDelta).toBeCloseTo(-0.1, 5);
|
|
173
|
+
expect(result.hasRegression).toBe(true);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it('should detect no regression when improvement occurs', async () => {
|
|
177
|
+
const baselineManifest = createMockManifest('baseline-run', 0.8, [
|
|
178
|
+
{ id: 'case-1', ok: true },
|
|
179
|
+
{ id: 'case-2', ok: true },
|
|
180
|
+
{ id: 'case-3', ok: false },
|
|
181
|
+
{ id: 'case-4', ok: false },
|
|
182
|
+
]);
|
|
183
|
+
|
|
184
|
+
const currentManifest = createMockManifest('current-run', 0.9, [
|
|
185
|
+
{ id: 'case-1', ok: true },
|
|
186
|
+
{ id: 'case-2', ok: true },
|
|
187
|
+
{ id: 'case-3', ok: true },
|
|
188
|
+
{ id: 'case-4', ok: false },
|
|
189
|
+
]);
|
|
190
|
+
|
|
191
|
+
mockStorageAdapter.load
|
|
192
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
193
|
+
.mockResolvedValueOnce(currentManifest);
|
|
194
|
+
|
|
195
|
+
const result = await kit.compare({
|
|
196
|
+
baseline: 'baseline-run',
|
|
197
|
+
current: 'current-run',
|
|
198
|
+
threshold: 0.05,
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
expect(result.comparison.successRateDelta).toBeCloseTo(0.1, 5);
|
|
202
|
+
expect(result.hasRegression).toBe(false);
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
it('should detect no regression within threshold', async () => {
|
|
206
|
+
const baselineManifest = createMockManifest('baseline-run', 0.9, [
|
|
207
|
+
{ id: 'case-1', ok: true },
|
|
208
|
+
{ id: 'case-2', ok: true },
|
|
209
|
+
{ id: 'case-3', ok: true },
|
|
210
|
+
{ id: 'case-4', ok: false },
|
|
211
|
+
]);
|
|
212
|
+
|
|
213
|
+
const currentManifest = createMockManifest('current-run', 0.87, [
|
|
214
|
+
{ id: 'case-1', ok: true },
|
|
215
|
+
{ id: 'case-2', ok: true },
|
|
216
|
+
{ id: 'case-3', ok: false },
|
|
217
|
+
{ id: 'case-4', ok: true },
|
|
218
|
+
]);
|
|
219
|
+
|
|
220
|
+
mockStorageAdapter.load
|
|
221
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
222
|
+
.mockResolvedValueOnce(currentManifest);
|
|
223
|
+
|
|
224
|
+
const result = await kit.compare({
|
|
225
|
+
baseline: 'baseline-run',
|
|
226
|
+
current: 'current-run',
|
|
227
|
+
threshold: 0.05, // 5% threshold
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
// 3% drop is within 5% threshold
|
|
231
|
+
expect(result.comparison.successRateDelta).toBeCloseTo(-0.03, 1);
|
|
232
|
+
expect(result.hasRegression).toBe(false);
|
|
233
|
+
});
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
describe('case-level comparison', () => {
|
|
237
|
+
it('should identify new failures', async () => {
|
|
238
|
+
const baselineManifest = createMockManifest('baseline-run', 1.0, [
|
|
239
|
+
{ id: 'case-1', ok: true },
|
|
240
|
+
{ id: 'case-2', ok: true },
|
|
241
|
+
]);
|
|
242
|
+
|
|
243
|
+
const currentManifest = createMockManifest('current-run', 0.5, [
|
|
244
|
+
{ id: 'case-1', ok: true },
|
|
245
|
+
{ id: 'case-2', ok: false },
|
|
246
|
+
]);
|
|
247
|
+
|
|
248
|
+
mockStorageAdapter.load
|
|
249
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
250
|
+
.mockResolvedValueOnce(currentManifest);
|
|
251
|
+
|
|
252
|
+
const result = await kit.compare({
|
|
253
|
+
baseline: 'baseline-run',
|
|
254
|
+
current: 'current-run',
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
expect(result.comparison.newFailures).toHaveLength(1);
|
|
258
|
+
expect(result.comparison.newFailures[0].caseId).toBe('case-2');
|
|
259
|
+
expect(result.comparison.newFailures[0].baselineStatus).toBe('passed');
|
|
260
|
+
expect(result.comparison.newFailures[0].currentStatus).toBe('failed');
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
it('should identify new passes', async () => {
|
|
264
|
+
const baselineManifest = createMockManifest('baseline-run', 0.5, [
|
|
265
|
+
{ id: 'case-1', ok: true },
|
|
266
|
+
{ id: 'case-2', ok: false },
|
|
267
|
+
]);
|
|
268
|
+
|
|
269
|
+
const currentManifest = createMockManifest('current-run', 1.0, [
|
|
270
|
+
{ id: 'case-1', ok: true },
|
|
271
|
+
{ id: 'case-2', ok: true },
|
|
272
|
+
]);
|
|
273
|
+
|
|
274
|
+
mockStorageAdapter.load
|
|
275
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
276
|
+
.mockResolvedValueOnce(currentManifest);
|
|
277
|
+
|
|
278
|
+
const result = await kit.compare({
|
|
279
|
+
baseline: 'baseline-run',
|
|
280
|
+
current: 'current-run',
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
expect(result.comparison.newPasses).toHaveLength(1);
|
|
284
|
+
expect(result.comparison.newPasses[0].caseId).toBe('case-2');
|
|
285
|
+
expect(result.comparison.newPasses[0].baselineStatus).toBe('failed');
|
|
286
|
+
expect(result.comparison.newPasses[0].currentStatus).toBe('passed');
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
it('should identify unchanged cases', async () => {
|
|
290
|
+
const baselineManifest = createMockManifest('baseline-run', 0.5, [
|
|
291
|
+
{ id: 'case-1', ok: true },
|
|
292
|
+
{ id: 'case-2', ok: false },
|
|
293
|
+
]);
|
|
294
|
+
|
|
295
|
+
const currentManifest = createMockManifest('current-run', 0.5, [
|
|
296
|
+
{ id: 'case-1', ok: true },
|
|
297
|
+
{ id: 'case-2', ok: false },
|
|
298
|
+
]);
|
|
299
|
+
|
|
300
|
+
mockStorageAdapter.load
|
|
301
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
302
|
+
.mockResolvedValueOnce(currentManifest);
|
|
303
|
+
|
|
304
|
+
const result = await kit.compare({
|
|
305
|
+
baseline: 'baseline-run',
|
|
306
|
+
current: 'current-run',
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
expect(result.comparison.unchanged).toHaveLength(2);
|
|
310
|
+
expect(result.comparison.newFailures).toHaveLength(0);
|
|
311
|
+
expect(result.comparison.newPasses).toHaveLength(0);
|
|
312
|
+
});
|
|
313
|
+
});
|
|
314
|
+
|
|
315
|
+
describe('threshold configuration', () => {
|
|
316
|
+
it('should use default threshold of 0.05', async () => {
|
|
317
|
+
const baselineManifest = createMockManifest('baseline-run', 0.9, [
|
|
318
|
+
{ id: 'case-1', ok: true },
|
|
319
|
+
]);
|
|
320
|
+
// 4% drop (0.86 - 0.9 = -0.04) should be within 5% threshold
|
|
321
|
+
const currentManifest = createMockManifest('current-run', 0.86, [
|
|
322
|
+
{ id: 'case-1', ok: false },
|
|
323
|
+
]);
|
|
324
|
+
|
|
325
|
+
mockStorageAdapter.load
|
|
326
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
327
|
+
.mockResolvedValueOnce(currentManifest);
|
|
328
|
+
|
|
329
|
+
const result = await kit.compare({
|
|
330
|
+
baseline: 'baseline-run',
|
|
331
|
+
current: 'current-run',
|
|
332
|
+
// No threshold specified, should default to 0.05
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
expect(result.threshold).toBe(0.05);
|
|
336
|
+
// 4% drop is within 5% threshold - not a regression
|
|
337
|
+
expect(result.hasRegression).toBe(false);
|
|
338
|
+
});
|
|
339
|
+
|
|
340
|
+
it('should respect custom threshold', async () => {
|
|
341
|
+
const baselineManifest = createMockManifest('baseline-run', 0.9, [
|
|
342
|
+
{ id: 'case-1', ok: true },
|
|
343
|
+
]);
|
|
344
|
+
const currentManifest = createMockManifest('current-run', 0.88, [
|
|
345
|
+
{ id: 'case-1', ok: false },
|
|
346
|
+
]);
|
|
347
|
+
|
|
348
|
+
mockStorageAdapter.load
|
|
349
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
350
|
+
.mockResolvedValueOnce(currentManifest);
|
|
351
|
+
|
|
352
|
+
const result = await kit.compare({
|
|
353
|
+
baseline: 'baseline-run',
|
|
354
|
+
current: 'current-run',
|
|
355
|
+
threshold: 0.01, // 1% threshold - more strict
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
expect(result.threshold).toBe(0.01);
|
|
359
|
+
// 2% drop exceeds 1% threshold
|
|
360
|
+
expect(result.hasRegression).toBe(true);
|
|
361
|
+
});
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
describe('special baseline values', () => {
|
|
365
|
+
it('should handle "latest" as baseline', async () => {
|
|
366
|
+
const latestRun = {
|
|
367
|
+
runId: 'latest-run-123',
|
|
368
|
+
project: 'test-project',
|
|
369
|
+
scenario: 'test-scenario',
|
|
370
|
+
type: 'run' as const,
|
|
371
|
+
startTime: '2026-03-17T10:00:00Z',
|
|
372
|
+
endTime: '2026-03-17T10:01:00Z',
|
|
373
|
+
durationMs: 60000,
|
|
374
|
+
successRate: 0.9,
|
|
375
|
+
totalCases: 4,
|
|
376
|
+
};
|
|
377
|
+
|
|
378
|
+
mockStorageAdapter.list.mockResolvedValue([latestRun]);
|
|
379
|
+
|
|
380
|
+
const baselineManifest = createMockManifest('latest-run-123', 0.9, [
|
|
381
|
+
{ id: 'case-1', ok: true },
|
|
382
|
+
]);
|
|
383
|
+
const currentManifest = createMockManifest('current-run', 0.8, [{ id: 'case-1', ok: false }]);
|
|
384
|
+
|
|
385
|
+
mockStorageAdapter.load
|
|
386
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
387
|
+
.mockResolvedValueOnce(currentManifest);
|
|
388
|
+
|
|
389
|
+
const result = await kit.compare({
|
|
390
|
+
baseline: 'latest',
|
|
391
|
+
current: 'current-run',
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
expect(result.baseline.runId).toBe('latest-run-123');
|
|
395
|
+
expect(mockStorageAdapter.list).toHaveBeenCalled();
|
|
396
|
+
});
|
|
397
|
+
});
|
|
398
|
+
|
|
399
|
+
describe('error handling', () => {
|
|
400
|
+
it('should throw error when baseline run not found', async () => {
|
|
401
|
+
mockStorageAdapter.load.mockRejectedValue(new Error('Run not found: invalid-id'));
|
|
402
|
+
|
|
403
|
+
await expect(
|
|
404
|
+
kit.compare({
|
|
405
|
+
baseline: 'invalid-id',
|
|
406
|
+
current: 'current-run',
|
|
407
|
+
})
|
|
408
|
+
).rejects.toThrow('Run not found');
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
it('should throw error when current run not found', async () => {
|
|
412
|
+
const baselineManifest = createMockManifest('baseline-run', 0.9, []);
|
|
413
|
+
mockStorageAdapter.load
|
|
414
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
415
|
+
.mockRejectedValueOnce(new Error('Run not found: invalid-id'));
|
|
416
|
+
|
|
417
|
+
await expect(
|
|
418
|
+
kit.compare({
|
|
419
|
+
baseline: 'baseline-run',
|
|
420
|
+
current: 'invalid-id',
|
|
421
|
+
})
|
|
422
|
+
).rejects.toThrow('Run not found');
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
it('should throw error when storage is not configured', async () => {
|
|
426
|
+
const kitWithoutStorage = new ArtemisKit({
|
|
427
|
+
project: 'test-project',
|
|
428
|
+
provider: 'openai',
|
|
429
|
+
model: 'gpt-4',
|
|
430
|
+
// No storage configuration
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
await expect(
|
|
434
|
+
kitWithoutStorage.compare({
|
|
435
|
+
baseline: 'baseline-run',
|
|
436
|
+
current: 'current-run',
|
|
437
|
+
})
|
|
438
|
+
).rejects.toThrow(/storage/i);
|
|
439
|
+
});
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
describe('progress events', () => {
|
|
443
|
+
it('should emit progress events during comparison', async () => {
|
|
444
|
+
const baselineManifest = createMockManifest('baseline-run', 0.9, [
|
|
445
|
+
{ id: 'case-1', ok: true },
|
|
446
|
+
]);
|
|
447
|
+
const currentManifest = createMockManifest('current-run', 0.9, [{ id: 'case-1', ok: true }]);
|
|
448
|
+
|
|
449
|
+
mockStorageAdapter.load
|
|
450
|
+
.mockResolvedValueOnce(baselineManifest)
|
|
451
|
+
.mockResolvedValueOnce(currentManifest);
|
|
452
|
+
|
|
453
|
+
const progressHandler = vi.fn();
|
|
454
|
+
kit.onProgress(progressHandler);
|
|
455
|
+
|
|
456
|
+
await kit.compare({
|
|
457
|
+
baseline: 'baseline-run',
|
|
458
|
+
current: 'current-run',
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
expect(progressHandler).toHaveBeenCalled();
|
|
462
|
+
expect(progressHandler).toHaveBeenCalledWith(
|
|
463
|
+
expect.objectContaining({
|
|
464
|
+
phase: expect.any(String),
|
|
465
|
+
})
|
|
466
|
+
);
|
|
467
|
+
});
|
|
468
|
+
});
|
|
469
|
+
});
|