@artemiskit/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CHANGELOG.md +213 -0
  2. package/README.md +44 -1
  3. package/dist/artemiskit.d.ts +149 -0
  4. package/dist/artemiskit.d.ts.map +1 -0
  5. package/dist/builders/index.d.ts +350 -0
  6. package/dist/builders/index.d.ts.map +1 -0
  7. package/dist/builders/index.js +398 -0
  8. package/dist/contracts/index.d.ts +253 -0
  9. package/dist/contracts/index.d.ts.map +1 -0
  10. package/dist/contracts/index.js +99 -0
  11. package/dist/guardian/action-validator.d.ts +92 -0
  12. package/dist/guardian/action-validator.d.ts.map +1 -0
  13. package/dist/guardian/circuit-breaker.d.ts +211 -0
  14. package/dist/guardian/circuit-breaker.d.ts.map +1 -0
  15. package/dist/guardian/guardian.d.ts +213 -0
  16. package/dist/guardian/guardian.d.ts.map +1 -0
  17. package/dist/guardian/guardrails.d.ts +107 -0
  18. package/dist/guardian/guardrails.d.ts.map +1 -0
  19. package/dist/guardian/index.d.ts +35 -0
  20. package/dist/guardian/index.d.ts.map +1 -0
  21. package/dist/guardian/intent-classifier.d.ts +81 -0
  22. package/dist/guardian/intent-classifier.d.ts.map +1 -0
  23. package/dist/guardian/interceptor.d.ts +141 -0
  24. package/dist/guardian/interceptor.d.ts.map +1 -0
  25. package/dist/guardian/policy.d.ts +52 -0
  26. package/dist/guardian/policy.d.ts.map +1 -0
  27. package/dist/guardian/semantic-validator.d.ts +61 -0
  28. package/dist/guardian/semantic-validator.d.ts.map +1 -0
  29. package/dist/guardian/types.d.ts +653 -0
  30. package/dist/guardian/types.d.ts.map +1 -0
  31. package/dist/index.d.ts +23 -0
  32. package/dist/index.d.ts.map +1 -0
  33. package/dist/index.js +24847 -13653
  34. package/dist/matchers/core.d.ts +81 -0
  35. package/dist/matchers/core.d.ts.map +1 -0
  36. package/dist/matchers/index.d.ts +6 -0
  37. package/dist/matchers/index.d.ts.map +1 -0
  38. package/dist/matchers/jest.d.ts +46 -0
  39. package/dist/matchers/jest.d.ts.map +1 -0
  40. package/dist/matchers/vitest.d.ts +24 -0
  41. package/dist/matchers/vitest.d.ts.map +1 -0
  42. package/dist/types-only.d.ts +31 -0
  43. package/dist/types-only.d.ts.map +1 -0
  44. package/dist/types-only.js +1 -0
  45. package/dist/types.d.ts +377 -0
  46. package/dist/types.d.ts.map +1 -0
  47. package/dist/utils/index.d.ts +112 -0
  48. package/dist/utils/index.d.ts.map +1 -0
  49. package/dist/utils/index.js +104 -0
  50. package/package.json +17 -2
  51. package/src/__tests__/builders.test.ts +445 -0
  52. package/src/__tests__/circuit-breaker.test.ts +704 -0
  53. package/src/__tests__/compare.test.ts +469 -0
  54. package/src/__tests__/contracts.test.ts +354 -0
  55. package/src/__tests__/guardian-modes.test.ts +215 -0
  56. package/src/__tests__/guardrails.test.ts +404 -0
  57. package/src/__tests__/semantic-validator.test.ts +421 -0
  58. package/src/__tests__/utils.test.ts +324 -0
  59. package/src/__tests__/validate.test.ts +307 -0
  60. package/src/artemiskit.ts +478 -5
  61. package/src/builders/index.ts +698 -0
  62. package/src/contracts/index.ts +357 -0
  63. package/src/guardian/circuit-breaker.ts +74 -28
  64. package/src/guardian/guardian.ts +307 -20
  65. package/src/guardian/guardrails.ts +143 -1
  66. package/src/guardian/index.ts +35 -1
  67. package/src/guardian/interceptor.ts +87 -11
  68. package/src/guardian/semantic-validator.ts +399 -0
  69. package/src/guardian/types.ts +413 -2
  70. package/src/index.ts +258 -14
  71. package/src/types-only.ts +276 -0
  72. package/src/types.ts +183 -0
  73. package/src/utils/index.ts +225 -0
  74. package/tsconfig.json +2 -1
@@ -0,0 +1,445 @@
1
+ /**
2
+ * Tests for SDK builders - scenario and test case construction
3
+ */
4
+
5
+ import { describe, expect, it } from 'bun:test';
6
+ import {
7
+ ScenarioBuilder,
8
+ TestCaseBuilder,
9
+ allOf,
10
+ anyOf,
11
+ contains,
12
+ containsCase,
13
+ exact,
14
+ exactCase,
15
+ fuzzy,
16
+ gradedCase,
17
+ inline,
18
+ jsonCase,
19
+ jsonSchema,
20
+ llmGrade,
21
+ notContains,
22
+ regex,
23
+ regexCase,
24
+ scenario,
25
+ similarity,
26
+ testCase,
27
+ } from '../builders';
28
+
29
+ describe('Builders - TestCaseBuilder', () => {
30
+ describe('Basic construction', () => {
31
+ it('should create a test case with contains expectation', () => {
32
+ const tc = testCase('test-1').prompt('What is 2 + 2?').expectContains(['4']).build();
33
+
34
+ expect(tc.id).toBe('test-1');
35
+ expect(tc.prompt).toBe('What is 2 + 2?');
36
+ expect(tc.expected.type).toBe('contains');
37
+ });
38
+
39
+ it('should create a test case with exact expectation', () => {
40
+ const tc = testCase('exact-1').prompt('Say hello').expectExact('Hello!').build();
41
+
42
+ expect(tc.expected.type).toBe('exact');
43
+ expect((tc.expected as any).value).toBe('Hello!');
44
+ });
45
+
46
+ it('should create a test case with regex expectation', () => {
47
+ const tc = testCase('regex-1').prompt('Give me a number').expectRegex('\\d+').build();
48
+
49
+ expect(tc.expected.type).toBe('regex');
50
+ expect((tc.expected as any).pattern).toBe('\\d+');
51
+ });
52
+
53
+ it('should set optional properties', () => {
54
+ const tc = testCase('full-1')
55
+ .name('Full Test Case')
56
+ .description('A comprehensive test case')
57
+ .prompt('Test prompt')
58
+ .expectContains(['test'])
59
+ .tags('smoke', 'critical')
60
+ .metadata({ priority: 1 })
61
+ .timeout(5000)
62
+ .retries(3)
63
+ .provider('anthropic')
64
+ .model('claude-3-opus')
65
+ .build();
66
+
67
+ expect(tc.name).toBe('Full Test Case');
68
+ expect(tc.description).toBe('A comprehensive test case');
69
+ expect(tc.tags).toContain('smoke');
70
+ expect(tc.tags).toContain('critical');
71
+ expect(tc.metadata).toEqual({ priority: 1 });
72
+ expect(tc.timeout).toBe(5000);
73
+ expect(tc.retries).toBe(3);
74
+ expect(tc.provider).toBe('anthropic');
75
+ expect(tc.model).toBe('claude-3-opus');
76
+ });
77
+ });
78
+
79
+ describe('Prompt types', () => {
80
+ it('should accept string prompts', () => {
81
+ const tc = testCase('string-prompt').prompt('Simple prompt').expectContains(['test']).build();
82
+
83
+ expect(typeof tc.prompt).toBe('string');
84
+ });
85
+
86
+ it('should accept chat message prompts', () => {
87
+ const tc = testCase('chat-prompt')
88
+ .messages([
89
+ { role: 'system', content: 'You are helpful' },
90
+ { role: 'user', content: 'Hello' },
91
+ ])
92
+ .expectContains(['test'])
93
+ .build();
94
+
95
+ expect(Array.isArray(tc.prompt)).toBe(true);
96
+ expect(tc.prompt as any[]).toHaveLength(2);
97
+ });
98
+
99
+ it('should accept system and user shorthand', () => {
100
+ const tc = testCase('system-user')
101
+ .systemAndUser('Be helpful', 'What is AI?')
102
+ .expectContains(['test'])
103
+ .build();
104
+
105
+ expect(Array.isArray(tc.prompt)).toBe(true);
106
+ const messages = tc.prompt as any[];
107
+ expect(messages[0].role).toBe('system');
108
+ expect(messages[1].role).toBe('user');
109
+ });
110
+ });
111
+
112
+ describe('Expectation types', () => {
113
+ it('should create fuzzy expectation', () => {
114
+ const tc = testCase('fuzzy-1').prompt('Test').expectFuzzy('expected value', 0.9).build();
115
+
116
+ expect(tc.expected.type).toBe('fuzzy');
117
+ expect((tc.expected as any).threshold).toBe(0.9);
118
+ });
119
+
120
+ it('should create JSON schema expectation', () => {
121
+ const schema = {
122
+ type: 'object',
123
+ required: ['name'],
124
+ properties: { name: { type: 'string' } },
125
+ };
126
+ const tc = testCase('json-1').prompt('Test').expectJsonSchema(schema).build();
127
+
128
+ expect(tc.expected.type).toBe('json_schema');
129
+ expect((tc.expected as any).schema).toEqual(schema);
130
+ });
131
+
132
+ it('should create LLM grader expectation', () => {
133
+ const tc = testCase('graded-1')
134
+ .prompt('Test')
135
+ .expectLLMGrade('Response should be helpful and accurate', {
136
+ threshold: 0.8,
137
+ model: 'gpt-4',
138
+ })
139
+ .build();
140
+
141
+ expect(tc.expected.type).toBe('llm_grader');
142
+ expect((tc.expected as any).rubric).toBe('Response should be helpful and accurate');
143
+ expect((tc.expected as any).threshold).toBe(0.8);
144
+ });
145
+
146
+ it('should create similarity expectation', () => {
147
+ const tc = testCase('sim-1')
148
+ .prompt('Test')
149
+ .expectSimilarity('expected text', { threshold: 0.85, mode: 'embedding' })
150
+ .build();
151
+
152
+ expect(tc.expected.type).toBe('similarity');
153
+ expect((tc.expected as any).mode).toBe('embedding');
154
+ });
155
+
156
+ it('should create inline expectation', () => {
157
+ const tc = testCase('inline-1').prompt('Test').expectInline('response.length > 10').build();
158
+
159
+ expect(tc.expected.type).toBe('inline');
160
+ expect((tc.expected as any).expression).toBe('response.length > 10');
161
+ });
162
+
163
+ it('should create combined AND expectation', () => {
164
+ const tc = testCase('combined-and')
165
+ .prompt('Test')
166
+ .expectAll(contains(['hello']), notContains(['error']))
167
+ .build();
168
+
169
+ expect(tc.expected.type).toBe('combined');
170
+ expect((tc.expected as any).operator).toBe('and');
171
+ expect((tc.expected as any).expectations).toHaveLength(2);
172
+ });
173
+
174
+ it('should create combined OR expectation', () => {
175
+ const tc = testCase('combined-or')
176
+ .prompt('Test')
177
+ .expectAny(exact('yes'), exact('no'))
178
+ .build();
179
+
180
+ expect(tc.expected.type).toBe('combined');
181
+ expect((tc.expected as any).operator).toBe('or');
182
+ });
183
+
184
+ it('should allow setting raw expected', () => {
185
+ const customExpected = { type: 'custom' as const, evaluator: 'my-evaluator' };
186
+ const tc = testCase('custom-1')
187
+ .prompt('Test')
188
+ .expect(customExpected as any)
189
+ .build();
190
+
191
+ expect(tc.expected).toEqual(customExpected);
192
+ });
193
+ });
194
+
195
+ describe('Validation', () => {
196
+ it('should throw if no expectation is set', () => {
197
+ expect(() => testCase('no-expect').prompt('Test').build()).toThrow(
198
+ 'must have an expectation'
199
+ );
200
+ });
201
+
202
+ it('should throw if no prompt is set', () => {
203
+ expect(() => testCase('no-prompt').expectContains(['test']).build()).toThrow(
204
+ 'must have a prompt'
205
+ );
206
+ });
207
+ });
208
+ });
209
+
210
+ describe('Builders - ScenarioBuilder', () => {
211
+ describe('Basic construction', () => {
212
+ it('should create a minimal scenario', () => {
213
+ const s = scenario('Basic Test')
214
+ .case(testCase('case-1').prompt('Test').expectContains(['pass']))
215
+ .build();
216
+
217
+ expect(s.name).toBe('Basic Test');
218
+ expect(s.cases).toHaveLength(1);
219
+ });
220
+
221
+ it('should set all optional properties', () => {
222
+ const s = scenario('Full Scenario')
223
+ .description('A comprehensive test scenario')
224
+ .version('2.0')
225
+ .provider('openai')
226
+ .model('gpt-4')
227
+ .seed(42)
228
+ .temperature(0.7)
229
+ .maxTokens(1000)
230
+ .tags('integration', 'api')
231
+ .variables({ topic: 'AI' })
232
+ .systemPrompt('You are a helpful assistant')
233
+ .case(testCase('case-1').prompt('Test').expectContains(['pass']))
234
+ .build();
235
+
236
+ expect(s.description).toBe('A comprehensive test scenario');
237
+ expect(s.version).toBe('2.0');
238
+ expect(s.provider).toBe('openai');
239
+ expect(s.model).toBe('gpt-4');
240
+ expect(s.seed).toBe(42);
241
+ expect(s.temperature).toBe(0.7);
242
+ expect(s.maxTokens).toBe(1000);
243
+ expect(s.tags).toContain('integration');
244
+ expect(s.variables).toEqual({ topic: 'AI' });
245
+ expect(s.setup?.systemPrompt).toBe('You are a helpful assistant');
246
+ });
247
+ });
248
+
249
+ describe('Adding cases', () => {
250
+ it('should add multiple cases with case()', () => {
251
+ const s = scenario('Multi-case')
252
+ .case(testCase('case-1').prompt('Test 1').expectContains(['1']))
253
+ .case(testCase('case-2').prompt('Test 2').expectContains(['2']))
254
+ .case(testCase('case-3').prompt('Test 3').expectContains(['3']))
255
+ .build();
256
+
257
+ expect(s.cases).toHaveLength(3);
258
+ });
259
+
260
+ it('should add multiple cases with cases()', () => {
261
+ const s = scenario('Batch cases')
262
+ .cases(
263
+ testCase('case-1').prompt('Test 1').expectContains(['1']),
264
+ testCase('case-2').prompt('Test 2').expectContains(['2'])
265
+ )
266
+ .build();
267
+
268
+ expect(s.cases).toHaveLength(2);
269
+ });
270
+
271
+ it('should add raw test case objects', () => {
272
+ const rawCase = {
273
+ id: 'raw-case',
274
+ prompt: 'Raw prompt',
275
+ expected: { type: 'contains' as const, values: ['test'], mode: 'all' as const },
276
+ tags: [],
277
+ metadata: {},
278
+ retries: 0,
279
+ };
280
+
281
+ const s = scenario('Raw case').addCase(rawCase).build();
282
+
283
+ expect(s.cases).toHaveLength(1);
284
+ expect(s.cases[0].id).toBe('raw-case');
285
+ });
286
+ });
287
+
288
+ describe('Provider configuration', () => {
289
+ it('should set provider config', () => {
290
+ const s = scenario('Configured')
291
+ .provider('azure-openai')
292
+ .providerConfig({
293
+ resourceName: 'my-resource',
294
+ deploymentName: 'gpt-4',
295
+ })
296
+ .case(testCase('case-1').prompt('Test').expectContains(['pass']))
297
+ .build();
298
+
299
+ expect(s.providerConfig).toEqual({
300
+ resourceName: 'my-resource',
301
+ deploymentName: 'gpt-4',
302
+ });
303
+ });
304
+ });
305
+
306
+ describe('Validation', () => {
307
+ it('should throw if no cases are added', () => {
308
+ expect(() => scenario('Empty').build()).toThrow('must have at least one test case');
309
+ });
310
+ });
311
+ });
312
+
313
+ describe('Builders - Quick Helpers', () => {
314
+ it('containsCase should create a case with contains expectation', () => {
315
+ const tc = containsCase('quick-1', 'Prompt', ['value1', 'value2']);
316
+
317
+ expect(tc.id).toBe('quick-1');
318
+ expect(tc.expected.type).toBe('contains');
319
+ });
320
+
321
+ it('exactCase should create a case with exact expectation', () => {
322
+ const tc = exactCase('quick-2', 'Prompt', 'exact value');
323
+
324
+ expect(tc.expected.type).toBe('exact');
325
+ expect((tc.expected as any).value).toBe('exact value');
326
+ });
327
+
328
+ it('regexCase should create a case with regex expectation', () => {
329
+ const tc = regexCase('quick-3', 'Prompt', '\\d+', 'g');
330
+
331
+ expect(tc.expected.type).toBe('regex');
332
+ expect((tc.expected as any).flags).toBe('g');
333
+ });
334
+
335
+ it('jsonCase should create a case with json_schema expectation', () => {
336
+ const schema = { type: 'object' };
337
+ const tc = jsonCase('quick-4', 'Prompt', schema);
338
+
339
+ expect(tc.expected.type).toBe('json_schema');
340
+ });
341
+
342
+ it('gradedCase should create a case with llm_grader expectation', () => {
343
+ const tc = gradedCase('quick-5', 'Prompt', 'Be helpful', 0.9);
344
+
345
+ expect(tc.expected.type).toBe('llm_grader');
346
+ expect((tc.expected as any).threshold).toBe(0.9);
347
+ });
348
+ });
349
+
350
+ describe('Builders - Expectation Helpers', () => {
351
+ it('exact() should create exact expectation', () => {
352
+ const exp = exact('value', false);
353
+ expect(exp).toEqual({ type: 'exact', value: 'value', caseSensitive: false });
354
+ });
355
+
356
+ it('contains() should create contains expectation', () => {
357
+ const exp = contains(['a', 'b'], 'any');
358
+ expect(exp).toEqual({ type: 'contains', values: ['a', 'b'], mode: 'any' });
359
+ });
360
+
361
+ it('notContains() should create not_contains expectation', () => {
362
+ const exp = notContains(['error']);
363
+ expect(exp).toEqual({ type: 'not_contains', values: ['error'], mode: 'all' });
364
+ });
365
+
366
+ it('regex() should create regex expectation', () => {
367
+ const exp = regex('\\w+', 'gi');
368
+ expect(exp).toEqual({ type: 'regex', pattern: '\\w+', flags: 'gi' });
369
+ });
370
+
371
+ it('fuzzy() should create fuzzy expectation', () => {
372
+ const exp = fuzzy('value', 0.9);
373
+ expect(exp).toEqual({ type: 'fuzzy', value: 'value', threshold: 0.9 });
374
+ });
375
+
376
+ it('jsonSchema() should create json_schema expectation', () => {
377
+ const schema = { type: 'object' };
378
+ const exp = jsonSchema(schema);
379
+ expect(exp).toEqual({ type: 'json_schema', schema });
380
+ });
381
+
382
+ it('llmGrade() should create llm_grader expectation', () => {
383
+ const exp = llmGrade('Be accurate', { threshold: 0.8 });
384
+ expect(exp.type).toBe('llm_grader');
385
+ expect((exp as any).rubric).toBe('Be accurate');
386
+ });
387
+
388
+ it('similarity() should create similarity expectation', () => {
389
+ const exp = similarity('expected', { mode: 'llm' });
390
+ expect(exp.type).toBe('similarity');
391
+ expect((exp as any).mode).toBe('llm');
392
+ });
393
+
394
+ it('inline() should create inline expectation', () => {
395
+ const exp = inline('response.includes("test")');
396
+ expect(exp).toEqual({
397
+ type: 'inline',
398
+ expression: 'response.includes("test")',
399
+ value: undefined,
400
+ });
401
+ });
402
+
403
+ it('allOf() should create combined AND expectation', () => {
404
+ const exp = allOf(contains(['a']), contains(['b']));
405
+ expect(exp.type).toBe('combined');
406
+ expect((exp as any).operator).toBe('and');
407
+ });
408
+
409
+ it('anyOf() should create combined OR expectation', () => {
410
+ const exp = anyOf(exact('yes'), exact('no'));
411
+ expect(exp.type).toBe('combined');
412
+ expect((exp as any).operator).toBe('or');
413
+ });
414
+ });
415
+
416
+ describe('Builders - Integration', () => {
417
+ it('should build a complete scenario programmatically', () => {
418
+ const mathScenario = scenario('Math Tests')
419
+ .description('Test basic math operations')
420
+ .provider('openai')
421
+ .model('gpt-4')
422
+ .temperature(0)
423
+ .tags('math', 'smoke')
424
+ .case(testCase('addition').prompt('What is 2 + 2?').expectContains(['4']).tags('basic'))
425
+ .case(
426
+ testCase('multiplication')
427
+ .prompt('What is 6 * 7?')
428
+ .expectAll(contains(['42']), notContains(['error']))
429
+ .tags('basic')
430
+ )
431
+ .case(
432
+ testCase('complex')
433
+ .prompt('Solve: (3 + 5) * 2 - 4')
434
+ .expectLLMGrade('The answer should be 12 and the solution should show work')
435
+ .tags('advanced')
436
+ )
437
+ .build();
438
+
439
+ expect(mathScenario.name).toBe('Math Tests');
440
+ expect(mathScenario.cases).toHaveLength(3);
441
+ expect(mathScenario.tags).toContain('math');
442
+ expect(mathScenario.cases[0].tags).toContain('basic');
443
+ expect(mathScenario.cases[2].expected.type).toBe('llm_grader');
444
+ });
445
+ });