@artemiskit/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +229 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +352 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +354 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +53 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +44918 -28644
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/index.js +13 -59
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/jest.js +0 -46
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/matchers/vitest.js +0 -46
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +379 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +58 -0
- package/package.json +19 -4
- package/src/__tests__/builders.test.ts +450 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +310 -0
- package/src/artemiskit.ts +479 -5
- package/src/builders/index.ts +700 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +186 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
- package/adapters/openai/dist/index.js +0 -5625
|
@@ -0,0 +1,700 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Fluent builders for constructing scenarios and test cases
|
|
4
|
+
*
|
|
5
|
+
* Provides a type-safe, fluent API for building ArtemisKit scenarios
|
|
6
|
+
* programmatically without manually constructing complex objects.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type {
|
|
10
|
+
ChatMessageType,
|
|
11
|
+
Expected,
|
|
12
|
+
Provider,
|
|
13
|
+
Scenario,
|
|
14
|
+
TestCase,
|
|
15
|
+
Variables,
|
|
16
|
+
} from '@artemiskit/core';
|
|
17
|
+
import type { RedactionConfig } from '@artemiskit/core';
|
|
18
|
+
|
|
19
|
+
// ============================================================================
|
|
20
|
+
// Test Case Builder
|
|
21
|
+
// ============================================================================
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Fluent builder for constructing test cases
|
|
25
|
+
*/
|
|
26
|
+
export class TestCaseBuilder {
|
|
27
|
+
private _id: string;
|
|
28
|
+
private _name?: string;
|
|
29
|
+
private _description?: string;
|
|
30
|
+
private _prompt: string | ChatMessageType[] = '';
|
|
31
|
+
private _expected?: Expected;
|
|
32
|
+
private _tags: string[] = [];
|
|
33
|
+
private _metadata: Record<string, unknown> = {};
|
|
34
|
+
private _timeout?: number;
|
|
35
|
+
private _retries = 0;
|
|
36
|
+
private _provider?: Provider;
|
|
37
|
+
private _model?: string;
|
|
38
|
+
private _variables?: Variables;
|
|
39
|
+
private _redaction?: RedactionConfig;
|
|
40
|
+
|
|
41
|
+
constructor(id: string) {
|
|
42
|
+
this._id = id;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Set the test case name (display name)
|
|
47
|
+
*/
|
|
48
|
+
name(name: string): this {
|
|
49
|
+
this._name = name;
|
|
50
|
+
return this;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Set the test case description
|
|
55
|
+
*/
|
|
56
|
+
description(description: string): this {
|
|
57
|
+
this._description = description;
|
|
58
|
+
return this;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Set the prompt as a simple string
|
|
63
|
+
*/
|
|
64
|
+
prompt(prompt: string): this {
|
|
65
|
+
this._prompt = prompt;
|
|
66
|
+
return this;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Set the prompt as chat messages
|
|
71
|
+
*/
|
|
72
|
+
messages(messages: ChatMessageType[]): this {
|
|
73
|
+
this._prompt = messages;
|
|
74
|
+
return this;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Add a system message followed by a user message
|
|
79
|
+
*/
|
|
80
|
+
systemAndUser(systemPrompt: string, userPrompt: string): this {
|
|
81
|
+
this._prompt = [
|
|
82
|
+
{ role: 'system', content: systemPrompt },
|
|
83
|
+
{ role: 'user', content: userPrompt },
|
|
84
|
+
];
|
|
85
|
+
return this;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Expect an exact match
|
|
90
|
+
*/
|
|
91
|
+
expectExact(value: string, caseSensitive = true): this {
|
|
92
|
+
this._expected = { type: 'exact', value, caseSensitive };
|
|
93
|
+
return this;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Expect the response to contain specific values
|
|
98
|
+
*/
|
|
99
|
+
expectContains(values: string[], mode: 'all' | 'any' = 'all'): this {
|
|
100
|
+
this._expected = { type: 'contains', values, mode };
|
|
101
|
+
return this;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Expect the response to NOT contain specific values
|
|
106
|
+
*/
|
|
107
|
+
expectNotContains(values: string[], mode: 'all' | 'any' = 'all'): this {
|
|
108
|
+
this._expected = { type: 'not_contains', values, mode };
|
|
109
|
+
return this;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Expect a regex match
|
|
114
|
+
*/
|
|
115
|
+
expectRegex(pattern: string, flags?: string): this {
|
|
116
|
+
this._expected = { type: 'regex', pattern, flags };
|
|
117
|
+
return this;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Expect a fuzzy match with similarity threshold
|
|
122
|
+
*/
|
|
123
|
+
expectFuzzy(value: string, threshold = 0.8): this {
|
|
124
|
+
this._expected = { type: 'fuzzy', value, threshold };
|
|
125
|
+
return this;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Expect a JSON response matching a schema
|
|
130
|
+
*/
|
|
131
|
+
expectJsonSchema(schema: Record<string, unknown>): this {
|
|
132
|
+
this._expected = { type: 'json_schema', schema };
|
|
133
|
+
return this;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Expect response to pass LLM grading with a rubric
|
|
138
|
+
*/
|
|
139
|
+
expectLLMGrade(
|
|
140
|
+
rubric: string,
|
|
141
|
+
options?: { threshold?: number; model?: string; provider?: Provider; strict?: boolean }
|
|
142
|
+
): this {
|
|
143
|
+
this._expected = {
|
|
144
|
+
type: 'llm_grader',
|
|
145
|
+
rubric,
|
|
146
|
+
threshold: options?.threshold ?? 0.7,
|
|
147
|
+
model: options?.model,
|
|
148
|
+
provider: options?.provider,
|
|
149
|
+
strict: options?.strict ?? false,
|
|
150
|
+
};
|
|
151
|
+
return this;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Expect semantic similarity
|
|
156
|
+
*/
|
|
157
|
+
expectSimilarity(
|
|
158
|
+
value: string,
|
|
159
|
+
options?: {
|
|
160
|
+
threshold?: number;
|
|
161
|
+
mode?: 'embedding' | 'llm';
|
|
162
|
+
model?: string;
|
|
163
|
+
embeddingModel?: string;
|
|
164
|
+
}
|
|
165
|
+
): this {
|
|
166
|
+
this._expected = {
|
|
167
|
+
type: 'similarity',
|
|
168
|
+
value,
|
|
169
|
+
threshold: options?.threshold ?? 0.75,
|
|
170
|
+
mode: options?.mode,
|
|
171
|
+
model: options?.model,
|
|
172
|
+
embeddingModel: options?.embeddingModel,
|
|
173
|
+
};
|
|
174
|
+
return this;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Expect an inline expression to evaluate to true
|
|
179
|
+
*/
|
|
180
|
+
expectInline(expression: string, value?: string): this {
|
|
181
|
+
this._expected = { type: 'inline', expression, value };
|
|
182
|
+
return this;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Combine multiple expectations with AND logic
|
|
187
|
+
* Note: Pass base expectations (not combined) for proper type inference
|
|
188
|
+
*/
|
|
189
|
+
expectAll(...expectations: Expected[]): this {
|
|
190
|
+
// Type assertion needed because TypeScript can't infer combined type correctly
|
|
191
|
+
this._expected = { type: 'combined', operator: 'and', expectations } as unknown as Expected;
|
|
192
|
+
return this;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Combine multiple expectations with OR logic
|
|
197
|
+
* Note: Pass base expectations (not combined) for proper type inference
|
|
198
|
+
*/
|
|
199
|
+
expectAny(...expectations: Expected[]): this {
|
|
200
|
+
// Type assertion needed because TypeScript can't infer combined type correctly
|
|
201
|
+
this._expected = { type: 'combined', operator: 'or', expectations } as unknown as Expected;
|
|
202
|
+
return this;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Set a custom expectation
|
|
207
|
+
*/
|
|
208
|
+
expect(expected: Expected): this {
|
|
209
|
+
this._expected = expected;
|
|
210
|
+
return this;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Add tags to the test case
|
|
215
|
+
*/
|
|
216
|
+
tags(...tags: string[]): this {
|
|
217
|
+
this._tags.push(...tags);
|
|
218
|
+
return this;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Add metadata to the test case
|
|
223
|
+
*/
|
|
224
|
+
metadata(data: Record<string, unknown>): this {
|
|
225
|
+
this._metadata = { ...this._metadata, ...data };
|
|
226
|
+
return this;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Set timeout for this test case
|
|
231
|
+
*/
|
|
232
|
+
timeout(ms: number): this {
|
|
233
|
+
this._timeout = ms;
|
|
234
|
+
return this;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Set number of retries for this test case
|
|
239
|
+
*/
|
|
240
|
+
retries(count: number): this {
|
|
241
|
+
this._retries = count;
|
|
242
|
+
return this;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Override provider for this test case
|
|
247
|
+
*/
|
|
248
|
+
provider(provider: Provider): this {
|
|
249
|
+
this._provider = provider;
|
|
250
|
+
return this;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Override model for this test case
|
|
255
|
+
*/
|
|
256
|
+
model(model: string): this {
|
|
257
|
+
this._model = model;
|
|
258
|
+
return this;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Set variables for this test case
|
|
263
|
+
*/
|
|
264
|
+
variables(vars: Variables): this {
|
|
265
|
+
this._variables = vars;
|
|
266
|
+
return this;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Enable redaction for this test case
|
|
271
|
+
*/
|
|
272
|
+
redact(config: RedactionConfig): this {
|
|
273
|
+
this._redaction = config;
|
|
274
|
+
return this;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Build the test case object
|
|
279
|
+
*/
|
|
280
|
+
build(): TestCase {
|
|
281
|
+
if (!this._expected) {
|
|
282
|
+
throw new Error(
|
|
283
|
+
`TestCase "${this._id}" must have an expectation. Call one of the expect* methods.`
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Check for empty prompt: empty string or empty array
|
|
288
|
+
const hasEmptyPrompt =
|
|
289
|
+
!this._prompt ||
|
|
290
|
+
(typeof this._prompt === 'string' && this._prompt.length === 0) ||
|
|
291
|
+
(Array.isArray(this._prompt) && this._prompt.length === 0);
|
|
292
|
+
|
|
293
|
+
if (hasEmptyPrompt) {
|
|
294
|
+
throw new Error(`TestCase "${this._id}" must have a prompt.`);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
return {
|
|
298
|
+
id: this._id,
|
|
299
|
+
name: this._name,
|
|
300
|
+
description: this._description,
|
|
301
|
+
prompt: this._prompt,
|
|
302
|
+
expected: this._expected,
|
|
303
|
+
tags: this._tags,
|
|
304
|
+
metadata: this._metadata,
|
|
305
|
+
timeout: this._timeout,
|
|
306
|
+
retries: this._retries,
|
|
307
|
+
provider: this._provider,
|
|
308
|
+
model: this._model,
|
|
309
|
+
variables: this._variables,
|
|
310
|
+
redaction: this._redaction,
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// ============================================================================
|
|
316
|
+
// Scenario Builder
|
|
317
|
+
// ============================================================================
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* Fluent builder for constructing scenarios
|
|
321
|
+
*/
|
|
322
|
+
export class ScenarioBuilder {
|
|
323
|
+
private _name: string;
|
|
324
|
+
private _description?: string;
|
|
325
|
+
private _version = '1.0';
|
|
326
|
+
private _provider?: Provider;
|
|
327
|
+
private _model?: string;
|
|
328
|
+
private _providerConfig?: Record<string, unknown>;
|
|
329
|
+
private _seed?: number;
|
|
330
|
+
private _temperature?: number;
|
|
331
|
+
private _maxTokens?: number;
|
|
332
|
+
private _tags: string[] = [];
|
|
333
|
+
private _variables?: Variables;
|
|
334
|
+
private _redaction?: RedactionConfig;
|
|
335
|
+
private _systemPrompt?: string;
|
|
336
|
+
private _cases: TestCase[] = [];
|
|
337
|
+
|
|
338
|
+
constructor(name: string) {
|
|
339
|
+
this._name = name;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Set the scenario description
|
|
344
|
+
*/
|
|
345
|
+
description(description: string): this {
|
|
346
|
+
this._description = description;
|
|
347
|
+
return this;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Set the scenario version
|
|
352
|
+
*/
|
|
353
|
+
version(version: string): this {
|
|
354
|
+
this._version = version;
|
|
355
|
+
return this;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Set the default provider
|
|
360
|
+
*/
|
|
361
|
+
provider(provider: Provider): this {
|
|
362
|
+
this._provider = provider;
|
|
363
|
+
return this;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Set the default model
|
|
368
|
+
*/
|
|
369
|
+
model(model: string): this {
|
|
370
|
+
this._model = model;
|
|
371
|
+
return this;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* Set provider configuration
|
|
376
|
+
*/
|
|
377
|
+
providerConfig(config: Record<string, unknown>): this {
|
|
378
|
+
this._providerConfig = config;
|
|
379
|
+
return this;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Set the random seed for reproducibility
|
|
384
|
+
*/
|
|
385
|
+
seed(seed: number): this {
|
|
386
|
+
this._seed = seed;
|
|
387
|
+
return this;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* Set the temperature
|
|
392
|
+
*/
|
|
393
|
+
temperature(temp: number): this {
|
|
394
|
+
this._temperature = temp;
|
|
395
|
+
return this;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Set max tokens
|
|
400
|
+
*/
|
|
401
|
+
maxTokens(tokens: number): this {
|
|
402
|
+
this._maxTokens = tokens;
|
|
403
|
+
return this;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/**
|
|
407
|
+
* Add tags to the scenario
|
|
408
|
+
*/
|
|
409
|
+
tags(...tags: string[]): this {
|
|
410
|
+
this._tags.push(...tags);
|
|
411
|
+
return this;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* Set variables for template substitution
|
|
416
|
+
*/
|
|
417
|
+
variables(vars: Variables): this {
|
|
418
|
+
this._variables = vars;
|
|
419
|
+
return this;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* Set redaction configuration
|
|
424
|
+
*/
|
|
425
|
+
redact(config: RedactionConfig): this {
|
|
426
|
+
this._redaction = config;
|
|
427
|
+
return this;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/**
|
|
431
|
+
* Set a system prompt for all test cases
|
|
432
|
+
*/
|
|
433
|
+
systemPrompt(prompt: string): this {
|
|
434
|
+
this._systemPrompt = prompt;
|
|
435
|
+
return this;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* Add a test case using a builder
|
|
440
|
+
*/
|
|
441
|
+
case(caseBuilder: TestCaseBuilder): this {
|
|
442
|
+
this._cases.push(caseBuilder.build());
|
|
443
|
+
return this;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* Add a test case directly
|
|
448
|
+
*/
|
|
449
|
+
addCase(testCase: TestCase): this {
|
|
450
|
+
this._cases.push(testCase);
|
|
451
|
+
return this;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Add multiple test cases
|
|
456
|
+
*/
|
|
457
|
+
cases(...caseBuilders: TestCaseBuilder[]): this {
|
|
458
|
+
for (const builder of caseBuilders) {
|
|
459
|
+
this._cases.push(builder.build());
|
|
460
|
+
}
|
|
461
|
+
return this;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
/**
|
|
465
|
+
* Build the scenario object
|
|
466
|
+
*/
|
|
467
|
+
build(): Scenario {
|
|
468
|
+
if (this._cases.length === 0) {
|
|
469
|
+
throw new Error(`Scenario "${this._name}" must have at least one test case.`);
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
return {
|
|
473
|
+
name: this._name,
|
|
474
|
+
description: this._description,
|
|
475
|
+
version: this._version,
|
|
476
|
+
provider: this._provider,
|
|
477
|
+
model: this._model,
|
|
478
|
+
providerConfig: this._providerConfig,
|
|
479
|
+
seed: this._seed,
|
|
480
|
+
temperature: this._temperature,
|
|
481
|
+
maxTokens: this._maxTokens,
|
|
482
|
+
tags: this._tags,
|
|
483
|
+
variables: this._variables,
|
|
484
|
+
redaction: this._redaction,
|
|
485
|
+
setup: this._systemPrompt ? { systemPrompt: this._systemPrompt } : undefined,
|
|
486
|
+
cases: this._cases,
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
// ============================================================================
|
|
492
|
+
// Factory Functions
|
|
493
|
+
// ============================================================================
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* Create a new scenario builder
|
|
497
|
+
*
|
|
498
|
+
* @example
|
|
499
|
+
* ```typescript
|
|
500
|
+
* import { scenario, testCase } from '@artemiskit/sdk/builders'
|
|
501
|
+
*
|
|
502
|
+
* const myScenario = scenario('Math Tests')
|
|
503
|
+
* .provider('openai')
|
|
504
|
+
* .model('gpt-4')
|
|
505
|
+
* .description('Basic math verification tests')
|
|
506
|
+
* .case(
|
|
507
|
+
* testCase('addition')
|
|
508
|
+
* .prompt('What is 2 + 2?')
|
|
509
|
+
* .expectContains(['4'])
|
|
510
|
+
* )
|
|
511
|
+
* .case(
|
|
512
|
+
* testCase('multiplication')
|
|
513
|
+
* .prompt('What is 6 * 7?')
|
|
514
|
+
* .expectContains(['42'])
|
|
515
|
+
* )
|
|
516
|
+
* .build()
|
|
517
|
+
* ```
|
|
518
|
+
*/
|
|
519
|
+
export function scenario(name: string): ScenarioBuilder {
|
|
520
|
+
return new ScenarioBuilder(name);
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/**
|
|
524
|
+
* Create a new test case builder
|
|
525
|
+
*
|
|
526
|
+
* @example
|
|
527
|
+
* ```typescript
|
|
528
|
+
* import { testCase } from '@artemiskit/sdk/builders'
|
|
529
|
+
*
|
|
530
|
+
* const case1 = testCase('json-output')
|
|
531
|
+
* .prompt('Return a JSON object with name and age')
|
|
532
|
+
* .expectJsonSchema({
|
|
533
|
+
* type: 'object',
|
|
534
|
+
* required: ['name', 'age'],
|
|
535
|
+
* properties: {
|
|
536
|
+
* name: { type: 'string' },
|
|
537
|
+
* age: { type: 'number' }
|
|
538
|
+
* }
|
|
539
|
+
* })
|
|
540
|
+
* .tags('json', 'structured')
|
|
541
|
+
* .build()
|
|
542
|
+
* ```
|
|
543
|
+
*/
|
|
544
|
+
export function testCase(id: string): TestCaseBuilder {
|
|
545
|
+
return new TestCaseBuilder(id);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// ============================================================================
|
|
549
|
+
// Quick Helpers
|
|
550
|
+
// ============================================================================
|
|
551
|
+
|
|
552
|
+
/**
|
|
553
|
+
* Create a simple test case with contains expectation
|
|
554
|
+
*/
|
|
555
|
+
export function containsCase(
|
|
556
|
+
id: string,
|
|
557
|
+
prompt: string,
|
|
558
|
+
values: string[],
|
|
559
|
+
mode: 'all' | 'any' = 'all'
|
|
560
|
+
): TestCase {
|
|
561
|
+
return testCase(id).prompt(prompt).expectContains(values, mode).build();
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* Create a simple test case with exact match expectation
|
|
566
|
+
*/
|
|
567
|
+
export function exactCase(id: string, prompt: string, value: string): TestCase {
|
|
568
|
+
return testCase(id).prompt(prompt).expectExact(value).build();
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* Create a simple test case with regex expectation
|
|
573
|
+
*/
|
|
574
|
+
export function regexCase(id: string, prompt: string, pattern: string, flags?: string): TestCase {
|
|
575
|
+
return testCase(id).prompt(prompt).expectRegex(pattern, flags).build();
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* Create a simple test case with JSON schema expectation
|
|
580
|
+
*/
|
|
581
|
+
export function jsonCase(id: string, prompt: string, schema: Record<string, unknown>): TestCase {
|
|
582
|
+
return testCase(id).prompt(prompt).expectJsonSchema(schema).build();
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* Create a simple test case with LLM grading
|
|
587
|
+
*/
|
|
588
|
+
export function gradedCase(id: string, prompt: string, rubric: string, threshold = 0.7): TestCase {
|
|
589
|
+
return testCase(id).prompt(prompt).expectLLMGrade(rubric, { threshold }).build();
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
// ============================================================================
|
|
593
|
+
// Expectation Helpers
|
|
594
|
+
// ============================================================================
|
|
595
|
+
|
|
596
|
+
/**
|
|
597
|
+
* Create an exact match expectation
|
|
598
|
+
*/
|
|
599
|
+
export function exact(value: string, caseSensitive = true): Expected {
|
|
600
|
+
return { type: 'exact', value, caseSensitive };
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* Create a contains expectation
|
|
605
|
+
*/
|
|
606
|
+
export function contains(values: string[], mode: 'all' | 'any' = 'all'): Expected {
|
|
607
|
+
return { type: 'contains', values, mode };
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/**
|
|
611
|
+
* Create a not_contains expectation
|
|
612
|
+
*/
|
|
613
|
+
export function notContains(values: string[], mode: 'all' | 'any' = 'all'): Expected {
|
|
614
|
+
return { type: 'not_contains', values, mode };
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Create a regex expectation
|
|
619
|
+
*/
|
|
620
|
+
export function regex(pattern: string, flags?: string): Expected {
|
|
621
|
+
return { type: 'regex', pattern, flags };
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
/**
|
|
625
|
+
* Create a fuzzy match expectation
|
|
626
|
+
*/
|
|
627
|
+
export function fuzzy(value: string, threshold = 0.8): Expected {
|
|
628
|
+
return { type: 'fuzzy', value, threshold };
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Create a JSON schema expectation
|
|
633
|
+
*/
|
|
634
|
+
export function jsonSchema(schema: Record<string, unknown>): Expected {
|
|
635
|
+
return { type: 'json_schema', schema };
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
/**
|
|
639
|
+
* Create an LLM grader expectation
|
|
640
|
+
*/
|
|
641
|
+
export function llmGrade(
|
|
642
|
+
rubric: string,
|
|
643
|
+
options?: { threshold?: number; model?: string; provider?: Provider; strict?: boolean }
|
|
644
|
+
): Expected {
|
|
645
|
+
return {
|
|
646
|
+
type: 'llm_grader',
|
|
647
|
+
rubric,
|
|
648
|
+
threshold: options?.threshold ?? 0.7,
|
|
649
|
+
model: options?.model,
|
|
650
|
+
provider: options?.provider,
|
|
651
|
+
strict: options?.strict ?? false,
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/**
|
|
656
|
+
* Create a similarity expectation
|
|
657
|
+
*/
|
|
658
|
+
export function similarity(
|
|
659
|
+
value: string,
|
|
660
|
+
options?: {
|
|
661
|
+
threshold?: number;
|
|
662
|
+
mode?: 'embedding' | 'llm';
|
|
663
|
+
model?: string;
|
|
664
|
+
embeddingModel?: string;
|
|
665
|
+
}
|
|
666
|
+
): Expected {
|
|
667
|
+
return {
|
|
668
|
+
type: 'similarity',
|
|
669
|
+
value,
|
|
670
|
+
threshold: options?.threshold ?? 0.75,
|
|
671
|
+
mode: options?.mode,
|
|
672
|
+
model: options?.model,
|
|
673
|
+
embeddingModel: options?.embeddingModel,
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/**
|
|
678
|
+
* Create an inline expression expectation
|
|
679
|
+
*/
|
|
680
|
+
export function inline(expression: string, value?: string): Expected {
|
|
681
|
+
return { type: 'inline', expression, value };
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/**
|
|
685
|
+
* Combine expectations with AND logic
|
|
686
|
+
* Note: Pass base expectations (not combined) for proper type inference
|
|
687
|
+
*/
|
|
688
|
+
export function allOf(...expectations: Expected[]): Expected {
|
|
689
|
+
// Type assertion needed because TypeScript can't infer combined type correctly
|
|
690
|
+
return { type: 'combined', operator: 'and', expectations } as unknown as Expected;
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
/**
|
|
694
|
+
* Combine expectations with OR logic
|
|
695
|
+
* Note: Pass base expectations (not combined) for proper type inference
|
|
696
|
+
*/
|
|
697
|
+
export function anyOf(...expectations: Expected[]): Expected {
|
|
698
|
+
// Type assertion needed because TypeScript can't infer combined type correctly
|
|
699
|
+
return { type: 'combined', operator: 'or', expectations } as unknown as Expected;
|
|
700
|
+
}
|