@artemiskit/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +229 -0
  2. package/README.md +44 -1
  3. package/dist/artemiskit.d.ts +149 -0
  4. package/dist/artemiskit.d.ts.map +1 -0
  5. package/dist/builders/index.d.ts +352 -0
  6. package/dist/builders/index.d.ts.map +1 -0
  7. package/dist/builders/index.js +354 -0
  8. package/dist/contracts/index.d.ts +253 -0
  9. package/dist/contracts/index.d.ts.map +1 -0
  10. package/dist/contracts/index.js +53 -0
  11. package/dist/guardian/action-validator.d.ts +92 -0
  12. package/dist/guardian/action-validator.d.ts.map +1 -0
  13. package/dist/guardian/circuit-breaker.d.ts +211 -0
  14. package/dist/guardian/circuit-breaker.d.ts.map +1 -0
  15. package/dist/guardian/guardian.d.ts +213 -0
  16. package/dist/guardian/guardian.d.ts.map +1 -0
  17. package/dist/guardian/guardrails.d.ts +107 -0
  18. package/dist/guardian/guardrails.d.ts.map +1 -0
  19. package/dist/guardian/index.d.ts +35 -0
  20. package/dist/guardian/index.d.ts.map +1 -0
  21. package/dist/guardian/intent-classifier.d.ts +81 -0
  22. package/dist/guardian/intent-classifier.d.ts.map +1 -0
  23. package/dist/guardian/interceptor.d.ts +141 -0
  24. package/dist/guardian/interceptor.d.ts.map +1 -0
  25. package/dist/guardian/policy.d.ts +52 -0
  26. package/dist/guardian/policy.d.ts.map +1 -0
  27. package/dist/guardian/semantic-validator.d.ts +61 -0
  28. package/dist/guardian/semantic-validator.d.ts.map +1 -0
  29. package/dist/guardian/types.d.ts +653 -0
  30. package/dist/guardian/types.d.ts.map +1 -0
  31. package/dist/index.d.ts +23 -0
  32. package/dist/index.d.ts.map +1 -0
  33. package/dist/index.js +44918 -28644
  34. package/dist/matchers/core.d.ts +81 -0
  35. package/dist/matchers/core.d.ts.map +1 -0
  36. package/dist/matchers/index.d.ts +6 -0
  37. package/dist/matchers/index.d.ts.map +1 -0
  38. package/dist/matchers/index.js +13 -59
  39. package/dist/matchers/jest.d.ts +46 -0
  40. package/dist/matchers/jest.d.ts.map +1 -0
  41. package/dist/matchers/jest.js +0 -46
  42. package/dist/matchers/vitest.d.ts +24 -0
  43. package/dist/matchers/vitest.d.ts.map +1 -0
  44. package/dist/matchers/vitest.js +0 -46
  45. package/dist/types-only.d.ts +31 -0
  46. package/dist/types-only.d.ts.map +1 -0
  47. package/dist/types-only.js +1 -0
  48. package/dist/types.d.ts +379 -0
  49. package/dist/types.d.ts.map +1 -0
  50. package/dist/utils/index.d.ts +112 -0
  51. package/dist/utils/index.d.ts.map +1 -0
  52. package/dist/utils/index.js +58 -0
  53. package/package.json +19 -4
  54. package/src/__tests__/builders.test.ts +450 -0
  55. package/src/__tests__/circuit-breaker.test.ts +704 -0
  56. package/src/__tests__/compare.test.ts +469 -0
  57. package/src/__tests__/contracts.test.ts +354 -0
  58. package/src/__tests__/guardian-modes.test.ts +215 -0
  59. package/src/__tests__/guardrails.test.ts +404 -0
  60. package/src/__tests__/semantic-validator.test.ts +421 -0
  61. package/src/__tests__/utils.test.ts +324 -0
  62. package/src/__tests__/validate.test.ts +310 -0
  63. package/src/artemiskit.ts +479 -5
  64. package/src/builders/index.ts +700 -0
  65. package/src/contracts/index.ts +357 -0
  66. package/src/guardian/circuit-breaker.ts +74 -28
  67. package/src/guardian/guardian.ts +307 -20
  68. package/src/guardian/guardrails.ts +143 -1
  69. package/src/guardian/index.ts +35 -1
  70. package/src/guardian/interceptor.ts +87 -11
  71. package/src/guardian/semantic-validator.ts +399 -0
  72. package/src/guardian/types.ts +413 -2
  73. package/src/index.ts +258 -14
  74. package/src/types-only.ts +276 -0
  75. package/src/types.ts +186 -0
  76. package/src/utils/index.ts +225 -0
  77. package/tsconfig.json +2 -1
  78. package/adapters/openai/dist/index.js +0 -5625
@@ -0,0 +1,352 @@
1
+ /**
2
+ * @artemiskit/sdk
3
+ * Fluent builders for constructing scenarios and test cases
4
+ *
5
+ * Provides a type-safe, fluent API for building ArtemisKit scenarios
6
+ * programmatically without manually constructing complex objects.
7
+ */
8
+ import type { ChatMessageType, Expected, Provider, Scenario, TestCase, Variables } from '@artemiskit/core';
9
+ import type { RedactionConfig } from '@artemiskit/core';
10
+ /**
11
+ * Fluent builder for constructing test cases
12
+ */
13
+ export declare class TestCaseBuilder {
14
+ private _id;
15
+ private _name?;
16
+ private _description?;
17
+ private _prompt;
18
+ private _expected?;
19
+ private _tags;
20
+ private _metadata;
21
+ private _timeout?;
22
+ private _retries;
23
+ private _provider?;
24
+ private _model?;
25
+ private _variables?;
26
+ private _redaction?;
27
+ constructor(id: string);
28
+ /**
29
+ * Set the test case name (display name)
30
+ */
31
+ name(name: string): this;
32
+ /**
33
+ * Set the test case description
34
+ */
35
+ description(description: string): this;
36
+ /**
37
+ * Set the prompt as a simple string
38
+ */
39
+ prompt(prompt: string): this;
40
+ /**
41
+ * Set the prompt as chat messages
42
+ */
43
+ messages(messages: ChatMessageType[]): this;
44
+ /**
45
+ * Add a system message followed by a user message
46
+ */
47
+ systemAndUser(systemPrompt: string, userPrompt: string): this;
48
+ /**
49
+ * Expect an exact match
50
+ */
51
+ expectExact(value: string, caseSensitive?: boolean): this;
52
+ /**
53
+ * Expect the response to contain specific values
54
+ */
55
+ expectContains(values: string[], mode?: 'all' | 'any'): this;
56
+ /**
57
+ * Expect the response to NOT contain specific values
58
+ */
59
+ expectNotContains(values: string[], mode?: 'all' | 'any'): this;
60
+ /**
61
+ * Expect a regex match
62
+ */
63
+ expectRegex(pattern: string, flags?: string): this;
64
+ /**
65
+ * Expect a fuzzy match with similarity threshold
66
+ */
67
+ expectFuzzy(value: string, threshold?: number): this;
68
+ /**
69
+ * Expect a JSON response matching a schema
70
+ */
71
+ expectJsonSchema(schema: Record<string, unknown>): this;
72
+ /**
73
+ * Expect response to pass LLM grading with a rubric
74
+ */
75
+ expectLLMGrade(rubric: string, options?: {
76
+ threshold?: number;
77
+ model?: string;
78
+ provider?: Provider;
79
+ strict?: boolean;
80
+ }): this;
81
+ /**
82
+ * Expect semantic similarity
83
+ */
84
+ expectSimilarity(value: string, options?: {
85
+ threshold?: number;
86
+ mode?: 'embedding' | 'llm';
87
+ model?: string;
88
+ embeddingModel?: string;
89
+ }): this;
90
+ /**
91
+ * Expect an inline expression to evaluate to true
92
+ */
93
+ expectInline(expression: string, value?: string): this;
94
+ /**
95
+ * Combine multiple expectations with AND logic
96
+ * Note: Pass base expectations (not combined) for proper type inference
97
+ */
98
+ expectAll(...expectations: Expected[]): this;
99
+ /**
100
+ * Combine multiple expectations with OR logic
101
+ * Note: Pass base expectations (not combined) for proper type inference
102
+ */
103
+ expectAny(...expectations: Expected[]): this;
104
+ /**
105
+ * Set a custom expectation
106
+ */
107
+ expect(expected: Expected): this;
108
+ /**
109
+ * Add tags to the test case
110
+ */
111
+ tags(...tags: string[]): this;
112
+ /**
113
+ * Add metadata to the test case
114
+ */
115
+ metadata(data: Record<string, unknown>): this;
116
+ /**
117
+ * Set timeout for this test case
118
+ */
119
+ timeout(ms: number): this;
120
+ /**
121
+ * Set number of retries for this test case
122
+ */
123
+ retries(count: number): this;
124
+ /**
125
+ * Override provider for this test case
126
+ */
127
+ provider(provider: Provider): this;
128
+ /**
129
+ * Override model for this test case
130
+ */
131
+ model(model: string): this;
132
+ /**
133
+ * Set variables for this test case
134
+ */
135
+ variables(vars: Variables): this;
136
+ /**
137
+ * Enable redaction for this test case
138
+ */
139
+ redact(config: RedactionConfig): this;
140
+ /**
141
+ * Build the test case object
142
+ */
143
+ build(): TestCase;
144
+ }
145
+ /**
146
+ * Fluent builder for constructing scenarios
147
+ */
148
+ export declare class ScenarioBuilder {
149
+ private _name;
150
+ private _description?;
151
+ private _version;
152
+ private _provider?;
153
+ private _model?;
154
+ private _providerConfig?;
155
+ private _seed?;
156
+ private _temperature?;
157
+ private _maxTokens?;
158
+ private _tags;
159
+ private _variables?;
160
+ private _redaction?;
161
+ private _systemPrompt?;
162
+ private _cases;
163
+ constructor(name: string);
164
+ /**
165
+ * Set the scenario description
166
+ */
167
+ description(description: string): this;
168
+ /**
169
+ * Set the scenario version
170
+ */
171
+ version(version: string): this;
172
+ /**
173
+ * Set the default provider
174
+ */
175
+ provider(provider: Provider): this;
176
+ /**
177
+ * Set the default model
178
+ */
179
+ model(model: string): this;
180
+ /**
181
+ * Set provider configuration
182
+ */
183
+ providerConfig(config: Record<string, unknown>): this;
184
+ /**
185
+ * Set the random seed for reproducibility
186
+ */
187
+ seed(seed: number): this;
188
+ /**
189
+ * Set the temperature
190
+ */
191
+ temperature(temp: number): this;
192
+ /**
193
+ * Set max tokens
194
+ */
195
+ maxTokens(tokens: number): this;
196
+ /**
197
+ * Add tags to the scenario
198
+ */
199
+ tags(...tags: string[]): this;
200
+ /**
201
+ * Set variables for template substitution
202
+ */
203
+ variables(vars: Variables): this;
204
+ /**
205
+ * Set redaction configuration
206
+ */
207
+ redact(config: RedactionConfig): this;
208
+ /**
209
+ * Set a system prompt for all test cases
210
+ */
211
+ systemPrompt(prompt: string): this;
212
+ /**
213
+ * Add a test case using a builder
214
+ */
215
+ case(caseBuilder: TestCaseBuilder): this;
216
+ /**
217
+ * Add a test case directly
218
+ */
219
+ addCase(testCase: TestCase): this;
220
+ /**
221
+ * Add multiple test cases
222
+ */
223
+ cases(...caseBuilders: TestCaseBuilder[]): this;
224
+ /**
225
+ * Build the scenario object
226
+ */
227
+ build(): Scenario;
228
+ }
229
+ /**
230
+ * Create a new scenario builder
231
+ *
232
+ * @example
233
+ * ```typescript
234
+ * import { scenario, testCase } from '@artemiskit/sdk/builders'
235
+ *
236
+ * const myScenario = scenario('Math Tests')
237
+ * .provider('openai')
238
+ * .model('gpt-4')
239
+ * .description('Basic math verification tests')
240
+ * .case(
241
+ * testCase('addition')
242
+ * .prompt('What is 2 + 2?')
243
+ * .expectContains(['4'])
244
+ * )
245
+ * .case(
246
+ * testCase('multiplication')
247
+ * .prompt('What is 6 * 7?')
248
+ * .expectContains(['42'])
249
+ * )
250
+ * .build()
251
+ * ```
252
+ */
253
+ export declare function scenario(name: string): ScenarioBuilder;
254
+ /**
255
+ * Create a new test case builder
256
+ *
257
+ * @example
258
+ * ```typescript
259
+ * import { testCase } from '@artemiskit/sdk/builders'
260
+ *
261
+ * const case1 = testCase('json-output')
262
+ * .prompt('Return a JSON object with name and age')
263
+ * .expectJsonSchema({
264
+ * type: 'object',
265
+ * required: ['name', 'age'],
266
+ * properties: {
267
+ * name: { type: 'string' },
268
+ * age: { type: 'number' }
269
+ * }
270
+ * })
271
+ * .tags('json', 'structured')
272
+ * .build()
273
+ * ```
274
+ */
275
+ export declare function testCase(id: string): TestCaseBuilder;
276
+ /**
277
+ * Create a simple test case with contains expectation
278
+ */
279
+ export declare function containsCase(id: string, prompt: string, values: string[], mode?: 'all' | 'any'): TestCase;
280
+ /**
281
+ * Create a simple test case with exact match expectation
282
+ */
283
+ export declare function exactCase(id: string, prompt: string, value: string): TestCase;
284
+ /**
285
+ * Create a simple test case with regex expectation
286
+ */
287
+ export declare function regexCase(id: string, prompt: string, pattern: string, flags?: string): TestCase;
288
+ /**
289
+ * Create a simple test case with JSON schema expectation
290
+ */
291
+ export declare function jsonCase(id: string, prompt: string, schema: Record<string, unknown>): TestCase;
292
+ /**
293
+ * Create a simple test case with LLM grading
294
+ */
295
+ export declare function gradedCase(id: string, prompt: string, rubric: string, threshold?: number): TestCase;
296
+ /**
297
+ * Create an exact match expectation
298
+ */
299
+ export declare function exact(value: string, caseSensitive?: boolean): Expected;
300
+ /**
301
+ * Create a contains expectation
302
+ */
303
+ export declare function contains(values: string[], mode?: 'all' | 'any'): Expected;
304
+ /**
305
+ * Create a not_contains expectation
306
+ */
307
+ export declare function notContains(values: string[], mode?: 'all' | 'any'): Expected;
308
+ /**
309
+ * Create a regex expectation
310
+ */
311
+ export declare function regex(pattern: string, flags?: string): Expected;
312
+ /**
313
+ * Create a fuzzy match expectation
314
+ */
315
+ export declare function fuzzy(value: string, threshold?: number): Expected;
316
+ /**
317
+ * Create a JSON schema expectation
318
+ */
319
+ export declare function jsonSchema(schema: Record<string, unknown>): Expected;
320
+ /**
321
+ * Create an LLM grader expectation
322
+ */
323
+ export declare function llmGrade(rubric: string, options?: {
324
+ threshold?: number;
325
+ model?: string;
326
+ provider?: Provider;
327
+ strict?: boolean;
328
+ }): Expected;
329
+ /**
330
+ * Create a similarity expectation
331
+ */
332
+ export declare function similarity(value: string, options?: {
333
+ threshold?: number;
334
+ mode?: 'embedding' | 'llm';
335
+ model?: string;
336
+ embeddingModel?: string;
337
+ }): Expected;
338
+ /**
339
+ * Create an inline expression expectation
340
+ */
341
+ export declare function inline(expression: string, value?: string): Expected;
342
+ /**
343
+ * Combine expectations with AND logic
344
+ * Note: Pass base expectations (not combined) for proper type inference
345
+ */
346
+ export declare function allOf(...expectations: Expected[]): Expected;
347
+ /**
348
+ * Combine expectations with OR logic
349
+ * Note: Pass base expectations (not combined) for proper type inference
350
+ */
351
+ export declare function anyOf(...expectations: Expected[]): Expected;
352
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/builders/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,SAAS,EACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD;;GAEG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,KAAK,CAAC,CAAS;IACvB,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,SAAS,CAAC,CAAW;IAC7B,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,SAAS,CAAC,CAAW;IAC7B,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,UAAU,CAAC,CAAY;IAC/B,OAAO,CAAC,UAAU,CAAC,CAAkB;gBAEzB,EAAE,EAAE,MAAM;IAItB;;OAEG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAKxB;;OAEG;IACH,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAKtC;;OAEG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAK5B;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,IAAI;IAK3C;;OAEG;IACH,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAQ7D;;OAEG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,UAAO,GAAG,IAAI;IAKtD;;OAEG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,KAAK,GAAG,KAAa,GAAG,IAAI;IAKnE;;OAEG;IACH,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,KAAK,GAAG,KAAa,GAAG,IAAI;IAKtE;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI;IAKlD;;OAEG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,IAAI;IAKjD;;OAEG;IACH,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAKvD;;OAEG;IACH,cAAc,CACZ,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,GACtF,IAAI;IAYP;;OAEG;IACH,gBAAgB,CACd,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QACR,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,IAAI,CAAC,EAAE,WAAW,GAAG,KAAK,CAAC;QAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GACA,IAAI;IAYP;;OAEG;IACH,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI;IAKtD;;;OAGG;IACH,SAAS,CAAC,GAAG,YAAY,EAAE,QAAQ,EAAE,GAAG,IAAI;IAM5C;;;OAGG;IACH,SAAS,CAAC,GAAG,YAAY,EAAE,QAAQ,EAAE,GAAG,IAAI;IAM5C;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAKhC;;OAEG;IACH,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI;IAK7B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAK7C;;OAEG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAKzB;;OAEG;IACH,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAK5B;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAKlC;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAK1B;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAKhC;;OAEG;IACH,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI;IAKrC;;OAEG;IACH,KAAK,IAAI,QAAQ;CAiClB;AAMD;;GAEG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,SAAS,CAAC,CAAW;IAC7B,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,eAAe,CAAC,CAA0B;IAClD,OAAO,CAAC,KAAK,CAAC,CAAS;IACvB,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,UAAU,CAAC,CAAS;IAC5B,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,UAAU,CAAC,CAAY;IAC/B,OAAO,CAAC,UAAU,CAAC,CAAkB;IACrC,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,MAAM,CAAkB;gBAEpB,IAAI,EAAE,MAAM;IAIxB;;OAEG;IACH,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAKtC;;OAEG;IACH,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAK9B;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAKlC;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAK1B;;OAEG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAKrD;;OAEG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAKxB;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAK/B;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAK/B;;OAEG;IACH,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI;IAK7B;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAKhC;;OAEG;IACH,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI;IAKrC;;OAEG;IACH,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAKlC;;OAEG;IACH,IAAI,CAAC,WAAW,EAAE,eAAe,GAAG,IAAI;IAKxC;;OAEG;IACH,OAAO,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAKjC;;OAEG;IACH,KAAK,CAAC,GAAG,YAAY,EAAE,eAAe,EAAE,GAAG,IAAI;IAO/C;;OAEG;IACH,KAAK,IAAI,QAAQ;CAsBlB;AAMD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAEtD;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,eAAe,CAEpD;AAMD;;GAEG;AACH,wBAAgB,YAAY,CAC1B,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EAAE,EAChB,IAAI,GAAE,KAAK,GAAG,KAAa,GAC1B,QAAQ,CAEV;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,QAAQ,CAE7E;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,CAE/F;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,QAAQ,CAE9F;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,QAAQ,CAEhG;AAMD;;GAEG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,UAAO,GAAG,QAAQ,CAEnE;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,KAAK,GAAG,KAAa,GAAG,QAAQ,CAEhF;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,KAAK,GAAG,KAAa,GAAG,QAAQ,CAEnF;AAED;;GAEG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,CAE/D;AAED;;GAEG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,QAAQ,CAE9D;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,QAAQ,CAEpE;AAED;;GAEG;AACH,wBAAgB,QAAQ,CACtB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,GACtF,QAAQ,CASV;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;IACR,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,WAAW,GAAG,KAAK,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,GACA,QAAQ,CASV;AAED;;GAEG;AACH,wBAAgB,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,CAEnE;AAED;;;GAGG;AACH,wBAAgB,KAAK,CAAC,GAAG,YAAY,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAG3D;AAED;;;GAGG;AACH,wBAAgB,KAAK,CAAC,GAAG,YAAY,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAG3D"}
@@ -0,0 +1,354 @@
1
+ // @bun
2
+ // src/builders/index.ts
3
+ class TestCaseBuilder {
4
+ _id;
5
+ _name;
6
+ _description;
7
+ _prompt = "";
8
+ _expected;
9
+ _tags = [];
10
+ _metadata = {};
11
+ _timeout;
12
+ _retries = 0;
13
+ _provider;
14
+ _model;
15
+ _variables;
16
+ _redaction;
17
+ constructor(id) {
18
+ this._id = id;
19
+ }
20
+ name(name) {
21
+ this._name = name;
22
+ return this;
23
+ }
24
+ description(description) {
25
+ this._description = description;
26
+ return this;
27
+ }
28
+ prompt(prompt) {
29
+ this._prompt = prompt;
30
+ return this;
31
+ }
32
+ messages(messages) {
33
+ this._prompt = messages;
34
+ return this;
35
+ }
36
+ systemAndUser(systemPrompt, userPrompt) {
37
+ this._prompt = [
38
+ { role: "system", content: systemPrompt },
39
+ { role: "user", content: userPrompt }
40
+ ];
41
+ return this;
42
+ }
43
+ expectExact(value, caseSensitive = true) {
44
+ this._expected = { type: "exact", value, caseSensitive };
45
+ return this;
46
+ }
47
+ expectContains(values, mode = "all") {
48
+ this._expected = { type: "contains", values, mode };
49
+ return this;
50
+ }
51
+ expectNotContains(values, mode = "all") {
52
+ this._expected = { type: "not_contains", values, mode };
53
+ return this;
54
+ }
55
+ expectRegex(pattern, flags) {
56
+ this._expected = { type: "regex", pattern, flags };
57
+ return this;
58
+ }
59
+ expectFuzzy(value, threshold = 0.8) {
60
+ this._expected = { type: "fuzzy", value, threshold };
61
+ return this;
62
+ }
63
+ expectJsonSchema(schema) {
64
+ this._expected = { type: "json_schema", schema };
65
+ return this;
66
+ }
67
+ expectLLMGrade(rubric, options) {
68
+ this._expected = {
69
+ type: "llm_grader",
70
+ rubric,
71
+ threshold: options?.threshold ?? 0.7,
72
+ model: options?.model,
73
+ provider: options?.provider,
74
+ strict: options?.strict ?? false
75
+ };
76
+ return this;
77
+ }
78
+ expectSimilarity(value, options) {
79
+ this._expected = {
80
+ type: "similarity",
81
+ value,
82
+ threshold: options?.threshold ?? 0.75,
83
+ mode: options?.mode,
84
+ model: options?.model,
85
+ embeddingModel: options?.embeddingModel
86
+ };
87
+ return this;
88
+ }
89
+ expectInline(expression, value) {
90
+ this._expected = { type: "inline", expression, value };
91
+ return this;
92
+ }
93
+ expectAll(...expectations) {
94
+ this._expected = { type: "combined", operator: "and", expectations };
95
+ return this;
96
+ }
97
+ expectAny(...expectations) {
98
+ this._expected = { type: "combined", operator: "or", expectations };
99
+ return this;
100
+ }
101
+ expect(expected) {
102
+ this._expected = expected;
103
+ return this;
104
+ }
105
+ tags(...tags) {
106
+ this._tags.push(...tags);
107
+ return this;
108
+ }
109
+ metadata(data) {
110
+ this._metadata = { ...this._metadata, ...data };
111
+ return this;
112
+ }
113
+ timeout(ms) {
114
+ this._timeout = ms;
115
+ return this;
116
+ }
117
+ retries(count) {
118
+ this._retries = count;
119
+ return this;
120
+ }
121
+ provider(provider) {
122
+ this._provider = provider;
123
+ return this;
124
+ }
125
+ model(model) {
126
+ this._model = model;
127
+ return this;
128
+ }
129
+ variables(vars) {
130
+ this._variables = vars;
131
+ return this;
132
+ }
133
+ redact(config) {
134
+ this._redaction = config;
135
+ return this;
136
+ }
137
+ build() {
138
+ if (!this._expected) {
139
+ throw new Error(`TestCase "${this._id}" must have an expectation. Call one of the expect* methods.`);
140
+ }
141
+ const hasEmptyPrompt = !this._prompt || typeof this._prompt === "string" && this._prompt.length === 0 || Array.isArray(this._prompt) && this._prompt.length === 0;
142
+ if (hasEmptyPrompt) {
143
+ throw new Error(`TestCase "${this._id}" must have a prompt.`);
144
+ }
145
+ return {
146
+ id: this._id,
147
+ name: this._name,
148
+ description: this._description,
149
+ prompt: this._prompt,
150
+ expected: this._expected,
151
+ tags: this._tags,
152
+ metadata: this._metadata,
153
+ timeout: this._timeout,
154
+ retries: this._retries,
155
+ provider: this._provider,
156
+ model: this._model,
157
+ variables: this._variables,
158
+ redaction: this._redaction
159
+ };
160
+ }
161
+ }
162
+
163
+ class ScenarioBuilder {
164
+ _name;
165
+ _description;
166
+ _version = "1.0";
167
+ _provider;
168
+ _model;
169
+ _providerConfig;
170
+ _seed;
171
+ _temperature;
172
+ _maxTokens;
173
+ _tags = [];
174
+ _variables;
175
+ _redaction;
176
+ _systemPrompt;
177
+ _cases = [];
178
+ constructor(name) {
179
+ this._name = name;
180
+ }
181
+ description(description) {
182
+ this._description = description;
183
+ return this;
184
+ }
185
+ version(version) {
186
+ this._version = version;
187
+ return this;
188
+ }
189
+ provider(provider) {
190
+ this._provider = provider;
191
+ return this;
192
+ }
193
+ model(model) {
194
+ this._model = model;
195
+ return this;
196
+ }
197
+ providerConfig(config) {
198
+ this._providerConfig = config;
199
+ return this;
200
+ }
201
+ seed(seed) {
202
+ this._seed = seed;
203
+ return this;
204
+ }
205
+ temperature(temp) {
206
+ this._temperature = temp;
207
+ return this;
208
+ }
209
+ maxTokens(tokens) {
210
+ this._maxTokens = tokens;
211
+ return this;
212
+ }
213
+ tags(...tags) {
214
+ this._tags.push(...tags);
215
+ return this;
216
+ }
217
+ variables(vars) {
218
+ this._variables = vars;
219
+ return this;
220
+ }
221
+ redact(config) {
222
+ this._redaction = config;
223
+ return this;
224
+ }
225
+ systemPrompt(prompt) {
226
+ this._systemPrompt = prompt;
227
+ return this;
228
+ }
229
+ case(caseBuilder) {
230
+ this._cases.push(caseBuilder.build());
231
+ return this;
232
+ }
233
+ addCase(testCase) {
234
+ this._cases.push(testCase);
235
+ return this;
236
+ }
237
+ cases(...caseBuilders) {
238
+ for (const builder of caseBuilders) {
239
+ this._cases.push(builder.build());
240
+ }
241
+ return this;
242
+ }
243
+ build() {
244
+ if (this._cases.length === 0) {
245
+ throw new Error(`Scenario "${this._name}" must have at least one test case.`);
246
+ }
247
+ return {
248
+ name: this._name,
249
+ description: this._description,
250
+ version: this._version,
251
+ provider: this._provider,
252
+ model: this._model,
253
+ providerConfig: this._providerConfig,
254
+ seed: this._seed,
255
+ temperature: this._temperature,
256
+ maxTokens: this._maxTokens,
257
+ tags: this._tags,
258
+ variables: this._variables,
259
+ redaction: this._redaction,
260
+ setup: this._systemPrompt ? { systemPrompt: this._systemPrompt } : undefined,
261
+ cases: this._cases
262
+ };
263
+ }
264
+ }
265
+ function scenario(name) {
266
+ return new ScenarioBuilder(name);
267
+ }
268
+ function testCase(id) {
269
+ return new TestCaseBuilder(id);
270
+ }
271
+ function containsCase(id, prompt, values, mode = "all") {
272
+ return testCase(id).prompt(prompt).expectContains(values, mode).build();
273
+ }
274
+ function exactCase(id, prompt, value) {
275
+ return testCase(id).prompt(prompt).expectExact(value).build();
276
+ }
277
+ function regexCase(id, prompt, pattern, flags) {
278
+ return testCase(id).prompt(prompt).expectRegex(pattern, flags).build();
279
+ }
280
+ function jsonCase(id, prompt, schema) {
281
+ return testCase(id).prompt(prompt).expectJsonSchema(schema).build();
282
+ }
283
+ function gradedCase(id, prompt, rubric, threshold = 0.7) {
284
+ return testCase(id).prompt(prompt).expectLLMGrade(rubric, { threshold }).build();
285
+ }
286
+ function exact(value, caseSensitive = true) {
287
+ return { type: "exact", value, caseSensitive };
288
+ }
289
+ function contains(values, mode = "all") {
290
+ return { type: "contains", values, mode };
291
+ }
292
+ function notContains(values, mode = "all") {
293
+ return { type: "not_contains", values, mode };
294
+ }
295
+ function regex(pattern, flags) {
296
+ return { type: "regex", pattern, flags };
297
+ }
298
+ function fuzzy(value, threshold = 0.8) {
299
+ return { type: "fuzzy", value, threshold };
300
+ }
301
+ function jsonSchema(schema) {
302
+ return { type: "json_schema", schema };
303
+ }
304
+ function llmGrade(rubric, options) {
305
+ return {
306
+ type: "llm_grader",
307
+ rubric,
308
+ threshold: options?.threshold ?? 0.7,
309
+ model: options?.model,
310
+ provider: options?.provider,
311
+ strict: options?.strict ?? false
312
+ };
313
+ }
314
+ function similarity(value, options) {
315
+ return {
316
+ type: "similarity",
317
+ value,
318
+ threshold: options?.threshold ?? 0.75,
319
+ mode: options?.mode,
320
+ model: options?.model,
321
+ embeddingModel: options?.embeddingModel
322
+ };
323
+ }
324
+ function inline(expression, value) {
325
+ return { type: "inline", expression, value };
326
+ }
327
+ function allOf(...expectations) {
328
+ return { type: "combined", operator: "and", expectations };
329
+ }
330
+ function anyOf(...expectations) {
331
+ return { type: "combined", operator: "or", expectations };
332
+ }
333
+ export {
334
+ ScenarioBuilder,
335
+ TestCaseBuilder,
336
+ allOf,
337
+ anyOf,
338
+ contains,
339
+ containsCase,
340
+ exact,
341
+ exactCase,
342
+ fuzzy,
343
+ gradedCase,
344
+ inline,
345
+ jsonCase,
346
+ jsonSchema,
347
+ llmGrade,
348
+ notContains,
349
+ regex,
350
+ regexCase,
351
+ scenario,
352
+ similarity,
353
+ testCase
354
+ };