@kindlm/core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/index.cjs +40 -0
  2. package/dist/index.cjs.map +1 -0
  3. package/dist/index.d.cts +3061 -0
  4. package/dist/index.d.ts +3061 -0
  5. package/dist/index.js +40 -0
  6. package/dist/index.js.map +1 -0
  7. package/package.json +57 -0
package/dist/index.d.ts ADDED
@@ -0,0 +1,3061 @@
1
+ import { z } from 'zod';
2
+
3
+ interface HttpClient {
4
+ fetch(url: string, init: HttpRequestInit): Promise<HttpResponse>;
5
+ }
6
+ interface HttpRequestInit {
7
+ method: string;
8
+ headers: Record<string, string>;
9
+ body: string;
10
+ timeoutMs?: number;
11
+ }
12
+ interface HttpResponse {
13
+ ok: boolean;
14
+ status: number;
15
+ json(): Promise<unknown>;
16
+ }
17
+ interface ProviderMessage {
18
+ role: "system" | "user" | "assistant" | "tool";
19
+ content: string;
20
+ toolCallId?: string;
21
+ toolName?: string;
22
+ toolCalls?: ProviderToolCall[];
23
+ }
24
+ interface ProviderToolDefinition {
25
+ name: string;
26
+ description?: string;
27
+ parameters?: Record<string, unknown>;
28
+ }
29
+ interface ProviderToolCall {
30
+ id: string;
31
+ name: string;
32
+ arguments: Record<string, unknown>;
33
+ }
34
+ interface ProviderRequest {
35
+ model: string;
36
+ messages: ProviderMessage[];
37
+ params: {
38
+ temperature: number;
39
+ maxTokens: number;
40
+ topP?: number;
41
+ stopSequences?: string[];
42
+ seed?: number;
43
+ };
44
+ tools?: ProviderToolDefinition[];
45
+ toolChoice?: "auto" | "required" | "none";
46
+ }
47
+ interface ProviderResponse {
48
+ text: string;
49
+ toolCalls: ProviderToolCall[];
50
+ usage: {
51
+ inputTokens: number;
52
+ outputTokens: number;
53
+ totalTokens: number;
54
+ };
55
+ raw: unknown;
56
+ latencyMs: number;
57
+ modelId: string;
58
+ finishReason: "stop" | "max_tokens" | "tool_calls" | "error" | "unknown";
59
+ }
60
+ type ProviderErrorCode = "AUTH_FAILED" | "RATE_LIMITED" | "TIMEOUT" | "MODEL_NOT_FOUND" | "CONTEXT_LENGTH" | "CONTENT_FILTERED" | "NETWORK_ERROR" | "PROVIDER_ERROR" | "UNKNOWN";
61
+ declare class ProviderError extends Error {
62
+ code: ProviderErrorCode;
63
+ statusCode?: number | undefined;
64
+ retryable: boolean;
65
+ raw?: unknown | undefined;
66
+ constructor(code: ProviderErrorCode, message: string, statusCode?: number | undefined, retryable?: boolean, raw?: unknown | undefined);
67
+ }
68
+ interface ProviderAdapter {
69
+ readonly name: string;
70
+ initialize(config: ProviderAdapterConfig): Promise<void>;
71
+ complete(request: ProviderRequest): Promise<ProviderResponse>;
72
+ estimateCost(model: string, usage: ProviderResponse["usage"]): number | null;
73
+ supportsTools(model: string): boolean;
74
+ }
75
+ interface ProviderAdapterConfig {
76
+ apiKey: string;
77
+ baseUrl?: string;
78
+ organization?: string;
79
+ timeoutMs: number;
80
+ maxRetries: number;
81
+ }
82
+ interface ConversationTurn {
83
+ request: ProviderRequest;
84
+ response: ProviderResponse;
85
+ }
86
+ interface ConversationResult {
87
+ turns: ConversationTurn[];
88
+ finalText: string;
89
+ allToolCalls: ProviderToolCall[];
90
+ totalUsage: ProviderResponse["usage"];
91
+ totalLatencyMs: number;
92
+ }
93
+
94
+ type Result<T, E = KindlmError> = {
95
+ success: true;
96
+ data: T;
97
+ } | {
98
+ success: false;
99
+ error: E;
100
+ };
101
+ interface KindlmError {
102
+ code: ErrorCode;
103
+ message: string;
104
+ details?: Record<string, unknown>;
105
+ cause?: Error;
106
+ }
107
+ type ErrorCode = "CONFIG_NOT_FOUND" | "CONFIG_PARSE_ERROR" | "CONFIG_VALIDATION_ERROR" | "CONFIG_FILE_REF_ERROR" | "CONFIG_TOO_LARGE" | "PATH_TRAVERSAL" | "PROVIDER_NOT_FOUND" | "PROVIDER_AUTH_ERROR" | "PROVIDER_RATE_LIMIT" | "PROVIDER_TIMEOUT" | "PROVIDER_API_ERROR" | "PROVIDER_NETWORK_ERROR" | "ASSERTION_EVAL_ERROR" | "SCHEMA_FILE_ERROR" | "JUDGE_EVAL_ERROR" | "ENGINE_MAX_TURNS" | "ENGINE_EMPTY_RESPONSE" | "BASELINE_NOT_FOUND" | "BASELINE_CORRUPT" | "BASELINE_VERSION_MISMATCH" | "CLOUD_AUTH_ERROR" | "CLOUD_UPLOAD_ERROR" | "CLOUD_PLAN_LIMIT" | "CLOUD_RATE_LIMIT" | "UNKNOWN_ERROR";
108
+ declare function ok<T>(data: T): Result<T, never>;
109
+ declare function err<E = KindlmError>(error: E): Result<never, E>;
110
+
111
+ declare const ProviderConfigSchema: z.ZodObject<{
112
+ apiKeyEnv: z.ZodString;
113
+ baseUrl: z.ZodOptional<z.ZodString>;
114
+ organization: z.ZodOptional<z.ZodString>;
115
+ }, "strip", z.ZodTypeAny, {
116
+ apiKeyEnv: string;
117
+ baseUrl?: string | undefined;
118
+ organization?: string | undefined;
119
+ }, {
120
+ apiKeyEnv: string;
121
+ baseUrl?: string | undefined;
122
+ organization?: string | undefined;
123
+ }>;
124
+ declare const ModelSchema: z.ZodObject<{
125
+ id: z.ZodString;
126
+ provider: z.ZodEnum<["openai", "anthropic", "ollama", "gemini", "mistral", "cohere"]>;
127
+ model: z.ZodString;
128
+ params: z.ZodDefault<z.ZodObject<{
129
+ temperature: z.ZodDefault<z.ZodNumber>;
130
+ maxTokens: z.ZodDefault<z.ZodNumber>;
131
+ topP: z.ZodOptional<z.ZodNumber>;
132
+ stopSequences: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
133
+ seed: z.ZodOptional<z.ZodNumber>;
134
+ }, "strip", z.ZodTypeAny, {
135
+ temperature: number;
136
+ maxTokens: number;
137
+ topP?: number | undefined;
138
+ stopSequences?: string[] | undefined;
139
+ seed?: number | undefined;
140
+ }, {
141
+ temperature?: number | undefined;
142
+ maxTokens?: number | undefined;
143
+ topP?: number | undefined;
144
+ stopSequences?: string[] | undefined;
145
+ seed?: number | undefined;
146
+ }>>;
147
+ }, "strip", z.ZodTypeAny, {
148
+ params: {
149
+ temperature: number;
150
+ maxTokens: number;
151
+ topP?: number | undefined;
152
+ stopSequences?: string[] | undefined;
153
+ seed?: number | undefined;
154
+ };
155
+ id: string;
156
+ provider: "openai" | "anthropic" | "ollama" | "gemini" | "mistral" | "cohere";
157
+ model: string;
158
+ }, {
159
+ id: string;
160
+ provider: "openai" | "anthropic" | "ollama" | "gemini" | "mistral" | "cohere";
161
+ model: string;
162
+ params?: {
163
+ temperature?: number | undefined;
164
+ maxTokens?: number | undefined;
165
+ topP?: number | undefined;
166
+ stopSequences?: string[] | undefined;
167
+ seed?: number | undefined;
168
+ } | undefined;
169
+ }>;
170
+ declare const JudgeCriterionSchema: z.ZodObject<{
171
+ criteria: z.ZodString;
172
+ minScore: z.ZodDefault<z.ZodNumber>;
173
+ model: z.ZodOptional<z.ZodString>;
174
+ rubric: z.ZodOptional<z.ZodString>;
175
+ }, "strip", z.ZodTypeAny, {
176
+ criteria: string;
177
+ minScore: number;
178
+ model?: string | undefined;
179
+ rubric?: string | undefined;
180
+ }, {
181
+ criteria: string;
182
+ model?: string | undefined;
183
+ minScore?: number | undefined;
184
+ rubric?: string | undefined;
185
+ }>;
186
+ declare const ToolCallExpectSchema: z.ZodObject<{
187
+ tool: z.ZodString;
188
+ shouldNotCall: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
189
+ argsMatch: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
190
+ argsSchema: z.ZodOptional<z.ZodString>;
191
+ order: z.ZodOptional<z.ZodNumber>;
192
+ responseContains: z.ZodOptional<z.ZodString>;
193
+ }, "strip", z.ZodTypeAny, {
194
+ tool: string;
195
+ shouldNotCall: boolean;
196
+ argsMatch?: Record<string, unknown> | undefined;
197
+ argsSchema?: string | undefined;
198
+ order?: number | undefined;
199
+ responseContains?: string | undefined;
200
+ }, {
201
+ tool: string;
202
+ shouldNotCall?: boolean | undefined;
203
+ argsMatch?: Record<string, unknown> | undefined;
204
+ argsSchema?: string | undefined;
205
+ order?: number | undefined;
206
+ responseContains?: string | undefined;
207
+ }>;
208
+ declare const ExpectSchema: z.ZodObject<{
209
+ output: z.ZodOptional<z.ZodEffects<z.ZodObject<{
210
+ format: z.ZodDefault<z.ZodEnum<["text", "json"]>>;
211
+ schemaFile: z.ZodOptional<z.ZodString>;
212
+ contains: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
213
+ notContains: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
214
+ maxLength: z.ZodOptional<z.ZodNumber>;
215
+ }, "strip", z.ZodTypeAny, {
216
+ format: "text" | "json";
217
+ schemaFile?: string | undefined;
218
+ contains?: string[] | undefined;
219
+ notContains?: string[] | undefined;
220
+ maxLength?: number | undefined;
221
+ }, {
222
+ format?: "text" | "json" | undefined;
223
+ schemaFile?: string | undefined;
224
+ contains?: string[] | undefined;
225
+ notContains?: string[] | undefined;
226
+ maxLength?: number | undefined;
227
+ }>, {
228
+ format: "text" | "json";
229
+ schemaFile?: string | undefined;
230
+ contains?: string[] | undefined;
231
+ notContains?: string[] | undefined;
232
+ maxLength?: number | undefined;
233
+ }, {
234
+ format?: "text" | "json" | undefined;
235
+ schemaFile?: string | undefined;
236
+ contains?: string[] | undefined;
237
+ notContains?: string[] | undefined;
238
+ maxLength?: number | undefined;
239
+ }>>;
240
+ guardrails: z.ZodOptional<z.ZodObject<{
241
+ pii: z.ZodOptional<z.ZodObject<{
242
+ enabled: z.ZodDefault<z.ZodBoolean>;
243
+ denyPatterns: z.ZodDefault<z.ZodArray<z.ZodEffects<z.ZodString, string, string>, "many">>;
244
+ customPatterns: z.ZodOptional<z.ZodArray<z.ZodObject<{
245
+ name: z.ZodString;
246
+ pattern: z.ZodEffects<z.ZodString, string, string>;
247
+ }, "strip", z.ZodTypeAny, {
248
+ name: string;
249
+ pattern: string;
250
+ }, {
251
+ name: string;
252
+ pattern: string;
253
+ }>, "many">>;
254
+ }, "strip", z.ZodTypeAny, {
255
+ enabled: boolean;
256
+ denyPatterns: string[];
257
+ customPatterns?: {
258
+ name: string;
259
+ pattern: string;
260
+ }[] | undefined;
261
+ }, {
262
+ enabled?: boolean | undefined;
263
+ denyPatterns?: string[] | undefined;
264
+ customPatterns?: {
265
+ name: string;
266
+ pattern: string;
267
+ }[] | undefined;
268
+ }>>;
269
+ keywords: z.ZodOptional<z.ZodObject<{
270
+ deny: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
271
+ allow: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
272
+ }, "strip", z.ZodTypeAny, {
273
+ deny: string[];
274
+ allow?: string[] | undefined;
275
+ }, {
276
+ deny?: string[] | undefined;
277
+ allow?: string[] | undefined;
278
+ }>>;
279
+ }, "strip", z.ZodTypeAny, {
280
+ pii?: {
281
+ enabled: boolean;
282
+ denyPatterns: string[];
283
+ customPatterns?: {
284
+ name: string;
285
+ pattern: string;
286
+ }[] | undefined;
287
+ } | undefined;
288
+ keywords?: {
289
+ deny: string[];
290
+ allow?: string[] | undefined;
291
+ } | undefined;
292
+ }, {
293
+ pii?: {
294
+ enabled?: boolean | undefined;
295
+ denyPatterns?: string[] | undefined;
296
+ customPatterns?: {
297
+ name: string;
298
+ pattern: string;
299
+ }[] | undefined;
300
+ } | undefined;
301
+ keywords?: {
302
+ deny?: string[] | undefined;
303
+ allow?: string[] | undefined;
304
+ } | undefined;
305
+ }>>;
306
+ judge: z.ZodOptional<z.ZodArray<z.ZodObject<{
307
+ criteria: z.ZodString;
308
+ minScore: z.ZodDefault<z.ZodNumber>;
309
+ model: z.ZodOptional<z.ZodString>;
310
+ rubric: z.ZodOptional<z.ZodString>;
311
+ }, "strip", z.ZodTypeAny, {
312
+ criteria: string;
313
+ minScore: number;
314
+ model?: string | undefined;
315
+ rubric?: string | undefined;
316
+ }, {
317
+ criteria: string;
318
+ model?: string | undefined;
319
+ minScore?: number | undefined;
320
+ rubric?: string | undefined;
321
+ }>, "many">>;
322
+ toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
323
+ tool: z.ZodString;
324
+ shouldNotCall: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
325
+ argsMatch: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
326
+ argsSchema: z.ZodOptional<z.ZodString>;
327
+ order: z.ZodOptional<z.ZodNumber>;
328
+ responseContains: z.ZodOptional<z.ZodString>;
329
+ }, "strip", z.ZodTypeAny, {
330
+ tool: string;
331
+ shouldNotCall: boolean;
332
+ argsMatch?: Record<string, unknown> | undefined;
333
+ argsSchema?: string | undefined;
334
+ order?: number | undefined;
335
+ responseContains?: string | undefined;
336
+ }, {
337
+ tool: string;
338
+ shouldNotCall?: boolean | undefined;
339
+ argsMatch?: Record<string, unknown> | undefined;
340
+ argsSchema?: string | undefined;
341
+ order?: number | undefined;
342
+ responseContains?: string | undefined;
343
+ }>, "many">>;
344
+ baseline: z.ZodOptional<z.ZodObject<{
345
+ drift: z.ZodOptional<z.ZodObject<{
346
+ maxScore: z.ZodDefault<z.ZodNumber>;
347
+ method: z.ZodDefault<z.ZodEnum<["judge", "embedding", "field-diff"]>>;
348
+ fields: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
349
+ }, "strip", z.ZodTypeAny, {
350
+ maxScore: number;
351
+ method: "judge" | "embedding" | "field-diff";
352
+ fields?: string[] | undefined;
353
+ }, {
354
+ maxScore?: number | undefined;
355
+ method?: "judge" | "embedding" | "field-diff" | undefined;
356
+ fields?: string[] | undefined;
357
+ }>>;
358
+ }, "strip", z.ZodTypeAny, {
359
+ drift?: {
360
+ maxScore: number;
361
+ method: "judge" | "embedding" | "field-diff";
362
+ fields?: string[] | undefined;
363
+ } | undefined;
364
+ }, {
365
+ drift?: {
366
+ maxScore?: number | undefined;
367
+ method?: "judge" | "embedding" | "field-diff" | undefined;
368
+ fields?: string[] | undefined;
369
+ } | undefined;
370
+ }>>;
371
+ }, "strip", z.ZodTypeAny, {
372
+ judge?: {
373
+ criteria: string;
374
+ minScore: number;
375
+ model?: string | undefined;
376
+ rubric?: string | undefined;
377
+ }[] | undefined;
378
+ output?: {
379
+ format: "text" | "json";
380
+ schemaFile?: string | undefined;
381
+ contains?: string[] | undefined;
382
+ notContains?: string[] | undefined;
383
+ maxLength?: number | undefined;
384
+ } | undefined;
385
+ guardrails?: {
386
+ pii?: {
387
+ enabled: boolean;
388
+ denyPatterns: string[];
389
+ customPatterns?: {
390
+ name: string;
391
+ pattern: string;
392
+ }[] | undefined;
393
+ } | undefined;
394
+ keywords?: {
395
+ deny: string[];
396
+ allow?: string[] | undefined;
397
+ } | undefined;
398
+ } | undefined;
399
+ toolCalls?: {
400
+ tool: string;
401
+ shouldNotCall: boolean;
402
+ argsMatch?: Record<string, unknown> | undefined;
403
+ argsSchema?: string | undefined;
404
+ order?: number | undefined;
405
+ responseContains?: string | undefined;
406
+ }[] | undefined;
407
+ baseline?: {
408
+ drift?: {
409
+ maxScore: number;
410
+ method: "judge" | "embedding" | "field-diff";
411
+ fields?: string[] | undefined;
412
+ } | undefined;
413
+ } | undefined;
414
+ }, {
415
+ judge?: {
416
+ criteria: string;
417
+ model?: string | undefined;
418
+ minScore?: number | undefined;
419
+ rubric?: string | undefined;
420
+ }[] | undefined;
421
+ output?: {
422
+ format?: "text" | "json" | undefined;
423
+ schemaFile?: string | undefined;
424
+ contains?: string[] | undefined;
425
+ notContains?: string[] | undefined;
426
+ maxLength?: number | undefined;
427
+ } | undefined;
428
+ guardrails?: {
429
+ pii?: {
430
+ enabled?: boolean | undefined;
431
+ denyPatterns?: string[] | undefined;
432
+ customPatterns?: {
433
+ name: string;
434
+ pattern: string;
435
+ }[] | undefined;
436
+ } | undefined;
437
+ keywords?: {
438
+ deny?: string[] | undefined;
439
+ allow?: string[] | undefined;
440
+ } | undefined;
441
+ } | undefined;
442
+ toolCalls?: {
443
+ tool: string;
444
+ shouldNotCall?: boolean | undefined;
445
+ argsMatch?: Record<string, unknown> | undefined;
446
+ argsSchema?: string | undefined;
447
+ order?: number | undefined;
448
+ responseContains?: string | undefined;
449
+ }[] | undefined;
450
+ baseline?: {
451
+ drift?: {
452
+ maxScore?: number | undefined;
453
+ method?: "judge" | "embedding" | "field-diff" | undefined;
454
+ fields?: string[] | undefined;
455
+ } | undefined;
456
+ } | undefined;
457
+ }>;
458
+ declare const ToolSimulationSchema: z.ZodObject<{
459
+ name: z.ZodString;
460
+ description: z.ZodOptional<z.ZodString>;
461
+ parameters: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
462
+ responses: z.ZodOptional<z.ZodArray<z.ZodObject<{
463
+ when: z.ZodRecord<z.ZodString, z.ZodUnknown>;
464
+ then: z.ZodUnknown;
465
+ }, "strip", z.ZodTypeAny, {
466
+ when: Record<string, unknown>;
467
+ then?: unknown;
468
+ }, {
469
+ when: Record<string, unknown>;
470
+ then?: unknown;
471
+ }>, "many">>;
472
+ defaultResponse: z.ZodOptional<z.ZodUnknown>;
473
+ }, "strip", z.ZodTypeAny, {
474
+ name: string;
475
+ description?: string | undefined;
476
+ parameters?: Record<string, unknown> | undefined;
477
+ responses?: {
478
+ when: Record<string, unknown>;
479
+ then?: unknown;
480
+ }[] | undefined;
481
+ defaultResponse?: unknown;
482
+ }, {
483
+ name: string;
484
+ description?: string | undefined;
485
+ parameters?: Record<string, unknown> | undefined;
486
+ responses?: {
487
+ when: Record<string, unknown>;
488
+ then?: unknown;
489
+ }[] | undefined;
490
+ defaultResponse?: unknown;
491
+ }>;
492
+ declare const TestCaseSchema: z.ZodEffects<z.ZodObject<{
493
+ name: z.ZodString;
494
+ prompt: z.ZodOptional<z.ZodString>;
495
+ command: z.ZodOptional<z.ZodString>;
496
+ vars: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
497
+ models: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
498
+ repeat: z.ZodOptional<z.ZodNumber>;
499
+ tools: z.ZodOptional<z.ZodArray<z.ZodObject<{
500
+ name: z.ZodString;
501
+ description: z.ZodOptional<z.ZodString>;
502
+ parameters: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
503
+ responses: z.ZodOptional<z.ZodArray<z.ZodObject<{
504
+ when: z.ZodRecord<z.ZodString, z.ZodUnknown>;
505
+ then: z.ZodUnknown;
506
+ }, "strip", z.ZodTypeAny, {
507
+ when: Record<string, unknown>;
508
+ then?: unknown;
509
+ }, {
510
+ when: Record<string, unknown>;
511
+ then?: unknown;
512
+ }>, "many">>;
513
+ defaultResponse: z.ZodOptional<z.ZodUnknown>;
514
+ }, "strip", z.ZodTypeAny, {
515
+ name: string;
516
+ description?: string | undefined;
517
+ parameters?: Record<string, unknown> | undefined;
518
+ responses?: {
519
+ when: Record<string, unknown>;
520
+ then?: unknown;
521
+ }[] | undefined;
522
+ defaultResponse?: unknown;
523
+ }, {
524
+ name: string;
525
+ description?: string | undefined;
526
+ parameters?: Record<string, unknown> | undefined;
527
+ responses?: {
528
+ when: Record<string, unknown>;
529
+ then?: unknown;
530
+ }[] | undefined;
531
+ defaultResponse?: unknown;
532
+ }>, "many">>;
533
+ expect: z.ZodObject<{
534
+ output: z.ZodOptional<z.ZodEffects<z.ZodObject<{
535
+ format: z.ZodDefault<z.ZodEnum<["text", "json"]>>;
536
+ schemaFile: z.ZodOptional<z.ZodString>;
537
+ contains: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
538
+ notContains: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
539
+ maxLength: z.ZodOptional<z.ZodNumber>;
540
+ }, "strip", z.ZodTypeAny, {
541
+ format: "text" | "json";
542
+ schemaFile?: string | undefined;
543
+ contains?: string[] | undefined;
544
+ notContains?: string[] | undefined;
545
+ maxLength?: number | undefined;
546
+ }, {
547
+ format?: "text" | "json" | undefined;
548
+ schemaFile?: string | undefined;
549
+ contains?: string[] | undefined;
550
+ notContains?: string[] | undefined;
551
+ maxLength?: number | undefined;
552
+ }>, {
553
+ format: "text" | "json";
554
+ schemaFile?: string | undefined;
555
+ contains?: string[] | undefined;
556
+ notContains?: string[] | undefined;
557
+ maxLength?: number | undefined;
558
+ }, {
559
+ format?: "text" | "json" | undefined;
560
+ schemaFile?: string | undefined;
561
+ contains?: string[] | undefined;
562
+ notContains?: string[] | undefined;
563
+ maxLength?: number | undefined;
564
+ }>>;
565
+ guardrails: z.ZodOptional<z.ZodObject<{
566
+ pii: z.ZodOptional<z.ZodObject<{
567
+ enabled: z.ZodDefault<z.ZodBoolean>;
568
+ denyPatterns: z.ZodDefault<z.ZodArray<z.ZodEffects<z.ZodString, string, string>, "many">>;
569
+ customPatterns: z.ZodOptional<z.ZodArray<z.ZodObject<{
570
+ name: z.ZodString;
571
+ pattern: z.ZodEffects<z.ZodString, string, string>;
572
+ }, "strip", z.ZodTypeAny, {
573
+ name: string;
574
+ pattern: string;
575
+ }, {
576
+ name: string;
577
+ pattern: string;
578
+ }>, "many">>;
579
+ }, "strip", z.ZodTypeAny, {
580
+ enabled: boolean;
581
+ denyPatterns: string[];
582
+ customPatterns?: {
583
+ name: string;
584
+ pattern: string;
585
+ }[] | undefined;
586
+ }, {
587
+ enabled?: boolean | undefined;
588
+ denyPatterns?: string[] | undefined;
589
+ customPatterns?: {
590
+ name: string;
591
+ pattern: string;
592
+ }[] | undefined;
593
+ }>>;
594
+ keywords: z.ZodOptional<z.ZodObject<{
595
+ deny: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
596
+ allow: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
597
+ }, "strip", z.ZodTypeAny, {
598
+ deny: string[];
599
+ allow?: string[] | undefined;
600
+ }, {
601
+ deny?: string[] | undefined;
602
+ allow?: string[] | undefined;
603
+ }>>;
604
+ }, "strip", z.ZodTypeAny, {
605
+ pii?: {
606
+ enabled: boolean;
607
+ denyPatterns: string[];
608
+ customPatterns?: {
609
+ name: string;
610
+ pattern: string;
611
+ }[] | undefined;
612
+ } | undefined;
613
+ keywords?: {
614
+ deny: string[];
615
+ allow?: string[] | undefined;
616
+ } | undefined;
617
+ }, {
618
+ pii?: {
619
+ enabled?: boolean | undefined;
620
+ denyPatterns?: string[] | undefined;
621
+ customPatterns?: {
622
+ name: string;
623
+ pattern: string;
624
+ }[] | undefined;
625
+ } | undefined;
626
+ keywords?: {
627
+ deny?: string[] | undefined;
628
+ allow?: string[] | undefined;
629
+ } | undefined;
630
+ }>>;
631
+ judge: z.ZodOptional<z.ZodArray<z.ZodObject<{
632
+ criteria: z.ZodString;
633
+ minScore: z.ZodDefault<z.ZodNumber>;
634
+ model: z.ZodOptional<z.ZodString>;
635
+ rubric: z.ZodOptional<z.ZodString>;
636
+ }, "strip", z.ZodTypeAny, {
637
+ criteria: string;
638
+ minScore: number;
639
+ model?: string | undefined;
640
+ rubric?: string | undefined;
641
+ }, {
642
+ criteria: string;
643
+ model?: string | undefined;
644
+ minScore?: number | undefined;
645
+ rubric?: string | undefined;
646
+ }>, "many">>;
647
+ toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
648
+ tool: z.ZodString;
649
+ shouldNotCall: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
650
+ argsMatch: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
651
+ argsSchema: z.ZodOptional<z.ZodString>;
652
+ order: z.ZodOptional<z.ZodNumber>;
653
+ responseContains: z.ZodOptional<z.ZodString>;
654
+ }, "strip", z.ZodTypeAny, {
655
+ tool: string;
656
+ shouldNotCall: boolean;
657
+ argsMatch?: Record<string, unknown> | undefined;
658
+ argsSchema?: string | undefined;
659
+ order?: number | undefined;
660
+ responseContains?: string | undefined;
661
+ }, {
662
+ tool: string;
663
+ shouldNotCall?: boolean | undefined;
664
+ argsMatch?: Record<string, unknown> | undefined;
665
+ argsSchema?: string | undefined;
666
+ order?: number | undefined;
667
+ responseContains?: string | undefined;
668
+ }>, "many">>;
669
+ baseline: z.ZodOptional<z.ZodObject<{
670
+ drift: z.ZodOptional<z.ZodObject<{
671
+ maxScore: z.ZodDefault<z.ZodNumber>;
672
+ method: z.ZodDefault<z.ZodEnum<["judge", "embedding", "field-diff"]>>;
673
+ fields: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
674
+ }, "strip", z.ZodTypeAny, {
675
+ maxScore: number;
676
+ method: "judge" | "embedding" | "field-diff";
677
+ fields?: string[] | undefined;
678
+ }, {
679
+ maxScore?: number | undefined;
680
+ method?: "judge" | "embedding" | "field-diff" | undefined;
681
+ fields?: string[] | undefined;
682
+ }>>;
683
+ }, "strip", z.ZodTypeAny, {
684
+ drift?: {
685
+ maxScore: number;
686
+ method: "judge" | "embedding" | "field-diff";
687
+ fields?: string[] | undefined;
688
+ } | undefined;
689
+ }, {
690
+ drift?: {
691
+ maxScore?: number | undefined;
692
+ method?: "judge" | "embedding" | "field-diff" | undefined;
693
+ fields?: string[] | undefined;
694
+ } | undefined;
695
+ }>>;
696
+ }, "strip", z.ZodTypeAny, {
697
+ judge?: {
698
+ criteria: string;
699
+ minScore: number;
700
+ model?: string | undefined;
701
+ rubric?: string | undefined;
702
+ }[] | undefined;
703
+ output?: {
704
+ format: "text" | "json";
705
+ schemaFile?: string | undefined;
706
+ contains?: string[] | undefined;
707
+ notContains?: string[] | undefined;
708
+ maxLength?: number | undefined;
709
+ } | undefined;
710
+ guardrails?: {
711
+ pii?: {
712
+ enabled: boolean;
713
+ denyPatterns: string[];
714
+ customPatterns?: {
715
+ name: string;
716
+ pattern: string;
717
+ }[] | undefined;
718
+ } | undefined;
719
+ keywords?: {
720
+ deny: string[];
721
+ allow?: string[] | undefined;
722
+ } | undefined;
723
+ } | undefined;
724
+ toolCalls?: {
725
+ tool: string;
726
+ shouldNotCall: boolean;
727
+ argsMatch?: Record<string, unknown> | undefined;
728
+ argsSchema?: string | undefined;
729
+ order?: number | undefined;
730
+ responseContains?: string | undefined;
731
+ }[] | undefined;
732
+ baseline?: {
733
+ drift?: {
734
+ maxScore: number;
735
+ method: "judge" | "embedding" | "field-diff";
736
+ fields?: string[] | undefined;
737
+ } | undefined;
738
+ } | undefined;
739
+ }, {
740
+ judge?: {
741
+ criteria: string;
742
+ model?: string | undefined;
743
+ minScore?: number | undefined;
744
+ rubric?: string | undefined;
745
+ }[] | undefined;
746
+ output?: {
747
+ format?: "text" | "json" | undefined;
748
+ schemaFile?: string | undefined;
749
+ contains?: string[] | undefined;
750
+ notContains?: string[] | undefined;
751
+ maxLength?: number | undefined;
752
+ } | undefined;
753
+ guardrails?: {
754
+ pii?: {
755
+ enabled?: boolean | undefined;
756
+ denyPatterns?: string[] | undefined;
757
+ customPatterns?: {
758
+ name: string;
759
+ pattern: string;
760
+ }[] | undefined;
761
+ } | undefined;
762
+ keywords?: {
763
+ deny?: string[] | undefined;
764
+ allow?: string[] | undefined;
765
+ } | undefined;
766
+ } | undefined;
767
+ toolCalls?: {
768
+ tool: string;
769
+ shouldNotCall?: boolean | undefined;
770
+ argsMatch?: Record<string, unknown> | undefined;
771
+ argsSchema?: string | undefined;
772
+ order?: number | undefined;
773
+ responseContains?: string | undefined;
774
+ }[] | undefined;
775
+ baseline?: {
776
+ drift?: {
777
+ maxScore?: number | undefined;
778
+ method?: "judge" | "embedding" | "field-diff" | undefined;
779
+ fields?: string[] | undefined;
780
+ } | undefined;
781
+ } | undefined;
782
+ }>;
783
+ tags: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
784
+ skip: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
785
+ }, "strip", z.ZodTypeAny, {
786
+ name: string;
787
+ vars: Record<string, string>;
788
+ expect: {
789
+ judge?: {
790
+ criteria: string;
791
+ minScore: number;
792
+ model?: string | undefined;
793
+ rubric?: string | undefined;
794
+ }[] | undefined;
795
+ output?: {
796
+ format: "text" | "json";
797
+ schemaFile?: string | undefined;
798
+ contains?: string[] | undefined;
799
+ notContains?: string[] | undefined;
800
+ maxLength?: number | undefined;
801
+ } | undefined;
802
+ guardrails?: {
803
+ pii?: {
804
+ enabled: boolean;
805
+ denyPatterns: string[];
806
+ customPatterns?: {
807
+ name: string;
808
+ pattern: string;
809
+ }[] | undefined;
810
+ } | undefined;
811
+ keywords?: {
812
+ deny: string[];
813
+ allow?: string[] | undefined;
814
+ } | undefined;
815
+ } | undefined;
816
+ toolCalls?: {
817
+ tool: string;
818
+ shouldNotCall: boolean;
819
+ argsMatch?: Record<string, unknown> | undefined;
820
+ argsSchema?: string | undefined;
821
+ order?: number | undefined;
822
+ responseContains?: string | undefined;
823
+ }[] | undefined;
824
+ baseline?: {
825
+ drift?: {
826
+ maxScore: number;
827
+ method: "judge" | "embedding" | "field-diff";
828
+ fields?: string[] | undefined;
829
+ } | undefined;
830
+ } | undefined;
831
+ };
832
+ skip: boolean;
833
+ prompt?: string | undefined;
834
+ command?: string | undefined;
835
+ models?: string[] | undefined;
836
+ repeat?: number | undefined;
837
+ tools?: {
838
+ name: string;
839
+ description?: string | undefined;
840
+ parameters?: Record<string, unknown> | undefined;
841
+ responses?: {
842
+ when: Record<string, unknown>;
843
+ then?: unknown;
844
+ }[] | undefined;
845
+ defaultResponse?: unknown;
846
+ }[] | undefined;
847
+ tags?: string[] | undefined;
848
+ }, {
849
+ name: string;
850
+ expect: {
851
+ judge?: {
852
+ criteria: string;
853
+ model?: string | undefined;
854
+ minScore?: number | undefined;
855
+ rubric?: string | undefined;
856
+ }[] | undefined;
857
+ output?: {
858
+ format?: "text" | "json" | undefined;
859
+ schemaFile?: string | undefined;
860
+ contains?: string[] | undefined;
861
+ notContains?: string[] | undefined;
862
+ maxLength?: number | undefined;
863
+ } | undefined;
864
+ guardrails?: {
865
+ pii?: {
866
+ enabled?: boolean | undefined;
867
+ denyPatterns?: string[] | undefined;
868
+ customPatterns?: {
869
+ name: string;
870
+ pattern: string;
871
+ }[] | undefined;
872
+ } | undefined;
873
+ keywords?: {
874
+ deny?: string[] | undefined;
875
+ allow?: string[] | undefined;
876
+ } | undefined;
877
+ } | undefined;
878
+ toolCalls?: {
879
+ tool: string;
880
+ shouldNotCall?: boolean | undefined;
881
+ argsMatch?: Record<string, unknown> | undefined;
882
+ argsSchema?: string | undefined;
883
+ order?: number | undefined;
884
+ responseContains?: string | undefined;
885
+ }[] | undefined;
886
+ baseline?: {
887
+ drift?: {
888
+ maxScore?: number | undefined;
889
+ method?: "judge" | "embedding" | "field-diff" | undefined;
890
+ fields?: string[] | undefined;
891
+ } | undefined;
892
+ } | undefined;
893
+ };
894
+ prompt?: string | undefined;
895
+ command?: string | undefined;
896
+ vars?: Record<string, string> | undefined;
897
+ models?: string[] | undefined;
898
+ repeat?: number | undefined;
899
+ tools?: {
900
+ name: string;
901
+ description?: string | undefined;
902
+ parameters?: Record<string, unknown> | undefined;
903
+ responses?: {
904
+ when: Record<string, unknown>;
905
+ then?: unknown;
906
+ }[] | undefined;
907
+ defaultResponse?: unknown;
908
+ }[] | undefined;
909
+ tags?: string[] | undefined;
910
+ skip?: boolean | undefined;
911
+ }>, {
912
+ name: string;
913
+ vars: Record<string, string>;
914
+ expect: {
915
+ judge?: {
916
+ criteria: string;
917
+ minScore: number;
918
+ model?: string | undefined;
919
+ rubric?: string | undefined;
920
+ }[] | undefined;
921
+ output?: {
922
+ format: "text" | "json";
923
+ schemaFile?: string | undefined;
924
+ contains?: string[] | undefined;
925
+ notContains?: string[] | undefined;
926
+ maxLength?: number | undefined;
927
+ } | undefined;
928
+ guardrails?: {
929
+ pii?: {
930
+ enabled: boolean;
931
+ denyPatterns: string[];
932
+ customPatterns?: {
933
+ name: string;
934
+ pattern: string;
935
+ }[] | undefined;
936
+ } | undefined;
937
+ keywords?: {
938
+ deny: string[];
939
+ allow?: string[] | undefined;
940
+ } | undefined;
941
+ } | undefined;
942
+ toolCalls?: {
943
+ tool: string;
944
+ shouldNotCall: boolean;
945
+ argsMatch?: Record<string, unknown> | undefined;
946
+ argsSchema?: string | undefined;
947
+ order?: number | undefined;
948
+ responseContains?: string | undefined;
949
+ }[] | undefined;
950
+ baseline?: {
951
+ drift?: {
952
+ maxScore: number;
953
+ method: "judge" | "embedding" | "field-diff";
954
+ fields?: string[] | undefined;
955
+ } | undefined;
956
+ } | undefined;
957
+ };
958
+ skip: boolean;
959
+ prompt?: string | undefined;
960
+ command?: string | undefined;
961
+ models?: string[] | undefined;
962
+ repeat?: number | undefined;
963
+ tools?: {
964
+ name: string;
965
+ description?: string | undefined;
966
+ parameters?: Record<string, unknown> | undefined;
967
+ responses?: {
968
+ when: Record<string, unknown>;
969
+ then?: unknown;
970
+ }[] | undefined;
971
+ defaultResponse?: unknown;
972
+ }[] | undefined;
973
+ tags?: string[] | undefined;
974
+ }, {
975
+ name: string;
976
+ expect: {
977
+ judge?: {
978
+ criteria: string;
979
+ model?: string | undefined;
980
+ minScore?: number | undefined;
981
+ rubric?: string | undefined;
982
+ }[] | undefined;
983
+ output?: {
984
+ format?: "text" | "json" | undefined;
985
+ schemaFile?: string | undefined;
986
+ contains?: string[] | undefined;
987
+ notContains?: string[] | undefined;
988
+ maxLength?: number | undefined;
989
+ } | undefined;
990
+ guardrails?: {
991
+ pii?: {
992
+ enabled?: boolean | undefined;
993
+ denyPatterns?: string[] | undefined;
994
+ customPatterns?: {
995
+ name: string;
996
+ pattern: string;
997
+ }[] | undefined;
998
+ } | undefined;
999
+ keywords?: {
1000
+ deny?: string[] | undefined;
1001
+ allow?: string[] | undefined;
1002
+ } | undefined;
1003
+ } | undefined;
1004
+ toolCalls?: {
1005
+ tool: string;
1006
+ shouldNotCall?: boolean | undefined;
1007
+ argsMatch?: Record<string, unknown> | undefined;
1008
+ argsSchema?: string | undefined;
1009
+ order?: number | undefined;
1010
+ responseContains?: string | undefined;
1011
+ }[] | undefined;
1012
+ baseline?: {
1013
+ drift?: {
1014
+ maxScore?: number | undefined;
1015
+ method?: "judge" | "embedding" | "field-diff" | undefined;
1016
+ fields?: string[] | undefined;
1017
+ } | undefined;
1018
+ } | undefined;
1019
+ };
1020
+ prompt?: string | undefined;
1021
+ command?: string | undefined;
1022
+ vars?: Record<string, string> | undefined;
1023
+ models?: string[] | undefined;
1024
+ repeat?: number | undefined;
1025
+ tools?: {
1026
+ name: string;
1027
+ description?: string | undefined;
1028
+ parameters?: Record<string, unknown> | undefined;
1029
+ responses?: {
1030
+ when: Record<string, unknown>;
1031
+ then?: unknown;
1032
+ }[] | undefined;
1033
+ defaultResponse?: unknown;
1034
+ }[] | undefined;
1035
+ tags?: string[] | undefined;
1036
+ skip?: boolean | undefined;
1037
+ }>;
1038
+ declare const GatesSchema: z.ZodObject<{
1039
+ passRateMin: z.ZodDefault<z.ZodNumber>;
1040
+ schemaFailuresMax: z.ZodDefault<z.ZodNumber>;
1041
+ judgeAvgMin: z.ZodOptional<z.ZodNumber>;
1042
+ driftScoreMax: z.ZodOptional<z.ZodNumber>;
1043
+ piiFailuresMax: z.ZodDefault<z.ZodNumber>;
1044
+ keywordFailuresMax: z.ZodDefault<z.ZodNumber>;
1045
+ costMaxUsd: z.ZodOptional<z.ZodNumber>;
1046
+ latencyMaxMs: z.ZodOptional<z.ZodNumber>;
1047
+ deterministicPassRate: z.ZodOptional<z.ZodNumber>;
1048
+ probabilisticPassRate: z.ZodOptional<z.ZodNumber>;
1049
+ }, "strip", z.ZodTypeAny, {
1050
+ passRateMin: number;
1051
+ schemaFailuresMax: number;
1052
+ piiFailuresMax: number;
1053
+ keywordFailuresMax: number;
1054
+ judgeAvgMin?: number | undefined;
1055
+ driftScoreMax?: number | undefined;
1056
+ costMaxUsd?: number | undefined;
1057
+ latencyMaxMs?: number | undefined;
1058
+ deterministicPassRate?: number | undefined;
1059
+ probabilisticPassRate?: number | undefined;
1060
+ }, {
1061
+ passRateMin?: number | undefined;
1062
+ schemaFailuresMax?: number | undefined;
1063
+ judgeAvgMin?: number | undefined;
1064
+ driftScoreMax?: number | undefined;
1065
+ piiFailuresMax?: number | undefined;
1066
+ keywordFailuresMax?: number | undefined;
1067
+ costMaxUsd?: number | undefined;
1068
+ latencyMaxMs?: number | undefined;
1069
+ deterministicPassRate?: number | undefined;
1070
+ probabilisticPassRate?: number | undefined;
1071
+ }>;
1072
+ declare const ComplianceSchema: z.ZodObject<{
1073
+ enabled: z.ZodDefault<z.ZodBoolean>;
1074
+ framework: z.ZodDefault<z.ZodEnum<["eu-ai-act", "custom"]>>;
1075
+ outputDir: z.ZodDefault<z.ZodString>;
1076
+ metadata: z.ZodOptional<z.ZodObject<{
1077
+ systemName: z.ZodOptional<z.ZodString>;
1078
+ systemVersion: z.ZodOptional<z.ZodString>;
1079
+ riskLevel: z.ZodOptional<z.ZodEnum<["high", "limited", "minimal"]>>;
1080
+ operator: z.ZodOptional<z.ZodString>;
1081
+ intendedPurpose: z.ZodOptional<z.ZodString>;
1082
+ dataGovernanceNotes: z.ZodOptional<z.ZodString>;
1083
+ }, "strip", z.ZodTypeAny, {
1084
+ systemName?: string | undefined;
1085
+ systemVersion?: string | undefined;
1086
+ riskLevel?: "high" | "limited" | "minimal" | undefined;
1087
+ operator?: string | undefined;
1088
+ intendedPurpose?: string | undefined;
1089
+ dataGovernanceNotes?: string | undefined;
1090
+ }, {
1091
+ systemName?: string | undefined;
1092
+ systemVersion?: string | undefined;
1093
+ riskLevel?: "high" | "limited" | "minimal" | undefined;
1094
+ operator?: string | undefined;
1095
+ intendedPurpose?: string | undefined;
1096
+ dataGovernanceNotes?: string | undefined;
1097
+ }>>;
1098
+ }, "strip", z.ZodTypeAny, {
1099
+ enabled: boolean;
1100
+ framework: "custom" | "eu-ai-act";
1101
+ outputDir: string;
1102
+ metadata?: {
1103
+ systemName?: string | undefined;
1104
+ systemVersion?: string | undefined;
1105
+ riskLevel?: "high" | "limited" | "minimal" | undefined;
1106
+ operator?: string | undefined;
1107
+ intendedPurpose?: string | undefined;
1108
+ dataGovernanceNotes?: string | undefined;
1109
+ } | undefined;
1110
+ }, {
1111
+ enabled?: boolean | undefined;
1112
+ framework?: "custom" | "eu-ai-act" | undefined;
1113
+ outputDir?: string | undefined;
1114
+ metadata?: {
1115
+ systemName?: string | undefined;
1116
+ systemVersion?: string | undefined;
1117
+ riskLevel?: "high" | "limited" | "minimal" | undefined;
1118
+ operator?: string | undefined;
1119
+ intendedPurpose?: string | undefined;
1120
+ dataGovernanceNotes?: string | undefined;
1121
+ } | undefined;
1122
+ }>;
1123
+ declare const KindLMConfigSchema: z.ZodObject<{
1124
+ kindlm: z.ZodLiteral<1>;
1125
+ project: z.ZodString;
1126
+ suite: z.ZodObject<{
1127
+ name: z.ZodString;
1128
+ description: z.ZodOptional<z.ZodString>;
1129
+ tags: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
1130
+ }, "strip", z.ZodTypeAny, {
1131
+ name: string;
1132
+ description?: string | undefined;
1133
+ tags?: string[] | undefined;
1134
+ }, {
1135
+ name: string;
1136
+ description?: string | undefined;
1137
+ tags?: string[] | undefined;
1138
+ }>;
1139
+ providers: z.ZodEffects<z.ZodObject<{
1140
+ openai: z.ZodOptional<z.ZodObject<{
1141
+ apiKeyEnv: z.ZodString;
1142
+ baseUrl: z.ZodOptional<z.ZodString>;
1143
+ organization: z.ZodOptional<z.ZodString>;
1144
+ }, "strip", z.ZodTypeAny, {
1145
+ apiKeyEnv: string;
1146
+ baseUrl?: string | undefined;
1147
+ organization?: string | undefined;
1148
+ }, {
1149
+ apiKeyEnv: string;
1150
+ baseUrl?: string | undefined;
1151
+ organization?: string | undefined;
1152
+ }>>;
1153
+ anthropic: z.ZodOptional<z.ZodObject<{
1154
+ apiKeyEnv: z.ZodString;
1155
+ baseUrl: z.ZodOptional<z.ZodString>;
1156
+ organization: z.ZodOptional<z.ZodString>;
1157
+ }, "strip", z.ZodTypeAny, {
1158
+ apiKeyEnv: string;
1159
+ baseUrl?: string | undefined;
1160
+ organization?: string | undefined;
1161
+ }, {
1162
+ apiKeyEnv: string;
1163
+ baseUrl?: string | undefined;
1164
+ organization?: string | undefined;
1165
+ }>>;
1166
+ ollama: z.ZodOptional<z.ZodObject<{
1167
+ apiKeyEnv: z.ZodOptional<z.ZodString>;
1168
+ baseUrl: z.ZodOptional<z.ZodString>;
1169
+ }, "strip", z.ZodTypeAny, {
1170
+ apiKeyEnv?: string | undefined;
1171
+ baseUrl?: string | undefined;
1172
+ }, {
1173
+ apiKeyEnv?: string | undefined;
1174
+ baseUrl?: string | undefined;
1175
+ }>>;
1176
+ gemini: z.ZodOptional<z.ZodObject<{
1177
+ apiKeyEnv: z.ZodString;
1178
+ baseUrl: z.ZodOptional<z.ZodString>;
1179
+ organization: z.ZodOptional<z.ZodString>;
1180
+ }, "strip", z.ZodTypeAny, {
1181
+ apiKeyEnv: string;
1182
+ baseUrl?: string | undefined;
1183
+ organization?: string | undefined;
1184
+ }, {
1185
+ apiKeyEnv: string;
1186
+ baseUrl?: string | undefined;
1187
+ organization?: string | undefined;
1188
+ }>>;
1189
+ mistral: z.ZodOptional<z.ZodObject<{
1190
+ apiKeyEnv: z.ZodString;
1191
+ baseUrl: z.ZodOptional<z.ZodString>;
1192
+ organization: z.ZodOptional<z.ZodString>;
1193
+ }, "strip", z.ZodTypeAny, {
1194
+ apiKeyEnv: string;
1195
+ baseUrl?: string | undefined;
1196
+ organization?: string | undefined;
1197
+ }, {
1198
+ apiKeyEnv: string;
1199
+ baseUrl?: string | undefined;
1200
+ organization?: string | undefined;
1201
+ }>>;
1202
+ cohere: z.ZodOptional<z.ZodObject<{
1203
+ apiKeyEnv: z.ZodString;
1204
+ baseUrl: z.ZodOptional<z.ZodString>;
1205
+ organization: z.ZodOptional<z.ZodString>;
1206
+ }, "strip", z.ZodTypeAny, {
1207
+ apiKeyEnv: string;
1208
+ baseUrl?: string | undefined;
1209
+ organization?: string | undefined;
1210
+ }, {
1211
+ apiKeyEnv: string;
1212
+ baseUrl?: string | undefined;
1213
+ organization?: string | undefined;
1214
+ }>>;
1215
+ }, "strip", z.ZodTypeAny, {
1216
+ openai?: {
1217
+ apiKeyEnv: string;
1218
+ baseUrl?: string | undefined;
1219
+ organization?: string | undefined;
1220
+ } | undefined;
1221
+ anthropic?: {
1222
+ apiKeyEnv: string;
1223
+ baseUrl?: string | undefined;
1224
+ organization?: string | undefined;
1225
+ } | undefined;
1226
+ ollama?: {
1227
+ apiKeyEnv?: string | undefined;
1228
+ baseUrl?: string | undefined;
1229
+ } | undefined;
1230
+ gemini?: {
1231
+ apiKeyEnv: string;
1232
+ baseUrl?: string | undefined;
1233
+ organization?: string | undefined;
1234
+ } | undefined;
1235
+ mistral?: {
1236
+ apiKeyEnv: string;
1237
+ baseUrl?: string | undefined;
1238
+ organization?: string | undefined;
1239
+ } | undefined;
1240
+ cohere?: {
1241
+ apiKeyEnv: string;
1242
+ baseUrl?: string | undefined;
1243
+ organization?: string | undefined;
1244
+ } | undefined;
1245
+ }, {
1246
+ openai?: {
1247
+ apiKeyEnv: string;
1248
+ baseUrl?: string | undefined;
1249
+ organization?: string | undefined;
1250
+ } | undefined;
1251
+ anthropic?: {
1252
+ apiKeyEnv: string;
1253
+ baseUrl?: string | undefined;
1254
+ organization?: string | undefined;
1255
+ } | undefined;
1256
+ ollama?: {
1257
+ apiKeyEnv?: string | undefined;
1258
+ baseUrl?: string | undefined;
1259
+ } | undefined;
1260
+ gemini?: {
1261
+ apiKeyEnv: string;
1262
+ baseUrl?: string | undefined;
1263
+ organization?: string | undefined;
1264
+ } | undefined;
1265
+ mistral?: {
1266
+ apiKeyEnv: string;
1267
+ baseUrl?: string | undefined;
1268
+ organization?: string | undefined;
1269
+ } | undefined;
1270
+ cohere?: {
1271
+ apiKeyEnv: string;
1272
+ baseUrl?: string | undefined;
1273
+ organization?: string | undefined;
1274
+ } | undefined;
1275
+ }>, {
1276
+ openai?: {
1277
+ apiKeyEnv: string;
1278
+ baseUrl?: string | undefined;
1279
+ organization?: string | undefined;
1280
+ } | undefined;
1281
+ anthropic?: {
1282
+ apiKeyEnv: string;
1283
+ baseUrl?: string | undefined;
1284
+ organization?: string | undefined;
1285
+ } | undefined;
1286
+ ollama?: {
1287
+ apiKeyEnv?: string | undefined;
1288
+ baseUrl?: string | undefined;
1289
+ } | undefined;
1290
+ gemini?: {
1291
+ apiKeyEnv: string;
1292
+ baseUrl?: string | undefined;
1293
+ organization?: string | undefined;
1294
+ } | undefined;
1295
+ mistral?: {
1296
+ apiKeyEnv: string;
1297
+ baseUrl?: string | undefined;
1298
+ organization?: string | undefined;
1299
+ } | undefined;
1300
+ cohere?: {
1301
+ apiKeyEnv: string;
1302
+ baseUrl?: string | undefined;
1303
+ organization?: string | undefined;
1304
+ } | undefined;
1305
+ }, {
1306
+ openai?: {
1307
+ apiKeyEnv: string;
1308
+ baseUrl?: string | undefined;
1309
+ organization?: string | undefined;
1310
+ } | undefined;
1311
+ anthropic?: {
1312
+ apiKeyEnv: string;
1313
+ baseUrl?: string | undefined;
1314
+ organization?: string | undefined;
1315
+ } | undefined;
1316
+ ollama?: {
1317
+ apiKeyEnv?: string | undefined;
1318
+ baseUrl?: string | undefined;
1319
+ } | undefined;
1320
+ gemini?: {
1321
+ apiKeyEnv: string;
1322
+ baseUrl?: string | undefined;
1323
+ organization?: string | undefined;
1324
+ } | undefined;
1325
+ mistral?: {
1326
+ apiKeyEnv: string;
1327
+ baseUrl?: string | undefined;
1328
+ organization?: string | undefined;
1329
+ } | undefined;
1330
+ cohere?: {
1331
+ apiKeyEnv: string;
1332
+ baseUrl?: string | undefined;
1333
+ organization?: string | undefined;
1334
+ } | undefined;
1335
+ }>;
1336
+ models: z.ZodArray<z.ZodObject<{
1337
+ id: z.ZodString;
1338
+ provider: z.ZodEnum<["openai", "anthropic", "ollama", "gemini", "mistral", "cohere"]>;
1339
+ model: z.ZodString;
1340
+ params: z.ZodDefault<z.ZodObject<{
1341
+ temperature: z.ZodDefault<z.ZodNumber>;
1342
+ maxTokens: z.ZodDefault<z.ZodNumber>;
1343
+ topP: z.ZodOptional<z.ZodNumber>;
1344
+ stopSequences: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
1345
+ seed: z.ZodOptional<z.ZodNumber>;
1346
+ }, "strip", z.ZodTypeAny, {
1347
+ temperature: number;
1348
+ maxTokens: number;
1349
+ topP?: number | undefined;
1350
+ stopSequences?: string[] | undefined;
1351
+ seed?: number | undefined;
1352
+ }, {
1353
+ temperature?: number | undefined;
1354
+ maxTokens?: number | undefined;
1355
+ topP?: number | undefined;
1356
+ stopSequences?: string[] | undefined;
1357
+ seed?: number | undefined;
1358
+ }>>;
1359
+ }, "strip", z.ZodTypeAny, {
1360
+ params: {
1361
+ temperature: number;
1362
+ maxTokens: number;
1363
+ topP?: number | undefined;
1364
+ stopSequences?: string[] | undefined;
1365
+ seed?: number | undefined;
1366
+ };
1367
+ id: string;
1368
+ provider: "openai" | "anthropic" | "ollama" | "gemini" | "mistral" | "cohere";
1369
+ model: string;
1370
+ }, {
1371
+ id: string;
1372
+ provider: "openai" | "anthropic" | "ollama" | "gemini" | "mistral" | "cohere";
1373
+ model: string;
1374
+ params?: {
1375
+ temperature?: number | undefined;
1376
+ maxTokens?: number | undefined;
1377
+ topP?: number | undefined;
1378
+ stopSequences?: string[] | undefined;
1379
+ seed?: number | undefined;
1380
+ } | undefined;
1381
+ }>, "many">;
1382
+ prompts: z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodObject<{
1383
+ system: z.ZodOptional<z.ZodString>;
1384
+ user: z.ZodString;
1385
+ assistant: z.ZodOptional<z.ZodString>;
1386
+ }, "strip", z.ZodTypeAny, {
1387
+ user: string;
1388
+ system?: string | undefined;
1389
+ assistant?: string | undefined;
1390
+ }, {
1391
+ user: string;
1392
+ system?: string | undefined;
1393
+ assistant?: string | undefined;
1394
+ }>>, Record<string, {
1395
+ user: string;
1396
+ system?: string | undefined;
1397
+ assistant?: string | undefined;
1398
+ }>, Record<string, {
1399
+ user: string;
1400
+ system?: string | undefined;
1401
+ assistant?: string | undefined;
1402
+ }>>;
1403
+ tests: z.ZodArray<z.ZodEffects<z.ZodObject<{
1404
+ name: z.ZodString;
1405
+ prompt: z.ZodOptional<z.ZodString>;
1406
+ command: z.ZodOptional<z.ZodString>;
1407
+ vars: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
1408
+ models: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
1409
+ repeat: z.ZodOptional<z.ZodNumber>;
1410
+ tools: z.ZodOptional<z.ZodArray<z.ZodObject<{
1411
+ name: z.ZodString;
1412
+ description: z.ZodOptional<z.ZodString>;
1413
+ parameters: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
1414
+ responses: z.ZodOptional<z.ZodArray<z.ZodObject<{
1415
+ when: z.ZodRecord<z.ZodString, z.ZodUnknown>;
1416
+ then: z.ZodUnknown;
1417
+ }, "strip", z.ZodTypeAny, {
1418
+ when: Record<string, unknown>;
1419
+ then?: unknown;
1420
+ }, {
1421
+ when: Record<string, unknown>;
1422
+ then?: unknown;
1423
+ }>, "many">>;
1424
+ defaultResponse: z.ZodOptional<z.ZodUnknown>;
1425
+ }, "strip", z.ZodTypeAny, {
1426
+ name: string;
1427
+ description?: string | undefined;
1428
+ parameters?: Record<string, unknown> | undefined;
1429
+ responses?: {
1430
+ when: Record<string, unknown>;
1431
+ then?: unknown;
1432
+ }[] | undefined;
1433
+ defaultResponse?: unknown;
1434
+ }, {
1435
+ name: string;
1436
+ description?: string | undefined;
1437
+ parameters?: Record<string, unknown> | undefined;
1438
+ responses?: {
1439
+ when: Record<string, unknown>;
1440
+ then?: unknown;
1441
+ }[] | undefined;
1442
+ defaultResponse?: unknown;
1443
+ }>, "many">>;
1444
+ expect: z.ZodObject<{
1445
+ output: z.ZodOptional<z.ZodEffects<z.ZodObject<{
1446
+ format: z.ZodDefault<z.ZodEnum<["text", "json"]>>;
1447
+ schemaFile: z.ZodOptional<z.ZodString>;
1448
+ contains: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
1449
+ notContains: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
1450
+ maxLength: z.ZodOptional<z.ZodNumber>;
1451
+ }, "strip", z.ZodTypeAny, {
1452
+ format: "text" | "json";
1453
+ schemaFile?: string | undefined;
1454
+ contains?: string[] | undefined;
1455
+ notContains?: string[] | undefined;
1456
+ maxLength?: number | undefined;
1457
+ }, {
1458
+ format?: "text" | "json" | undefined;
1459
+ schemaFile?: string | undefined;
1460
+ contains?: string[] | undefined;
1461
+ notContains?: string[] | undefined;
1462
+ maxLength?: number | undefined;
1463
+ }>, {
1464
+ format: "text" | "json";
1465
+ schemaFile?: string | undefined;
1466
+ contains?: string[] | undefined;
1467
+ notContains?: string[] | undefined;
1468
+ maxLength?: number | undefined;
1469
+ }, {
1470
+ format?: "text" | "json" | undefined;
1471
+ schemaFile?: string | undefined;
1472
+ contains?: string[] | undefined;
1473
+ notContains?: string[] | undefined;
1474
+ maxLength?: number | undefined;
1475
+ }>>;
1476
+ guardrails: z.ZodOptional<z.ZodObject<{
1477
+ pii: z.ZodOptional<z.ZodObject<{
1478
+ enabled: z.ZodDefault<z.ZodBoolean>;
1479
+ denyPatterns: z.ZodDefault<z.ZodArray<z.ZodEffects<z.ZodString, string, string>, "many">>;
1480
+ customPatterns: z.ZodOptional<z.ZodArray<z.ZodObject<{
1481
+ name: z.ZodString;
1482
+ pattern: z.ZodEffects<z.ZodString, string, string>;
1483
+ }, "strip", z.ZodTypeAny, {
1484
+ name: string;
1485
+ pattern: string;
1486
+ }, {
1487
+ name: string;
1488
+ pattern: string;
1489
+ }>, "many">>;
1490
+ }, "strip", z.ZodTypeAny, {
1491
+ enabled: boolean;
1492
+ denyPatterns: string[];
1493
+ customPatterns?: {
1494
+ name: string;
1495
+ pattern: string;
1496
+ }[] | undefined;
1497
+ }, {
1498
+ enabled?: boolean | undefined;
1499
+ denyPatterns?: string[] | undefined;
1500
+ customPatterns?: {
1501
+ name: string;
1502
+ pattern: string;
1503
+ }[] | undefined;
1504
+ }>>;
1505
+ keywords: z.ZodOptional<z.ZodObject<{
1506
+ deny: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
1507
+ allow: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
1508
+ }, "strip", z.ZodTypeAny, {
1509
+ deny: string[];
1510
+ allow?: string[] | undefined;
1511
+ }, {
1512
+ deny?: string[] | undefined;
1513
+ allow?: string[] | undefined;
1514
+ }>>;
1515
+ }, "strip", z.ZodTypeAny, {
1516
+ pii?: {
1517
+ enabled: boolean;
1518
+ denyPatterns: string[];
1519
+ customPatterns?: {
1520
+ name: string;
1521
+ pattern: string;
1522
+ }[] | undefined;
1523
+ } | undefined;
1524
+ keywords?: {
1525
+ deny: string[];
1526
+ allow?: string[] | undefined;
1527
+ } | undefined;
1528
+ }, {
1529
+ pii?: {
1530
+ enabled?: boolean | undefined;
1531
+ denyPatterns?: string[] | undefined;
1532
+ customPatterns?: {
1533
+ name: string;
1534
+ pattern: string;
1535
+ }[] | undefined;
1536
+ } | undefined;
1537
+ keywords?: {
1538
+ deny?: string[] | undefined;
1539
+ allow?: string[] | undefined;
1540
+ } | undefined;
1541
+ }>>;
1542
+ judge: z.ZodOptional<z.ZodArray<z.ZodObject<{
1543
+ criteria: z.ZodString;
1544
+ minScore: z.ZodDefault<z.ZodNumber>;
1545
+ model: z.ZodOptional<z.ZodString>;
1546
+ rubric: z.ZodOptional<z.ZodString>;
1547
+ }, "strip", z.ZodTypeAny, {
1548
+ criteria: string;
1549
+ minScore: number;
1550
+ model?: string | undefined;
1551
+ rubric?: string | undefined;
1552
+ }, {
1553
+ criteria: string;
1554
+ model?: string | undefined;
1555
+ minScore?: number | undefined;
1556
+ rubric?: string | undefined;
1557
+ }>, "many">>;
1558
+ toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
1559
+ tool: z.ZodString;
1560
+ shouldNotCall: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
1561
+ argsMatch: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
1562
+ argsSchema: z.ZodOptional<z.ZodString>;
1563
+ order: z.ZodOptional<z.ZodNumber>;
1564
+ responseContains: z.ZodOptional<z.ZodString>;
1565
+ }, "strip", z.ZodTypeAny, {
1566
+ tool: string;
1567
+ shouldNotCall: boolean;
1568
+ argsMatch?: Record<string, unknown> | undefined;
1569
+ argsSchema?: string | undefined;
1570
+ order?: number | undefined;
1571
+ responseContains?: string | undefined;
1572
+ }, {
1573
+ tool: string;
1574
+ shouldNotCall?: boolean | undefined;
1575
+ argsMatch?: Record<string, unknown> | undefined;
1576
+ argsSchema?: string | undefined;
1577
+ order?: number | undefined;
1578
+ responseContains?: string | undefined;
1579
+ }>, "many">>;
1580
+ baseline: z.ZodOptional<z.ZodObject<{
1581
+ drift: z.ZodOptional<z.ZodObject<{
1582
+ maxScore: z.ZodDefault<z.ZodNumber>;
1583
+ method: z.ZodDefault<z.ZodEnum<["judge", "embedding", "field-diff"]>>;
1584
+ fields: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
1585
+ }, "strip", z.ZodTypeAny, {
1586
+ maxScore: number;
1587
+ method: "judge" | "embedding" | "field-diff";
1588
+ fields?: string[] | undefined;
1589
+ }, {
1590
+ maxScore?: number | undefined;
1591
+ method?: "judge" | "embedding" | "field-diff" | undefined;
1592
+ fields?: string[] | undefined;
1593
+ }>>;
1594
+ }, "strip", z.ZodTypeAny, {
1595
+ drift?: {
1596
+ maxScore: number;
1597
+ method: "judge" | "embedding" | "field-diff";
1598
+ fields?: string[] | undefined;
1599
+ } | undefined;
1600
+ }, {
1601
+ drift?: {
1602
+ maxScore?: number | undefined;
1603
+ method?: "judge" | "embedding" | "field-diff" | undefined;
1604
+ fields?: string[] | undefined;
1605
+ } | undefined;
1606
+ }>>;
1607
+ }, "strip", z.ZodTypeAny, {
1608
+ judge?: {
1609
+ criteria: string;
1610
+ minScore: number;
1611
+ model?: string | undefined;
1612
+ rubric?: string | undefined;
1613
+ }[] | undefined;
1614
+ output?: {
1615
+ format: "text" | "json";
1616
+ schemaFile?: string | undefined;
1617
+ contains?: string[] | undefined;
1618
+ notContains?: string[] | undefined;
1619
+ maxLength?: number | undefined;
1620
+ } | undefined;
1621
+ guardrails?: {
1622
+ pii?: {
1623
+ enabled: boolean;
1624
+ denyPatterns: string[];
1625
+ customPatterns?: {
1626
+ name: string;
1627
+ pattern: string;
1628
+ }[] | undefined;
1629
+ } | undefined;
1630
+ keywords?: {
1631
+ deny: string[];
1632
+ allow?: string[] | undefined;
1633
+ } | undefined;
1634
+ } | undefined;
1635
+ toolCalls?: {
1636
+ tool: string;
1637
+ shouldNotCall: boolean;
1638
+ argsMatch?: Record<string, unknown> | undefined;
1639
+ argsSchema?: string | undefined;
1640
+ order?: number | undefined;
1641
+ responseContains?: string | undefined;
1642
+ }[] | undefined;
1643
+ baseline?: {
1644
+ drift?: {
1645
+ maxScore: number;
1646
+ method: "judge" | "embedding" | "field-diff";
1647
+ fields?: string[] | undefined;
1648
+ } | undefined;
1649
+ } | undefined;
1650
+ }, {
1651
+ judge?: {
1652
+ criteria: string;
1653
+ model?: string | undefined;
1654
+ minScore?: number | undefined;
1655
+ rubric?: string | undefined;
1656
+ }[] | undefined;
1657
+ output?: {
1658
+ format?: "text" | "json" | undefined;
1659
+ schemaFile?: string | undefined;
1660
+ contains?: string[] | undefined;
1661
+ notContains?: string[] | undefined;
1662
+ maxLength?: number | undefined;
1663
+ } | undefined;
1664
+ guardrails?: {
1665
+ pii?: {
1666
+ enabled?: boolean | undefined;
1667
+ denyPatterns?: string[] | undefined;
1668
+ customPatterns?: {
1669
+ name: string;
1670
+ pattern: string;
1671
+ }[] | undefined;
1672
+ } | undefined;
1673
+ keywords?: {
1674
+ deny?: string[] | undefined;
1675
+ allow?: string[] | undefined;
1676
+ } | undefined;
1677
+ } | undefined;
1678
+ toolCalls?: {
1679
+ tool: string;
1680
+ shouldNotCall?: boolean | undefined;
1681
+ argsMatch?: Record<string, unknown> | undefined;
1682
+ argsSchema?: string | undefined;
1683
+ order?: number | undefined;
1684
+ responseContains?: string | undefined;
1685
+ }[] | undefined;
1686
+ baseline?: {
1687
+ drift?: {
1688
+ maxScore?: number | undefined;
1689
+ method?: "judge" | "embedding" | "field-diff" | undefined;
1690
+ fields?: string[] | undefined;
1691
+ } | undefined;
1692
+ } | undefined;
1693
+ }>;
1694
+ tags: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
1695
+ skip: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
1696
+ }, "strip", z.ZodTypeAny, {
1697
+ name: string;
1698
+ vars: Record<string, string>;
1699
+ expect: {
1700
+ judge?: {
1701
+ criteria: string;
1702
+ minScore: number;
1703
+ model?: string | undefined;
1704
+ rubric?: string | undefined;
1705
+ }[] | undefined;
1706
+ output?: {
1707
+ format: "text" | "json";
1708
+ schemaFile?: string | undefined;
1709
+ contains?: string[] | undefined;
1710
+ notContains?: string[] | undefined;
1711
+ maxLength?: number | undefined;
1712
+ } | undefined;
1713
+ guardrails?: {
1714
+ pii?: {
1715
+ enabled: boolean;
1716
+ denyPatterns: string[];
1717
+ customPatterns?: {
1718
+ name: string;
1719
+ pattern: string;
1720
+ }[] | undefined;
1721
+ } | undefined;
1722
+ keywords?: {
1723
+ deny: string[];
1724
+ allow?: string[] | undefined;
1725
+ } | undefined;
1726
+ } | undefined;
1727
+ toolCalls?: {
1728
+ tool: string;
1729
+ shouldNotCall: boolean;
1730
+ argsMatch?: Record<string, unknown> | undefined;
1731
+ argsSchema?: string | undefined;
1732
+ order?: number | undefined;
1733
+ responseContains?: string | undefined;
1734
+ }[] | undefined;
1735
+ baseline?: {
1736
+ drift?: {
1737
+ maxScore: number;
1738
+ method: "judge" | "embedding" | "field-diff";
1739
+ fields?: string[] | undefined;
1740
+ } | undefined;
1741
+ } | undefined;
1742
+ };
1743
+ skip: boolean;
1744
+ prompt?: string | undefined;
1745
+ command?: string | undefined;
1746
+ models?: string[] | undefined;
1747
+ repeat?: number | undefined;
1748
+ tools?: {
1749
+ name: string;
1750
+ description?: string | undefined;
1751
+ parameters?: Record<string, unknown> | undefined;
1752
+ responses?: {
1753
+ when: Record<string, unknown>;
1754
+ then?: unknown;
1755
+ }[] | undefined;
1756
+ defaultResponse?: unknown;
1757
+ }[] | undefined;
1758
+ tags?: string[] | undefined;
1759
+ }, {
1760
+ name: string;
1761
+ expect: {
1762
+ judge?: {
1763
+ criteria: string;
1764
+ model?: string | undefined;
1765
+ minScore?: number | undefined;
1766
+ rubric?: string | undefined;
1767
+ }[] | undefined;
1768
+ output?: {
1769
+ format?: "text" | "json" | undefined;
1770
+ schemaFile?: string | undefined;
1771
+ contains?: string[] | undefined;
1772
+ notContains?: string[] | undefined;
1773
+ maxLength?: number | undefined;
1774
+ } | undefined;
1775
+ guardrails?: {
1776
+ pii?: {
1777
+ enabled?: boolean | undefined;
1778
+ denyPatterns?: string[] | undefined;
1779
+ customPatterns?: {
1780
+ name: string;
1781
+ pattern: string;
1782
+ }[] | undefined;
1783
+ } | undefined;
1784
+ keywords?: {
1785
+ deny?: string[] | undefined;
1786
+ allow?: string[] | undefined;
1787
+ } | undefined;
1788
+ } | undefined;
1789
+ toolCalls?: {
1790
+ tool: string;
1791
+ shouldNotCall?: boolean | undefined;
1792
+ argsMatch?: Record<string, unknown> | undefined;
1793
+ argsSchema?: string | undefined;
1794
+ order?: number | undefined;
1795
+ responseContains?: string | undefined;
1796
+ }[] | undefined;
1797
+ baseline?: {
1798
+ drift?: {
1799
+ maxScore?: number | undefined;
1800
+ method?: "judge" | "embedding" | "field-diff" | undefined;
1801
+ fields?: string[] | undefined;
1802
+ } | undefined;
1803
+ } | undefined;
1804
+ };
1805
+ prompt?: string | undefined;
1806
+ command?: string | undefined;
1807
+ vars?: Record<string, string> | undefined;
1808
+ models?: string[] | undefined;
1809
+ repeat?: number | undefined;
1810
+ tools?: {
1811
+ name: string;
1812
+ description?: string | undefined;
1813
+ parameters?: Record<string, unknown> | undefined;
1814
+ responses?: {
1815
+ when: Record<string, unknown>;
1816
+ then?: unknown;
1817
+ }[] | undefined;
1818
+ defaultResponse?: unknown;
1819
+ }[] | undefined;
1820
+ tags?: string[] | undefined;
1821
+ skip?: boolean | undefined;
1822
+ }>, {
1823
+ name: string;
1824
+ vars: Record<string, string>;
1825
+ expect: {
1826
+ judge?: {
1827
+ criteria: string;
1828
+ minScore: number;
1829
+ model?: string | undefined;
1830
+ rubric?: string | undefined;
1831
+ }[] | undefined;
1832
+ output?: {
1833
+ format: "text" | "json";
1834
+ schemaFile?: string | undefined;
1835
+ contains?: string[] | undefined;
1836
+ notContains?: string[] | undefined;
1837
+ maxLength?: number | undefined;
1838
+ } | undefined;
1839
+ guardrails?: {
1840
+ pii?: {
1841
+ enabled: boolean;
1842
+ denyPatterns: string[];
1843
+ customPatterns?: {
1844
+ name: string;
1845
+ pattern: string;
1846
+ }[] | undefined;
1847
+ } | undefined;
1848
+ keywords?: {
1849
+ deny: string[];
1850
+ allow?: string[] | undefined;
1851
+ } | undefined;
1852
+ } | undefined;
1853
+ toolCalls?: {
1854
+ tool: string;
1855
+ shouldNotCall: boolean;
1856
+ argsMatch?: Record<string, unknown> | undefined;
1857
+ argsSchema?: string | undefined;
1858
+ order?: number | undefined;
1859
+ responseContains?: string | undefined;
1860
+ }[] | undefined;
1861
+ baseline?: {
1862
+ drift?: {
1863
+ maxScore: number;
1864
+ method: "judge" | "embedding" | "field-diff";
1865
+ fields?: string[] | undefined;
1866
+ } | undefined;
1867
+ } | undefined;
1868
+ };
1869
+ skip: boolean;
1870
+ prompt?: string | undefined;
1871
+ command?: string | undefined;
1872
+ models?: string[] | undefined;
1873
+ repeat?: number | undefined;
1874
+ tools?: {
1875
+ name: string;
1876
+ description?: string | undefined;
1877
+ parameters?: Record<string, unknown> | undefined;
1878
+ responses?: {
1879
+ when: Record<string, unknown>;
1880
+ then?: unknown;
1881
+ }[] | undefined;
1882
+ defaultResponse?: unknown;
1883
+ }[] | undefined;
1884
+ tags?: string[] | undefined;
1885
+ }, {
1886
+ name: string;
1887
+ expect: {
1888
+ judge?: {
1889
+ criteria: string;
1890
+ model?: string | undefined;
1891
+ minScore?: number | undefined;
1892
+ rubric?: string | undefined;
1893
+ }[] | undefined;
1894
+ output?: {
1895
+ format?: "text" | "json" | undefined;
1896
+ schemaFile?: string | undefined;
1897
+ contains?: string[] | undefined;
1898
+ notContains?: string[] | undefined;
1899
+ maxLength?: number | undefined;
1900
+ } | undefined;
1901
+ guardrails?: {
1902
+ pii?: {
1903
+ enabled?: boolean | undefined;
1904
+ denyPatterns?: string[] | undefined;
1905
+ customPatterns?: {
1906
+ name: string;
1907
+ pattern: string;
1908
+ }[] | undefined;
1909
+ } | undefined;
1910
+ keywords?: {
1911
+ deny?: string[] | undefined;
1912
+ allow?: string[] | undefined;
1913
+ } | undefined;
1914
+ } | undefined;
1915
+ toolCalls?: {
1916
+ tool: string;
1917
+ shouldNotCall?: boolean | undefined;
1918
+ argsMatch?: Record<string, unknown> | undefined;
1919
+ argsSchema?: string | undefined;
1920
+ order?: number | undefined;
1921
+ responseContains?: string | undefined;
1922
+ }[] | undefined;
1923
+ baseline?: {
1924
+ drift?: {
1925
+ maxScore?: number | undefined;
1926
+ method?: "judge" | "embedding" | "field-diff" | undefined;
1927
+ fields?: string[] | undefined;
1928
+ } | undefined;
1929
+ } | undefined;
1930
+ };
1931
+ prompt?: string | undefined;
1932
+ command?: string | undefined;
1933
+ vars?: Record<string, string> | undefined;
1934
+ models?: string[] | undefined;
1935
+ repeat?: number | undefined;
1936
+ tools?: {
1937
+ name: string;
1938
+ description?: string | undefined;
1939
+ parameters?: Record<string, unknown> | undefined;
1940
+ responses?: {
1941
+ when: Record<string, unknown>;
1942
+ then?: unknown;
1943
+ }[] | undefined;
1944
+ defaultResponse?: unknown;
1945
+ }[] | undefined;
1946
+ tags?: string[] | undefined;
1947
+ skip?: boolean | undefined;
1948
+ }>, "many">;
1949
+ gates: z.ZodDefault<z.ZodObject<{
1950
+ passRateMin: z.ZodDefault<z.ZodNumber>;
1951
+ schemaFailuresMax: z.ZodDefault<z.ZodNumber>;
1952
+ judgeAvgMin: z.ZodOptional<z.ZodNumber>;
1953
+ driftScoreMax: z.ZodOptional<z.ZodNumber>;
1954
+ piiFailuresMax: z.ZodDefault<z.ZodNumber>;
1955
+ keywordFailuresMax: z.ZodDefault<z.ZodNumber>;
1956
+ costMaxUsd: z.ZodOptional<z.ZodNumber>;
1957
+ latencyMaxMs: z.ZodOptional<z.ZodNumber>;
1958
+ deterministicPassRate: z.ZodOptional<z.ZodNumber>;
1959
+ probabilisticPassRate: z.ZodOptional<z.ZodNumber>;
1960
+ }, "strip", z.ZodTypeAny, {
1961
+ passRateMin: number;
1962
+ schemaFailuresMax: number;
1963
+ piiFailuresMax: number;
1964
+ keywordFailuresMax: number;
1965
+ judgeAvgMin?: number | undefined;
1966
+ driftScoreMax?: number | undefined;
1967
+ costMaxUsd?: number | undefined;
1968
+ latencyMaxMs?: number | undefined;
1969
+ deterministicPassRate?: number | undefined;
1970
+ probabilisticPassRate?: number | undefined;
1971
+ }, {
1972
+ passRateMin?: number | undefined;
1973
+ schemaFailuresMax?: number | undefined;
1974
+ judgeAvgMin?: number | undefined;
1975
+ driftScoreMax?: number | undefined;
1976
+ piiFailuresMax?: number | undefined;
1977
+ keywordFailuresMax?: number | undefined;
1978
+ costMaxUsd?: number | undefined;
1979
+ latencyMaxMs?: number | undefined;
1980
+ deterministicPassRate?: number | undefined;
1981
+ probabilisticPassRate?: number | undefined;
1982
+ }>>;
1983
+ compliance: z.ZodOptional<z.ZodObject<{
1984
+ enabled: z.ZodDefault<z.ZodBoolean>;
1985
+ framework: z.ZodDefault<z.ZodEnum<["eu-ai-act", "custom"]>>;
1986
+ outputDir: z.ZodDefault<z.ZodString>;
1987
+ metadata: z.ZodOptional<z.ZodObject<{
1988
+ systemName: z.ZodOptional<z.ZodString>;
1989
+ systemVersion: z.ZodOptional<z.ZodString>;
1990
+ riskLevel: z.ZodOptional<z.ZodEnum<["high", "limited", "minimal"]>>;
1991
+ operator: z.ZodOptional<z.ZodString>;
1992
+ intendedPurpose: z.ZodOptional<z.ZodString>;
1993
+ dataGovernanceNotes: z.ZodOptional<z.ZodString>;
1994
+ }, "strip", z.ZodTypeAny, {
1995
+ systemName?: string | undefined;
1996
+ systemVersion?: string | undefined;
1997
+ riskLevel?: "high" | "limited" | "minimal" | undefined;
1998
+ operator?: string | undefined;
1999
+ intendedPurpose?: string | undefined;
2000
+ dataGovernanceNotes?: string | undefined;
2001
+ }, {
2002
+ systemName?: string | undefined;
2003
+ systemVersion?: string | undefined;
2004
+ riskLevel?: "high" | "limited" | "minimal" | undefined;
2005
+ operator?: string | undefined;
2006
+ intendedPurpose?: string | undefined;
2007
+ dataGovernanceNotes?: string | undefined;
2008
+ }>>;
2009
+ }, "strip", z.ZodTypeAny, {
2010
+ enabled: boolean;
2011
+ framework: "custom" | "eu-ai-act";
2012
+ outputDir: string;
2013
+ metadata?: {
2014
+ systemName?: string | undefined;
2015
+ systemVersion?: string | undefined;
2016
+ riskLevel?: "high" | "limited" | "minimal" | undefined;
2017
+ operator?: string | undefined;
2018
+ intendedPurpose?: string | undefined;
2019
+ dataGovernanceNotes?: string | undefined;
2020
+ } | undefined;
2021
+ }, {
2022
+ enabled?: boolean | undefined;
2023
+ framework?: "custom" | "eu-ai-act" | undefined;
2024
+ outputDir?: string | undefined;
2025
+ metadata?: {
2026
+ systemName?: string | undefined;
2027
+ systemVersion?: string | undefined;
2028
+ riskLevel?: "high" | "limited" | "minimal" | undefined;
2029
+ operator?: string | undefined;
2030
+ intendedPurpose?: string | undefined;
2031
+ dataGovernanceNotes?: string | undefined;
2032
+ } | undefined;
2033
+ }>>;
2034
+ trace: z.ZodOptional<z.ZodObject<{
2035
+ port: z.ZodDefault<z.ZodNumber>;
2036
+ timeoutMs: z.ZodDefault<z.ZodNumber>;
2037
+ spanMapping: z.ZodDefault<z.ZodObject<{
2038
+ outputTextAttr: z.ZodDefault<z.ZodString>;
2039
+ modelAttr: z.ZodDefault<z.ZodString>;
2040
+ systemAttr: z.ZodDefault<z.ZodString>;
2041
+ inputTokensAttr: z.ZodDefault<z.ZodString>;
2042
+ outputTokensAttr: z.ZodDefault<z.ZodString>;
2043
+ }, "strip", z.ZodTypeAny, {
2044
+ outputTextAttr: string;
2045
+ modelAttr: string;
2046
+ systemAttr: string;
2047
+ inputTokensAttr: string;
2048
+ outputTokensAttr: string;
2049
+ }, {
2050
+ outputTextAttr?: string | undefined;
2051
+ modelAttr?: string | undefined;
2052
+ systemAttr?: string | undefined;
2053
+ inputTokensAttr?: string | undefined;
2054
+ outputTokensAttr?: string | undefined;
2055
+ }>>;
2056
+ spanFilter: z.ZodOptional<z.ZodObject<{
2057
+ namePattern: z.ZodOptional<z.ZodString>;
2058
+ attributeMatch: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
2059
+ minDurationMs: z.ZodOptional<z.ZodNumber>;
2060
+ }, "strip", z.ZodTypeAny, {
2061
+ namePattern?: string | undefined;
2062
+ attributeMatch?: Record<string, string> | undefined;
2063
+ minDurationMs?: number | undefined;
2064
+ }, {
2065
+ namePattern?: string | undefined;
2066
+ attributeMatch?: Record<string, string> | undefined;
2067
+ minDurationMs?: number | undefined;
2068
+ }>>;
2069
+ }, "strip", z.ZodTypeAny, {
2070
+ port: number;
2071
+ timeoutMs: number;
2072
+ spanMapping: {
2073
+ outputTextAttr: string;
2074
+ modelAttr: string;
2075
+ systemAttr: string;
2076
+ inputTokensAttr: string;
2077
+ outputTokensAttr: string;
2078
+ };
2079
+ spanFilter?: {
2080
+ namePattern?: string | undefined;
2081
+ attributeMatch?: Record<string, string> | undefined;
2082
+ minDurationMs?: number | undefined;
2083
+ } | undefined;
2084
+ }, {
2085
+ port?: number | undefined;
2086
+ timeoutMs?: number | undefined;
2087
+ spanMapping?: {
2088
+ outputTextAttr?: string | undefined;
2089
+ modelAttr?: string | undefined;
2090
+ systemAttr?: string | undefined;
2091
+ inputTokensAttr?: string | undefined;
2092
+ outputTokensAttr?: string | undefined;
2093
+ } | undefined;
2094
+ spanFilter?: {
2095
+ namePattern?: string | undefined;
2096
+ attributeMatch?: Record<string, string> | undefined;
2097
+ minDurationMs?: number | undefined;
2098
+ } | undefined;
2099
+ }>>;
2100
+ upload: z.ZodDefault<z.ZodObject<{
2101
+ enabled: z.ZodDefault<z.ZodBoolean>;
2102
+ includeArtifacts: z.ZodDefault<z.ZodBoolean>;
2103
+ redactPatterns: z.ZodOptional<z.ZodArray<z.ZodEffects<z.ZodString, string, string>, "many">>;
2104
+ apiUrl: z.ZodDefault<z.ZodString>;
2105
+ }, "strip", z.ZodTypeAny, {
2106
+ enabled: boolean;
2107
+ includeArtifacts: boolean;
2108
+ apiUrl: string;
2109
+ redactPatterns?: string[] | undefined;
2110
+ }, {
2111
+ enabled?: boolean | undefined;
2112
+ includeArtifacts?: boolean | undefined;
2113
+ redactPatterns?: string[] | undefined;
2114
+ apiUrl?: string | undefined;
2115
+ }>>;
2116
+ defaults: z.ZodDefault<z.ZodObject<{
2117
+ repeat: z.ZodDefault<z.ZodNumber>;
2118
+ concurrency: z.ZodDefault<z.ZodNumber>;
2119
+ timeoutMs: z.ZodDefault<z.ZodNumber>;
2120
+ judgeModel: z.ZodOptional<z.ZodString>;
2121
+ }, "strip", z.ZodTypeAny, {
2122
+ timeoutMs: number;
2123
+ repeat: number;
2124
+ concurrency: number;
2125
+ judgeModel?: string | undefined;
2126
+ }, {
2127
+ timeoutMs?: number | undefined;
2128
+ repeat?: number | undefined;
2129
+ concurrency?: number | undefined;
2130
+ judgeModel?: string | undefined;
2131
+ }>>;
2132
+ }, "strip", z.ZodTypeAny, {
2133
+ models: {
2134
+ params: {
2135
+ temperature: number;
2136
+ maxTokens: number;
2137
+ topP?: number | undefined;
2138
+ stopSequences?: string[] | undefined;
2139
+ seed?: number | undefined;
2140
+ };
2141
+ id: string;
2142
+ provider: "openai" | "anthropic" | "ollama" | "gemini" | "mistral" | "cohere";
2143
+ model: string;
2144
+ }[];
2145
+ kindlm: 1;
2146
+ project: string;
2147
+ suite: {
2148
+ name: string;
2149
+ description?: string | undefined;
2150
+ tags?: string[] | undefined;
2151
+ };
2152
+ providers: {
2153
+ openai?: {
2154
+ apiKeyEnv: string;
2155
+ baseUrl?: string | undefined;
2156
+ organization?: string | undefined;
2157
+ } | undefined;
2158
+ anthropic?: {
2159
+ apiKeyEnv: string;
2160
+ baseUrl?: string | undefined;
2161
+ organization?: string | undefined;
2162
+ } | undefined;
2163
+ ollama?: {
2164
+ apiKeyEnv?: string | undefined;
2165
+ baseUrl?: string | undefined;
2166
+ } | undefined;
2167
+ gemini?: {
2168
+ apiKeyEnv: string;
2169
+ baseUrl?: string | undefined;
2170
+ organization?: string | undefined;
2171
+ } | undefined;
2172
+ mistral?: {
2173
+ apiKeyEnv: string;
2174
+ baseUrl?: string | undefined;
2175
+ organization?: string | undefined;
2176
+ } | undefined;
2177
+ cohere?: {
2178
+ apiKeyEnv: string;
2179
+ baseUrl?: string | undefined;
2180
+ organization?: string | undefined;
2181
+ } | undefined;
2182
+ };
2183
+ prompts: Record<string, {
2184
+ user: string;
2185
+ system?: string | undefined;
2186
+ assistant?: string | undefined;
2187
+ }>;
2188
+ tests: {
2189
+ name: string;
2190
+ vars: Record<string, string>;
2191
+ expect: {
2192
+ judge?: {
2193
+ criteria: string;
2194
+ minScore: number;
2195
+ model?: string | undefined;
2196
+ rubric?: string | undefined;
2197
+ }[] | undefined;
2198
+ output?: {
2199
+ format: "text" | "json";
2200
+ schemaFile?: string | undefined;
2201
+ contains?: string[] | undefined;
2202
+ notContains?: string[] | undefined;
2203
+ maxLength?: number | undefined;
2204
+ } | undefined;
2205
+ guardrails?: {
2206
+ pii?: {
2207
+ enabled: boolean;
2208
+ denyPatterns: string[];
2209
+ customPatterns?: {
2210
+ name: string;
2211
+ pattern: string;
2212
+ }[] | undefined;
2213
+ } | undefined;
2214
+ keywords?: {
2215
+ deny: string[];
2216
+ allow?: string[] | undefined;
2217
+ } | undefined;
2218
+ } | undefined;
2219
+ toolCalls?: {
2220
+ tool: string;
2221
+ shouldNotCall: boolean;
2222
+ argsMatch?: Record<string, unknown> | undefined;
2223
+ argsSchema?: string | undefined;
2224
+ order?: number | undefined;
2225
+ responseContains?: string | undefined;
2226
+ }[] | undefined;
2227
+ baseline?: {
2228
+ drift?: {
2229
+ maxScore: number;
2230
+ method: "judge" | "embedding" | "field-diff";
2231
+ fields?: string[] | undefined;
2232
+ } | undefined;
2233
+ } | undefined;
2234
+ };
2235
+ skip: boolean;
2236
+ prompt?: string | undefined;
2237
+ command?: string | undefined;
2238
+ models?: string[] | undefined;
2239
+ repeat?: number | undefined;
2240
+ tools?: {
2241
+ name: string;
2242
+ description?: string | undefined;
2243
+ parameters?: Record<string, unknown> | undefined;
2244
+ responses?: {
2245
+ when: Record<string, unknown>;
2246
+ then?: unknown;
2247
+ }[] | undefined;
2248
+ defaultResponse?: unknown;
2249
+ }[] | undefined;
2250
+ tags?: string[] | undefined;
2251
+ }[];
2252
+ gates: {
2253
+ passRateMin: number;
2254
+ schemaFailuresMax: number;
2255
+ piiFailuresMax: number;
2256
+ keywordFailuresMax: number;
2257
+ judgeAvgMin?: number | undefined;
2258
+ driftScoreMax?: number | undefined;
2259
+ costMaxUsd?: number | undefined;
2260
+ latencyMaxMs?: number | undefined;
2261
+ deterministicPassRate?: number | undefined;
2262
+ probabilisticPassRate?: number | undefined;
2263
+ };
2264
+ upload: {
2265
+ enabled: boolean;
2266
+ includeArtifacts: boolean;
2267
+ apiUrl: string;
2268
+ redactPatterns?: string[] | undefined;
2269
+ };
2270
+ defaults: {
2271
+ timeoutMs: number;
2272
+ repeat: number;
2273
+ concurrency: number;
2274
+ judgeModel?: string | undefined;
2275
+ };
2276
+ compliance?: {
2277
+ enabled: boolean;
2278
+ framework: "custom" | "eu-ai-act";
2279
+ outputDir: string;
2280
+ metadata?: {
2281
+ systemName?: string | undefined;
2282
+ systemVersion?: string | undefined;
2283
+ riskLevel?: "high" | "limited" | "minimal" | undefined;
2284
+ operator?: string | undefined;
2285
+ intendedPurpose?: string | undefined;
2286
+ dataGovernanceNotes?: string | undefined;
2287
+ } | undefined;
2288
+ } | undefined;
2289
+ trace?: {
2290
+ port: number;
2291
+ timeoutMs: number;
2292
+ spanMapping: {
2293
+ outputTextAttr: string;
2294
+ modelAttr: string;
2295
+ systemAttr: string;
2296
+ inputTokensAttr: string;
2297
+ outputTokensAttr: string;
2298
+ };
2299
+ spanFilter?: {
2300
+ namePattern?: string | undefined;
2301
+ attributeMatch?: Record<string, string> | undefined;
2302
+ minDurationMs?: number | undefined;
2303
+ } | undefined;
2304
+ } | undefined;
2305
+ }, {
2306
+ models: {
2307
+ id: string;
2308
+ provider: "openai" | "anthropic" | "ollama" | "gemini" | "mistral" | "cohere";
2309
+ model: string;
2310
+ params?: {
2311
+ temperature?: number | undefined;
2312
+ maxTokens?: number | undefined;
2313
+ topP?: number | undefined;
2314
+ stopSequences?: string[] | undefined;
2315
+ seed?: number | undefined;
2316
+ } | undefined;
2317
+ }[];
2318
+ kindlm: 1;
2319
+ project: string;
2320
+ suite: {
2321
+ name: string;
2322
+ description?: string | undefined;
2323
+ tags?: string[] | undefined;
2324
+ };
2325
+ providers: {
2326
+ openai?: {
2327
+ apiKeyEnv: string;
2328
+ baseUrl?: string | undefined;
2329
+ organization?: string | undefined;
2330
+ } | undefined;
2331
+ anthropic?: {
2332
+ apiKeyEnv: string;
2333
+ baseUrl?: string | undefined;
2334
+ organization?: string | undefined;
2335
+ } | undefined;
2336
+ ollama?: {
2337
+ apiKeyEnv?: string | undefined;
2338
+ baseUrl?: string | undefined;
2339
+ } | undefined;
2340
+ gemini?: {
2341
+ apiKeyEnv: string;
2342
+ baseUrl?: string | undefined;
2343
+ organization?: string | undefined;
2344
+ } | undefined;
2345
+ mistral?: {
2346
+ apiKeyEnv: string;
2347
+ baseUrl?: string | undefined;
2348
+ organization?: string | undefined;
2349
+ } | undefined;
2350
+ cohere?: {
2351
+ apiKeyEnv: string;
2352
+ baseUrl?: string | undefined;
2353
+ organization?: string | undefined;
2354
+ } | undefined;
2355
+ };
2356
+ prompts: Record<string, {
2357
+ user: string;
2358
+ system?: string | undefined;
2359
+ assistant?: string | undefined;
2360
+ }>;
2361
+ tests: {
2362
+ name: string;
2363
+ expect: {
2364
+ judge?: {
2365
+ criteria: string;
2366
+ model?: string | undefined;
2367
+ minScore?: number | undefined;
2368
+ rubric?: string | undefined;
2369
+ }[] | undefined;
2370
+ output?: {
2371
+ format?: "text" | "json" | undefined;
2372
+ schemaFile?: string | undefined;
2373
+ contains?: string[] | undefined;
2374
+ notContains?: string[] | undefined;
2375
+ maxLength?: number | undefined;
2376
+ } | undefined;
2377
+ guardrails?: {
2378
+ pii?: {
2379
+ enabled?: boolean | undefined;
2380
+ denyPatterns?: string[] | undefined;
2381
+ customPatterns?: {
2382
+ name: string;
2383
+ pattern: string;
2384
+ }[] | undefined;
2385
+ } | undefined;
2386
+ keywords?: {
2387
+ deny?: string[] | undefined;
2388
+ allow?: string[] | undefined;
2389
+ } | undefined;
2390
+ } | undefined;
2391
+ toolCalls?: {
2392
+ tool: string;
2393
+ shouldNotCall?: boolean | undefined;
2394
+ argsMatch?: Record<string, unknown> | undefined;
2395
+ argsSchema?: string | undefined;
2396
+ order?: number | undefined;
2397
+ responseContains?: string | undefined;
2398
+ }[] | undefined;
2399
+ baseline?: {
2400
+ drift?: {
2401
+ maxScore?: number | undefined;
2402
+ method?: "judge" | "embedding" | "field-diff" | undefined;
2403
+ fields?: string[] | undefined;
2404
+ } | undefined;
2405
+ } | undefined;
2406
+ };
2407
+ prompt?: string | undefined;
2408
+ command?: string | undefined;
2409
+ vars?: Record<string, string> | undefined;
2410
+ models?: string[] | undefined;
2411
+ repeat?: number | undefined;
2412
+ tools?: {
2413
+ name: string;
2414
+ description?: string | undefined;
2415
+ parameters?: Record<string, unknown> | undefined;
2416
+ responses?: {
2417
+ when: Record<string, unknown>;
2418
+ then?: unknown;
2419
+ }[] | undefined;
2420
+ defaultResponse?: unknown;
2421
+ }[] | undefined;
2422
+ tags?: string[] | undefined;
2423
+ skip?: boolean | undefined;
2424
+ }[];
2425
+ gates?: {
2426
+ passRateMin?: number | undefined;
2427
+ schemaFailuresMax?: number | undefined;
2428
+ judgeAvgMin?: number | undefined;
2429
+ driftScoreMax?: number | undefined;
2430
+ piiFailuresMax?: number | undefined;
2431
+ keywordFailuresMax?: number | undefined;
2432
+ costMaxUsd?: number | undefined;
2433
+ latencyMaxMs?: number | undefined;
2434
+ deterministicPassRate?: number | undefined;
2435
+ probabilisticPassRate?: number | undefined;
2436
+ } | undefined;
2437
+ compliance?: {
2438
+ enabled?: boolean | undefined;
2439
+ framework?: "custom" | "eu-ai-act" | undefined;
2440
+ outputDir?: string | undefined;
2441
+ metadata?: {
2442
+ systemName?: string | undefined;
2443
+ systemVersion?: string | undefined;
2444
+ riskLevel?: "high" | "limited" | "minimal" | undefined;
2445
+ operator?: string | undefined;
2446
+ intendedPurpose?: string | undefined;
2447
+ dataGovernanceNotes?: string | undefined;
2448
+ } | undefined;
2449
+ } | undefined;
2450
+ trace?: {
2451
+ port?: number | undefined;
2452
+ timeoutMs?: number | undefined;
2453
+ spanMapping?: {
2454
+ outputTextAttr?: string | undefined;
2455
+ modelAttr?: string | undefined;
2456
+ systemAttr?: string | undefined;
2457
+ inputTokensAttr?: string | undefined;
2458
+ outputTokensAttr?: string | undefined;
2459
+ } | undefined;
2460
+ spanFilter?: {
2461
+ namePattern?: string | undefined;
2462
+ attributeMatch?: Record<string, string> | undefined;
2463
+ minDurationMs?: number | undefined;
2464
+ } | undefined;
2465
+ } | undefined;
2466
+ upload?: {
2467
+ enabled?: boolean | undefined;
2468
+ includeArtifacts?: boolean | undefined;
2469
+ redactPatterns?: string[] | undefined;
2470
+ apiUrl?: string | undefined;
2471
+ } | undefined;
2472
+ defaults?: {
2473
+ timeoutMs?: number | undefined;
2474
+ repeat?: number | undefined;
2475
+ concurrency?: number | undefined;
2476
+ judgeModel?: string | undefined;
2477
+ } | undefined;
2478
+ }>;
2479
+ type KindLMConfig = z.infer<typeof KindLMConfigSchema>;
2480
+ type TestCase = z.infer<typeof TestCaseSchema>;
2481
+ type Expect = z.infer<typeof ExpectSchema>;
2482
+ type ModelConfig = z.infer<typeof ModelSchema>;
2483
+ type ProviderConfig = z.infer<typeof ProviderConfigSchema>;
2484
+ type GatesConfig = z.infer<typeof GatesSchema>;
2485
+ type JudgeCriterion = z.infer<typeof JudgeCriterionSchema>;
2486
+ type ToolCallExpect = z.infer<typeof ToolCallExpectSchema>;
2487
+ type ToolSimulation = z.infer<typeof ToolSimulationSchema>;
2488
+ type ComplianceConfig = z.infer<typeof ComplianceSchema>;
2489
+ declare function validateConfig(raw: unknown): Result<KindLMConfig, KindlmError>;
2490
+
2491
+ interface OtlpAttribute {
2492
+ key: string;
2493
+ value: OtlpAttributeValue;
2494
+ }
2495
+ interface OtlpAttributeValue {
2496
+ stringValue?: string;
2497
+ intValue?: string;
2498
+ doubleValue?: number;
2499
+ boolValue?: boolean;
2500
+ arrayValue?: {
2501
+ values: OtlpAttributeValue[];
2502
+ };
2503
+ }
2504
+ interface OtlpSpan {
2505
+ traceId: string;
2506
+ spanId: string;
2507
+ parentSpanId?: string;
2508
+ name: string;
2509
+ kind: number;
2510
+ startTimeUnixNano: string;
2511
+ endTimeUnixNano: string;
2512
+ attributes?: OtlpAttribute[];
2513
+ status?: {
2514
+ code?: number;
2515
+ message?: string;
2516
+ };
2517
+ }
2518
+ interface OtlpScopeSpans {
2519
+ scope?: {
2520
+ name?: string;
2521
+ version?: string;
2522
+ };
2523
+ spans: OtlpSpan[];
2524
+ }
2525
+ interface OtlpResourceSpans {
2526
+ resource?: {
2527
+ attributes?: OtlpAttribute[];
2528
+ };
2529
+ scopeSpans: OtlpScopeSpans[];
2530
+ }
2531
+ interface OtlpTracesPayload {
2532
+ resourceSpans: OtlpResourceSpans[];
2533
+ }
2534
+ interface ParsedSpan {
2535
+ traceId: string;
2536
+ spanId: string;
2537
+ parentSpanId?: string;
2538
+ name: string;
2539
+ kind: number;
2540
+ startTimeMs: number;
2541
+ endTimeMs: number;
2542
+ durationMs: number;
2543
+ attributes: Record<string, string | number | boolean>;
2544
+ resourceAttributes: Record<string, string | number | boolean>;
2545
+ statusCode?: number;
2546
+ statusMessage?: string;
2547
+ }
2548
+ interface SpanMappingResult {
2549
+ outputText: string;
2550
+ toolCalls: {
2551
+ id: string;
2552
+ name: string;
2553
+ arguments: Record<string, unknown>;
2554
+ }[];
2555
+ latencyMs: number;
2556
+ inputTokens: number;
2557
+ outputTokens: number;
2558
+ model?: string;
2559
+ system?: string;
2560
+ }
2561
+ declare const SpanMappingSchema: z.ZodObject<{
2562
+ outputTextAttr: z.ZodDefault<z.ZodString>;
2563
+ modelAttr: z.ZodDefault<z.ZodString>;
2564
+ systemAttr: z.ZodDefault<z.ZodString>;
2565
+ inputTokensAttr: z.ZodDefault<z.ZodString>;
2566
+ outputTokensAttr: z.ZodDefault<z.ZodString>;
2567
+ }, "strip", z.ZodTypeAny, {
2568
+ outputTextAttr: string;
2569
+ modelAttr: string;
2570
+ systemAttr: string;
2571
+ inputTokensAttr: string;
2572
+ outputTokensAttr: string;
2573
+ }, {
2574
+ outputTextAttr?: string | undefined;
2575
+ modelAttr?: string | undefined;
2576
+ systemAttr?: string | undefined;
2577
+ inputTokensAttr?: string | undefined;
2578
+ outputTokensAttr?: string | undefined;
2579
+ }>;
2580
+ declare const SpanFilterSchema: z.ZodObject<{
2581
+ namePattern: z.ZodOptional<z.ZodString>;
2582
+ attributeMatch: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
2583
+ minDurationMs: z.ZodOptional<z.ZodNumber>;
2584
+ }, "strip", z.ZodTypeAny, {
2585
+ namePattern?: string | undefined;
2586
+ attributeMatch?: Record<string, string> | undefined;
2587
+ minDurationMs?: number | undefined;
2588
+ }, {
2589
+ namePattern?: string | undefined;
2590
+ attributeMatch?: Record<string, string> | undefined;
2591
+ minDurationMs?: number | undefined;
2592
+ }>;
2593
+ declare const TraceConfigSchema: z.ZodObject<{
2594
+ port: z.ZodDefault<z.ZodNumber>;
2595
+ timeoutMs: z.ZodDefault<z.ZodNumber>;
2596
+ spanMapping: z.ZodDefault<z.ZodObject<{
2597
+ outputTextAttr: z.ZodDefault<z.ZodString>;
2598
+ modelAttr: z.ZodDefault<z.ZodString>;
2599
+ systemAttr: z.ZodDefault<z.ZodString>;
2600
+ inputTokensAttr: z.ZodDefault<z.ZodString>;
2601
+ outputTokensAttr: z.ZodDefault<z.ZodString>;
2602
+ }, "strip", z.ZodTypeAny, {
2603
+ outputTextAttr: string;
2604
+ modelAttr: string;
2605
+ systemAttr: string;
2606
+ inputTokensAttr: string;
2607
+ outputTokensAttr: string;
2608
+ }, {
2609
+ outputTextAttr?: string | undefined;
2610
+ modelAttr?: string | undefined;
2611
+ systemAttr?: string | undefined;
2612
+ inputTokensAttr?: string | undefined;
2613
+ outputTokensAttr?: string | undefined;
2614
+ }>>;
2615
+ spanFilter: z.ZodOptional<z.ZodObject<{
2616
+ namePattern: z.ZodOptional<z.ZodString>;
2617
+ attributeMatch: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
2618
+ minDurationMs: z.ZodOptional<z.ZodNumber>;
2619
+ }, "strip", z.ZodTypeAny, {
2620
+ namePattern?: string | undefined;
2621
+ attributeMatch?: Record<string, string> | undefined;
2622
+ minDurationMs?: number | undefined;
2623
+ }, {
2624
+ namePattern?: string | undefined;
2625
+ attributeMatch?: Record<string, string> | undefined;
2626
+ minDurationMs?: number | undefined;
2627
+ }>>;
2628
+ }, "strip", z.ZodTypeAny, {
2629
+ port: number;
2630
+ timeoutMs: number;
2631
+ spanMapping: {
2632
+ outputTextAttr: string;
2633
+ modelAttr: string;
2634
+ systemAttr: string;
2635
+ inputTokensAttr: string;
2636
+ outputTokensAttr: string;
2637
+ };
2638
+ spanFilter?: {
2639
+ namePattern?: string | undefined;
2640
+ attributeMatch?: Record<string, string> | undefined;
2641
+ minDurationMs?: number | undefined;
2642
+ } | undefined;
2643
+ }, {
2644
+ port?: number | undefined;
2645
+ timeoutMs?: number | undefined;
2646
+ spanMapping?: {
2647
+ outputTextAttr?: string | undefined;
2648
+ modelAttr?: string | undefined;
2649
+ systemAttr?: string | undefined;
2650
+ inputTokensAttr?: string | undefined;
2651
+ outputTokensAttr?: string | undefined;
2652
+ } | undefined;
2653
+ spanFilter?: {
2654
+ namePattern?: string | undefined;
2655
+ attributeMatch?: Record<string, string> | undefined;
2656
+ minDurationMs?: number | undefined;
2657
+ } | undefined;
2658
+ }>;
2659
+ type TraceConfig = z.infer<typeof TraceConfigSchema>;
2660
+ type SpanMapping = z.infer<typeof SpanMappingSchema>;
2661
+ type SpanFilter = z.infer<typeof SpanFilterSchema>;
2662
+
2663
+ interface ComplianceReport {
2664
+ title: string;
2665
+ generatedAt: string;
2666
+ version: string;
2667
+ hash: string;
2668
+ sections: ComplianceSection[];
2669
+ }
2670
+ interface ComplianceSection {
2671
+ articleRef: string;
2672
+ title: string;
2673
+ content: string;
2674
+ evidence: ComplianceEvidence[];
2675
+ }
2676
+ interface ComplianceEvidence {
2677
+ testName: string;
2678
+ suiteName: string;
2679
+ passed: boolean;
2680
+ details: string;
2681
+ }
2682
+ interface JUnitReport {
2683
+ name: string;
2684
+ tests: number;
2685
+ failures: number;
2686
+ errors: number;
2687
+ time: number;
2688
+ testSuites: JUnitTestSuite[];
2689
+ }
2690
+ interface JUnitTestSuite {
2691
+ name: string;
2692
+ tests: number;
2693
+ failures: number;
2694
+ errors: number;
2695
+ time: number;
2696
+ testCases: JUnitTestCase[];
2697
+ }
2698
+ interface JUnitTestCase {
2699
+ name: string;
2700
+ classname: string;
2701
+ time: number;
2702
+ failure?: {
2703
+ message: string;
2704
+ type: string;
2705
+ content: string;
2706
+ };
2707
+ error?: {
2708
+ message: string;
2709
+ type: string;
2710
+ content: string;
2711
+ };
2712
+ }
2713
+
2714
+ type FailureCode = "SCHEMA_INVALID" | "SCHEMA_PARSE_ERROR" | "PII_DETECTED" | "KEYWORD_DENIED" | "KEYWORD_MISSING" | "CONTAINS_FAILED" | "NOT_CONTAINS_FAILED" | "MAX_LENGTH_EXCEEDED" | "JUDGE_BELOW_THRESHOLD" | "TOOL_CALL_MISSING" | "TOOL_CALL_UNEXPECTED" | "TOOL_CALL_ARGS_MISMATCH" | "TOOL_CALL_ORDER_WRONG" | "TOOL_CALL_ARGS_SCHEMA_INVALID" | "DRIFT_EXCEEDED" | "PROVIDER_TIMEOUT" | "PROVIDER_AUTH_FAILED" | "PROVIDER_ERROR" | "INVALID_PATTERN" | "INTERNAL_ERROR";
2715
+ interface AssertionResult {
2716
+ assertionType: string;
2717
+ label: string;
2718
+ passed: boolean;
2719
+ score: number;
2720
+ failureCode?: FailureCode;
2721
+ failureMessage?: string;
2722
+ metadata?: Record<string, unknown>;
2723
+ }
2724
+ interface AssertionContext {
2725
+ outputText: string;
2726
+ outputJson?: unknown;
2727
+ toolCalls: ProviderToolCall[];
2728
+ baselineText?: string;
2729
+ baselineJson?: unknown;
2730
+ judgeAdapter?: ProviderAdapter;
2731
+ judgeModel?: string;
2732
+ configDir: string;
2733
+ latencyMs?: number;
2734
+ costUsd?: number;
2735
+ }
2736
+ interface Assertion {
2737
+ readonly type: string;
2738
+ evaluate(context: AssertionContext): Promise<AssertionResult[]>;
2739
+ }
2740
+
2741
+ interface ToolCallExpectation {
2742
+ tool: string;
2743
+ shouldNotCall?: boolean;
2744
+ argsMatch?: Record<string, unknown>;
2745
+ argsSchema?: string;
2746
+ order?: number;
2747
+ }
2748
+ declare function createToolCalledAssertion(tool: string, argsMatch?: Record<string, unknown>): Assertion;
2749
+ declare function createToolNotCalledAssertion(tool: string): Assertion;
2750
+ declare function createToolOrderAssertion(expectations: ToolCallExpectation[]): Assertion;
2751
+
2752
+ interface SchemaAssertionConfig {
2753
+ format: "text" | "json";
2754
+ schemaFile?: string;
2755
+ schemaContent?: Record<string, unknown>;
2756
+ contains?: string[];
2757
+ notContains?: string[];
2758
+ maxLength?: number;
2759
+ }
2760
+ declare function createSchemaAssertion(config: SchemaAssertionConfig): Assertion;
2761
+
2762
+ interface PiiAssertionConfig {
2763
+ denyPatterns: string[];
2764
+ customPatterns?: Array<{
2765
+ name: string;
2766
+ pattern: string;
2767
+ }>;
2768
+ }
2769
+ declare function createPiiAssertion(config: PiiAssertionConfig): Assertion;
2770
+
2771
+ declare function createKeywordsPresentAssertion(keywords: string[]): Assertion;
2772
+ declare function createKeywordsAbsentAssertion(keywords: string[]): Assertion;
2773
+
2774
+ interface JudgeAssertionConfig {
2775
+ criteria: string;
2776
+ minScore: number;
2777
+ rubric?: string;
2778
+ }
2779
+ declare function createJudgeAssertion(config: JudgeAssertionConfig): Assertion;
2780
+
2781
+ interface DriftAssertionConfig {
2782
+ maxScore: number;
2783
+ method: "judge" | "embedding" | "field-diff";
2784
+ fields?: string[];
2785
+ }
2786
+ declare function createDriftAssertion(config: DriftAssertionConfig): Assertion;
2787
+
2788
+ interface LatencyAssertionConfig {
2789
+ maxMs: number;
2790
+ }
2791
+ declare function createLatencyAssertion(config: LatencyAssertionConfig): Assertion;
2792
+
2793
+ interface CostAssertionConfig {
2794
+ maxUsd: number;
2795
+ }
2796
+ declare function createCostAssertion(config: CostAssertionConfig): Assertion;
2797
+
2798
+ type AssertionFactory = (config: Expect) => Assertion;
2799
+ interface AssertionOverrides {
2800
+ schemaContent?: Record<string, unknown>;
2801
+ }
2802
+ declare function createAssertionsFromExpect(expect: Expect, overrides?: AssertionOverrides): Assertion[];
2803
+ declare function createAssertionRegistry(): Map<string, AssertionFactory>;
2804
+
2805
+ type AssertionCategory = "deterministic" | "probabilistic";
2806
+ declare function classifyAssertion(assertionType: string): AssertionCategory;
2807
+ declare function isDeterministic(assertionType: string): boolean;
2808
+ declare function isProbabilistic(assertionType: string): boolean;
2809
+
2810
+ declare function createOpenAIAdapter(httpClient: HttpClient): ProviderAdapter;
2811
+
2812
+ declare function createAnthropicAdapter(httpClient: HttpClient): ProviderAdapter;
2813
+
2814
+ declare function createOllamaAdapter(httpClient: HttpClient): ProviderAdapter;
2815
+
2816
+ declare function createGeminiAdapter(httpClient: HttpClient): ProviderAdapter;
2817
+
2818
+ declare function createMistralAdapter(httpClient: HttpClient): ProviderAdapter;
2819
+
2820
+ declare function createCohereAdapter(httpClient: HttpClient): ProviderAdapter;
2821
+
2822
+ declare function createProvider(name: string, httpClient: HttpClient): ProviderAdapter;
2823
+
2824
+ declare function runConversation(adapter: ProviderAdapter, initialRequest: ProviderRequest, toolSimulations: ToolSimulation[], options?: {
2825
+ maxTurns?: number;
2826
+ }): Promise<ConversationResult>;
2827
+
2828
+ interface RetryOptions {
2829
+ maxRetries: number;
2830
+ shouldRetry: (error: unknown) => boolean;
2831
+ baseDelayMs?: number;
2832
+ }
2833
+ declare function withRetry<T>(fn: () => Promise<T>, options: RetryOptions): Promise<T>;
2834
+
2835
+ interface FileReader {
2836
+ readFile(path: string): Result<string>;
2837
+ }
2838
+ interface ParseOptions {
2839
+ configDir: string;
2840
+ fileReader?: FileReader;
2841
+ }
2842
+ declare function parseConfig(yamlContent: string, options: ParseOptions): Result<KindLMConfig>;
2843
+
2844
+ declare function interpolate(template: string, vars: Record<string, string>): Result<string>;
2845
+ declare function findMissingVars(template: string, vars: Record<string, string>): string[];
2846
+
2847
+ interface TestCaseRunResult {
2848
+ testCaseName: string;
2849
+ modelId: string;
2850
+ runIndex: number;
2851
+ outputText: string;
2852
+ assertions: AssertionResult[];
2853
+ latencyMs: number;
2854
+ tokenUsage: {
2855
+ inputTokens: number;
2856
+ outputTokens: number;
2857
+ totalTokens: number;
2858
+ };
2859
+ costEstimateUsd: number | null;
2860
+ }
2861
+ interface AggregatedTestResult {
2862
+ testCaseName: string;
2863
+ modelId: string;
2864
+ runCount: number;
2865
+ passed: boolean;
2866
+ passRate: number;
2867
+ assertionScores: Record<string, {
2868
+ mean: number;
2869
+ min: number;
2870
+ max: number;
2871
+ }>;
2872
+ failureCodes: string[];
2873
+ latencyAvgMs: number;
2874
+ totalCostUsd: number;
2875
+ totalTokens: number;
2876
+ runs: TestCaseRunResult[];
2877
+ }
2878
+ declare function aggregateRuns(runs: TestCaseRunResult[]): AggregatedTestResult;
2879
+
2880
+ declare const BASELINE_VERSION = "1";
2881
+ interface BaselineIO {
2882
+ read(suiteName: string): Result<string>;
2883
+ write(suiteName: string, content: string): Result<void>;
2884
+ list(): Result<string[]>;
2885
+ }
2886
+ interface BaselineTestEntry {
2887
+ passRate: number;
2888
+ outputText: string;
2889
+ failureCodes: string[];
2890
+ latencyAvgMs: number;
2891
+ costUsd: number;
2892
+ runCount: number;
2893
+ }
2894
+ interface BaselineData {
2895
+ version: string;
2896
+ suiteName: string;
2897
+ createdAt: string;
2898
+ results: Record<string, BaselineTestEntry>;
2899
+ }
2900
+ declare function serializeBaseline(data: BaselineData): string;
2901
+ declare function deserializeBaseline(raw: string): Result<BaselineData>;
2902
+ declare function readBaseline(suiteName: string, io: BaselineIO): Result<BaselineData>;
2903
+ declare function writeBaseline(data: BaselineData, io: BaselineIO): Result<void>;
2904
+ declare function listBaselines(io: BaselineIO): Result<string[]>;
2905
+
2906
+ interface CommandExecutor {
2907
+ execute(command: string, options: CommandExecuteOptions): Promise<Result<RawCommandOutput>>;
2908
+ }
2909
+ interface CommandExecuteOptions {
2910
+ timeoutMs: number;
2911
+ env?: Record<string, string>;
2912
+ cwd?: string;
2913
+ }
2914
+ interface RawCommandOutput {
2915
+ stdout: string;
2916
+ stderr: string;
2917
+ exitCode: number;
2918
+ }
2919
+ interface CommandResult {
2920
+ outputText: string;
2921
+ toolCalls: ProviderToolCall[];
2922
+ outputJson?: unknown;
2923
+ exitCode: number;
2924
+ stderr: string;
2925
+ }
2926
+ declare function parseCommandOutput(raw: RawCommandOutput): CommandResult;
2927
+
2928
+ interface RunnerDeps {
2929
+ adapters: Map<string, ProviderAdapter>;
2930
+ configDir: string;
2931
+ fileReader: FileReader;
2932
+ onProgress?: (event: ProgressEvent) => void;
2933
+ baselineData?: BaselineData;
2934
+ commandExecutor?: CommandExecutor;
2935
+ }
2936
+ type ProgressEvent = {
2937
+ type: "test_start";
2938
+ test: string;
2939
+ model: string;
2940
+ run: number;
2941
+ } | {
2942
+ type: "test_complete";
2943
+ test: string;
2944
+ model: string;
2945
+ run: number;
2946
+ passed: boolean;
2947
+ };
2948
+ interface RunResult {
2949
+ suites: SuiteRunResult[];
2950
+ totalTests: number;
2951
+ passed: number;
2952
+ failed: number;
2953
+ errored: number;
2954
+ skipped: number;
2955
+ durationMs: number;
2956
+ }
2957
+ interface SuiteRunResult {
2958
+ name: string;
2959
+ status: "passed" | "failed" | "errored" | "skipped";
2960
+ tests: TestRunResult[];
2961
+ error?: string;
2962
+ }
2963
+ interface TestRunResult {
2964
+ name: string;
2965
+ modelId: string;
2966
+ status: "passed" | "failed" | "errored" | "skipped";
2967
+ assertions: AssertionResult[];
2968
+ error?: KindlmError;
2969
+ latencyMs: number;
2970
+ costUsd: number;
2971
+ }
2972
+ interface RunnerResult {
2973
+ runResult: RunResult;
2974
+ aggregated: AggregatedTestResult[];
2975
+ }
2976
+ declare function createRunner(config: KindLMConfig, deps: RunnerDeps): {
2977
+ run(): Promise<Result<RunnerResult>>;
2978
+ };
2979
+
2980
+ interface GateResult {
2981
+ gateName: string;
2982
+ passed: boolean;
2983
+ actual: number;
2984
+ threshold: number;
2985
+ message: string;
2986
+ }
2987
+ interface GateEvaluation {
2988
+ passed: boolean;
2989
+ gates: GateResult[];
2990
+ }
2991
+ declare function evaluateGates(config: GatesConfig, results: AggregatedTestResult[]): GateEvaluation;
2992
+
2993
+ interface ReporterOutput {
2994
+ content: string;
2995
+ format: "text" | "json" | "xml" | "markdown";
2996
+ }
2997
+ interface Reporter {
2998
+ readonly name: string;
2999
+ generate(runResult: RunResult, gateEvaluation: GateEvaluation): ReporterOutput;
3000
+ }
3001
+ interface Colorize {
3002
+ bold: (text: string) => string;
3003
+ red: (text: string) => string;
3004
+ green: (text: string) => string;
3005
+ yellow: (text: string) => string;
3006
+ cyan: (text: string) => string;
3007
+ dim: (text: string) => string;
3008
+ greenBold: (text: string) => string;
3009
+ redBold: (text: string) => string;
3010
+ }
3011
+ declare const noColor: Colorize;
3012
+
3013
+ declare function createPrettyReporter(colorize?: Colorize): Reporter;
3014
+
3015
+ declare function createJsonReporter(): Reporter;
3016
+
3017
+ declare function createJunitReporter(): Reporter;
3018
+
3019
+ declare function createComplianceReporter(): Reporter;
3020
+
3021
+ declare function buildBaselineData(suiteName: string, aggregated: AggregatedTestResult[], timestamp: string): BaselineData;
3022
+
3023
+ interface BaselineComparison {
3024
+ suiteName: string;
3025
+ hasBaseline: boolean;
3026
+ regressions: BaselineRegression[];
3027
+ improvements: BaselineImprovement[];
3028
+ unchanged: BaselineUnchanged[];
3029
+ newTests: string[];
3030
+ removedTests: string[];
3031
+ }
3032
+ interface BaselineRegression {
3033
+ testName: string;
3034
+ baselinePassRate: number;
3035
+ currentPassRate: number;
3036
+ newFailureCodes: string[];
3037
+ }
3038
+ interface BaselineImprovement {
3039
+ testName: string;
3040
+ baselinePassRate: number;
3041
+ currentPassRate: number;
3042
+ }
3043
+ interface BaselineUnchanged {
3044
+ testName: string;
3045
+ passRate: number;
3046
+ }
3047
+ declare function compareBaseline(baseline: BaselineData, currentResults: Record<string, BaselineTestEntry>): BaselineComparison;
3048
+
3049
+ declare function parseOtlpPayload(raw: unknown): Result<ParsedSpan[]>;
3050
+
3051
+ declare function filterSpans(spans: ParsedSpan[], filter?: SpanFilter): ParsedSpan[];
3052
+ declare function mapSpansToResult(spans: ParsedSpan[], mapping: SpanMapping): SpanMappingResult;
3053
+ interface BuildContextOptions {
3054
+ configDir: string;
3055
+ judgeAdapter?: ProviderAdapter;
3056
+ judgeModel?: string;
3057
+ baselineText?: string;
3058
+ }
3059
+ declare function buildContextFromTrace(result: SpanMappingResult, options: BuildContextOptions): AssertionContext;
3060
+
3061
+ export { type AggregatedTestResult, type Assertion, type AssertionCategory, type AssertionContext, type AssertionFactory, type AssertionOverrides, type AssertionResult, BASELINE_VERSION, type BaselineComparison, type BaselineData, type BaselineIO, type BaselineImprovement, type BaselineRegression, type BaselineTestEntry, type BaselineUnchanged, type BuildContextOptions, type Colorize, type CommandExecuteOptions, type CommandExecutor, type CommandResult, type ComplianceConfig, type ComplianceEvidence, type ComplianceReport, type ComplianceSection, type ConversationResult, type ConversationTurn, type CostAssertionConfig, type DriftAssertionConfig, type ErrorCode, type Expect, type FailureCode, type FileReader, type GateEvaluation, type GateResult, type GatesConfig, type HttpClient, type HttpRequestInit, type HttpResponse, type JUnitReport, type JUnitTestCase, type JUnitTestSuite, type JudgeAssertionConfig, type JudgeCriterion, type KindLMConfig, KindLMConfigSchema, type KindLMConfig as KindlmConfig, type KindlmError, type LatencyAssertionConfig, type ModelConfig, type OtlpAttribute, type OtlpAttributeValue, type OtlpResourceSpans, type OtlpScopeSpans, type OtlpSpan, type OtlpTracesPayload, type ParseOptions, type ParsedSpan, type PiiAssertionConfig, type ProgressEvent, type ProviderAdapter, type ProviderAdapterConfig, type ProviderConfig, ProviderError, type ProviderErrorCode, type ProviderMessage, type ProviderRequest, type ProviderResponse, type ProviderToolCall, type ProviderToolDefinition, type RawCommandOutput, type Reporter, type ReporterOutput, type Result, type RetryOptions, type RunResult, type RunnerDeps, type RunnerResult, type SchemaAssertionConfig, type SpanFilter, SpanFilterSchema, type SpanMapping, type SpanMappingResult, SpanMappingSchema, type SuiteRunResult, type TestCase, type TestCaseRunResult, type TestRunResult, type ToolCallExpect, type ToolCallExpectation, type ToolSimulation, type TraceConfig, TraceConfigSchema, aggregateRuns, buildBaselineData, buildContextFromTrace, classifyAssertion, compareBaseline, createAnthropicAdapter, createAssertionRegistry, createAssertionsFromExpect, createCohereAdapter, createComplianceReporter, createCostAssertion, createDriftAssertion, createGeminiAdapter, createJsonReporter, createJudgeAssertion, createJunitReporter, createKeywordsAbsentAssertion, createKeywordsPresentAssertion, createLatencyAssertion, createMistralAdapter, createOllamaAdapter, createOpenAIAdapter, createPiiAssertion, createPrettyReporter, createProvider, createRunner, createSchemaAssertion, createToolCalledAssertion, createToolNotCalledAssertion, createToolOrderAssertion, deserializeBaseline, err, evaluateGates, filterSpans, findMissingVars, interpolate, isDeterministic, isProbabilistic, listBaselines, mapSpansToResult, noColor, ok, parseCommandOutput, parseConfig, parseOtlpPayload, readBaseline, runConversation, serializeBaseline, validateConfig, withRetry, writeBaseline };