@wix/evalforge-types 0.59.0 → 0.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +35 -51
- package/build/index.js.map +2 -2
- package/build/index.mjs +35 -45
- package/build/index.mjs.map +3 -3
- package/build/types/assertion/assertion.d.ts +22 -193
- package/build/types/assertion/index.d.ts +1 -1
- package/build/types/assertion/system-assertions.d.ts +1 -1
- package/build/types/scenario/test-scenario.d.ts +21 -3
- package/package.json +2 -2
|
@@ -6,7 +6,9 @@ import { z } from 'zod';
|
|
|
6
6
|
* - build_passed: Runs a command and checks exit code (deterministic, system-level)
|
|
7
7
|
* - time_limit: Checks that scenario completed within a duration threshold (deterministic, system-level)
|
|
8
8
|
* - cost: Checks that scenario LLM cost stays within a USD threshold (deterministic, system-level)
|
|
9
|
-
* - llm_judge: LLM evaluates output with a prompt (LLM-based,
|
|
9
|
+
* - llm_judge: LLM evaluates output with a prompt (LLM-based, system-level)
|
|
10
|
+
*
|
|
11
|
+
* Any assertion can be negated by setting `negate: true` to invert the pass/fail logic.
|
|
10
12
|
*/
|
|
11
13
|
export declare const AssertionTypeSchema: z.ZodEnum<{
|
|
12
14
|
skill_was_called: "skill_was_called";
|
|
@@ -72,12 +74,13 @@ export declare const CostConfigSchema: z.ZodObject<{
|
|
|
72
74
|
export type CostConfig = z.infer<typeof CostConfigSchema>;
|
|
73
75
|
/** Configuration for tool_called_with_param assertion type.
|
|
74
76
|
* Uses strictObject to reject objects with unknown keys.
|
|
77
|
+
* When expectedParams is omitted, the assertion checks only that the tool was called (or not, if negated).
|
|
75
78
|
*/
|
|
76
79
|
export declare const ToolCalledWithParamConfigSchema: z.ZodObject<{
|
|
77
80
|
/** Name of the tool that must have been called */
|
|
78
81
|
toolName: z.ZodString;
|
|
79
|
-
/** JSON string of key-value pairs for expected parameters (substring match) */
|
|
80
|
-
expectedParams: z.ZodString
|
|
82
|
+
/** JSON string of key-value pairs for expected parameters (substring match). Optional — when omitted, only checks tool presence. */
|
|
83
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
81
84
|
/** If true, the matching tool call must also have succeeded (step.success === true) */
|
|
82
85
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
83
86
|
}, z.core.$strict>;
|
|
@@ -103,7 +106,6 @@ export declare const TimeConfigSchema: z.ZodObject<{
|
|
|
103
106
|
export type TimeConfig = z.infer<typeof TimeConfigSchema>;
|
|
104
107
|
/**
|
|
105
108
|
* Configuration for llm_judge assertion type.
|
|
106
|
-
* User-created assertions with customizable parameters.
|
|
107
109
|
*/
|
|
108
110
|
export declare const LlmJudgeConfigSchema: z.ZodObject<{
|
|
109
111
|
prompt: z.ZodString;
|
|
@@ -127,24 +129,28 @@ export declare const LlmJudgeConfigSchema: z.ZodObject<{
|
|
|
127
129
|
export type LlmJudgeConfig = z.infer<typeof LlmJudgeConfigSchema>;
|
|
128
130
|
export declare const SkillWasCalledAssertionSchema: z.ZodObject<{
|
|
129
131
|
skillNames: z.ZodArray<z.ZodString>;
|
|
132
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
130
133
|
type: z.ZodLiteral<"skill_was_called">;
|
|
131
134
|
}, z.core.$strip>;
|
|
132
135
|
export type SkillWasCalledAssertion = z.infer<typeof SkillWasCalledAssertionSchema>;
|
|
133
136
|
export declare const ToolCalledWithParamAssertionSchema: z.ZodObject<{
|
|
134
137
|
toolName: z.ZodString;
|
|
135
|
-
expectedParams: z.ZodString
|
|
138
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
136
139
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
140
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
137
141
|
type: z.ZodLiteral<"tool_called_with_param">;
|
|
138
142
|
}, z.core.$strict>;
|
|
139
143
|
export type ToolCalledWithParamAssertion = z.infer<typeof ToolCalledWithParamAssertionSchema>;
|
|
140
144
|
export declare const BuildPassedAssertionSchema: z.ZodObject<{
|
|
141
145
|
command: z.ZodOptional<z.ZodString>;
|
|
142
146
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
147
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
143
148
|
type: z.ZodLiteral<"build_passed">;
|
|
144
149
|
}, z.core.$strict>;
|
|
145
150
|
export type BuildPassedAssertion = z.infer<typeof BuildPassedAssertionSchema>;
|
|
146
151
|
export declare const CostAssertionSchema: z.ZodObject<{
|
|
147
152
|
maxCostUsd: z.ZodNumber;
|
|
153
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
148
154
|
type: z.ZodLiteral<"cost">;
|
|
149
155
|
}, z.core.$strict>;
|
|
150
156
|
export type CostAssertion = z.infer<typeof CostAssertionSchema>;
|
|
@@ -166,11 +172,13 @@ export declare const LlmJudgeAssertionSchema: z.ZodObject<{
|
|
|
166
172
|
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
167
173
|
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
168
174
|
}, z.core.$strip>>>;
|
|
175
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
169
176
|
type: z.ZodLiteral<"llm_judge">;
|
|
170
177
|
}, z.core.$strip>;
|
|
171
178
|
export type LlmJudgeAssertion = z.infer<typeof LlmJudgeAssertionSchema>;
|
|
172
179
|
export declare const TimeAssertionSchema: z.ZodObject<{
|
|
173
180
|
maxDurationMs: z.ZodNumber;
|
|
181
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
174
182
|
type: z.ZodLiteral<"time_limit">;
|
|
175
183
|
}, z.core.$strict>;
|
|
176
184
|
export type TimeAssertion = z.infer<typeof TimeAssertionSchema>;
|
|
@@ -180,21 +188,26 @@ export type TimeAssertion = z.infer<typeof TimeAssertionSchema>;
|
|
|
180
188
|
*/
|
|
181
189
|
export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
182
190
|
skillNames: z.ZodArray<z.ZodString>;
|
|
191
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
183
192
|
type: z.ZodLiteral<"skill_was_called">;
|
|
184
193
|
}, z.core.$strip>, z.ZodObject<{
|
|
185
194
|
toolName: z.ZodString;
|
|
186
|
-
expectedParams: z.ZodString
|
|
195
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
187
196
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
197
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
188
198
|
type: z.ZodLiteral<"tool_called_with_param">;
|
|
189
199
|
}, z.core.$strict>, z.ZodObject<{
|
|
190
200
|
command: z.ZodOptional<z.ZodString>;
|
|
191
201
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
202
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
192
203
|
type: z.ZodLiteral<"build_passed">;
|
|
193
204
|
}, z.core.$strict>, z.ZodObject<{
|
|
194
205
|
maxDurationMs: z.ZodNumber;
|
|
206
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
195
207
|
type: z.ZodLiteral<"time_limit">;
|
|
196
208
|
}, z.core.$strict>, z.ZodObject<{
|
|
197
209
|
maxCostUsd: z.ZodNumber;
|
|
210
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
198
211
|
type: z.ZodLiteral<"cost">;
|
|
199
212
|
}, z.core.$strict>, z.ZodObject<{
|
|
200
213
|
prompt: z.ZodString;
|
|
@@ -214,6 +227,7 @@ export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
214
227
|
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
215
228
|
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
216
229
|
}, z.core.$strip>>>;
|
|
230
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
217
231
|
type: z.ZodLiteral<"llm_judge">;
|
|
218
232
|
}, z.core.$strip>]>;
|
|
219
233
|
export type Assertion = z.infer<typeof AssertionSchema>;
|
|
@@ -246,8 +260,8 @@ export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
246
260
|
}, z.core.$strip>, z.ZodObject<{
|
|
247
261
|
/** Name of the tool that must have been called */
|
|
248
262
|
toolName: z.ZodString;
|
|
249
|
-
/** JSON string of key-value pairs for expected parameters (substring match) */
|
|
250
|
-
expectedParams: z.ZodString
|
|
263
|
+
/** JSON string of key-value pairs for expected parameters (substring match). Optional — when omitted, only checks tool presence. */
|
|
264
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
251
265
|
/** If true, the matching tool call must also have succeeded (step.success === true) */
|
|
252
266
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
253
267
|
}, z.core.$strict>, z.ZodObject<{
|
|
@@ -263,193 +277,8 @@ export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
263
277
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
264
278
|
}, z.core.$strict>, z.ZodObject<{}, z.core.$strip>]>;
|
|
265
279
|
export type AssertionConfig = z.infer<typeof AssertionConfigSchema>;
|
|
266
|
-
/**
|
|
267
|
-
* Custom Assertion entity - stored in the database.
|
|
268
|
-
* Replaces inline assertions in test scenarios.
|
|
269
|
-
*/
|
|
270
|
-
export declare const CustomAssertionSchema: z.ZodObject<{
|
|
271
|
-
id: z.ZodString;
|
|
272
|
-
name: z.ZodString;
|
|
273
|
-
description: z.ZodString;
|
|
274
|
-
createdAt: z.ZodString;
|
|
275
|
-
updatedAt: z.ZodString;
|
|
276
|
-
deleted: z.ZodOptional<z.ZodBoolean>;
|
|
277
|
-
projectId: z.ZodString;
|
|
278
|
-
type: z.ZodEnum<{
|
|
279
|
-
skill_was_called: "skill_was_called";
|
|
280
|
-
tool_called_with_param: "tool_called_with_param";
|
|
281
|
-
build_passed: "build_passed";
|
|
282
|
-
time_limit: "time_limit";
|
|
283
|
-
cost: "cost";
|
|
284
|
-
llm_judge: "llm_judge";
|
|
285
|
-
}>;
|
|
286
|
-
config: z.ZodUnion<readonly [z.ZodObject<{
|
|
287
|
-
prompt: z.ZodString;
|
|
288
|
-
minScore: z.ZodOptional<z.ZodNumber>;
|
|
289
|
-
model: z.ZodOptional<z.ZodString>;
|
|
290
|
-
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
291
|
-
temperature: z.ZodOptional<z.ZodNumber>;
|
|
292
|
-
parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
293
|
-
name: z.ZodString;
|
|
294
|
-
label: z.ZodString;
|
|
295
|
-
type: z.ZodEnum<{
|
|
296
|
-
string: "string";
|
|
297
|
-
number: "number";
|
|
298
|
-
boolean: "boolean";
|
|
299
|
-
}>;
|
|
300
|
-
required: z.ZodBoolean;
|
|
301
|
-
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
302
|
-
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
303
|
-
}, z.core.$strip>>>;
|
|
304
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
305
|
-
skillNames: z.ZodArray<z.ZodString>;
|
|
306
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
307
|
-
/** Name of the tool that must have been called */
|
|
308
|
-
toolName: z.ZodString;
|
|
309
|
-
/** JSON string of key-value pairs for expected parameters (substring match) */
|
|
310
|
-
expectedParams: z.ZodString;
|
|
311
|
-
/** If true, the matching tool call must also have succeeded (step.success === true) */
|
|
312
|
-
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
313
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
314
|
-
/** Maximum allowed duration in milliseconds */
|
|
315
|
-
maxDurationMs: z.ZodNumber;
|
|
316
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
317
|
-
/** Maximum allowed cost in USD */
|
|
318
|
-
maxCostUsd: z.ZodNumber;
|
|
319
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
320
|
-
/** Command to run (default: "yarn build") */
|
|
321
|
-
command: z.ZodOptional<z.ZodString>;
|
|
322
|
-
/** Expected exit code (default: 0) */
|
|
323
|
-
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
324
|
-
}, z.core.$strict>, z.ZodObject<{}, z.core.$strip>]>;
|
|
325
|
-
}, z.core.$strip>;
|
|
326
|
-
export type CustomAssertion = z.infer<typeof CustomAssertionSchema>;
|
|
327
|
-
/**
|
|
328
|
-
* Input schema for creating a new CustomAssertion.
|
|
329
|
-
*/
|
|
330
|
-
export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
|
|
331
|
-
type: z.ZodEnum<{
|
|
332
|
-
skill_was_called: "skill_was_called";
|
|
333
|
-
tool_called_with_param: "tool_called_with_param";
|
|
334
|
-
build_passed: "build_passed";
|
|
335
|
-
time_limit: "time_limit";
|
|
336
|
-
cost: "cost";
|
|
337
|
-
llm_judge: "llm_judge";
|
|
338
|
-
}>;
|
|
339
|
-
name: z.ZodString;
|
|
340
|
-
description: z.ZodString;
|
|
341
|
-
projectId: z.ZodString;
|
|
342
|
-
config: z.ZodUnion<readonly [z.ZodObject<{
|
|
343
|
-
prompt: z.ZodString;
|
|
344
|
-
minScore: z.ZodOptional<z.ZodNumber>;
|
|
345
|
-
model: z.ZodOptional<z.ZodString>;
|
|
346
|
-
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
347
|
-
temperature: z.ZodOptional<z.ZodNumber>;
|
|
348
|
-
parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
349
|
-
name: z.ZodString;
|
|
350
|
-
label: z.ZodString;
|
|
351
|
-
type: z.ZodEnum<{
|
|
352
|
-
string: "string";
|
|
353
|
-
number: "number";
|
|
354
|
-
boolean: "boolean";
|
|
355
|
-
}>;
|
|
356
|
-
required: z.ZodBoolean;
|
|
357
|
-
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
358
|
-
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
359
|
-
}, z.core.$strip>>>;
|
|
360
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
361
|
-
skillNames: z.ZodArray<z.ZodString>;
|
|
362
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
363
|
-
/** Name of the tool that must have been called */
|
|
364
|
-
toolName: z.ZodString;
|
|
365
|
-
/** JSON string of key-value pairs for expected parameters (substring match) */
|
|
366
|
-
expectedParams: z.ZodString;
|
|
367
|
-
/** If true, the matching tool call must also have succeeded (step.success === true) */
|
|
368
|
-
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
369
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
370
|
-
/** Maximum allowed duration in milliseconds */
|
|
371
|
-
maxDurationMs: z.ZodNumber;
|
|
372
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
373
|
-
/** Maximum allowed cost in USD */
|
|
374
|
-
maxCostUsd: z.ZodNumber;
|
|
375
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
376
|
-
/** Command to run (default: "yarn build") */
|
|
377
|
-
command: z.ZodOptional<z.ZodString>;
|
|
378
|
-
/** Expected exit code (default: 0) */
|
|
379
|
-
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
380
|
-
}, z.core.$strict>, z.ZodObject<{}, z.core.$strip>]>;
|
|
381
|
-
}, z.core.$strip>;
|
|
382
|
-
export type CreateCustomAssertionInput = z.infer<typeof CreateCustomAssertionInputSchema>;
|
|
383
|
-
/**
|
|
384
|
-
* Input schema for updating a CustomAssertion.
|
|
385
|
-
*/
|
|
386
|
-
export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
|
|
387
|
-
type: z.ZodOptional<z.ZodEnum<{
|
|
388
|
-
skill_was_called: "skill_was_called";
|
|
389
|
-
tool_called_with_param: "tool_called_with_param";
|
|
390
|
-
build_passed: "build_passed";
|
|
391
|
-
time_limit: "time_limit";
|
|
392
|
-
cost: "cost";
|
|
393
|
-
llm_judge: "llm_judge";
|
|
394
|
-
}>>;
|
|
395
|
-
name: z.ZodOptional<z.ZodString>;
|
|
396
|
-
description: z.ZodOptional<z.ZodString>;
|
|
397
|
-
projectId: z.ZodOptional<z.ZodString>;
|
|
398
|
-
config: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
399
|
-
prompt: z.ZodString;
|
|
400
|
-
minScore: z.ZodOptional<z.ZodNumber>;
|
|
401
|
-
model: z.ZodOptional<z.ZodString>;
|
|
402
|
-
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
403
|
-
temperature: z.ZodOptional<z.ZodNumber>;
|
|
404
|
-
parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
405
|
-
name: z.ZodString;
|
|
406
|
-
label: z.ZodString;
|
|
407
|
-
type: z.ZodEnum<{
|
|
408
|
-
string: "string";
|
|
409
|
-
number: "number";
|
|
410
|
-
boolean: "boolean";
|
|
411
|
-
}>;
|
|
412
|
-
required: z.ZodBoolean;
|
|
413
|
-
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
414
|
-
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
415
|
-
}, z.core.$strip>>>;
|
|
416
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
417
|
-
skillNames: z.ZodArray<z.ZodString>;
|
|
418
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
419
|
-
/** Name of the tool that must have been called */
|
|
420
|
-
toolName: z.ZodString;
|
|
421
|
-
/** JSON string of key-value pairs for expected parameters (substring match) */
|
|
422
|
-
expectedParams: z.ZodString;
|
|
423
|
-
/** If true, the matching tool call must also have succeeded (step.success === true) */
|
|
424
|
-
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
425
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
426
|
-
/** Maximum allowed duration in milliseconds */
|
|
427
|
-
maxDurationMs: z.ZodNumber;
|
|
428
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
429
|
-
/** Maximum allowed cost in USD */
|
|
430
|
-
maxCostUsd: z.ZodNumber;
|
|
431
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
432
|
-
/** Command to run (default: "yarn build") */
|
|
433
|
-
command: z.ZodOptional<z.ZodString>;
|
|
434
|
-
/** Expected exit code (default: 0) */
|
|
435
|
-
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
436
|
-
}, z.core.$strict>, z.ZodObject<{}, z.core.$strip>]>>;
|
|
437
|
-
}, z.core.$strip>;
|
|
438
|
-
export type UpdateCustomAssertionInput = z.infer<typeof UpdateCustomAssertionInputSchema>;
|
|
439
280
|
/**
|
|
440
281
|
* Helper function to validate config based on assertion type.
|
|
441
282
|
* Returns true if config is valid for the given type.
|
|
442
283
|
*/
|
|
443
284
|
export declare function validateAssertionConfig(type: AssertionType, config: unknown): boolean;
|
|
444
|
-
/**
|
|
445
|
-
* Get typed config for skill_was_called assertion.
|
|
446
|
-
*/
|
|
447
|
-
export declare function getSkillWasCalledConfig(assertion: CustomAssertion): SkillWasCalledConfig | null;
|
|
448
|
-
/**
|
|
449
|
-
* Get typed config for build_passed assertion.
|
|
450
|
-
*/
|
|
451
|
-
export declare function getBuildPassedConfig(assertion: CustomAssertion): BuildPassedConfig | null;
|
|
452
|
-
/**
|
|
453
|
-
* Get typed config for llm_judge assertion.
|
|
454
|
-
*/
|
|
455
|
-
export declare function getLlmJudgeConfig(assertion: CustomAssertion): LlmJudgeConfig | null;
|
|
@@ -16,7 +16,7 @@ export interface SystemAssertion {
|
|
|
16
16
|
parameters: AssertionParameter[];
|
|
17
17
|
}
|
|
18
18
|
/**
|
|
19
|
-
* System assertion IDs - prefixed with 'system:'
|
|
19
|
+
* System assertion IDs - prefixed with 'system:'.
|
|
20
20
|
*/
|
|
21
21
|
export declare const SYSTEM_ASSERTION_IDS: {
|
|
22
22
|
readonly SKILL_WAS_CALLED: "system:skill_was_called";
|
|
@@ -27,21 +27,26 @@ export declare const TestScenarioSchema: z.ZodObject<{
|
|
|
27
27
|
templateId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
28
28
|
assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
29
29
|
skillNames: z.ZodArray<z.ZodString>;
|
|
30
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
30
31
|
type: z.ZodLiteral<"skill_was_called">;
|
|
31
32
|
}, z.core.$strip>, z.ZodObject<{
|
|
32
33
|
toolName: z.ZodString;
|
|
33
|
-
expectedParams: z.ZodString
|
|
34
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
34
35
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
36
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
35
37
|
type: z.ZodLiteral<"tool_called_with_param">;
|
|
36
38
|
}, z.core.$strict>, z.ZodObject<{
|
|
37
39
|
command: z.ZodOptional<z.ZodString>;
|
|
38
40
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
41
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
39
42
|
type: z.ZodLiteral<"build_passed">;
|
|
40
43
|
}, z.core.$strict>, z.ZodObject<{
|
|
41
44
|
maxDurationMs: z.ZodNumber;
|
|
45
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
42
46
|
type: z.ZodLiteral<"time_limit">;
|
|
43
47
|
}, z.core.$strict>, z.ZodObject<{
|
|
44
48
|
maxCostUsd: z.ZodNumber;
|
|
49
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
45
50
|
type: z.ZodLiteral<"cost">;
|
|
46
51
|
}, z.core.$strict>, z.ZodObject<{
|
|
47
52
|
prompt: z.ZodString;
|
|
@@ -61,6 +66,7 @@ export declare const TestScenarioSchema: z.ZodObject<{
|
|
|
61
66
|
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
62
67
|
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
63
68
|
}, z.core.$strip>>>;
|
|
69
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
64
70
|
type: z.ZodLiteral<"llm_judge">;
|
|
65
71
|
}, z.core.$strip>]>>>;
|
|
66
72
|
assertionIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
@@ -82,21 +88,26 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
|
|
|
82
88
|
triggerPrompt: z.ZodString;
|
|
83
89
|
assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
84
90
|
skillNames: z.ZodArray<z.ZodString>;
|
|
91
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
85
92
|
type: z.ZodLiteral<"skill_was_called">;
|
|
86
93
|
}, z.core.$strip>, z.ZodObject<{
|
|
87
94
|
toolName: z.ZodString;
|
|
88
|
-
expectedParams: z.ZodString
|
|
95
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
89
96
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
97
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
90
98
|
type: z.ZodLiteral<"tool_called_with_param">;
|
|
91
99
|
}, z.core.$strict>, z.ZodObject<{
|
|
92
100
|
command: z.ZodOptional<z.ZodString>;
|
|
93
101
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
102
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
94
103
|
type: z.ZodLiteral<"build_passed">;
|
|
95
104
|
}, z.core.$strict>, z.ZodObject<{
|
|
96
105
|
maxDurationMs: z.ZodNumber;
|
|
106
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
97
107
|
type: z.ZodLiteral<"time_limit">;
|
|
98
108
|
}, z.core.$strict>, z.ZodObject<{
|
|
99
109
|
maxCostUsd: z.ZodNumber;
|
|
110
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
100
111
|
type: z.ZodLiteral<"cost">;
|
|
101
112
|
}, z.core.$strict>, z.ZodObject<{
|
|
102
113
|
prompt: z.ZodString;
|
|
@@ -116,6 +127,7 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
|
|
|
116
127
|
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
117
128
|
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
118
129
|
}, z.core.$strip>>>;
|
|
130
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
119
131
|
type: z.ZodLiteral<"llm_judge">;
|
|
120
132
|
}, z.core.$strip>]>>>;
|
|
121
133
|
assertionIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
@@ -137,21 +149,26 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
|
|
|
137
149
|
triggerPrompt: z.ZodOptional<z.ZodString>;
|
|
138
150
|
assertions: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
139
151
|
skillNames: z.ZodArray<z.ZodString>;
|
|
152
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
140
153
|
type: z.ZodLiteral<"skill_was_called">;
|
|
141
154
|
}, z.core.$strip>, z.ZodObject<{
|
|
142
155
|
toolName: z.ZodString;
|
|
143
|
-
expectedParams: z.ZodString
|
|
156
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
144
157
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
158
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
145
159
|
type: z.ZodLiteral<"tool_called_with_param">;
|
|
146
160
|
}, z.core.$strict>, z.ZodObject<{
|
|
147
161
|
command: z.ZodOptional<z.ZodString>;
|
|
148
162
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
163
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
149
164
|
type: z.ZodLiteral<"build_passed">;
|
|
150
165
|
}, z.core.$strict>, z.ZodObject<{
|
|
151
166
|
maxDurationMs: z.ZodNumber;
|
|
167
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
152
168
|
type: z.ZodLiteral<"time_limit">;
|
|
153
169
|
}, z.core.$strict>, z.ZodObject<{
|
|
154
170
|
maxCostUsd: z.ZodNumber;
|
|
171
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
155
172
|
type: z.ZodLiteral<"cost">;
|
|
156
173
|
}, z.core.$strict>, z.ZodObject<{
|
|
157
174
|
prompt: z.ZodString;
|
|
@@ -171,6 +188,7 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
|
|
|
171
188
|
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
172
189
|
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
173
190
|
}, z.core.$strip>>>;
|
|
191
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
174
192
|
type: z.ZodLiteral<"llm_judge">;
|
|
175
193
|
}, z.core.$strip>]>>>>;
|
|
176
194
|
assertionIds: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodString>>>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wix/evalforge-types",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.61.0",
|
|
4
4
|
"description": "Unified types for EvalForge agent evaluation system",
|
|
5
5
|
"files": [
|
|
6
6
|
"build"
|
|
@@ -46,5 +46,5 @@
|
|
|
46
46
|
"artifactId": "evalforge-types"
|
|
47
47
|
}
|
|
48
48
|
},
|
|
49
|
-
"falconPackageHash": "
|
|
49
|
+
"falconPackageHash": "1829a0f485cce6f1adaf0ec93b47528012e55ab746b19e3533f5b605"
|
|
50
50
|
}
|