@wix/evalforge-types 0.60.0 → 0.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +34 -10
- package/build/index.js.map +2 -2
- package/build/index.mjs +34 -10
- package/build/index.mjs.map +2 -2
- package/build/types/assertion/assertion.d.ts +21 -6
- package/build/types/scenario/test-scenario.d.ts +21 -3
- package/package.json +2 -2
|
@@ -7,6 +7,8 @@ import { z } from 'zod';
|
|
|
7
7
|
* - time_limit: Checks that scenario completed within a duration threshold (deterministic, system-level)
|
|
8
8
|
* - cost: Checks that scenario LLM cost stays within a USD threshold (deterministic, system-level)
|
|
9
9
|
* - llm_judge: LLM evaluates output with a prompt (LLM-based, system-level)
|
|
10
|
+
*
|
|
11
|
+
* Any assertion can be negated by setting `negate: true` to invert the pass/fail logic.
|
|
10
12
|
*/
|
|
11
13
|
export declare const AssertionTypeSchema: z.ZodEnum<{
|
|
12
14
|
skill_was_called: "skill_was_called";
|
|
@@ -72,12 +74,13 @@ export declare const CostConfigSchema: z.ZodObject<{
|
|
|
72
74
|
export type CostConfig = z.infer<typeof CostConfigSchema>;
|
|
73
75
|
/** Configuration for tool_called_with_param assertion type.
|
|
74
76
|
* Uses strictObject to reject objects with unknown keys.
|
|
77
|
+
* When expectedParams is omitted, the assertion checks only that the tool was called (or not, if negated).
|
|
75
78
|
*/
|
|
76
79
|
export declare const ToolCalledWithParamConfigSchema: z.ZodObject<{
|
|
77
80
|
/** Name of the tool that must have been called */
|
|
78
81
|
toolName: z.ZodString;
|
|
79
|
-
/** JSON string of key-value pairs for expected parameters (substring match) */
|
|
80
|
-
expectedParams: z.ZodString
|
|
82
|
+
/** JSON string of key-value pairs for expected parameters (substring match). Optional — when omitted, only checks tool presence. */
|
|
83
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
81
84
|
/** If true, the matching tool call must also have succeeded (step.success === true) */
|
|
82
85
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
83
86
|
}, z.core.$strict>;
|
|
@@ -126,24 +129,28 @@ export declare const LlmJudgeConfigSchema: z.ZodObject<{
|
|
|
126
129
|
export type LlmJudgeConfig = z.infer<typeof LlmJudgeConfigSchema>;
|
|
127
130
|
export declare const SkillWasCalledAssertionSchema: z.ZodObject<{
|
|
128
131
|
skillNames: z.ZodArray<z.ZodString>;
|
|
132
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
129
133
|
type: z.ZodLiteral<"skill_was_called">;
|
|
130
134
|
}, z.core.$strip>;
|
|
131
135
|
export type SkillWasCalledAssertion = z.infer<typeof SkillWasCalledAssertionSchema>;
|
|
132
136
|
export declare const ToolCalledWithParamAssertionSchema: z.ZodObject<{
|
|
133
137
|
toolName: z.ZodString;
|
|
134
|
-
expectedParams: z.ZodString
|
|
138
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
135
139
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
140
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
136
141
|
type: z.ZodLiteral<"tool_called_with_param">;
|
|
137
142
|
}, z.core.$strict>;
|
|
138
143
|
export type ToolCalledWithParamAssertion = z.infer<typeof ToolCalledWithParamAssertionSchema>;
|
|
139
144
|
export declare const BuildPassedAssertionSchema: z.ZodObject<{
|
|
140
145
|
command: z.ZodOptional<z.ZodString>;
|
|
141
146
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
147
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
142
148
|
type: z.ZodLiteral<"build_passed">;
|
|
143
149
|
}, z.core.$strict>;
|
|
144
150
|
export type BuildPassedAssertion = z.infer<typeof BuildPassedAssertionSchema>;
|
|
145
151
|
export declare const CostAssertionSchema: z.ZodObject<{
|
|
146
152
|
maxCostUsd: z.ZodNumber;
|
|
153
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
147
154
|
type: z.ZodLiteral<"cost">;
|
|
148
155
|
}, z.core.$strict>;
|
|
149
156
|
export type CostAssertion = z.infer<typeof CostAssertionSchema>;
|
|
@@ -165,11 +172,13 @@ export declare const LlmJudgeAssertionSchema: z.ZodObject<{
|
|
|
165
172
|
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
166
173
|
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
167
174
|
}, z.core.$strip>>>;
|
|
175
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
168
176
|
type: z.ZodLiteral<"llm_judge">;
|
|
169
177
|
}, z.core.$strip>;
|
|
170
178
|
export type LlmJudgeAssertion = z.infer<typeof LlmJudgeAssertionSchema>;
|
|
171
179
|
export declare const TimeAssertionSchema: z.ZodObject<{
|
|
172
180
|
maxDurationMs: z.ZodNumber;
|
|
181
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
173
182
|
type: z.ZodLiteral<"time_limit">;
|
|
174
183
|
}, z.core.$strict>;
|
|
175
184
|
export type TimeAssertion = z.infer<typeof TimeAssertionSchema>;
|
|
@@ -179,21 +188,26 @@ export type TimeAssertion = z.infer<typeof TimeAssertionSchema>;
|
|
|
179
188
|
*/
|
|
180
189
|
export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
181
190
|
skillNames: z.ZodArray<z.ZodString>;
|
|
191
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
182
192
|
type: z.ZodLiteral<"skill_was_called">;
|
|
183
193
|
}, z.core.$strip>, z.ZodObject<{
|
|
184
194
|
toolName: z.ZodString;
|
|
185
|
-
expectedParams: z.ZodString
|
|
195
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
186
196
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
197
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
187
198
|
type: z.ZodLiteral<"tool_called_with_param">;
|
|
188
199
|
}, z.core.$strict>, z.ZodObject<{
|
|
189
200
|
command: z.ZodOptional<z.ZodString>;
|
|
190
201
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
202
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
191
203
|
type: z.ZodLiteral<"build_passed">;
|
|
192
204
|
}, z.core.$strict>, z.ZodObject<{
|
|
193
205
|
maxDurationMs: z.ZodNumber;
|
|
206
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
194
207
|
type: z.ZodLiteral<"time_limit">;
|
|
195
208
|
}, z.core.$strict>, z.ZodObject<{
|
|
196
209
|
maxCostUsd: z.ZodNumber;
|
|
210
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
197
211
|
type: z.ZodLiteral<"cost">;
|
|
198
212
|
}, z.core.$strict>, z.ZodObject<{
|
|
199
213
|
prompt: z.ZodString;
|
|
@@ -213,6 +227,7 @@ export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
213
227
|
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
214
228
|
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
215
229
|
}, z.core.$strip>>>;
|
|
230
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
216
231
|
type: z.ZodLiteral<"llm_judge">;
|
|
217
232
|
}, z.core.$strip>]>;
|
|
218
233
|
export type Assertion = z.infer<typeof AssertionSchema>;
|
|
@@ -245,8 +260,8 @@ export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
245
260
|
}, z.core.$strip>, z.ZodObject<{
|
|
246
261
|
/** Name of the tool that must have been called */
|
|
247
262
|
toolName: z.ZodString;
|
|
248
|
-
/** JSON string of key-value pairs for expected parameters (substring match) */
|
|
249
|
-
expectedParams: z.ZodString
|
|
263
|
+
/** JSON string of key-value pairs for expected parameters (substring match). Optional — when omitted, only checks tool presence. */
|
|
264
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
250
265
|
/** If true, the matching tool call must also have succeeded (step.success === true) */
|
|
251
266
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
252
267
|
}, z.core.$strict>, z.ZodObject<{
|
|
@@ -27,21 +27,26 @@ export declare const TestScenarioSchema: z.ZodObject<{
|
|
|
27
27
|
templateId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
28
28
|
assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
29
29
|
skillNames: z.ZodArray<z.ZodString>;
|
|
30
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
30
31
|
type: z.ZodLiteral<"skill_was_called">;
|
|
31
32
|
}, z.core.$strip>, z.ZodObject<{
|
|
32
33
|
toolName: z.ZodString;
|
|
33
|
-
expectedParams: z.ZodString
|
|
34
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
34
35
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
36
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
35
37
|
type: z.ZodLiteral<"tool_called_with_param">;
|
|
36
38
|
}, z.core.$strict>, z.ZodObject<{
|
|
37
39
|
command: z.ZodOptional<z.ZodString>;
|
|
38
40
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
41
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
39
42
|
type: z.ZodLiteral<"build_passed">;
|
|
40
43
|
}, z.core.$strict>, z.ZodObject<{
|
|
41
44
|
maxDurationMs: z.ZodNumber;
|
|
45
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
42
46
|
type: z.ZodLiteral<"time_limit">;
|
|
43
47
|
}, z.core.$strict>, z.ZodObject<{
|
|
44
48
|
maxCostUsd: z.ZodNumber;
|
|
49
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
45
50
|
type: z.ZodLiteral<"cost">;
|
|
46
51
|
}, z.core.$strict>, z.ZodObject<{
|
|
47
52
|
prompt: z.ZodString;
|
|
@@ -61,6 +66,7 @@ export declare const TestScenarioSchema: z.ZodObject<{
|
|
|
61
66
|
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
62
67
|
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
63
68
|
}, z.core.$strip>>>;
|
|
69
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
64
70
|
type: z.ZodLiteral<"llm_judge">;
|
|
65
71
|
}, z.core.$strip>]>>>;
|
|
66
72
|
assertionIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
@@ -82,21 +88,26 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
|
|
|
82
88
|
triggerPrompt: z.ZodString;
|
|
83
89
|
assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
84
90
|
skillNames: z.ZodArray<z.ZodString>;
|
|
91
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
85
92
|
type: z.ZodLiteral<"skill_was_called">;
|
|
86
93
|
}, z.core.$strip>, z.ZodObject<{
|
|
87
94
|
toolName: z.ZodString;
|
|
88
|
-
expectedParams: z.ZodString
|
|
95
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
89
96
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
97
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
90
98
|
type: z.ZodLiteral<"tool_called_with_param">;
|
|
91
99
|
}, z.core.$strict>, z.ZodObject<{
|
|
92
100
|
command: z.ZodOptional<z.ZodString>;
|
|
93
101
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
102
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
94
103
|
type: z.ZodLiteral<"build_passed">;
|
|
95
104
|
}, z.core.$strict>, z.ZodObject<{
|
|
96
105
|
maxDurationMs: z.ZodNumber;
|
|
106
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
97
107
|
type: z.ZodLiteral<"time_limit">;
|
|
98
108
|
}, z.core.$strict>, z.ZodObject<{
|
|
99
109
|
maxCostUsd: z.ZodNumber;
|
|
110
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
100
111
|
type: z.ZodLiteral<"cost">;
|
|
101
112
|
}, z.core.$strict>, z.ZodObject<{
|
|
102
113
|
prompt: z.ZodString;
|
|
@@ -116,6 +127,7 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
|
|
|
116
127
|
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
117
128
|
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
118
129
|
}, z.core.$strip>>>;
|
|
130
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
119
131
|
type: z.ZodLiteral<"llm_judge">;
|
|
120
132
|
}, z.core.$strip>]>>>;
|
|
121
133
|
assertionIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
@@ -137,21 +149,26 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
|
|
|
137
149
|
triggerPrompt: z.ZodOptional<z.ZodString>;
|
|
138
150
|
assertions: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
139
151
|
skillNames: z.ZodArray<z.ZodString>;
|
|
152
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
140
153
|
type: z.ZodLiteral<"skill_was_called">;
|
|
141
154
|
}, z.core.$strip>, z.ZodObject<{
|
|
142
155
|
toolName: z.ZodString;
|
|
143
|
-
expectedParams: z.ZodString
|
|
156
|
+
expectedParams: z.ZodOptional<z.ZodString>;
|
|
144
157
|
requireSuccess: z.ZodOptional<z.ZodBoolean>;
|
|
158
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
145
159
|
type: z.ZodLiteral<"tool_called_with_param">;
|
|
146
160
|
}, z.core.$strict>, z.ZodObject<{
|
|
147
161
|
command: z.ZodOptional<z.ZodString>;
|
|
148
162
|
expectedExitCode: z.ZodOptional<z.ZodNumber>;
|
|
163
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
149
164
|
type: z.ZodLiteral<"build_passed">;
|
|
150
165
|
}, z.core.$strict>, z.ZodObject<{
|
|
151
166
|
maxDurationMs: z.ZodNumber;
|
|
167
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
152
168
|
type: z.ZodLiteral<"time_limit">;
|
|
153
169
|
}, z.core.$strict>, z.ZodObject<{
|
|
154
170
|
maxCostUsd: z.ZodNumber;
|
|
171
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
155
172
|
type: z.ZodLiteral<"cost">;
|
|
156
173
|
}, z.core.$strict>, z.ZodObject<{
|
|
157
174
|
prompt: z.ZodString;
|
|
@@ -171,6 +188,7 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
|
|
|
171
188
|
defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
|
|
172
189
|
advanced: z.ZodOptional<z.ZodBoolean>;
|
|
173
190
|
}, z.core.$strip>>>;
|
|
191
|
+
negate: z.ZodOptional<z.ZodBoolean>;
|
|
174
192
|
type: z.ZodLiteral<"llm_judge">;
|
|
175
193
|
}, z.core.$strip>]>>>>;
|
|
176
194
|
assertionIds: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodString>>>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wix/evalforge-types",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.61.0",
|
|
4
4
|
"description": "Unified types for EvalForge agent evaluation system",
|
|
5
5
|
"files": [
|
|
6
6
|
"build"
|
|
@@ -46,5 +46,5 @@
|
|
|
46
46
|
"artifactId": "evalforge-types"
|
|
47
47
|
}
|
|
48
48
|
},
|
|
49
|
-
"falconPackageHash": "
|
|
49
|
+
"falconPackageHash": "1829a0f485cce6f1adaf0ec93b47528012e55ab746b19e3533f5b605"
|
|
50
50
|
}
|