@wix/evalforge-types 0.90.0 → 0.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +589 -452
- package/build/index.js.map +4 -4
- package/build/index.mjs +575 -451
- package/build/index.mjs.map +4 -4
- package/build/types/assertion/build-passed-command.d.ts +1 -1
- package/build/types/scenario/index.d.ts +2 -0
- package/build/types/scenario/site-setup.d.ts +132 -0
- package/build/types/scenario/test-scenario.d.ts +120 -0
- package/build/types/scenario/wix-origin-template-ids.d.ts +5 -0
- package/package.json +2 -2
package/build/index.js
CHANGED
|
@@ -159,6 +159,7 @@ __export(index_exports, {
|
|
|
159
159
|
PresetSchema: () => PresetSchema,
|
|
160
160
|
ProjectSchema: () => ProjectSchema,
|
|
161
161
|
PromptResultSchema: () => PromptResultSchema,
|
|
162
|
+
ProvisionedSiteSchema: () => ProvisionedSiteSchema,
|
|
162
163
|
RUN_COMMAND_LABELS: () => RUN_COMMAND_LABELS,
|
|
163
164
|
ReasoningEffortSchema: () => ReasoningEffortSchema,
|
|
164
165
|
RuleSchema: () => RuleSchema,
|
|
@@ -166,13 +167,20 @@ __export(index_exports, {
|
|
|
166
167
|
RunAnalysisFindingSchema: () => RunAnalysisFindingSchema,
|
|
167
168
|
RunAnalysisSchema: () => RunAnalysisSchema,
|
|
168
169
|
SEMVER_REGEX: () => SEMVER_REGEX,
|
|
170
|
+
SITE_SETUP_EXCLUSIVE_VARIABLES: () => SITE_SETUP_EXCLUSIVE_VARIABLES,
|
|
169
171
|
SKILL_FOLDER_NAME_REGEX: () => SKILL_FOLDER_NAME_REGEX,
|
|
170
172
|
SYSTEM_ASSERTIONS: () => SYSTEM_ASSERTIONS,
|
|
171
173
|
SYSTEM_ASSERTION_IDS: () => SYSTEM_ASSERTION_IDS,
|
|
172
174
|
ScenarioAssertionLinkSchema: () => ScenarioAssertionLinkSchema,
|
|
173
175
|
ScenarioConversationSchema: () => ScenarioConversationSchema,
|
|
174
176
|
SimpleAgentConfigSchema: () => SimpleAgentConfigSchema,
|
|
177
|
+
SiteBootstrapHttpMethodSchema: () => SiteBootstrapHttpMethodSchema,
|
|
178
|
+
SiteBootstrapResultSchema: () => SiteBootstrapResultSchema,
|
|
179
|
+
SiteBootstrapSchema: () => SiteBootstrapSchema,
|
|
180
|
+
SiteBootstrapStepResultSchema: () => SiteBootstrapStepResultSchema,
|
|
181
|
+
SiteBootstrapStepSchema: () => SiteBootstrapStepSchema,
|
|
175
182
|
SiteConfigTestSchema: () => SiteConfigTestSchema,
|
|
183
|
+
SiteSetupConfigSchema: () => SiteSetupConfigSchema,
|
|
176
184
|
SkillFileSchema: () => SkillFileSchema,
|
|
177
185
|
SkillMetadataSchema: () => SkillMetadataSchema,
|
|
178
186
|
SkillSchema: () => SkillSchema,
|
|
@@ -226,6 +234,7 @@ __export(index_exports, {
|
|
|
226
234
|
UpdateTestScenarioInputSchema: () => UpdateTestScenarioInputSchema,
|
|
227
235
|
UpdateTestSuiteInputSchema: () => UpdateTestSuiteInputSchema,
|
|
228
236
|
VitestTestSchema: () => VitestTestSchema,
|
|
237
|
+
WixSiteSummarySchema: () => WixSiteSummarySchema,
|
|
229
238
|
capabilityToMcp: () => capabilityToMcp,
|
|
230
239
|
capabilityToRule: () => capabilityToRule,
|
|
231
240
|
capabilityToSkill: () => capabilityToSkill,
|
|
@@ -233,6 +242,7 @@ __export(index_exports, {
|
|
|
233
242
|
capabilityToSubAgent: () => capabilityToSubAgent,
|
|
234
243
|
capabilityVersionToSkillVersion: () => capabilityVersionToSkillVersion,
|
|
235
244
|
classifyAssertionRef: () => classifyAssertionRef,
|
|
245
|
+
extractVariableNamesFromPrompt: () => extractVariableNamesFromPrompt,
|
|
236
246
|
formatTraceEventLine: () => formatTraceEventLine,
|
|
237
247
|
getSystemAssertion: () => getSystemAssertion,
|
|
238
248
|
getSystemAssertions: () => getSystemAssertions,
|
|
@@ -245,8 +255,11 @@ __export(index_exports, {
|
|
|
245
255
|
normalizeModelId: () => normalizeModelId,
|
|
246
256
|
parseBuildCommandToArgv: () => parseBuildCommandToArgv,
|
|
247
257
|
parseTraceEventLine: () => parseTraceEventLine,
|
|
258
|
+
promptUsesSiteSetupExclusiveVariables: () => promptUsesSiteSetupExclusiveVariables,
|
|
259
|
+
resolveWixOriginTemplateId: () => resolveWixOriginTemplateId,
|
|
248
260
|
validateAssertionConfig: () => validateAssertionConfig,
|
|
249
|
-
validateBuildPassedParamsInAssertionLinks: () => validateBuildPassedParamsInAssertionLinks
|
|
261
|
+
validateBuildPassedParamsInAssertionLinks: () => validateBuildPassedParamsInAssertionLinks,
|
|
262
|
+
validateSiteSetupExclusivity: () => validateSiteSetupExclusivity
|
|
250
263
|
});
|
|
251
264
|
module.exports = __toCommonJS(index_exports);
|
|
252
265
|
|
|
@@ -1072,21 +1085,72 @@ var EnvironmentSchema = import_zod22.z.object({
|
|
|
1072
1085
|
metaSite: MetaSiteConfigSchema.optional()
|
|
1073
1086
|
});
|
|
1074
1087
|
|
|
1088
|
+
// src/scenario/site-setup.ts
|
|
1089
|
+
var import_zod23 = require("zod");
|
|
1090
|
+
var SiteBootstrapHttpMethodSchema = import_zod23.z.enum([
|
|
1091
|
+
"get",
|
|
1092
|
+
"post",
|
|
1093
|
+
"put",
|
|
1094
|
+
"patch",
|
|
1095
|
+
"delete"
|
|
1096
|
+
]);
|
|
1097
|
+
var SiteBootstrapStepSchema = import_zod23.z.object({
|
|
1098
|
+
label: import_zod23.z.string().optional(),
|
|
1099
|
+
method: SiteBootstrapHttpMethodSchema,
|
|
1100
|
+
url: import_zod23.z.string().min(1),
|
|
1101
|
+
body: import_zod23.z.record(import_zod23.z.string(), import_zod23.z.unknown()).optional()
|
|
1102
|
+
});
|
|
1103
|
+
var SiteBootstrapSchema = import_zod23.z.object({
|
|
1104
|
+
steps: import_zod23.z.array(SiteBootstrapStepSchema).default([])
|
|
1105
|
+
});
|
|
1106
|
+
var SiteBootstrapStepResultSchema = import_zod23.z.object({
|
|
1107
|
+
label: import_zod23.z.string().optional(),
|
|
1108
|
+
statusCode: import_zod23.z.number().int(),
|
|
1109
|
+
ok: import_zod23.z.boolean(),
|
|
1110
|
+
error: import_zod23.z.string().optional()
|
|
1111
|
+
});
|
|
1112
|
+
var SiteBootstrapResultSchema = import_zod23.z.object({
|
|
1113
|
+
steps: import_zod23.z.array(SiteBootstrapStepResultSchema)
|
|
1114
|
+
});
|
|
1115
|
+
var SiteSetupConfigSchema = import_zod23.z.discriminatedUnion("mode", [
|
|
1116
|
+
import_zod23.z.object({ mode: import_zod23.z.literal("none") }),
|
|
1117
|
+
import_zod23.z.object({
|
|
1118
|
+
mode: import_zod23.z.literal("clone"),
|
|
1119
|
+
sourceSiteId: import_zod23.z.string().min(1),
|
|
1120
|
+
bootstrap: SiteBootstrapSchema.optional()
|
|
1121
|
+
}),
|
|
1122
|
+
import_zod23.z.object({
|
|
1123
|
+
mode: import_zod23.z.literal("template"),
|
|
1124
|
+
templateId: import_zod23.z.string().min(1),
|
|
1125
|
+
bootstrap: SiteBootstrapSchema.optional()
|
|
1126
|
+
})
|
|
1127
|
+
]);
|
|
1128
|
+
var WixSiteSummarySchema = import_zod23.z.object({
|
|
1129
|
+
id: import_zod23.z.string(),
|
|
1130
|
+
displayName: import_zod23.z.string(),
|
|
1131
|
+
url: import_zod23.z.string().optional()
|
|
1132
|
+
});
|
|
1133
|
+
var ProvisionedSiteSchema = import_zod23.z.object({
|
|
1134
|
+
id: import_zod23.z.string(),
|
|
1135
|
+
url: import_zod23.z.string().optional(),
|
|
1136
|
+
editorUrl: import_zod23.z.string().optional()
|
|
1137
|
+
});
|
|
1138
|
+
|
|
1075
1139
|
// src/scenario/test-scenario.ts
|
|
1076
|
-
var
|
|
1140
|
+
var import_zod26 = require("zod");
|
|
1077
1141
|
|
|
1078
1142
|
// src/assertion/assertion.ts
|
|
1079
|
-
var
|
|
1143
|
+
var import_zod25 = require("zod");
|
|
1080
1144
|
|
|
1081
1145
|
// src/assertion/build-passed-command.ts
|
|
1082
|
-
var
|
|
1146
|
+
var import_zod24 = require("zod");
|
|
1083
1147
|
var ALLOWED_BUILD_COMMANDS = [
|
|
1084
1148
|
"yarn build",
|
|
1085
1149
|
"npm run build",
|
|
1086
1150
|
"pnpm run build",
|
|
1087
1151
|
"pnpm build"
|
|
1088
1152
|
];
|
|
1089
|
-
var DEFAULT_BUILD_PASSED_COMMAND = "
|
|
1153
|
+
var DEFAULT_BUILD_PASSED_COMMAND = "npm run build";
|
|
1090
1154
|
var BUILD_COMMAND_ARGV = {
|
|
1091
1155
|
"yarn build": ["yarn", "build"],
|
|
1092
1156
|
"npm run build": ["npm", "run", "build"],
|
|
@@ -1105,10 +1169,10 @@ function parseBuildCommandToArgv(command) {
|
|
|
1105
1169
|
return BUILD_COMMAND_ARGV[trimmed];
|
|
1106
1170
|
}
|
|
1107
1171
|
var enumTuple = ALLOWED_BUILD_COMMANDS;
|
|
1108
|
-
var BuildPassedCommandStringSchema =
|
|
1172
|
+
var BuildPassedCommandStringSchema = import_zod24.z.enum(enumTuple);
|
|
1109
1173
|
|
|
1110
1174
|
// src/assertion/assertion.ts
|
|
1111
|
-
var AssertionTypeSchema =
|
|
1175
|
+
var AssertionTypeSchema = import_zod25.z.enum([
|
|
1112
1176
|
"skill_was_called",
|
|
1113
1177
|
"tool_called_with_param",
|
|
1114
1178
|
"build_passed",
|
|
@@ -1117,61 +1181,61 @@ var AssertionTypeSchema = import_zod24.z.enum([
|
|
|
1117
1181
|
"llm_judge",
|
|
1118
1182
|
"api_call"
|
|
1119
1183
|
]);
|
|
1120
|
-
var AssertionParameterTypeSchema =
|
|
1184
|
+
var AssertionParameterTypeSchema = import_zod25.z.enum([
|
|
1121
1185
|
"string",
|
|
1122
1186
|
"number",
|
|
1123
1187
|
"boolean"
|
|
1124
1188
|
]);
|
|
1125
|
-
var AssertionParameterSchema =
|
|
1189
|
+
var AssertionParameterSchema = import_zod25.z.object({
|
|
1126
1190
|
/** Parameter name (used as key in params object) */
|
|
1127
|
-
name:
|
|
1191
|
+
name: import_zod25.z.string().min(1),
|
|
1128
1192
|
/** Display label for the parameter */
|
|
1129
|
-
label:
|
|
1193
|
+
label: import_zod25.z.string().min(1),
|
|
1130
1194
|
/** Parameter type */
|
|
1131
1195
|
type: AssertionParameterTypeSchema,
|
|
1132
1196
|
/** Whether this parameter is required */
|
|
1133
|
-
required:
|
|
1197
|
+
required: import_zod25.z.boolean(),
|
|
1134
1198
|
/** Default value (optional, used when not provided) */
|
|
1135
|
-
defaultValue:
|
|
1199
|
+
defaultValue: import_zod25.z.union([import_zod25.z.string(), import_zod25.z.number(), import_zod25.z.boolean()]).optional(),
|
|
1136
1200
|
/** If true, parameter is hidden by default behind "Show advanced options" */
|
|
1137
|
-
advanced:
|
|
1201
|
+
advanced: import_zod25.z.boolean().optional()
|
|
1138
1202
|
});
|
|
1139
|
-
var ScenarioAssertionLinkSchema =
|
|
1203
|
+
var ScenarioAssertionLinkSchema = import_zod25.z.object({
|
|
1140
1204
|
/** ID of the system assertion (e.g., 'system:skill_was_called') */
|
|
1141
|
-
assertionId:
|
|
1205
|
+
assertionId: import_zod25.z.string(),
|
|
1142
1206
|
/** Parameter values for this assertion in this scenario */
|
|
1143
|
-
params:
|
|
1144
|
-
|
|
1145
|
-
|
|
1207
|
+
params: import_zod25.z.record(
|
|
1208
|
+
import_zod25.z.string(),
|
|
1209
|
+
import_zod25.z.union([import_zod25.z.string(), import_zod25.z.number(), import_zod25.z.boolean(), import_zod25.z.null()])
|
|
1146
1210
|
).optional()
|
|
1147
1211
|
});
|
|
1148
|
-
var SkillWasCalledConfigSchema =
|
|
1212
|
+
var SkillWasCalledConfigSchema = import_zod25.z.object({
|
|
1149
1213
|
/** Names of the skills that must have been called */
|
|
1150
|
-
skillNames:
|
|
1214
|
+
skillNames: import_zod25.z.array(import_zod25.z.string().min(1)).min(1)
|
|
1151
1215
|
});
|
|
1152
|
-
var CostConfigSchema =
|
|
1216
|
+
var CostConfigSchema = import_zod25.z.strictObject({
|
|
1153
1217
|
/** Maximum allowed cost in USD */
|
|
1154
|
-
maxCostUsd:
|
|
1218
|
+
maxCostUsd: import_zod25.z.number().positive()
|
|
1155
1219
|
});
|
|
1156
|
-
var ToolCalledWithParamConfigSchema =
|
|
1220
|
+
var ToolCalledWithParamConfigSchema = import_zod25.z.strictObject({
|
|
1157
1221
|
/** Name of the tool that must have been called */
|
|
1158
|
-
toolName:
|
|
1222
|
+
toolName: import_zod25.z.string().min(1),
|
|
1159
1223
|
/** JSON string of key-value pairs for expected parameters (substring match). Optional — when omitted, only checks tool presence. */
|
|
1160
|
-
expectedParams:
|
|
1224
|
+
expectedParams: import_zod25.z.string().min(1).optional(),
|
|
1161
1225
|
/** If true, the matching tool call must also have succeeded (step.success === true) */
|
|
1162
|
-
requireSuccess:
|
|
1226
|
+
requireSuccess: import_zod25.z.boolean().optional()
|
|
1163
1227
|
});
|
|
1164
|
-
var BuildPassedConfigSchema =
|
|
1165
|
-
/** Allowlisted command only (default at runtime: "
|
|
1228
|
+
var BuildPassedConfigSchema = import_zod25.z.strictObject({
|
|
1229
|
+
/** Allowlisted command only (default at runtime: "npm run build") */
|
|
1166
1230
|
command: BuildPassedCommandStringSchema.optional(),
|
|
1167
1231
|
/** Expected exit code (default: 0) */
|
|
1168
|
-
expectedExitCode:
|
|
1232
|
+
expectedExitCode: import_zod25.z.number().int().optional()
|
|
1169
1233
|
});
|
|
1170
|
-
var TimeConfigSchema =
|
|
1234
|
+
var TimeConfigSchema = import_zod25.z.strictObject({
|
|
1171
1235
|
/** Maximum allowed duration in milliseconds */
|
|
1172
|
-
maxDurationMs:
|
|
1236
|
+
maxDurationMs: import_zod25.z.number().int().positive()
|
|
1173
1237
|
});
|
|
1174
|
-
var LlmJudgeConfigSchema =
|
|
1238
|
+
var LlmJudgeConfigSchema = import_zod25.z.object({
|
|
1175
1239
|
/**
|
|
1176
1240
|
* Prompt template with placeholders:
|
|
1177
1241
|
* - {{output}}: agent's final output
|
|
@@ -1182,65 +1246,65 @@ var LlmJudgeConfigSchema = import_zod24.z.object({
|
|
|
1182
1246
|
* - {{trace}}: step-by-step trace of tool calls
|
|
1183
1247
|
* - Custom parameters defined in the parameters array
|
|
1184
1248
|
*/
|
|
1185
|
-
prompt:
|
|
1249
|
+
prompt: import_zod25.z.string().min(1),
|
|
1186
1250
|
/** Minimum score to pass (0-10, default 7) */
|
|
1187
|
-
minScore:
|
|
1251
|
+
minScore: import_zod25.z.number().int().min(0).max(10).optional(),
|
|
1188
1252
|
/** Model for the judge (e.g. claude-3-5-haiku-20241022) */
|
|
1189
|
-
model:
|
|
1253
|
+
model: import_zod25.z.string().optional(),
|
|
1190
1254
|
/** Max output tokens */
|
|
1191
|
-
maxTokens:
|
|
1255
|
+
maxTokens: import_zod25.z.number().int().optional(),
|
|
1192
1256
|
/** Temperature (0-1) */
|
|
1193
|
-
temperature:
|
|
1257
|
+
temperature: import_zod25.z.number().min(0).max(1).optional(),
|
|
1194
1258
|
/** User-defined parameters for this assertion */
|
|
1195
|
-
parameters:
|
|
1259
|
+
parameters: import_zod25.z.array(AssertionParameterSchema).optional()
|
|
1196
1260
|
});
|
|
1197
|
-
var ApiCallConfigSchema =
|
|
1261
|
+
var ApiCallConfigSchema = import_zod25.z.strictObject({
|
|
1198
1262
|
/** URL to call */
|
|
1199
|
-
url:
|
|
1263
|
+
url: import_zod25.z.string().min(1),
|
|
1200
1264
|
/** HTTP method (default GET) */
|
|
1201
|
-
method:
|
|
1265
|
+
method: import_zod25.z.enum(["GET", "POST"]).optional(),
|
|
1202
1266
|
/** Request body (JSON string, for POST requests) */
|
|
1203
|
-
requestBody:
|
|
1267
|
+
requestBody: import_zod25.z.string().optional(),
|
|
1204
1268
|
/** Expected JSON response to validate against (subset match — extra fields in actual are OK) */
|
|
1205
|
-
expectedResponse:
|
|
1269
|
+
expectedResponse: import_zod25.z.string().min(1),
|
|
1206
1270
|
/** Request headers as JSON string of key-value pairs */
|
|
1207
|
-
requestHeaders:
|
|
1271
|
+
requestHeaders: import_zod25.z.string().optional(),
|
|
1208
1272
|
/** Request timeout in milliseconds (default 30000) */
|
|
1209
|
-
timeoutMs:
|
|
1273
|
+
timeoutMs: import_zod25.z.number().int().positive().optional()
|
|
1210
1274
|
});
|
|
1211
1275
|
var AssertionBaseFields = {
|
|
1212
1276
|
/** When true, the assertion's pass/fail logic is inverted (NOT operator). */
|
|
1213
|
-
negate:
|
|
1277
|
+
negate: import_zod25.z.boolean().optional()
|
|
1214
1278
|
};
|
|
1215
1279
|
var SkillWasCalledAssertionSchema = SkillWasCalledConfigSchema.extend({
|
|
1216
|
-
type:
|
|
1280
|
+
type: import_zod25.z.literal("skill_was_called"),
|
|
1217
1281
|
...AssertionBaseFields
|
|
1218
1282
|
});
|
|
1219
1283
|
var ToolCalledWithParamAssertionSchema = ToolCalledWithParamConfigSchema.extend({
|
|
1220
|
-
type:
|
|
1284
|
+
type: import_zod25.z.literal("tool_called_with_param"),
|
|
1221
1285
|
...AssertionBaseFields
|
|
1222
1286
|
});
|
|
1223
1287
|
var BuildPassedAssertionSchema = BuildPassedConfigSchema.extend({
|
|
1224
|
-
type:
|
|
1288
|
+
type: import_zod25.z.literal("build_passed"),
|
|
1225
1289
|
...AssertionBaseFields
|
|
1226
1290
|
});
|
|
1227
1291
|
var CostAssertionSchema = CostConfigSchema.extend({
|
|
1228
|
-
type:
|
|
1292
|
+
type: import_zod25.z.literal("cost"),
|
|
1229
1293
|
...AssertionBaseFields
|
|
1230
1294
|
});
|
|
1231
1295
|
var LlmJudgeAssertionSchema = LlmJudgeConfigSchema.extend({
|
|
1232
|
-
type:
|
|
1296
|
+
type: import_zod25.z.literal("llm_judge"),
|
|
1233
1297
|
...AssertionBaseFields
|
|
1234
1298
|
});
|
|
1235
1299
|
var ApiCallAssertionSchema = ApiCallConfigSchema.extend({
|
|
1236
|
-
type:
|
|
1300
|
+
type: import_zod25.z.literal("api_call"),
|
|
1237
1301
|
...AssertionBaseFields
|
|
1238
1302
|
});
|
|
1239
1303
|
var TimeAssertionSchema = TimeConfigSchema.extend({
|
|
1240
|
-
type:
|
|
1304
|
+
type: import_zod25.z.literal("time_limit"),
|
|
1241
1305
|
...AssertionBaseFields
|
|
1242
1306
|
});
|
|
1243
|
-
var AssertionSchema =
|
|
1307
|
+
var AssertionSchema = import_zod25.z.union([
|
|
1244
1308
|
SkillWasCalledAssertionSchema,
|
|
1245
1309
|
ToolCalledWithParamAssertionSchema,
|
|
1246
1310
|
BuildPassedAssertionSchema,
|
|
@@ -1249,7 +1313,7 @@ var AssertionSchema = import_zod24.z.union([
|
|
|
1249
1313
|
LlmJudgeAssertionSchema,
|
|
1250
1314
|
ApiCallAssertionSchema
|
|
1251
1315
|
]);
|
|
1252
|
-
var AssertionConfigSchema =
|
|
1316
|
+
var AssertionConfigSchema = import_zod25.z.union([
|
|
1253
1317
|
LlmJudgeConfigSchema,
|
|
1254
1318
|
// requires prompt - check first
|
|
1255
1319
|
SkillWasCalledConfigSchema,
|
|
@@ -1264,7 +1328,7 @@ var AssertionConfigSchema = import_zod24.z.union([
|
|
|
1264
1328
|
// requires maxCostUsd, uses strictObject
|
|
1265
1329
|
BuildPassedConfigSchema,
|
|
1266
1330
|
// all optional, uses strictObject to reject unknown keys
|
|
1267
|
-
|
|
1331
|
+
import_zod25.z.object({})
|
|
1268
1332
|
// fallback empty config
|
|
1269
1333
|
]);
|
|
1270
1334
|
function validateAssertionConfig(type, config) {
|
|
@@ -1369,7 +1433,7 @@ var SYSTEM_ASSERTIONS = {
|
|
|
1369
1433
|
label: "Build Command",
|
|
1370
1434
|
type: "string",
|
|
1371
1435
|
required: false,
|
|
1372
|
-
defaultValue:
|
|
1436
|
+
defaultValue: DEFAULT_BUILD_PASSED_COMMAND
|
|
1373
1437
|
},
|
|
1374
1438
|
{
|
|
1375
1439
|
name: "expectedExitCode",
|
|
@@ -1510,36 +1574,67 @@ function getSystemAssertion(id) {
|
|
|
1510
1574
|
|
|
1511
1575
|
// src/scenario/test-scenario.ts
|
|
1512
1576
|
var MAX_IMAGE_BASE64_LENGTH = 4 * Math.ceil(2 * 1024 * 1024 / 3);
|
|
1513
|
-
var TriggerPromptImageSchema =
|
|
1577
|
+
var TriggerPromptImageSchema = import_zod26.z.object({
|
|
1514
1578
|
/** Base64-encoded image data (no data URL prefix) */
|
|
1515
|
-
base64:
|
|
1579
|
+
base64: import_zod26.z.string().max(MAX_IMAGE_BASE64_LENGTH, "Image exceeds 2 MB size limit"),
|
|
1516
1580
|
/** MIME type of the image */
|
|
1517
|
-
mediaType:
|
|
1581
|
+
mediaType: import_zod26.z.enum(["image/jpeg", "image/png", "image/gif", "image/webp"]),
|
|
1518
1582
|
/** Original filename of the image */
|
|
1519
|
-
name:
|
|
1583
|
+
name: import_zod26.z.string()
|
|
1520
1584
|
});
|
|
1521
|
-
var ExpectedFileSchema =
|
|
1585
|
+
var ExpectedFileSchema = import_zod26.z.object({
|
|
1522
1586
|
/** Relative path where the file should be created */
|
|
1523
|
-
path:
|
|
1587
|
+
path: import_zod26.z.string(),
|
|
1524
1588
|
/** Optional expected content */
|
|
1525
|
-
content:
|
|
1589
|
+
content: import_zod26.z.string().optional()
|
|
1526
1590
|
});
|
|
1527
1591
|
var TestScenarioSchema = TenantEntitySchema.extend({
|
|
1528
1592
|
/** The prompt sent to the agent to trigger the task */
|
|
1529
|
-
triggerPrompt:
|
|
1593
|
+
triggerPrompt: import_zod26.z.string().min(10),
|
|
1530
1594
|
/** ID of the template to use for this scenario (null = no template) */
|
|
1531
|
-
templateId:
|
|
1595
|
+
templateId: import_zod26.z.string().nullish(),
|
|
1532
1596
|
/** Inline assertions to evaluate for this scenario (legacy) */
|
|
1533
|
-
assertions:
|
|
1597
|
+
assertions: import_zod26.z.array(AssertionSchema).optional(),
|
|
1534
1598
|
/** IDs of saved assertions to evaluate (from assertions table) - legacy, use assertionLinks */
|
|
1535
|
-
assertionIds:
|
|
1599
|
+
assertionIds: import_zod26.z.array(import_zod26.z.string()).optional(),
|
|
1536
1600
|
/** Linked assertions with per-scenario parameter values */
|
|
1537
|
-
assertionLinks:
|
|
1601
|
+
assertionLinks: import_zod26.z.array(ScenarioAssertionLinkSchema).optional(),
|
|
1538
1602
|
/** Tags for categorisation and filtering */
|
|
1539
|
-
tags:
|
|
1603
|
+
tags: import_zod26.z.array(import_zod26.z.string()).optional(),
|
|
1540
1604
|
/** Base64-encoded images attached to the trigger prompt (max 3) */
|
|
1541
|
-
triggerPromptImages:
|
|
1542
|
-
|
|
1605
|
+
triggerPromptImages: import_zod26.z.array(TriggerPromptImageSchema).max(3).optional(),
|
|
1606
|
+
/** Optional per-scenario Wix site provisioning instructions. Absent ≡ no site. */
|
|
1607
|
+
siteSetup: SiteSetupConfigSchema.optional()
|
|
1608
|
+
});
|
|
1609
|
+
var SITE_SETUP_EXCLUSIVE_VARIABLES = ["site-id"];
|
|
1610
|
+
function extractVariableNamesFromPrompt(prompt) {
|
|
1611
|
+
const names = /* @__PURE__ */ new Set();
|
|
1612
|
+
for (const match of prompt.matchAll(/\{\{([\w-]+)\}\}/g)) {
|
|
1613
|
+
names.add(match[1]);
|
|
1614
|
+
}
|
|
1615
|
+
return [...names];
|
|
1616
|
+
}
|
|
1617
|
+
function promptUsesSiteSetupExclusiveVariables(prompt) {
|
|
1618
|
+
const names = extractVariableNamesFromPrompt(prompt);
|
|
1619
|
+
return SITE_SETUP_EXCLUSIVE_VARIABLES.some(
|
|
1620
|
+
(exclusive) => names.includes(exclusive)
|
|
1621
|
+
);
|
|
1622
|
+
}
|
|
1623
|
+
function hasActiveSiteSetup(siteSetup) {
|
|
1624
|
+
return siteSetup?.mode === "clone" || siteSetup?.mode === "template";
|
|
1625
|
+
}
|
|
1626
|
+
function validateSiteSetupExclusivity(data, ctx) {
|
|
1627
|
+
if (!hasActiveSiteSetup(data.siteSetup)) return;
|
|
1628
|
+
const prompt = data.triggerPrompt;
|
|
1629
|
+
if (prompt === void 0 || !promptUsesSiteSetupExclusiveVariables(prompt)) {
|
|
1630
|
+
return;
|
|
1631
|
+
}
|
|
1632
|
+
ctx.addIssue({
|
|
1633
|
+
code: import_zod26.z.ZodIssueCode.custom,
|
|
1634
|
+
message: "Site setup and {{site-id}} run variables cannot be used together. Remove {{site-id}} from the trigger prompt or disable site setup.",
|
|
1635
|
+
path: ["triggerPrompt"]
|
|
1636
|
+
});
|
|
1637
|
+
}
|
|
1543
1638
|
function validateBuildPassedParamsInAssertionLinks(links, ctx) {
|
|
1544
1639
|
if (!links) return;
|
|
1545
1640
|
for (let i = 0; i < links.length; i++) {
|
|
@@ -1549,7 +1644,7 @@ function validateBuildPassedParamsInAssertionLinks(links, ctx) {
|
|
|
1549
1644
|
if (cmd === void 0 || cmd === null) continue;
|
|
1550
1645
|
if (typeof cmd !== "string") {
|
|
1551
1646
|
ctx.addIssue({
|
|
1552
|
-
code:
|
|
1647
|
+
code: import_zod26.z.ZodIssueCode.custom,
|
|
1553
1648
|
message: "build_passed command must be a string",
|
|
1554
1649
|
path: ["assertionLinks", i, "params", "command"]
|
|
1555
1650
|
});
|
|
@@ -1557,7 +1652,7 @@ function validateBuildPassedParamsInAssertionLinks(links, ctx) {
|
|
|
1557
1652
|
}
|
|
1558
1653
|
if (!isAllowedBuildCommandString(cmd)) {
|
|
1559
1654
|
ctx.addIssue({
|
|
1560
|
-
code:
|
|
1655
|
+
code: import_zod26.z.ZodIssueCode.custom,
|
|
1561
1656
|
message: "Invalid build_passed command. Allowed: yarn build, npm run build, pnpm run build, pnpm build",
|
|
1562
1657
|
path: ["assertionLinks", i, "params", "command"]
|
|
1563
1658
|
});
|
|
@@ -1572,27 +1667,56 @@ var TestScenarioCreateBaseSchema = TestScenarioSchema.omit({
|
|
|
1572
1667
|
});
|
|
1573
1668
|
var CreateTestScenarioInputSchema = TestScenarioCreateBaseSchema.superRefine((data, ctx) => {
|
|
1574
1669
|
validateBuildPassedParamsInAssertionLinks(data.assertionLinks, ctx);
|
|
1670
|
+
validateSiteSetupExclusivity(data, ctx);
|
|
1575
1671
|
});
|
|
1576
1672
|
var UpdateTestScenarioInputSchema = TestScenarioCreateBaseSchema.partial().superRefine((data, ctx) => {
|
|
1577
1673
|
if (data.assertionLinks !== void 0) {
|
|
1578
1674
|
validateBuildPassedParamsInAssertionLinks(data.assertionLinks, ctx);
|
|
1579
1675
|
}
|
|
1676
|
+
validateSiteSetupExclusivity(data, ctx);
|
|
1580
1677
|
});
|
|
1581
1678
|
|
|
1679
|
+
// src/scenario/wix-origin-template-ids.ts
|
|
1680
|
+
var WIX_ORIGIN_TEMPLATE_ID_BY_ALIAS = {
|
|
1681
|
+
ecommerce: "e5da13f4-c01e-4b61-a9c7-55dacd961d54",
|
|
1682
|
+
default: "212b41cb-0da6-4401-9c72-7c579e6477a2",
|
|
1683
|
+
blog: "68fc7371-365f-44c6-8467-69d88bfc172e",
|
|
1684
|
+
astrowind: "9e9292c1-1a35-4ba0-8986-d06f2ecb5366",
|
|
1685
|
+
scheduler: "72ade0e3-1871-4c04-ac54-419ca874d9d3",
|
|
1686
|
+
registration: "e5d63bf1-cd06-48eb-ad77-0da9235adcf1",
|
|
1687
|
+
"picasso-studio": "61f05de1-b0ce-4873-b9f5-52241a6fd262",
|
|
1688
|
+
"picasso-ecom": "daa9187d-f010-4eb0-bd49-e658b5a5037a",
|
|
1689
|
+
picasso: "99b9a3c7-82ad-4e1b-9066-e490bb9863af",
|
|
1690
|
+
"ecom-editorless": "738c7c0b-046e-4bf0-87dd-9a06ee5a52c4"
|
|
1691
|
+
};
|
|
1692
|
+
var GUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
1693
|
+
function resolveWixOriginTemplateId(templateId) {
|
|
1694
|
+
if (GUID_PATTERN.test(templateId)) {
|
|
1695
|
+
return templateId;
|
|
1696
|
+
}
|
|
1697
|
+
const originTemplateId = WIX_ORIGIN_TEMPLATE_ID_BY_ALIAS[templateId];
|
|
1698
|
+
if (!originTemplateId) {
|
|
1699
|
+
throw new Error(
|
|
1700
|
+
`Unknown Wix site template alias "${templateId}". Use a GUID or one of: ${Object.keys(WIX_ORIGIN_TEMPLATE_ID_BY_ALIAS).join(", ")}`
|
|
1701
|
+
);
|
|
1702
|
+
}
|
|
1703
|
+
return originTemplateId;
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1582
1706
|
// src/scenario/batch-import.ts
|
|
1583
|
-
var
|
|
1707
|
+
var import_zod27 = require("zod");
|
|
1584
1708
|
var UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
1585
|
-
var BatchAssertionLinkSchema =
|
|
1586
|
-
|
|
1709
|
+
var BatchAssertionLinkSchema = import_zod27.z.union([
|
|
1710
|
+
import_zod27.z.string().min(1),
|
|
1587
1711
|
ScenarioAssertionLinkSchema
|
|
1588
1712
|
]);
|
|
1589
|
-
var BatchScenarioEntrySchema =
|
|
1590
|
-
name:
|
|
1591
|
-
description:
|
|
1592
|
-
triggerPrompt:
|
|
1593
|
-
templateId:
|
|
1594
|
-
tags:
|
|
1595
|
-
assertionLinks:
|
|
1713
|
+
var BatchScenarioEntrySchema = import_zod27.z.object({
|
|
1714
|
+
name: import_zod27.z.string().min(1, "name: Required"),
|
|
1715
|
+
description: import_zod27.z.string().optional().default(""),
|
|
1716
|
+
triggerPrompt: import_zod27.z.string().min(10, "triggerPrompt: Must be at least 10 characters"),
|
|
1717
|
+
templateId: import_zod27.z.string().nullish(),
|
|
1718
|
+
tags: import_zod27.z.array(import_zod27.z.string()).optional(),
|
|
1719
|
+
assertionLinks: import_zod27.z.array(BatchAssertionLinkSchema).optional()
|
|
1596
1720
|
}).superRefine((data, ctx) => {
|
|
1597
1721
|
if (!data.assertionLinks) return;
|
|
1598
1722
|
const objectLinks = data.assertionLinks.filter(
|
|
@@ -1602,8 +1726,8 @@ var BatchScenarioEntrySchema = import_zod26.z.object({
|
|
|
1602
1726
|
validateBuildPassedParamsInAssertionLinks(objectLinks, ctx);
|
|
1603
1727
|
}
|
|
1604
1728
|
});
|
|
1605
|
-
var BatchImportPayloadSchema =
|
|
1606
|
-
scenarios:
|
|
1729
|
+
var BatchImportPayloadSchema = import_zod27.z.object({
|
|
1730
|
+
scenarios: import_zod27.z.array(BatchScenarioEntrySchema).min(1, "scenarios array must contain at least one entry").max(100, "Maximum 100 scenarios per upload")
|
|
1607
1731
|
});
|
|
1608
1732
|
var BATCH_IMPORT_LIMITS = {
|
|
1609
1733
|
MAX_SCENARIOS: 100,
|
|
@@ -1625,29 +1749,29 @@ function normalizeBatchAssertionLink(link) {
|
|
|
1625
1749
|
}
|
|
1626
1750
|
return link;
|
|
1627
1751
|
}
|
|
1628
|
-
var BatchResultItemSchema =
|
|
1629
|
-
index:
|
|
1630
|
-
name:
|
|
1631
|
-
status:
|
|
1632
|
-
id:
|
|
1633
|
-
errors:
|
|
1634
|
-
});
|
|
1635
|
-
var BatchSummarySchema =
|
|
1636
|
-
total:
|
|
1637
|
-
valid:
|
|
1638
|
-
invalid:
|
|
1639
|
-
created:
|
|
1640
|
-
});
|
|
1641
|
-
var BatchImportResponseSchema =
|
|
1752
|
+
var BatchResultItemSchema = import_zod27.z.object({
|
|
1753
|
+
index: import_zod27.z.number(),
|
|
1754
|
+
name: import_zod27.z.string(),
|
|
1755
|
+
status: import_zod27.z.enum(["valid", "invalid"]),
|
|
1756
|
+
id: import_zod27.z.string().nullable().optional(),
|
|
1757
|
+
errors: import_zod27.z.array(import_zod27.z.string()).optional()
|
|
1758
|
+
});
|
|
1759
|
+
var BatchSummarySchema = import_zod27.z.object({
|
|
1760
|
+
total: import_zod27.z.number(),
|
|
1761
|
+
valid: import_zod27.z.number(),
|
|
1762
|
+
invalid: import_zod27.z.number(),
|
|
1763
|
+
created: import_zod27.z.number()
|
|
1764
|
+
});
|
|
1765
|
+
var BatchImportResponseSchema = import_zod27.z.object({
|
|
1642
1766
|
summary: BatchSummarySchema,
|
|
1643
|
-
results:
|
|
1767
|
+
results: import_zod27.z.array(BatchResultItemSchema)
|
|
1644
1768
|
});
|
|
1645
1769
|
|
|
1646
1770
|
// src/suite/test-suite.ts
|
|
1647
|
-
var
|
|
1771
|
+
var import_zod28 = require("zod");
|
|
1648
1772
|
var TestSuiteSchema = TenantEntitySchema.extend({
|
|
1649
1773
|
/** IDs of test scenarios in this suite */
|
|
1650
|
-
scenarioIds:
|
|
1774
|
+
scenarioIds: import_zod28.z.array(import_zod28.z.string())
|
|
1651
1775
|
});
|
|
1652
1776
|
var CreateTestSuiteInputSchema = TestSuiteSchema.omit({
|
|
1653
1777
|
id: true,
|
|
@@ -1658,21 +1782,21 @@ var CreateTestSuiteInputSchema = TestSuiteSchema.omit({
|
|
|
1658
1782
|
var UpdateTestSuiteInputSchema = CreateTestSuiteInputSchema.partial();
|
|
1659
1783
|
|
|
1660
1784
|
// src/evaluation/metrics.ts
|
|
1661
|
-
var
|
|
1662
|
-
var TokenUsageSchema =
|
|
1663
|
-
prompt:
|
|
1664
|
-
completion:
|
|
1665
|
-
total:
|
|
1666
|
-
});
|
|
1667
|
-
var EvalMetricsSchema =
|
|
1668
|
-
totalAssertions:
|
|
1669
|
-
passed:
|
|
1670
|
-
failed:
|
|
1671
|
-
skipped:
|
|
1672
|
-
errors:
|
|
1673
|
-
passRate:
|
|
1674
|
-
avgDuration:
|
|
1675
|
-
totalDuration:
|
|
1785
|
+
var import_zod29 = require("zod");
|
|
1786
|
+
var TokenUsageSchema = import_zod29.z.object({
|
|
1787
|
+
prompt: import_zod29.z.number(),
|
|
1788
|
+
completion: import_zod29.z.number(),
|
|
1789
|
+
total: import_zod29.z.number()
|
|
1790
|
+
});
|
|
1791
|
+
var EvalMetricsSchema = import_zod29.z.object({
|
|
1792
|
+
totalAssertions: import_zod29.z.number(),
|
|
1793
|
+
passed: import_zod29.z.number(),
|
|
1794
|
+
failed: import_zod29.z.number(),
|
|
1795
|
+
skipped: import_zod29.z.number(),
|
|
1796
|
+
errors: import_zod29.z.number(),
|
|
1797
|
+
passRate: import_zod29.z.number(),
|
|
1798
|
+
avgDuration: import_zod29.z.number(),
|
|
1799
|
+
totalDuration: import_zod29.z.number()
|
|
1676
1800
|
});
|
|
1677
1801
|
var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
|
|
1678
1802
|
EvalStatus2["PENDING"] = "pending";
|
|
@@ -1682,7 +1806,7 @@ var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
|
|
|
1682
1806
|
EvalStatus2["CANCELLED"] = "cancelled";
|
|
1683
1807
|
return EvalStatus2;
|
|
1684
1808
|
})(EvalStatus || {});
|
|
1685
|
-
var EvalStatusSchema =
|
|
1809
|
+
var EvalStatusSchema = import_zod29.z.enum(EvalStatus);
|
|
1686
1810
|
var LLMStepType = /* @__PURE__ */ ((LLMStepType2) => {
|
|
1687
1811
|
LLMStepType2["COMPLETION"] = "completion";
|
|
1688
1812
|
LLMStepType2["TOOL_USE"] = "tool_use";
|
|
@@ -1690,54 +1814,54 @@ var LLMStepType = /* @__PURE__ */ ((LLMStepType2) => {
|
|
|
1690
1814
|
LLMStepType2["THINKING"] = "thinking";
|
|
1691
1815
|
return LLMStepType2;
|
|
1692
1816
|
})(LLMStepType || {});
|
|
1693
|
-
var LLMTraceStepSchema =
|
|
1694
|
-
id:
|
|
1695
|
-
stepNumber:
|
|
1696
|
-
type:
|
|
1697
|
-
model:
|
|
1698
|
-
provider:
|
|
1699
|
-
startedAt:
|
|
1700
|
-
durationMs:
|
|
1817
|
+
var LLMTraceStepSchema = import_zod29.z.object({
|
|
1818
|
+
id: import_zod29.z.string(),
|
|
1819
|
+
stepNumber: import_zod29.z.number(),
|
|
1820
|
+
type: import_zod29.z.enum(LLMStepType),
|
|
1821
|
+
model: import_zod29.z.string(),
|
|
1822
|
+
provider: import_zod29.z.string(),
|
|
1823
|
+
startedAt: import_zod29.z.string(),
|
|
1824
|
+
durationMs: import_zod29.z.number(),
|
|
1701
1825
|
tokenUsage: TokenUsageSchema,
|
|
1702
|
-
costUsd:
|
|
1703
|
-
toolName:
|
|
1704
|
-
toolArguments:
|
|
1705
|
-
inputPreview:
|
|
1706
|
-
outputPreview:
|
|
1707
|
-
success:
|
|
1708
|
-
error:
|
|
1709
|
-
turnIndex:
|
|
1710
|
-
});
|
|
1711
|
-
var LLMBreakdownStatsSchema =
|
|
1712
|
-
count:
|
|
1713
|
-
durationMs:
|
|
1714
|
-
tokens:
|
|
1715
|
-
costUsd:
|
|
1716
|
-
});
|
|
1717
|
-
var LLMTraceSummarySchema =
|
|
1718
|
-
totalSteps:
|
|
1719
|
-
totalTurns:
|
|
1720
|
-
totalDurationMs:
|
|
1826
|
+
costUsd: import_zod29.z.number(),
|
|
1827
|
+
toolName: import_zod29.z.string().optional(),
|
|
1828
|
+
toolArguments: import_zod29.z.string().optional(),
|
|
1829
|
+
inputPreview: import_zod29.z.string().optional(),
|
|
1830
|
+
outputPreview: import_zod29.z.string().optional(),
|
|
1831
|
+
success: import_zod29.z.boolean(),
|
|
1832
|
+
error: import_zod29.z.string().optional(),
|
|
1833
|
+
turnIndex: import_zod29.z.number().optional()
|
|
1834
|
+
});
|
|
1835
|
+
var LLMBreakdownStatsSchema = import_zod29.z.object({
|
|
1836
|
+
count: import_zod29.z.number(),
|
|
1837
|
+
durationMs: import_zod29.z.number(),
|
|
1838
|
+
tokens: import_zod29.z.number(),
|
|
1839
|
+
costUsd: import_zod29.z.number()
|
|
1840
|
+
});
|
|
1841
|
+
var LLMTraceSummarySchema = import_zod29.z.object({
|
|
1842
|
+
totalSteps: import_zod29.z.number(),
|
|
1843
|
+
totalTurns: import_zod29.z.number().optional(),
|
|
1844
|
+
totalDurationMs: import_zod29.z.number(),
|
|
1721
1845
|
totalTokens: TokenUsageSchema,
|
|
1722
|
-
totalCostUsd:
|
|
1723
|
-
stepTypeBreakdown:
|
|
1724
|
-
modelBreakdown:
|
|
1725
|
-
modelsUsed:
|
|
1726
|
-
});
|
|
1727
|
-
var LLMTraceSchema =
|
|
1728
|
-
id:
|
|
1729
|
-
steps:
|
|
1846
|
+
totalCostUsd: import_zod29.z.number(),
|
|
1847
|
+
stepTypeBreakdown: import_zod29.z.record(import_zod29.z.string(), LLMBreakdownStatsSchema).optional(),
|
|
1848
|
+
modelBreakdown: import_zod29.z.record(import_zod29.z.string(), LLMBreakdownStatsSchema),
|
|
1849
|
+
modelsUsed: import_zod29.z.array(import_zod29.z.string())
|
|
1850
|
+
});
|
|
1851
|
+
var LLMTraceSchema = import_zod29.z.object({
|
|
1852
|
+
id: import_zod29.z.string(),
|
|
1853
|
+
steps: import_zod29.z.array(LLMTraceStepSchema),
|
|
1730
1854
|
summary: LLMTraceSummarySchema
|
|
1731
1855
|
});
|
|
1732
1856
|
|
|
1733
1857
|
// src/evaluation/eval-result.ts
|
|
1734
|
-
var
|
|
1858
|
+
var import_zod33 = require("zod");
|
|
1735
1859
|
|
|
1736
1860
|
// src/evaluation/eval-run.ts
|
|
1737
|
-
var
|
|
1861
|
+
var import_zod31 = require("zod");
|
|
1738
1862
|
|
|
1739
1863
|
// src/evaluation/live-trace.ts
|
|
1740
|
-
var
|
|
1864
|
+
var import_zod30 = require("zod");
|
|
1741
1865
|
var LiveTraceEventType = /* @__PURE__ */ ((LiveTraceEventType2) => {
|
|
1742
1866
|
LiveTraceEventType2["THINKING"] = "thinking";
|
|
1743
1867
|
LiveTraceEventType2["TOOL_USE"] = "tool_use";
|
|
@@ -1751,37 +1875,37 @@ var LiveTraceEventType = /* @__PURE__ */ ((LiveTraceEventType2) => {
|
|
|
1751
1875
|
LiveTraceEventType2["USER"] = "user";
|
|
1752
1876
|
return LiveTraceEventType2;
|
|
1753
1877
|
})(LiveTraceEventType || {});
|
|
1754
|
-
var LiveTraceEventSchema =
|
|
1878
|
+
var LiveTraceEventSchema = import_zod30.z.object({
|
|
1755
1879
|
/** The evaluation run ID */
|
|
1756
|
-
evalRunId:
|
|
1880
|
+
evalRunId: import_zod30.z.string(),
|
|
1757
1881
|
/** The scenario ID being executed */
|
|
1758
|
-
scenarioId:
|
|
1882
|
+
scenarioId: import_zod30.z.string(),
|
|
1759
1883
|
/** The scenario name for display */
|
|
1760
|
-
scenarioName:
|
|
1884
|
+
scenarioName: import_zod30.z.string(),
|
|
1761
1885
|
/** The target ID (skill, agent, etc.) */
|
|
1762
|
-
targetId:
|
|
1886
|
+
targetId: import_zod30.z.string(),
|
|
1763
1887
|
/** The target name for display */
|
|
1764
|
-
targetName:
|
|
1888
|
+
targetName: import_zod30.z.string(),
|
|
1765
1889
|
/** Step number in the current scenario execution */
|
|
1766
|
-
stepNumber:
|
|
1890
|
+
stepNumber: import_zod30.z.number(),
|
|
1767
1891
|
/** Type of trace event */
|
|
1768
|
-
type:
|
|
1892
|
+
type: import_zod30.z.enum(LiveTraceEventType),
|
|
1769
1893
|
/** Tool name if this is a tool_use event */
|
|
1770
|
-
toolName:
|
|
1894
|
+
toolName: import_zod30.z.string().optional(),
|
|
1771
1895
|
/** Tool arguments preview (truncated JSON) */
|
|
1772
|
-
toolArgs:
|
|
1896
|
+
toolArgs: import_zod30.z.string().optional(),
|
|
1773
1897
|
/** Output preview (truncated text) */
|
|
1774
|
-
outputPreview:
|
|
1898
|
+
outputPreview: import_zod30.z.string().optional(),
|
|
1775
1899
|
/** File path for file operations */
|
|
1776
|
-
filePath:
|
|
1900
|
+
filePath: import_zod30.z.string().optional(),
|
|
1777
1901
|
/** Elapsed time in milliseconds for progress events */
|
|
1778
|
-
elapsedMs:
|
|
1902
|
+
elapsedMs: import_zod30.z.number().optional(),
|
|
1779
1903
|
/** Thinking/reasoning text from Claude */
|
|
1780
|
-
thinking:
|
|
1904
|
+
thinking: import_zod30.z.string().optional(),
|
|
1781
1905
|
/** Timestamp when this event occurred */
|
|
1782
|
-
timestamp:
|
|
1906
|
+
timestamp: import_zod30.z.string(),
|
|
1783
1907
|
/** Whether this is the final event for this scenario */
|
|
1784
|
-
isComplete:
|
|
1908
|
+
isComplete: import_zod30.z.boolean()
|
|
1785
1909
|
});
|
|
1786
1910
|
var TRACE_EVENT_PREFIX = "TRACE_EVENT:";
|
|
1787
1911
|
function parseTraceEventLine(line) {
|
|
@@ -1810,40 +1934,40 @@ var TriggerType = /* @__PURE__ */ ((TriggerType2) => {
|
|
|
1810
1934
|
TriggerType2["SCHEDULED"] = "SCHEDULED";
|
|
1811
1935
|
return TriggerType2;
|
|
1812
1936
|
})(TriggerType || {});
|
|
1813
|
-
var TriggerMetadataSchema =
|
|
1814
|
-
version:
|
|
1815
|
-
resourceUpdated:
|
|
1816
|
-
scheduleId:
|
|
1937
|
+
var TriggerMetadataSchema = import_zod31.z.object({
|
|
1938
|
+
version: import_zod31.z.string().optional(),
|
|
1939
|
+
resourceUpdated: import_zod31.z.array(import_zod31.z.string()).optional(),
|
|
1940
|
+
scheduleId: import_zod31.z.string().optional()
|
|
1817
1941
|
});
|
|
1818
|
-
var TriggerSchema =
|
|
1819
|
-
id:
|
|
1942
|
+
var TriggerSchema = import_zod31.z.object({
|
|
1943
|
+
id: import_zod31.z.string(),
|
|
1820
1944
|
metadata: TriggerMetadataSchema.optional(),
|
|
1821
|
-
type:
|
|
1945
|
+
type: import_zod31.z.nativeEnum(TriggerType)
|
|
1822
1946
|
});
|
|
1823
|
-
var DiffLineTypeSchema =
|
|
1824
|
-
var DiffLineSchema =
|
|
1947
|
+
var DiffLineTypeSchema = import_zod31.z.enum(["added", "removed", "unchanged"]);
|
|
1948
|
+
var DiffLineSchema = import_zod31.z.object({
|
|
1825
1949
|
type: DiffLineTypeSchema,
|
|
1826
|
-
content:
|
|
1827
|
-
lineNumber:
|
|
1828
|
-
});
|
|
1829
|
-
var DiffContentSchema =
|
|
1830
|
-
path:
|
|
1831
|
-
expected:
|
|
1832
|
-
actual:
|
|
1833
|
-
diffLines:
|
|
1834
|
-
renamedFrom:
|
|
1950
|
+
content: import_zod31.z.string(),
|
|
1951
|
+
lineNumber: import_zod31.z.number()
|
|
1952
|
+
});
|
|
1953
|
+
var DiffContentSchema = import_zod31.z.object({
|
|
1954
|
+
path: import_zod31.z.string(),
|
|
1955
|
+
expected: import_zod31.z.string(),
|
|
1956
|
+
actual: import_zod31.z.string(),
|
|
1957
|
+
diffLines: import_zod31.z.array(DiffLineSchema),
|
|
1958
|
+
renamedFrom: import_zod31.z.string().optional(),
|
|
1835
1959
|
/** Whether this file is an infrastructure/config file (e.g. .claude/settings.json, .mcp.json) */
|
|
1836
|
-
isInfrastructure:
|
|
1960
|
+
isInfrastructure: import_zod31.z.boolean().optional()
|
|
1837
1961
|
});
|
|
1838
|
-
var CommandExecutionSchema =
|
|
1839
|
-
command:
|
|
1840
|
-
exitCode:
|
|
1841
|
-
output:
|
|
1842
|
-
duration:
|
|
1962
|
+
var CommandExecutionSchema = import_zod31.z.object({
|
|
1963
|
+
command: import_zod31.z.string(),
|
|
1964
|
+
exitCode: import_zod31.z.number(),
|
|
1965
|
+
output: import_zod31.z.string().optional(),
|
|
1966
|
+
duration: import_zod31.z.number()
|
|
1843
1967
|
});
|
|
1844
|
-
var FileModificationSchema =
|
|
1845
|
-
path:
|
|
1846
|
-
action:
|
|
1968
|
+
var FileModificationSchema = import_zod31.z.object({
|
|
1969
|
+
path: import_zod31.z.string(),
|
|
1970
|
+
action: import_zod31.z.enum(["created", "modified", "deleted"])
|
|
1847
1971
|
});
|
|
1848
1972
|
var TemplateFileStatus = /* @__PURE__ */ ((TemplateFileStatus2) => {
|
|
1849
1973
|
TemplateFileStatus2["NEW"] = "new";
|
|
@@ -1851,58 +1975,58 @@ var TemplateFileStatus = /* @__PURE__ */ ((TemplateFileStatus2) => {
|
|
|
1851
1975
|
TemplateFileStatus2["UNCHANGED"] = "unchanged";
|
|
1852
1976
|
return TemplateFileStatus2;
|
|
1853
1977
|
})(TemplateFileStatus || {});
|
|
1854
|
-
var TemplateFileSchema =
|
|
1978
|
+
var TemplateFileSchema = import_zod31.z.object({
|
|
1855
1979
|
/** Relative path within the template */
|
|
1856
|
-
path:
|
|
1980
|
+
path: import_zod31.z.string(),
|
|
1857
1981
|
/** Full file content after execution */
|
|
1858
|
-
content:
|
|
1982
|
+
content: import_zod31.z.string(),
|
|
1859
1983
|
/** File status (new, modified, unchanged) */
|
|
1860
|
-
status:
|
|
1984
|
+
status: import_zod31.z.enum(["new", "modified", "unchanged"]),
|
|
1861
1985
|
/** Whether this file is an infrastructure/config file (e.g. .claude/settings.json, .mcp.json) */
|
|
1862
|
-
isInfrastructure:
|
|
1986
|
+
isInfrastructure: import_zod31.z.boolean().optional()
|
|
1863
1987
|
});
|
|
1864
|
-
var ApiCallSchema =
|
|
1865
|
-
endpoint:
|
|
1866
|
-
tokensUsed:
|
|
1867
|
-
duration:
|
|
1988
|
+
var ApiCallSchema = import_zod31.z.object({
|
|
1989
|
+
endpoint: import_zod31.z.string(),
|
|
1990
|
+
tokensUsed: import_zod31.z.number(),
|
|
1991
|
+
duration: import_zod31.z.number()
|
|
1868
1992
|
});
|
|
1869
|
-
var ExecutionTraceSchema =
|
|
1870
|
-
commands:
|
|
1871
|
-
filesModified:
|
|
1872
|
-
apiCalls:
|
|
1873
|
-
totalDuration:
|
|
1993
|
+
var ExecutionTraceSchema = import_zod31.z.object({
|
|
1994
|
+
commands: import_zod31.z.array(CommandExecutionSchema),
|
|
1995
|
+
filesModified: import_zod31.z.array(FileModificationSchema),
|
|
1996
|
+
apiCalls: import_zod31.z.array(ApiCallSchema),
|
|
1997
|
+
totalDuration: import_zod31.z.number()
|
|
1874
1998
|
});
|
|
1875
|
-
var RunAnalysisFindingSchema =
|
|
1876
|
-
category:
|
|
1999
|
+
var RunAnalysisFindingSchema = import_zod31.z.object({
|
|
2000
|
+
category: import_zod31.z.enum([
|
|
1877
2001
|
"failure_pattern",
|
|
1878
2002
|
"cost_waste",
|
|
1879
2003
|
"flakiness",
|
|
1880
2004
|
"inefficiency",
|
|
1881
2005
|
"positive"
|
|
1882
2006
|
]),
|
|
1883
|
-
severity:
|
|
1884
|
-
description:
|
|
1885
|
-
affectedScenarios:
|
|
1886
|
-
recommendation:
|
|
2007
|
+
severity: import_zod31.z.enum(["high", "medium", "low"]),
|
|
2008
|
+
description: import_zod31.z.string(),
|
|
2009
|
+
affectedScenarios: import_zod31.z.array(import_zod31.z.string()),
|
|
2010
|
+
recommendation: import_zod31.z.string().optional()
|
|
1887
2011
|
});
|
|
1888
|
-
var RunAnalysisSchema =
|
|
1889
|
-
generatedAt:
|
|
1890
|
-
summary:
|
|
1891
|
-
findings:
|
|
2012
|
+
var RunAnalysisSchema = import_zod31.z.object({
|
|
2013
|
+
generatedAt: import_zod31.z.string(),
|
|
2014
|
+
summary: import_zod31.z.string(),
|
|
2015
|
+
findings: import_zod31.z.array(RunAnalysisFindingSchema)
|
|
1892
2016
|
});
|
|
1893
2017
|
var EvalRunSchema = TenantEntitySchema.extend({
|
|
1894
2018
|
/** Agent ID for this run */
|
|
1895
|
-
agentId:
|
|
2019
|
+
agentId: import_zod31.z.string().optional(),
|
|
1896
2020
|
/** Preset ID that originated this run (optional) */
|
|
1897
|
-
presetId:
|
|
2021
|
+
presetId: import_zod31.z.string().optional(),
|
|
1898
2022
|
/** Scenario IDs to run (always present — resolved server-side from tags when needed) */
|
|
1899
|
-
scenarioIds:
|
|
2023
|
+
scenarioIds: import_zod31.z.array(import_zod31.z.string()),
|
|
1900
2024
|
/** Current status */
|
|
1901
2025
|
status: EvalStatusSchema,
|
|
1902
2026
|
/** Progress percentage (0-100) */
|
|
1903
|
-
progress:
|
|
2027
|
+
progress: import_zod31.z.number(),
|
|
1904
2028
|
/** Results for each scenario/target combination (lazy to break eval-result ↔ eval-run cycle) */
|
|
1905
|
-
results:
|
|
2029
|
+
results: import_zod31.z.array(import_zod31.z.lazy(() => EvalRunResultSchema)),
|
|
1906
2030
|
/** Aggregated metrics across all results */
|
|
1907
2031
|
aggregateMetrics: EvalMetricsSchema,
|
|
1908
2032
|
/** Aggregated LLM trace summary */
|
|
@@ -1910,49 +2034,49 @@ var EvalRunSchema = TenantEntitySchema.extend({
|
|
|
1910
2034
|
/** What triggered this run */
|
|
1911
2035
|
trigger: TriggerSchema.optional(),
|
|
1912
2036
|
/** When the run started (set when evaluation is triggered) */
|
|
1913
|
-
startedAt:
|
|
2037
|
+
startedAt: import_zod31.z.string().optional(),
|
|
1914
2038
|
/** When the run completed */
|
|
1915
|
-
completedAt:
|
|
2039
|
+
completedAt: import_zod31.z.string().optional(),
|
|
1916
2040
|
/** Live trace events captured during execution (for playback on results page) */
|
|
1917
|
-
liveTraceEvents:
|
|
2041
|
+
liveTraceEvents: import_zod31.z.array(LiveTraceEventSchema).optional(),
|
|
1918
2042
|
/** Remote job ID for tracking execution in Dev Machines */
|
|
1919
|
-
jobId:
|
|
2043
|
+
jobId: import_zod31.z.string().optional(),
|
|
1920
2044
|
/** Remote job status from the Dev Machine API (PENDING, RUNNING, COMPLETED, FAILED, CANCELLED) */
|
|
1921
|
-
jobStatus:
|
|
2045
|
+
jobStatus: import_zod31.z.string().optional(),
|
|
1922
2046
|
/** Remote job error message if the job failed */
|
|
1923
|
-
jobError:
|
|
2047
|
+
jobError: import_zod31.z.string().optional(),
|
|
1924
2048
|
/** Timestamp of the last job status check */
|
|
1925
|
-
jobStatusCheckedAt:
|
|
2049
|
+
jobStatusCheckedAt: import_zod31.z.string().optional(),
|
|
1926
2050
|
/** Unified capability IDs */
|
|
1927
|
-
capabilityIds:
|
|
2051
|
+
capabilityIds: import_zod31.z.array(import_zod31.z.string()).optional(),
|
|
1928
2052
|
/** Map of capabilityId to capabilityVersionId for version pinning */
|
|
1929
|
-
capabilityVersions:
|
|
2053
|
+
capabilityVersions: import_zod31.z.record(import_zod31.z.string(), import_zod31.z.string()).optional(),
|
|
1930
2054
|
/** Tags used to select scenarios for this run (for traceability) */
|
|
1931
|
-
tags:
|
|
2055
|
+
tags: import_zod31.z.array(import_zod31.z.string()).optional(),
|
|
1932
2056
|
/** How many times each scenario is executed within this eval run. Default: 1. Max: 20. */
|
|
1933
|
-
runsPerScenario:
|
|
2057
|
+
runsPerScenario: import_zod31.z.number().int().min(1).max(20).optional(),
|
|
1934
2058
|
/** Variable values to substitute in scenario trigger prompts at runtime */
|
|
1935
|
-
variables:
|
|
2059
|
+
variables: import_zod31.z.record(import_zod31.z.string(), import_zod31.z.string()).optional(),
|
|
1936
2060
|
/** Snapshot of agent configuration captured at run creation time */
|
|
1937
|
-
agentSnapshot:
|
|
1938
|
-
name:
|
|
2061
|
+
agentSnapshot: import_zod31.z.object({
|
|
2062
|
+
name: import_zod31.z.string().optional(),
|
|
1939
2063
|
agentType: AgentTypeSchema.optional(),
|
|
1940
2064
|
runCommand: AgentRunCommandSchema.optional(),
|
|
1941
|
-
systemPrompt:
|
|
2065
|
+
systemPrompt: import_zod31.z.string().nullable().optional(),
|
|
1942
2066
|
/** @deprecated retained for backward compat with stored snapshots */
|
|
1943
2067
|
modelConfig: ModelConfigSchema.optional(),
|
|
1944
|
-
config:
|
|
2068
|
+
config: import_zod31.z.record(import_zod31.z.string(), import_zod31.z.unknown()).optional()
|
|
1945
2069
|
}).optional(),
|
|
1946
2070
|
/** UUID linking all runs in a comparison group */
|
|
1947
|
-
comparisonGroupId:
|
|
2071
|
+
comparisonGroupId: import_zod31.z.string().optional(),
|
|
1948
2072
|
/** Human-readable label for this variant (e.g., "MCP: Wix Stores") */
|
|
1949
|
-
comparisonLabel:
|
|
2073
|
+
comparisonLabel: import_zod31.z.string().optional(),
|
|
1950
2074
|
/** LLM-generated analysis of the completed run */
|
|
1951
2075
|
runAnalysis: RunAnalysisSchema.optional(),
|
|
1952
2076
|
/** IDs of folders this run belongs to (read-only, managed via AddRunToFolder / RemoveRunFromFolder) */
|
|
1953
|
-
folderIds:
|
|
2077
|
+
folderIds: import_zod31.z.array(import_zod31.z.string()).optional(),
|
|
1954
2078
|
/** ID of the schedule that triggered this run, if any (read-only) */
|
|
1955
|
-
scheduleId:
|
|
2079
|
+
scheduleId: import_zod31.z.string().optional()
|
|
1956
2080
|
});
|
|
1957
2081
|
var CreateEvalRunInputSchema = EvalRunSchema.omit({
|
|
1958
2082
|
id: true,
|
|
@@ -1968,60 +2092,60 @@ var CreateEvalRunInputSchema = EvalRunSchema.omit({
|
|
|
1968
2092
|
agentSnapshot: true
|
|
1969
2093
|
}).extend({
|
|
1970
2094
|
/** Optional on input — backend resolves from tags when not provided */
|
|
1971
|
-
scenarioIds:
|
|
2095
|
+
scenarioIds: import_zod31.z.array(import_zod31.z.string()).optional()
|
|
1972
2096
|
}).refine(
|
|
1973
2097
|
(data) => data.scenarioIds && data.scenarioIds.length > 0 || data.tags && data.tags.length > 0,
|
|
1974
2098
|
{ message: "Either scenarioIds or tags must be provided" }
|
|
1975
2099
|
);
|
|
1976
|
-
var EvaluationProgressSchema =
|
|
1977
|
-
runId:
|
|
1978
|
-
targetId:
|
|
1979
|
-
totalScenarios:
|
|
1980
|
-
completedScenarios:
|
|
1981
|
-
scenarioProgress:
|
|
1982
|
-
|
|
1983
|
-
scenarioId:
|
|
1984
|
-
currentStep:
|
|
1985
|
-
error:
|
|
2100
|
+
var EvaluationProgressSchema = import_zod31.z.object({
|
|
2101
|
+
runId: import_zod31.z.string(),
|
|
2102
|
+
targetId: import_zod31.z.string(),
|
|
2103
|
+
totalScenarios: import_zod31.z.number(),
|
|
2104
|
+
completedScenarios: import_zod31.z.number(),
|
|
2105
|
+
scenarioProgress: import_zod31.z.array(
|
|
2106
|
+
import_zod31.z.object({
|
|
2107
|
+
scenarioId: import_zod31.z.string(),
|
|
2108
|
+
currentStep: import_zod31.z.string(),
|
|
2109
|
+
error: import_zod31.z.string().optional()
|
|
1986
2110
|
})
|
|
1987
2111
|
),
|
|
1988
|
-
createdAt:
|
|
1989
|
-
});
|
|
1990
|
-
var EvaluationLogSchema =
|
|
1991
|
-
runId:
|
|
1992
|
-
scenarioId:
|
|
1993
|
-
log:
|
|
1994
|
-
level:
|
|
1995
|
-
message:
|
|
1996
|
-
args:
|
|
1997
|
-
error:
|
|
2112
|
+
createdAt: import_zod31.z.number()
|
|
2113
|
+
});
|
|
2114
|
+
var EvaluationLogSchema = import_zod31.z.object({
|
|
2115
|
+
runId: import_zod31.z.string(),
|
|
2116
|
+
scenarioId: import_zod31.z.string(),
|
|
2117
|
+
log: import_zod31.z.object({
|
|
2118
|
+
level: import_zod31.z.enum(["info", "error", "debug"]),
|
|
2119
|
+
message: import_zod31.z.string().optional(),
|
|
2120
|
+
args: import_zod31.z.array(import_zod31.z.any()).optional(),
|
|
2121
|
+
error: import_zod31.z.string().optional()
|
|
1998
2122
|
})
|
|
1999
2123
|
});
|
|
2000
2124
|
var LLM_TIMEOUT = 12e4;
|
|
2001
2125
|
|
|
2002
2126
|
// src/evaluation/conversation.ts
|
|
2003
|
-
var
|
|
2004
|
-
var TextBlockSchema =
|
|
2005
|
-
type:
|
|
2006
|
-
text:
|
|
2007
|
-
});
|
|
2008
|
-
var ThinkingBlockSchema =
|
|
2009
|
-
type:
|
|
2010
|
-
thinking:
|
|
2011
|
-
});
|
|
2012
|
-
var ToolUseBlockSchema =
|
|
2013
|
-
type:
|
|
2014
|
-
toolName:
|
|
2015
|
-
toolId:
|
|
2016
|
-
input:
|
|
2017
|
-
});
|
|
2018
|
-
var ToolResultBlockSchema =
|
|
2019
|
-
type:
|
|
2020
|
-
toolUseId:
|
|
2021
|
-
content:
|
|
2022
|
-
isError:
|
|
2023
|
-
});
|
|
2024
|
-
var ConversationBlockSchema =
|
|
2127
|
+
var import_zod32 = require("zod");
|
|
2128
|
+
var TextBlockSchema = import_zod32.z.object({
|
|
2129
|
+
type: import_zod32.z.literal("text"),
|
|
2130
|
+
text: import_zod32.z.string()
|
|
2131
|
+
});
|
|
2132
|
+
var ThinkingBlockSchema = import_zod32.z.object({
|
|
2133
|
+
type: import_zod32.z.literal("thinking"),
|
|
2134
|
+
thinking: import_zod32.z.string()
|
|
2135
|
+
});
|
|
2136
|
+
var ToolUseBlockSchema = import_zod32.z.object({
|
|
2137
|
+
type: import_zod32.z.literal("tool_use"),
|
|
2138
|
+
toolName: import_zod32.z.string(),
|
|
2139
|
+
toolId: import_zod32.z.string(),
|
|
2140
|
+
input: import_zod32.z.unknown()
|
|
2141
|
+
});
|
|
2142
|
+
var ToolResultBlockSchema = import_zod32.z.object({
|
|
2143
|
+
type: import_zod32.z.literal("tool_result"),
|
|
2144
|
+
toolUseId: import_zod32.z.string(),
|
|
2145
|
+
content: import_zod32.z.string(),
|
|
2146
|
+
isError: import_zod32.z.boolean().optional()
|
|
2147
|
+
});
|
|
2148
|
+
var ConversationBlockSchema = import_zod32.z.discriminatedUnion("type", [
|
|
2025
2149
|
TextBlockSchema,
|
|
2026
2150
|
ThinkingBlockSchema,
|
|
2027
2151
|
ToolUseBlockSchema,
|
|
@@ -2032,22 +2156,22 @@ var ConversationMessageRoles = [
|
|
|
2032
2156
|
"user",
|
|
2033
2157
|
"system"
|
|
2034
2158
|
];
|
|
2035
|
-
var ConversationMessageSchema =
|
|
2036
|
-
role:
|
|
2037
|
-
content:
|
|
2038
|
-
timestamp:
|
|
2159
|
+
var ConversationMessageSchema = import_zod32.z.object({
|
|
2160
|
+
role: import_zod32.z.enum(ConversationMessageRoles),
|
|
2161
|
+
content: import_zod32.z.array(ConversationBlockSchema),
|
|
2162
|
+
timestamp: import_zod32.z.string()
|
|
2039
2163
|
});
|
|
2040
|
-
var ScenarioConversationSchema =
|
|
2041
|
-
id:
|
|
2042
|
-
projectId:
|
|
2043
|
-
evalRunId:
|
|
2044
|
-
resultId:
|
|
2045
|
-
messages:
|
|
2046
|
-
createdAt:
|
|
2164
|
+
var ScenarioConversationSchema = import_zod32.z.object({
|
|
2165
|
+
id: import_zod32.z.string(),
|
|
2166
|
+
projectId: import_zod32.z.string(),
|
|
2167
|
+
evalRunId: import_zod32.z.string(),
|
|
2168
|
+
resultId: import_zod32.z.string(),
|
|
2169
|
+
messages: import_zod32.z.array(ConversationMessageSchema),
|
|
2170
|
+
createdAt: import_zod32.z.string()
|
|
2047
2171
|
});
|
|
2048
|
-
var ConversationResponseSchema =
|
|
2049
|
-
messages:
|
|
2050
|
-
isPartial:
|
|
2172
|
+
var ConversationResponseSchema = import_zod32.z.object({
|
|
2173
|
+
messages: import_zod32.z.array(ConversationMessageSchema),
|
|
2174
|
+
isPartial: import_zod32.z.boolean()
|
|
2051
2175
|
});
|
|
2052
2176
|
|
|
2053
2177
|
// src/evaluation/eval-result.ts
|
|
@@ -2058,98 +2182,98 @@ var AssertionResultStatus = /* @__PURE__ */ ((AssertionResultStatus2) => {
|
|
|
2058
2182
|
AssertionResultStatus2["ERROR"] = "error";
|
|
2059
2183
|
return AssertionResultStatus2;
|
|
2060
2184
|
})(AssertionResultStatus || {});
|
|
2061
|
-
var AssertionResultSchema =
|
|
2062
|
-
id:
|
|
2063
|
-
assertionId:
|
|
2064
|
-
assertionType:
|
|
2065
|
-
assertionName:
|
|
2066
|
-
status:
|
|
2067
|
-
message:
|
|
2068
|
-
expected:
|
|
2069
|
-
actual:
|
|
2070
|
-
duration:
|
|
2071
|
-
details:
|
|
2072
|
-
llmTraceSteps:
|
|
2073
|
-
});
|
|
2074
|
-
var EvalRunResultSchema =
|
|
2075
|
-
id:
|
|
2076
|
-
targetId:
|
|
2077
|
-
targetName:
|
|
2185
|
+
var AssertionResultSchema = import_zod33.z.object({
|
|
2186
|
+
id: import_zod33.z.string(),
|
|
2187
|
+
assertionId: import_zod33.z.string(),
|
|
2188
|
+
assertionType: import_zod33.z.string(),
|
|
2189
|
+
assertionName: import_zod33.z.string(),
|
|
2190
|
+
status: import_zod33.z.enum(AssertionResultStatus),
|
|
2191
|
+
message: import_zod33.z.string().optional(),
|
|
2192
|
+
expected: import_zod33.z.string().optional(),
|
|
2193
|
+
actual: import_zod33.z.string().optional(),
|
|
2194
|
+
duration: import_zod33.z.number().optional(),
|
|
2195
|
+
details: import_zod33.z.record(import_zod33.z.string(), import_zod33.z.unknown()).optional(),
|
|
2196
|
+
llmTraceSteps: import_zod33.z.array(LLMTraceStepSchema).optional()
|
|
2197
|
+
});
|
|
2198
|
+
var EvalRunResultSchema = import_zod33.z.object({
|
|
2199
|
+
id: import_zod33.z.string(),
|
|
2200
|
+
targetId: import_zod33.z.string(),
|
|
2201
|
+
targetName: import_zod33.z.string().optional(),
|
|
2078
2202
|
/** SkillVersion ID used for this evaluation (for version tracking) */
|
|
2079
|
-
skillVersionId:
|
|
2203
|
+
skillVersionId: import_zod33.z.string().optional(),
|
|
2080
2204
|
/** SkillVersion semver string (e.g., "1.0.0", "1.2.3") for display */
|
|
2081
|
-
skillVersion:
|
|
2082
|
-
scenarioId:
|
|
2083
|
-
scenarioName:
|
|
2205
|
+
skillVersion: import_zod33.z.string().optional(),
|
|
2206
|
+
scenarioId: import_zod33.z.string(),
|
|
2207
|
+
scenarioName: import_zod33.z.string(),
|
|
2084
2208
|
/** Snapshot of the trigger prompt used during the run (prevents stale display after edits) */
|
|
2085
|
-
triggerPrompt:
|
|
2209
|
+
triggerPrompt: import_zod33.z.string().optional(),
|
|
2086
2210
|
modelConfig: ModelConfigSchema.optional(),
|
|
2087
|
-
assertionResults:
|
|
2211
|
+
assertionResults: import_zod33.z.array(AssertionResultSchema),
|
|
2088
2212
|
metrics: EvalMetricsSchema.optional(),
|
|
2089
|
-
passed:
|
|
2090
|
-
failed:
|
|
2091
|
-
passRate:
|
|
2092
|
-
duration:
|
|
2093
|
-
outputText:
|
|
2094
|
-
files:
|
|
2095
|
-
fileDiffs:
|
|
2213
|
+
passed: import_zod33.z.number(),
|
|
2214
|
+
failed: import_zod33.z.number(),
|
|
2215
|
+
passRate: import_zod33.z.number(),
|
|
2216
|
+
duration: import_zod33.z.number(),
|
|
2217
|
+
outputText: import_zod33.z.string().optional(),
|
|
2218
|
+
files: import_zod33.z.array(ExpectedFileSchema).optional(),
|
|
2219
|
+
fileDiffs: import_zod33.z.array(DiffContentSchema).optional(),
|
|
2096
2220
|
/** Full template files after execution with status indicators */
|
|
2097
|
-
templateFiles:
|
|
2098
|
-
startedAt:
|
|
2099
|
-
completedAt:
|
|
2221
|
+
templateFiles: import_zod33.z.array(TemplateFileSchema).optional(),
|
|
2222
|
+
startedAt: import_zod33.z.string().optional(),
|
|
2223
|
+
completedAt: import_zod33.z.string().optional(),
|
|
2100
2224
|
llmTrace: LLMTraceSchema.optional(),
|
|
2101
2225
|
/** Full conversation messages (only present in transit; stripped before DB storage) */
|
|
2102
|
-
conversation:
|
|
2226
|
+
conversation: import_zod33.z.array(ConversationMessageSchema).optional(),
|
|
2103
2227
|
/** 0-based iteration index when a scenario is run multiple times within a single eval run */
|
|
2104
|
-
iterationIndex:
|
|
2105
|
-
});
|
|
2106
|
-
var PromptResultSchema =
|
|
2107
|
-
text:
|
|
2108
|
-
files:
|
|
2109
|
-
finishReason:
|
|
2110
|
-
reasoning:
|
|
2111
|
-
reasoningDetails:
|
|
2112
|
-
toolCalls:
|
|
2113
|
-
toolResults:
|
|
2114
|
-
warnings:
|
|
2115
|
-
sources:
|
|
2116
|
-
steps:
|
|
2117
|
-
generationTimeMs:
|
|
2118
|
-
prompt:
|
|
2119
|
-
systemPrompt:
|
|
2120
|
-
usage:
|
|
2121
|
-
totalTokens:
|
|
2122
|
-
totalMicrocentsSpent:
|
|
2228
|
+
iterationIndex: import_zod33.z.number().int().min(0).optional()
|
|
2229
|
+
});
|
|
2230
|
+
var PromptResultSchema = import_zod33.z.object({
|
|
2231
|
+
text: import_zod33.z.string(),
|
|
2232
|
+
files: import_zod33.z.array(import_zod33.z.unknown()).optional(),
|
|
2233
|
+
finishReason: import_zod33.z.string().optional(),
|
|
2234
|
+
reasoning: import_zod33.z.string().optional(),
|
|
2235
|
+
reasoningDetails: import_zod33.z.unknown().optional(),
|
|
2236
|
+
toolCalls: import_zod33.z.array(import_zod33.z.unknown()).optional(),
|
|
2237
|
+
toolResults: import_zod33.z.array(import_zod33.z.unknown()).optional(),
|
|
2238
|
+
warnings: import_zod33.z.array(import_zod33.z.unknown()).optional(),
|
|
2239
|
+
sources: import_zod33.z.array(import_zod33.z.unknown()).optional(),
|
|
2240
|
+
steps: import_zod33.z.array(import_zod33.z.unknown()),
|
|
2241
|
+
generationTimeMs: import_zod33.z.number(),
|
|
2242
|
+
prompt: import_zod33.z.string(),
|
|
2243
|
+
systemPrompt: import_zod33.z.string(),
|
|
2244
|
+
usage: import_zod33.z.object({
|
|
2245
|
+
totalTokens: import_zod33.z.number().optional(),
|
|
2246
|
+
totalMicrocentsSpent: import_zod33.z.number().optional()
|
|
2123
2247
|
})
|
|
2124
2248
|
});
|
|
2125
|
-
var EvaluationResultSchema =
|
|
2126
|
-
id:
|
|
2127
|
-
runId:
|
|
2128
|
-
timestamp:
|
|
2249
|
+
var EvaluationResultSchema = import_zod33.z.object({
|
|
2250
|
+
id: import_zod33.z.string(),
|
|
2251
|
+
runId: import_zod33.z.string(),
|
|
2252
|
+
timestamp: import_zod33.z.number(),
|
|
2129
2253
|
promptResult: PromptResultSchema,
|
|
2130
|
-
testResults:
|
|
2131
|
-
tags:
|
|
2132
|
-
feedback:
|
|
2133
|
-
score:
|
|
2134
|
-
suiteId:
|
|
2135
|
-
});
|
|
2136
|
-
var LeanEvaluationResultSchema =
|
|
2137
|
-
id:
|
|
2138
|
-
runId:
|
|
2139
|
-
timestamp:
|
|
2140
|
-
tags:
|
|
2141
|
-
scenarioId:
|
|
2142
|
-
scenarioVersion:
|
|
2143
|
-
targetId:
|
|
2144
|
-
targetVersion:
|
|
2145
|
-
suiteId:
|
|
2146
|
-
score:
|
|
2147
|
-
time:
|
|
2148
|
-
microcentsSpent:
|
|
2254
|
+
testResults: import_zod33.z.array(import_zod33.z.unknown()),
|
|
2255
|
+
tags: import_zod33.z.array(import_zod33.z.string()).optional(),
|
|
2256
|
+
feedback: import_zod33.z.string().optional(),
|
|
2257
|
+
score: import_zod33.z.number(),
|
|
2258
|
+
suiteId: import_zod33.z.string().optional()
|
|
2259
|
+
});
|
|
2260
|
+
var LeanEvaluationResultSchema = import_zod33.z.object({
|
|
2261
|
+
id: import_zod33.z.string(),
|
|
2262
|
+
runId: import_zod33.z.string(),
|
|
2263
|
+
timestamp: import_zod33.z.number(),
|
|
2264
|
+
tags: import_zod33.z.array(import_zod33.z.string()).optional(),
|
|
2265
|
+
scenarioId: import_zod33.z.string(),
|
|
2266
|
+
scenarioVersion: import_zod33.z.number().optional(),
|
|
2267
|
+
targetId: import_zod33.z.string(),
|
|
2268
|
+
targetVersion: import_zod33.z.number().optional(),
|
|
2269
|
+
suiteId: import_zod33.z.string().optional(),
|
|
2270
|
+
score: import_zod33.z.number(),
|
|
2271
|
+
time: import_zod33.z.number().optional(),
|
|
2272
|
+
microcentsSpent: import_zod33.z.number().optional()
|
|
2149
2273
|
});
|
|
2150
2274
|
|
|
2151
2275
|
// src/evaluation/eval-run-folder.ts
|
|
2152
|
-
var
|
|
2276
|
+
var import_zod34 = require("zod");
|
|
2153
2277
|
var EvalRunFolderSchema = TenantEntitySchema.extend({});
|
|
2154
2278
|
var CreateEvalRunFolderInputSchema = EvalRunFolderSchema.omit({
|
|
2155
2279
|
id: true,
|
|
@@ -2163,26 +2287,26 @@ var UpdateEvalRunFolderInputSchema = EvalRunFolderSchema.omit({
|
|
|
2163
2287
|
updatedAt: true,
|
|
2164
2288
|
deleted: true
|
|
2165
2289
|
}).partial();
|
|
2166
|
-
var EvalRunFolderMembershipSchema =
|
|
2167
|
-
folderId:
|
|
2168
|
-
evalRunId:
|
|
2169
|
-
projectId:
|
|
2170
|
-
createdAt:
|
|
2290
|
+
var EvalRunFolderMembershipSchema = import_zod34.z.object({
|
|
2291
|
+
folderId: import_zod34.z.string(),
|
|
2292
|
+
evalRunId: import_zod34.z.string(),
|
|
2293
|
+
projectId: import_zod34.z.string(),
|
|
2294
|
+
createdAt: import_zod34.z.string()
|
|
2171
2295
|
});
|
|
2172
2296
|
|
|
2173
2297
|
// src/project/project.ts
|
|
2174
|
-
var
|
|
2298
|
+
var import_zod35 = require("zod");
|
|
2175
2299
|
var ProjectSchema = BaseEntitySchema.extend({
|
|
2176
|
-
appId:
|
|
2177
|
-
scenarioTags:
|
|
2300
|
+
appId: import_zod35.z.string().optional().describe("The ID of the app in Dev Center"),
|
|
2301
|
+
scenarioTags: import_zod35.z.array(import_zod35.z.string()).optional().describe("Project-level tag vocabulary for scenarios"),
|
|
2178
2302
|
/** Per-project Wix auth token (write-only — never returned in GET responses). null = clear. */
|
|
2179
|
-
wixAuthToken:
|
|
2303
|
+
wixAuthToken: import_zod35.z.string().nullable().optional().describe("Wix auth token for CLI/MCP authentication (encrypted at rest)"),
|
|
2180
2304
|
/** Per-project Base44 auth file content (write-only — never returned in GET responses). null = clear. */
|
|
2181
|
-
base44AuthFile:
|
|
2305
|
+
base44AuthFile: import_zod35.z.string().nullable().optional().describe("Base64-encoded Base44 auth file content (encrypted at rest)"),
|
|
2182
2306
|
/** Resolved at runtime from the encrypted Wix auth token */
|
|
2183
|
-
wixAuthEmail:
|
|
2307
|
+
wixAuthEmail: import_zod35.z.string().optional().describe("Email associated with the Wix auth token (resolved at runtime)"),
|
|
2184
2308
|
/** Resolved at runtime from the encrypted Base44 auth file */
|
|
2185
|
-
base44AuthEmail:
|
|
2309
|
+
base44AuthEmail: import_zod35.z.string().optional().describe("Email from the Base44 auth file (resolved at runtime)")
|
|
2186
2310
|
});
|
|
2187
2311
|
var CreateProjectInputSchema = ProjectSchema.omit({
|
|
2188
2312
|
id: true,
|
|
@@ -2192,34 +2316,34 @@ var CreateProjectInputSchema = ProjectSchema.omit({
|
|
|
2192
2316
|
wixAuthEmail: true,
|
|
2193
2317
|
base44AuthEmail: true
|
|
2194
2318
|
}).extend({
|
|
2195
|
-
appId:
|
|
2319
|
+
appId: import_zod35.z.string().describe(
|
|
2196
2320
|
"Required: The ID of the app in Dev Center for credential scoping"
|
|
2197
2321
|
)
|
|
2198
2322
|
});
|
|
2199
2323
|
var UpdateProjectInputSchema = CreateProjectInputSchema.partial();
|
|
2200
2324
|
|
|
2201
2325
|
// src/template/template.ts
|
|
2202
|
-
var
|
|
2203
|
-
var SourceFileSchema =
|
|
2204
|
-
path:
|
|
2205
|
-
content:
|
|
2326
|
+
var import_zod36 = require("zod");
|
|
2327
|
+
var SourceFileSchema = import_zod36.z.object({
|
|
2328
|
+
path: import_zod36.z.string().min(1),
|
|
2329
|
+
content: import_zod36.z.string()
|
|
2206
2330
|
});
|
|
2207
|
-
var ExtraFileSchema =
|
|
2208
|
-
path:
|
|
2209
|
-
content:
|
|
2331
|
+
var ExtraFileSchema = import_zod36.z.object({
|
|
2332
|
+
path: import_zod36.z.string().min(1),
|
|
2333
|
+
content: import_zod36.z.string().optional(),
|
|
2210
2334
|
gitSource: GitHubSourceSchema.optional()
|
|
2211
2335
|
}).refine((ef) => ef.content !== void 0 || ef.gitSource !== void 0, {
|
|
2212
2336
|
message: "ExtraFile must have either content or gitSource"
|
|
2213
2337
|
});
|
|
2214
|
-
var TemplateFileEntrySchema =
|
|
2215
|
-
path:
|
|
2216
|
-
content:
|
|
2217
|
-
extra:
|
|
2338
|
+
var TemplateFileEntrySchema = import_zod36.z.object({
|
|
2339
|
+
path: import_zod36.z.string().min(1),
|
|
2340
|
+
content: import_zod36.z.string(),
|
|
2341
|
+
extra: import_zod36.z.boolean()
|
|
2218
2342
|
});
|
|
2219
2343
|
var TemplateSchema = TenantEntitySchema.extend({
|
|
2220
2344
|
source: GitHubSourceSchema.optional(),
|
|
2221
|
-
sourceFiles:
|
|
2222
|
-
extraFiles:
|
|
2345
|
+
sourceFiles: import_zod36.z.array(SourceFileSchema).optional(),
|
|
2346
|
+
extraFiles: import_zod36.z.array(ExtraFileSchema).optional()
|
|
2223
2347
|
});
|
|
2224
2348
|
var singleSourceKind = (t) => !(t.source && t.sourceFiles?.length);
|
|
2225
2349
|
var singleSourceKindError = {
|
|
@@ -2239,66 +2363,66 @@ var UpdateTemplateInputSchema = TemplateSchema.omit({
|
|
|
2239
2363
|
}).partial().refine(singleSourceKind, singleSourceKindError);
|
|
2240
2364
|
|
|
2241
2365
|
// src/agent/agent-config.ts
|
|
2242
|
-
var
|
|
2243
|
-
var BaseAgentConfigSchema =
|
|
2366
|
+
var import_zod37 = require("zod");
|
|
2367
|
+
var BaseAgentConfigSchema = import_zod37.z.object({
|
|
2244
2368
|
/** Model ID (Claude or OpenAI). */
|
|
2245
2369
|
model: AnyModelSchema.optional(),
|
|
2246
2370
|
/** Sampling temperature (0–1). */
|
|
2247
|
-
temperature:
|
|
2371
|
+
temperature: import_zod37.z.number().min(0).max(1).optional(),
|
|
2248
2372
|
/** Max output tokens per turn. */
|
|
2249
|
-
maxTokens:
|
|
2373
|
+
maxTokens: import_zod37.z.number().int().min(1).optional(),
|
|
2250
2374
|
/** Number of agentic turns. 0 = unlimited. */
|
|
2251
|
-
maxTurns:
|
|
2375
|
+
maxTurns: import_zod37.z.number().int().min(0).optional(),
|
|
2252
2376
|
/** Execution timeout in milliseconds. Overrides the default maxTurns-based calculation. */
|
|
2253
|
-
maxDurationMs:
|
|
2377
|
+
maxDurationMs: import_zod37.z.number().int().min(0).optional()
|
|
2254
2378
|
});
|
|
2255
|
-
var EffortLevelSchema =
|
|
2379
|
+
var EffortLevelSchema = import_zod37.z.enum(["low", "medium", "high", "max"]);
|
|
2256
2380
|
var ClaudeCodeConfigSchema = BaseAgentConfigSchema.extend({
|
|
2257
2381
|
/** Extended thinking token budget. */
|
|
2258
|
-
maxThinkingTokens:
|
|
2382
|
+
maxThinkingTokens: import_zod37.z.number().int().min(0).optional(),
|
|
2259
2383
|
/** Override the default allowedTools list passed to the SDK. */
|
|
2260
|
-
allowedTools:
|
|
2384
|
+
allowedTools: import_zod37.z.array(import_zod37.z.string()).optional(),
|
|
2261
2385
|
/** Tools to remove from the model's context entirely. */
|
|
2262
|
-
disallowedTools:
|
|
2386
|
+
disallowedTools: import_zod37.z.array(import_zod37.z.string()).optional(),
|
|
2263
2387
|
/** Controls thinking depth: low, medium, high, max. */
|
|
2264
2388
|
effort: EffortLevelSchema.optional(),
|
|
2265
2389
|
/** Maximum USD spend per run. Stops execution when reached. */
|
|
2266
|
-
maxBudgetUsd:
|
|
2390
|
+
maxBudgetUsd: import_zod37.z.number().min(0).optional()
|
|
2267
2391
|
});
|
|
2268
|
-
var PermissionValueSchema =
|
|
2269
|
-
var OpenCodePermissionSchema =
|
|
2270
|
-
|
|
2271
|
-
|
|
2392
|
+
var PermissionValueSchema = import_zod37.z.enum(["allow", "deny"]);
|
|
2393
|
+
var OpenCodePermissionSchema = import_zod37.z.record(
|
|
2394
|
+
import_zod37.z.string(),
|
|
2395
|
+
import_zod37.z.union([PermissionValueSchema, import_zod37.z.record(import_zod37.z.string(), PermissionValueSchema)])
|
|
2272
2396
|
);
|
|
2273
|
-
var ThinkingVariantSchema =
|
|
2397
|
+
var ThinkingVariantSchema = import_zod37.z.enum(["high", "low", "none"]);
|
|
2274
2398
|
var OpenCodeConfigSchema = BaseAgentConfigSchema.extend({
|
|
2275
2399
|
/** Permission overrides (defaults: allow-all). */
|
|
2276
2400
|
permission: OpenCodePermissionSchema.optional(),
|
|
2277
2401
|
/** Maps to `--variant` CLI flag. 'none' omits --thinking entirely. Default: 'high'. */
|
|
2278
2402
|
thinkingVariant: ThinkingVariantSchema.optional(),
|
|
2279
2403
|
/** Nucleus sampling (0–1). Alternative to temperature. */
|
|
2280
|
-
topP:
|
|
2404
|
+
topP: import_zod37.z.number().min(0).max(1).optional()
|
|
2281
2405
|
}).omit({ maxTokens: true });
|
|
2282
|
-
var ReasoningEffortSchema =
|
|
2406
|
+
var ReasoningEffortSchema = import_zod37.z.enum(["low", "medium", "high"]);
|
|
2283
2407
|
var SimpleAgentConfigSchema = BaseAgentConfigSchema.extend({
|
|
2284
2408
|
/** Anthropic thinking budget in tokens. Default: 10 000. */
|
|
2285
|
-
thinkingBudgetTokens:
|
|
2409
|
+
thinkingBudgetTokens: import_zod37.z.number().int().min(0).optional(),
|
|
2286
2410
|
/** Nucleus sampling (0–1). Alternative to temperature. */
|
|
2287
|
-
topP:
|
|
2411
|
+
topP: import_zod37.z.number().min(0).max(1).optional(),
|
|
2288
2412
|
/** Integer seed for deterministic/reproducible results (if model supports it). */
|
|
2289
|
-
seed:
|
|
2413
|
+
seed: import_zod37.z.number().int().optional(),
|
|
2290
2414
|
/** Stop sequences — model stops when generating any of these strings. */
|
|
2291
|
-
stopSequences:
|
|
2415
|
+
stopSequences: import_zod37.z.array(import_zod37.z.string()).optional(),
|
|
2292
2416
|
/** OpenAI reasoning effort level. Default: 'high'. */
|
|
2293
2417
|
reasoningEffort: ReasoningEffortSchema.optional(),
|
|
2294
2418
|
/** Frequency penalty (−2 to 2). Reduces repetition of same tokens. */
|
|
2295
|
-
frequencyPenalty:
|
|
2419
|
+
frequencyPenalty: import_zod37.z.number().min(-2).max(2).optional(),
|
|
2296
2420
|
/** Presence penalty (−2 to 2). Encourages topic diversity. */
|
|
2297
|
-
presencePenalty:
|
|
2421
|
+
presencePenalty: import_zod37.z.number().min(-2).max(2).optional()
|
|
2298
2422
|
});
|
|
2299
2423
|
|
|
2300
2424
|
// src/schedule/eval-schedule.ts
|
|
2301
|
-
var
|
|
2425
|
+
var import_zod38 = require("zod");
|
|
2302
2426
|
var FrequencyType = /* @__PURE__ */ ((FrequencyType2) => {
|
|
2303
2427
|
FrequencyType2["DAILY"] = "daily";
|
|
2304
2428
|
FrequencyType2["WEEKDAY"] = "weekday";
|
|
@@ -2308,31 +2432,31 @@ var FrequencyType = /* @__PURE__ */ ((FrequencyType2) => {
|
|
|
2308
2432
|
})(FrequencyType || {});
|
|
2309
2433
|
var EvalScheduleSchema = TenantEntitySchema.extend({
|
|
2310
2434
|
/** Whether the schedule is active */
|
|
2311
|
-
enabled:
|
|
2435
|
+
enabled: import_zod38.z.boolean(),
|
|
2312
2436
|
/** Test suite to run */
|
|
2313
|
-
suiteId:
|
|
2437
|
+
suiteId: import_zod38.z.string(),
|
|
2314
2438
|
/** Preset that provides agent + entities for this schedule */
|
|
2315
|
-
presetId:
|
|
2439
|
+
presetId: import_zod38.z.string(),
|
|
2316
2440
|
/** How often to run */
|
|
2317
|
-
frequencyType:
|
|
2441
|
+
frequencyType: import_zod38.z.nativeEnum(FrequencyType),
|
|
2318
2442
|
/** Time of day in 24h format (HH:MM), hours 00-23, minutes 00-59 */
|
|
2319
|
-
timeOfDay:
|
|
2443
|
+
timeOfDay: import_zod38.z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/),
|
|
2320
2444
|
/** Day of week (0=Sun, 6=Sat) for weekly schedules */
|
|
2321
|
-
dayOfWeek:
|
|
2445
|
+
dayOfWeek: import_zod38.z.number().min(0).max(6).optional(),
|
|
2322
2446
|
/** Day of month (1-31) for monthly schedules */
|
|
2323
|
-
dayOfMonth:
|
|
2447
|
+
dayOfMonth: import_zod38.z.number().min(1).max(31).optional(),
|
|
2324
2448
|
/** IANA timezone (e.g., 'America/New_York') */
|
|
2325
|
-
timezone:
|
|
2449
|
+
timezone: import_zod38.z.string(),
|
|
2326
2450
|
/** ID of the last eval run created by this schedule */
|
|
2327
|
-
lastRunId:
|
|
2451
|
+
lastRunId: import_zod38.z.string().optional(),
|
|
2328
2452
|
/** Denormalized status of the last run */
|
|
2329
|
-
lastRunStatus:
|
|
2453
|
+
lastRunStatus: import_zod38.z.string().optional(),
|
|
2330
2454
|
/** ISO timestamp of the last run */
|
|
2331
|
-
lastRunAt:
|
|
2455
|
+
lastRunAt: import_zod38.z.string().optional(),
|
|
2332
2456
|
/** Next scheduled run time in UTC (pre-computed for efficient querying, set by backend) */
|
|
2333
|
-
nextRunAt:
|
|
2457
|
+
nextRunAt: import_zod38.z.string().optional(),
|
|
2334
2458
|
/** Per-scenario variable values forwarded to runs triggered by this schedule (scenarioId → varName → value) */
|
|
2335
|
-
variables:
|
|
2459
|
+
variables: import_zod38.z.record(import_zod38.z.string(), import_zod38.z.record(import_zod38.z.string(), import_zod38.z.string())).optional()
|
|
2336
2460
|
});
|
|
2337
2461
|
function isValidTimezone(tz) {
|
|
2338
2462
|
try {
|
|
@@ -2345,14 +2469,14 @@ function isValidTimezone(tz) {
|
|
|
2345
2469
|
function validateScheduleFields(data, ctx, options) {
|
|
2346
2470
|
if (data.frequencyType === "weekly" /* WEEKLY */ && data.dayOfWeek == null) {
|
|
2347
2471
|
ctx.addIssue({
|
|
2348
|
-
code:
|
|
2472
|
+
code: import_zod38.z.ZodIssueCode.custom,
|
|
2349
2473
|
message: "dayOfWeek is required for weekly schedules",
|
|
2350
2474
|
path: ["dayOfWeek"]
|
|
2351
2475
|
});
|
|
2352
2476
|
}
|
|
2353
2477
|
if (data.frequencyType === "monthly" /* MONTHLY */ && data.dayOfMonth == null) {
|
|
2354
2478
|
ctx.addIssue({
|
|
2355
|
-
code:
|
|
2479
|
+
code: import_zod38.z.ZodIssueCode.custom,
|
|
2356
2480
|
message: "dayOfMonth is required for monthly schedules",
|
|
2357
2481
|
path: ["dayOfMonth"]
|
|
2358
2482
|
});
|
|
@@ -2360,7 +2484,7 @@ function validateScheduleFields(data, ctx, options) {
|
|
|
2360
2484
|
const shouldValidateTz = options.partial ? data.timezone !== void 0 : true;
|
|
2361
2485
|
if (shouldValidateTz && !isValidTimezone(data.timezone)) {
|
|
2362
2486
|
ctx.addIssue({
|
|
2363
|
-
code:
|
|
2487
|
+
code: import_zod38.z.ZodIssueCode.custom,
|
|
2364
2488
|
message: "Invalid IANA timezone",
|
|
2365
2489
|
path: ["timezone"]
|
|
2366
2490
|
});
|
|
@@ -2524,6 +2648,7 @@ var UpdateEvalScheduleInputSchema = BaseCreateScheduleSchema.partial().superRefi
|
|
|
2524
2648
|
PresetSchema,
|
|
2525
2649
|
ProjectSchema,
|
|
2526
2650
|
PromptResultSchema,
|
|
2651
|
+
ProvisionedSiteSchema,
|
|
2527
2652
|
RUN_COMMAND_LABELS,
|
|
2528
2653
|
ReasoningEffortSchema,
|
|
2529
2654
|
RuleSchema,
|
|
@@ -2531,13 +2656,20 @@ var UpdateEvalScheduleInputSchema = BaseCreateScheduleSchema.partial().superRefi
|
|
|
2531
2656
|
RunAnalysisFindingSchema,
|
|
2532
2657
|
RunAnalysisSchema,
|
|
2533
2658
|
SEMVER_REGEX,
|
|
2659
|
+
SITE_SETUP_EXCLUSIVE_VARIABLES,
|
|
2534
2660
|
SKILL_FOLDER_NAME_REGEX,
|
|
2535
2661
|
SYSTEM_ASSERTIONS,
|
|
2536
2662
|
SYSTEM_ASSERTION_IDS,
|
|
2537
2663
|
ScenarioAssertionLinkSchema,
|
|
2538
2664
|
ScenarioConversationSchema,
|
|
2539
2665
|
SimpleAgentConfigSchema,
|
|
2666
|
+
SiteBootstrapHttpMethodSchema,
|
|
2667
|
+
SiteBootstrapResultSchema,
|
|
2668
|
+
SiteBootstrapSchema,
|
|
2669
|
+
SiteBootstrapStepResultSchema,
|
|
2670
|
+
SiteBootstrapStepSchema,
|
|
2540
2671
|
SiteConfigTestSchema,
|
|
2672
|
+
SiteSetupConfigSchema,
|
|
2541
2673
|
SkillFileSchema,
|
|
2542
2674
|
SkillMetadataSchema,
|
|
2543
2675
|
SkillSchema,
|
|
@@ -2591,6 +2723,7 @@ var UpdateEvalScheduleInputSchema = BaseCreateScheduleSchema.partial().superRefi
|
|
|
2591
2723
|
UpdateTestScenarioInputSchema,
|
|
2592
2724
|
UpdateTestSuiteInputSchema,
|
|
2593
2725
|
VitestTestSchema,
|
|
2726
|
+
WixSiteSummarySchema,
|
|
2594
2727
|
capabilityToMcp,
|
|
2595
2728
|
capabilityToRule,
|
|
2596
2729
|
capabilityToSkill,
|
|
@@ -2598,6 +2731,7 @@ var UpdateEvalScheduleInputSchema = BaseCreateScheduleSchema.partial().superRefi
|
|
|
2598
2731
|
capabilityToSubAgent,
|
|
2599
2732
|
capabilityVersionToSkillVersion,
|
|
2600
2733
|
classifyAssertionRef,
|
|
2734
|
+
extractVariableNamesFromPrompt,
|
|
2601
2735
|
formatTraceEventLine,
|
|
2602
2736
|
getSystemAssertion,
|
|
2603
2737
|
getSystemAssertions,
|
|
@@ -2610,7 +2744,10 @@ var UpdateEvalScheduleInputSchema = BaseCreateScheduleSchema.partial().superRefi
|
|
|
2610
2744
|
normalizeModelId,
|
|
2611
2745
|
parseBuildCommandToArgv,
|
|
2612
2746
|
parseTraceEventLine,
|
|
2747
|
+
promptUsesSiteSetupExclusiveVariables,
|
|
2748
|
+
resolveWixOriginTemplateId,
|
|
2613
2749
|
validateAssertionConfig,
|
|
2614
|
-
validateBuildPassedParamsInAssertionLinks
|
|
2750
|
+
validateBuildPassedParamsInAssertionLinks,
|
|
2751
|
+
validateSiteSetupExclusivity
|
|
2615
2752
|
});
|
|
2616
2753
|
//# sourceMappingURL=index.js.map
|