@wix/evalforge-types 0.65.0 → 0.67.0

package/build/index.js

@@ -43,8 +43,15 @@ __export(index_exports, {
   AssertionResultStatus: () => AssertionResultStatus,
   AssertionSchema: () => AssertionSchema,
   AssertionTypeSchema: () => AssertionTypeSchema,
+  BATCH_IMPORT_LIMITS: () => BATCH_IMPORT_LIMITS,
   BaseEntitySchema: () => BaseEntitySchema,
   BaseTestSchema: () => BaseTestSchema,
+  BatchAssertionLinkSchema: () => BatchAssertionLinkSchema,
+  BatchImportPayloadSchema: () => BatchImportPayloadSchema,
+  BatchImportResponseSchema: () => BatchImportResponseSchema,
+  BatchResultItemSchema: () => BatchResultItemSchema,
+  BatchScenarioEntrySchema: () => BatchScenarioEntrySchema,
+  BatchSummarySchema: () => BatchSummarySchema,
   BuildCheckTestSchema: () => BuildCheckTestSchema,
   BuildPassedAssertionSchema: () => BuildPassedAssertionSchema,
   BuildPassedConfigSchema: () => BuildPassedConfigSchema,
@@ -187,11 +194,13 @@ __export(index_exports, {
   UpdateTestScenarioInputSchema: () => UpdateTestScenarioInputSchema,
   UpdateTestSuiteInputSchema: () => UpdateTestSuiteInputSchema,
   VitestTestSchema: () => VitestTestSchema,
+  classifyAssertionRef: () => classifyAssertionRef,
   formatTraceEventLine: () => formatTraceEventLine,
   getSystemAssertion: () => getSystemAssertion,
   getSystemAssertions: () => getSystemAssertions,
   isSystemAssertionId: () => isSystemAssertionId,
   isValidSkillFolderName: () => isValidSkillFolderName,
+  normalizeBatchAssertionLink: () => normalizeBatchAssertionLink,
   normalizeModelId: () => normalizeModelId,
   parseTraceEventLine: () => parseTraceEventLine,
   validateAssertionConfig: () => validateAssertionConfig
@@ -997,11 +1006,67 @@ var CreateTestScenarioInputSchema = TestScenarioSchema.omit({
 });
 var UpdateTestScenarioInputSchema = CreateTestScenarioInputSchema.partial();
 
-// src/suite/test-suite.ts
+// src/scenario/batch-import.ts
 var import_zod24 = require("zod");
+var UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+var BatchAssertionLinkSchema = import_zod24.z.union([
+  import_zod24.z.string().min(1),
+  ScenarioAssertionLinkSchema
+]);
+var BatchScenarioEntrySchema = import_zod24.z.object({
+  name: import_zod24.z.string().min(1, "name: Required"),
+  description: import_zod24.z.string().optional().default(""),
+  triggerPrompt: import_zod24.z.string().min(10, "triggerPrompt: Must be at least 10 characters"),
+  templateId: import_zod24.z.string().nullish(),
+  tags: import_zod24.z.array(import_zod24.z.string()).optional(),
+  assertionLinks: import_zod24.z.array(BatchAssertionLinkSchema).optional()
+});
+var BatchImportPayloadSchema = import_zod24.z.object({
+  scenarios: import_zod24.z.array(BatchScenarioEntrySchema).min(1, "scenarios array must contain at least one entry").max(100, "Maximum 100 scenarios per upload")
+});
+var BATCH_IMPORT_LIMITS = {
+  MAX_SCENARIOS: 100,
+  MAX_PAYLOAD_BYTES: 1048576
+  // 1 MB
+};
+function classifyAssertionRef(ref) {
+  if (ref.startsWith("system:")) {
+    return { type: "system", value: ref };
+  }
+  if (UUID_REGEX.test(ref)) {
+    return { type: "uuid", value: ref };
+  }
+  return { type: "name", value: ref };
+}
+function normalizeBatchAssertionLink(link) {
+  if (typeof link === "string") {
+    return { assertionId: link };
+  }
+  return link;
+}
+var BatchResultItemSchema = import_zod24.z.object({
+  index: import_zod24.z.number(),
+  name: import_zod24.z.string(),
+  status: import_zod24.z.enum(["valid", "invalid"]),
+  id: import_zod24.z.string().nullable().optional(),
+  errors: import_zod24.z.array(import_zod24.z.string()).optional()
+});
+var BatchSummarySchema = import_zod24.z.object({
+  total: import_zod24.z.number(),
+  valid: import_zod24.z.number(),
+  invalid: import_zod24.z.number(),
+  created: import_zod24.z.number()
+});
+var BatchImportResponseSchema = import_zod24.z.object({
+  summary: BatchSummarySchema,
+  results: import_zod24.z.array(BatchResultItemSchema)
+});
+
+// src/suite/test-suite.ts
+var import_zod25 = require("zod");
 var TestSuiteSchema = TenantEntitySchema.extend({
   /** IDs of test scenarios in this suite */
-  scenarioIds: import_zod24.z.array(import_zod24.z.string())
+  scenarioIds: import_zod25.z.array(import_zod25.z.string())
 });
 var CreateTestSuiteInputSchema = TestSuiteSchema.omit({
   id: true,
@@ -1012,21 +1077,21 @@ var CreateTestSuiteInputSchema = TestSuiteSchema.omit({
 var UpdateTestSuiteInputSchema = CreateTestSuiteInputSchema.partial();
 
 // src/evaluation/metrics.ts
-var import_zod25 = require("zod");
-var TokenUsageSchema = import_zod25.z.object({
-  prompt: import_zod25.z.number(),
-  completion: import_zod25.z.number(),
-  total: import_zod25.z.number()
-});
-var EvalMetricsSchema = import_zod25.z.object({
-  totalAssertions: import_zod25.z.number(),
-  passed: import_zod25.z.number(),
-  failed: import_zod25.z.number(),
-  skipped: import_zod25.z.number(),
-  errors: import_zod25.z.number(),
-  passRate: import_zod25.z.number(),
-  avgDuration: import_zod25.z.number(),
-  totalDuration: import_zod25.z.number()
+var import_zod26 = require("zod");
+var TokenUsageSchema = import_zod26.z.object({
+  prompt: import_zod26.z.number(),
+  completion: import_zod26.z.number(),
+  total: import_zod26.z.number()
+});
+var EvalMetricsSchema = import_zod26.z.object({
+  totalAssertions: import_zod26.z.number(),
+  passed: import_zod26.z.number(),
+  failed: import_zod26.z.number(),
+  skipped: import_zod26.z.number(),
+  errors: import_zod26.z.number(),
+  passRate: import_zod26.z.number(),
+  avgDuration: import_zod26.z.number(),
+  totalDuration: import_zod26.z.number()
 });
 var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
   EvalStatus2["PENDING"] = "pending";
@@ -1036,7 +1101,7 @@ var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
   EvalStatus2["CANCELLED"] = "cancelled";
   return EvalStatus2;
 })(EvalStatus || {});
-var EvalStatusSchema = import_zod25.z.enum(EvalStatus);
+var EvalStatusSchema = import_zod26.z.enum(EvalStatus);
 var LLMStepType = /* @__PURE__ */ ((LLMStepType2) => {
   LLMStepType2["COMPLETION"] = "completion";
   LLMStepType2["TOOL_USE"] = "tool_use";
@@ -1044,54 +1109,54 @@ var LLMStepType = /* @__PURE__ */ ((LLMStepType2) => {
   LLMStepType2["THINKING"] = "thinking";
   return LLMStepType2;
 })(LLMStepType || {});
-var LLMTraceStepSchema = import_zod25.z.object({
-  id: import_zod25.z.string(),
-  stepNumber: import_zod25.z.number(),
-  type: import_zod25.z.enum(LLMStepType),
-  model: import_zod25.z.string(),
-  provider: import_zod25.z.string(),
-  startedAt: import_zod25.z.string(),
-  durationMs: import_zod25.z.number(),
+var LLMTraceStepSchema = import_zod26.z.object({
+  id: import_zod26.z.string(),
+  stepNumber: import_zod26.z.number(),
+  type: import_zod26.z.enum(LLMStepType),
+  model: import_zod26.z.string(),
+  provider: import_zod26.z.string(),
+  startedAt: import_zod26.z.string(),
+  durationMs: import_zod26.z.number(),
   tokenUsage: TokenUsageSchema,
-  costUsd: import_zod25.z.number(),
-  toolName: import_zod25.z.string().optional(),
-  toolArguments: import_zod25.z.string().optional(),
-  inputPreview: import_zod25.z.string().optional(),
-  outputPreview: import_zod25.z.string().optional(),
-  success: import_zod25.z.boolean(),
-  error: import_zod25.z.string().optional(),
-  turnIndex: import_zod25.z.number().optional()
-});
-var LLMBreakdownStatsSchema = import_zod25.z.object({
-  count: import_zod25.z.number(),
-  durationMs: import_zod25.z.number(),
-  tokens: import_zod25.z.number(),
-  costUsd: import_zod25.z.number()
-});
-var LLMTraceSummarySchema = import_zod25.z.object({
-  totalSteps: import_zod25.z.number(),
-  totalTurns: import_zod25.z.number().optional(),
-  totalDurationMs: import_zod25.z.number(),
+  costUsd: import_zod26.z.number(),
+  toolName: import_zod26.z.string().optional(),
+  toolArguments: import_zod26.z.string().optional(),
+  inputPreview: import_zod26.z.string().optional(),
+  outputPreview: import_zod26.z.string().optional(),
+  success: import_zod26.z.boolean(),
+  error: import_zod26.z.string().optional(),
+  turnIndex: import_zod26.z.number().optional()
+});
+var LLMBreakdownStatsSchema = import_zod26.z.object({
+  count: import_zod26.z.number(),
+  durationMs: import_zod26.z.number(),
+  tokens: import_zod26.z.number(),
+  costUsd: import_zod26.z.number()
+});
+var LLMTraceSummarySchema = import_zod26.z.object({
+  totalSteps: import_zod26.z.number(),
+  totalTurns: import_zod26.z.number().optional(),
+  totalDurationMs: import_zod26.z.number(),
   totalTokens: TokenUsageSchema,
-  totalCostUsd: import_zod25.z.number(),
-  stepTypeBreakdown: import_zod25.z.record(import_zod25.z.string(), LLMBreakdownStatsSchema).optional(),
-  modelBreakdown: import_zod25.z.record(import_zod25.z.string(), LLMBreakdownStatsSchema),
-  modelsUsed: import_zod25.z.array(import_zod25.z.string())
-});
-var LLMTraceSchema = import_zod25.z.object({
-  id: import_zod25.z.string(),
-  steps: import_zod25.z.array(LLMTraceStepSchema),
+  totalCostUsd: import_zod26.z.number(),
+  stepTypeBreakdown: import_zod26.z.record(import_zod26.z.string(), LLMBreakdownStatsSchema).optional(),
+  modelBreakdown: import_zod26.z.record(import_zod26.z.string(), LLMBreakdownStatsSchema),
+  modelsUsed: import_zod26.z.array(import_zod26.z.string())
+});
+var LLMTraceSchema = import_zod26.z.object({
+  id: import_zod26.z.string(),
+  steps: import_zod26.z.array(LLMTraceStepSchema),
   summary: LLMTraceSummarySchema
 });
 
 // src/evaluation/eval-result.ts
-var import_zod29 = require("zod");
+var import_zod30 = require("zod");
 
 // src/evaluation/eval-run.ts
-var import_zod27 = require("zod");
+var import_zod28 = require("zod");
 
 // src/evaluation/live-trace.ts
-var import_zod26 = require("zod");
+var import_zod27 = require("zod");
 var LiveTraceEventType = /* @__PURE__ */ ((LiveTraceEventType2) => {
   LiveTraceEventType2["THINKING"] = "thinking";
   LiveTraceEventType2["TOOL_USE"] = "tool_use";
@@ -1105,37 +1170,37 @@ var LiveTraceEventType = /* @__PURE__ */ ((LiveTraceEventType2) => {
   LiveTraceEventType2["USER"] = "user";
   return LiveTraceEventType2;
 })(LiveTraceEventType || {});
-var LiveTraceEventSchema = import_zod26.z.object({
+var LiveTraceEventSchema = import_zod27.z.object({
   /** The evaluation run ID */
-  evalRunId: import_zod26.z.string(),
+  evalRunId: import_zod27.z.string(),
   /** The scenario ID being executed */
-  scenarioId: import_zod26.z.string(),
+  scenarioId: import_zod27.z.string(),
   /** The scenario name for display */
-  scenarioName: import_zod26.z.string(),
+  scenarioName: import_zod27.z.string(),
   /** The target ID (skill, agent, etc.) */
-  targetId: import_zod26.z.string(),
+  targetId: import_zod27.z.string(),
   /** The target name for display */
-  targetName: import_zod26.z.string(),
+  targetName: import_zod27.z.string(),
   /** Step number in the current scenario execution */
-  stepNumber: import_zod26.z.number(),
+  stepNumber: import_zod27.z.number(),
   /** Type of trace event */
-  type: import_zod26.z.enum(LiveTraceEventType),
+  type: import_zod27.z.enum(LiveTraceEventType),
   /** Tool name if this is a tool_use event */
-  toolName: import_zod26.z.string().optional(),
+  toolName: import_zod27.z.string().optional(),
   /** Tool arguments preview (truncated JSON) */
-  toolArgs: import_zod26.z.string().optional(),
+  toolArgs: import_zod27.z.string().optional(),
   /** Output preview (truncated text) */
-  outputPreview: import_zod26.z.string().optional(),
+  outputPreview: import_zod27.z.string().optional(),
   /** File path for file operations */
-  filePath: import_zod26.z.string().optional(),
+  filePath: import_zod27.z.string().optional(),
   /** Elapsed time in milliseconds for progress events */
-  elapsedMs: import_zod26.z.number().optional(),
+  elapsedMs: import_zod27.z.number().optional(),
   /** Thinking/reasoning text from Claude */
-  thinking: import_zod26.z.string().optional(),
+  thinking: import_zod27.z.string().optional(),
   /** Timestamp when this event occurred */
-  timestamp: import_zod26.z.string(),
+  timestamp: import_zod27.z.string(),
   /** Whether this is the final event for this scenario */
-  isComplete: import_zod26.z.boolean()
+  isComplete: import_zod27.z.boolean()
 });
 var TRACE_EVENT_PREFIX = "TRACE_EVENT:";
 function parseTraceEventLine(line) {
@@ -1164,15 +1229,15 @@ var TriggerType = /* @__PURE__ */ ((TriggerType2) => {
   TriggerType2["SCHEDULED"] = "SCHEDULED";
   return TriggerType2;
 })(TriggerType || {});
-var TriggerMetadataSchema = import_zod27.z.object({
-  version: import_zod27.z.string().optional(),
-  resourceUpdated: import_zod27.z.array(import_zod27.z.string()).optional(),
-  scheduleId: import_zod27.z.string().optional()
+var TriggerMetadataSchema = import_zod28.z.object({
+  version: import_zod28.z.string().optional(),
+  resourceUpdated: import_zod28.z.array(import_zod28.z.string()).optional(),
+  scheduleId: import_zod28.z.string().optional()
 });
-var TriggerSchema = import_zod27.z.object({
-  id: import_zod27.z.string(),
+var TriggerSchema = import_zod28.z.object({
+  id: import_zod28.z.string(),
   metadata: TriggerMetadataSchema.optional(),
-  type: import_zod27.z.nativeEnum(TriggerType)
+  type: import_zod28.z.nativeEnum(TriggerType)
 });
 var FailureCategory = /* @__PURE__ */ ((FailureCategory2) => {
   FailureCategory2["MISSING_FILE"] = "missing_file";
@@ -1190,30 +1255,30 @@ var FailureSeverity = /* @__PURE__ */ ((FailureSeverity2) => {
   FailureSeverity2["LOW"] = "low";
   return FailureSeverity2;
 })(FailureSeverity || {});
-var DiffLineTypeSchema = import_zod27.z.enum(["added", "removed", "unchanged"]);
-var DiffLineSchema = import_zod27.z.object({
+var DiffLineTypeSchema = import_zod28.z.enum(["added", "removed", "unchanged"]);
+var DiffLineSchema = import_zod28.z.object({
   type: DiffLineTypeSchema,
-  content: import_zod27.z.string(),
-  lineNumber: import_zod27.z.number()
-});
-var DiffContentSchema = import_zod27.z.object({
-  path: import_zod27.z.string(),
-  expected: import_zod27.z.string(),
-  actual: import_zod27.z.string(),
-  diffLines: import_zod27.z.array(DiffLineSchema),
-  renamedFrom: import_zod27.z.string().optional(),
+  content: import_zod28.z.string(),
+  lineNumber: import_zod28.z.number()
+});
+var DiffContentSchema = import_zod28.z.object({
+  path: import_zod28.z.string(),
+  expected: import_zod28.z.string(),
+  actual: import_zod28.z.string(),
+  diffLines: import_zod28.z.array(DiffLineSchema),
+  renamedFrom: import_zod28.z.string().optional(),
   /** Whether this file is an infrastructure/config file (e.g. .claude/settings.json, .mcp.json) */
-  isInfrastructure: import_zod27.z.boolean().optional()
+  isInfrastructure: import_zod28.z.boolean().optional()
 });
-var CommandExecutionSchema = import_zod27.z.object({
-  command: import_zod27.z.string(),
-  exitCode: import_zod27.z.number(),
-  output: import_zod27.z.string().optional(),
-  duration: import_zod27.z.number()
+var CommandExecutionSchema = import_zod28.z.object({
+  command: import_zod28.z.string(),
+  exitCode: import_zod28.z.number(),
+  output: import_zod28.z.string().optional(),
+  duration: import_zod28.z.number()
 });
-var FileModificationSchema = import_zod27.z.object({
-  path: import_zod27.z.string(),
-  action: import_zod27.z.enum(["created", "modified", "deleted"])
+var FileModificationSchema = import_zod28.z.object({
+  path: import_zod28.z.string(),
+  action: import_zod28.z.enum(["created", "modified", "deleted"])
 });
 var TemplateFileStatus = /* @__PURE__ */ ((TemplateFileStatus2) => {
   TemplateFileStatus2["NEW"] = "new";
@@ -1221,89 +1286,91 @@ var TemplateFileStatus = /* @__PURE__ */ ((TemplateFileStatus2) => {
   TemplateFileStatus2["UNCHANGED"] = "unchanged";
   return TemplateFileStatus2;
 })(TemplateFileStatus || {});
-var TemplateFileSchema = import_zod27.z.object({
+var TemplateFileSchema = import_zod28.z.object({
   /** Relative path within the template */
-  path: import_zod27.z.string(),
+  path: import_zod28.z.string(),
   /** Full file content after execution */
-  content: import_zod27.z.string(),
+  content: import_zod28.z.string(),
   /** File status (new, modified, unchanged) */
-  status: import_zod27.z.enum(["new", "modified", "unchanged"]),
+  status: import_zod28.z.enum(["new", "modified", "unchanged"]),
   /** Whether this file is an infrastructure/config file (e.g. .claude/settings.json, .mcp.json) */
-  isInfrastructure: import_zod27.z.boolean().optional()
-});
-var ApiCallSchema = import_zod27.z.object({
-  endpoint: import_zod27.z.string(),
-  tokensUsed: import_zod27.z.number(),
-  duration: import_zod27.z.number()
-});
-var ExecutionTraceSchema = import_zod27.z.object({
-  commands: import_zod27.z.array(CommandExecutionSchema),
-  filesModified: import_zod27.z.array(FileModificationSchema),
-  apiCalls: import_zod27.z.array(ApiCallSchema),
-  totalDuration: import_zod27.z.number()
-});
-var FailureAnalysisSchema = import_zod27.z.object({
-  category: import_zod27.z.enum(FailureCategory),
-  severity: import_zod27.z.enum(FailureSeverity),
-  summary: import_zod27.z.string(),
-  details: import_zod27.z.string(),
-  rootCause: import_zod27.z.string(),
-  suggestedFix: import_zod27.z.string(),
-  relatedAssertions: import_zod27.z.array(import_zod27.z.string()),
-  codeSnippet: import_zod27.z.string().optional(),
-  similarIssues: import_zod27.z.array(import_zod27.z.string()).optional(),
-  patternId: import_zod27.z.string().optional(),
+  isInfrastructure: import_zod28.z.boolean().optional()
+});
+var ApiCallSchema = import_zod28.z.object({
+  endpoint: import_zod28.z.string(),
+  tokensUsed: import_zod28.z.number(),
+  duration: import_zod28.z.number()
+});
+var ExecutionTraceSchema = import_zod28.z.object({
+  commands: import_zod28.z.array(CommandExecutionSchema),
+  filesModified: import_zod28.z.array(FileModificationSchema),
+  apiCalls: import_zod28.z.array(ApiCallSchema),
+  totalDuration: import_zod28.z.number()
+});
+var FailureAnalysisSchema = import_zod28.z.object({
+  category: import_zod28.z.enum(FailureCategory),
+  severity: import_zod28.z.enum(FailureSeverity),
+  summary: import_zod28.z.string(),
+  details: import_zod28.z.string(),
+  rootCause: import_zod28.z.string(),
+  suggestedFix: import_zod28.z.string(),
+  relatedAssertions: import_zod28.z.array(import_zod28.z.string()),
+  codeSnippet: import_zod28.z.string().optional(),
+  similarIssues: import_zod28.z.array(import_zod28.z.string()).optional(),
+  patternId: import_zod28.z.string().optional(),
   // Extended fields for detailed debugging
   diff: DiffContentSchema.optional(),
   executionTrace: ExecutionTraceSchema.optional()
 });
 var EvalRunSchema = TenantEntitySchema.extend({
   /** Agent ID for this run */
-  agentId: import_zod27.z.string().optional(),
+  agentId: import_zod28.z.string().optional(),
   /** Preset ID that originated this run (optional) */
-  presetId: import_zod27.z.string().optional(),
+  presetId: import_zod28.z.string().optional(),
   /** Skill IDs for this run */
-  skillIds: import_zod27.z.array(import_zod27.z.string()).optional(),
+  skillIds: import_zod28.z.array(import_zod28.z.string()).optional(),
   /** Map of skillId to skillVersionId for this run */
-  skillVersions: import_zod27.z.record(import_zod27.z.string(), import_zod27.z.string()).optional(),
+  skillVersions: import_zod28.z.record(import_zod28.z.string(), import_zod28.z.string()).optional(),
   /** Scenario IDs to run (always present — resolved server-side from tags when needed) */
-  scenarioIds: import_zod27.z.array(import_zod27.z.string()),
+  scenarioIds: import_zod28.z.array(import_zod28.z.string()),
   /** Current status */
   status: EvalStatusSchema,
   /** Progress percentage (0-100) */
-  progress: import_zod27.z.number(),
+  progress: import_zod28.z.number(),
   /** Results for each scenario/target combination (lazy to break eval-result ↔ eval-run cycle) */
-  results: import_zod27.z.array(import_zod27.z.lazy(() => EvalRunResultSchema)),
+  results: import_zod28.z.array(import_zod28.z.lazy(() => EvalRunResultSchema)),
   /** Aggregated metrics across all results */
   aggregateMetrics: EvalMetricsSchema,
   /** Failure analyses */
-  failureAnalyses: import_zod27.z.array(FailureAnalysisSchema).optional(),
+  failureAnalyses: import_zod28.z.array(FailureAnalysisSchema).optional(),
   /** Aggregated LLM trace summary */
   llmTraceSummary: LLMTraceSummarySchema.optional(),
   /** What triggered this run */
   trigger: TriggerSchema.optional(),
   /** When the run started (set when evaluation is triggered) */
-  startedAt: import_zod27.z.string().optional(),
+  startedAt: import_zod28.z.string().optional(),
   /** When the run completed */
-  completedAt: import_zod27.z.string().optional(),
+  completedAt: import_zod28.z.string().optional(),
   /** Live trace events captured during execution (for playback on results page) */
-  liveTraceEvents: import_zod27.z.array(LiveTraceEventSchema).optional(),
+  liveTraceEvents: import_zod28.z.array(LiveTraceEventSchema).optional(),
   /** Remote job ID for tracking execution in Dev Machines */
-  jobId: import_zod27.z.string().optional(),
+  jobId: import_zod28.z.string().optional(),
   /** Remote job status from the Dev Machine API (PENDING, RUNNING, COMPLETED, FAILED, CANCELLED) */
-  jobStatus: import_zod27.z.string().optional(),
+  jobStatus: import_zod28.z.string().optional(),
   /** Remote job error message if the job failed */
-  jobError: import_zod27.z.string().optional(),
+  jobError: import_zod28.z.string().optional(),
   /** Timestamp of the last job status check */
-  jobStatusCheckedAt: import_zod27.z.string().optional(),
+  jobStatusCheckedAt: import_zod28.z.string().optional(),
   /** MCP server IDs to enable for this run (optional) */
-  mcpIds: import_zod27.z.array(import_zod27.z.string()).optional(),
+  mcpIds: import_zod28.z.array(import_zod28.z.string()).optional(),
   /** Sub-agent IDs to enable for this run (optional) */
-  subAgentIds: import_zod27.z.array(import_zod27.z.string()).optional(),
+  subAgentIds: import_zod28.z.array(import_zod28.z.string()).optional(),
   /** Rule IDs to enable for this run (optional) */
-  ruleIds: import_zod27.z.array(import_zod27.z.string()).optional(),
+  ruleIds: import_zod28.z.array(import_zod28.z.string()).optional(),
   /** Tags used to select scenarios for this run (for traceability) */
-  tags: import_zod27.z.array(import_zod27.z.string()).optional()
+  tags: import_zod28.z.array(import_zod28.z.string()).optional(),
+  /** How many times each scenario is executed within this eval run. Default: 1. Max: 20. */
+  runsPerScenario: import_zod28.z.number().int().min(1).max(20).optional()
 });
 var CreateEvalRunInputSchema = EvalRunSchema.omit({
   id: true,
@@ -1318,60 +1385,60 @@ var CreateEvalRunInputSchema = EvalRunSchema.omit({
   scenarioIds: true
 }).extend({
   /** Optional on input — backend resolves from tags when not provided */
-  scenarioIds: import_zod27.z.array(import_zod27.z.string()).optional()
+  scenarioIds: import_zod28.z.array(import_zod28.z.string()).optional()
 }).refine(
   (data) => data.scenarioIds && data.scenarioIds.length > 0 || data.tags && data.tags.length > 0,
   { message: "Either scenarioIds or tags must be provided" }
 );
-var EvaluationProgressSchema = import_zod27.z.object({
-  runId: import_zod27.z.string(),
-  targetId: import_zod27.z.string(),
-  totalScenarios: import_zod27.z.number(),
-  completedScenarios: import_zod27.z.number(),
-  scenarioProgress: import_zod27.z.array(
-    import_zod27.z.object({
-      scenarioId: import_zod27.z.string(),
-      currentStep: import_zod27.z.string(),
-      error: import_zod27.z.string().optional()
+var EvaluationProgressSchema = import_zod28.z.object({
+  runId: import_zod28.z.string(),
+  targetId: import_zod28.z.string(),
+  totalScenarios: import_zod28.z.number(),
+  completedScenarios: import_zod28.z.number(),
+  scenarioProgress: import_zod28.z.array(
+    import_zod28.z.object({
+      scenarioId: import_zod28.z.string(),
+      currentStep: import_zod28.z.string(),
+      error: import_zod28.z.string().optional()
     })
   ),
-  createdAt: import_zod27.z.number()
-});
-var EvaluationLogSchema = import_zod27.z.object({
-  runId: import_zod27.z.string(),
-  scenarioId: import_zod27.z.string(),
-  log: import_zod27.z.object({
-    level: import_zod27.z.enum(["info", "error", "debug"]),
-    message: import_zod27.z.string().optional(),
-    args: import_zod27.z.array(import_zod27.z.any()).optional(),
-    error: import_zod27.z.string().optional()
+  createdAt: import_zod28.z.number()
+});
+var EvaluationLogSchema = import_zod28.z.object({
+  runId: import_zod28.z.string(),
+  scenarioId: import_zod28.z.string(),
+  log: import_zod28.z.object({
+    level: import_zod28.z.enum(["info", "error", "debug"]),
+    message: import_zod28.z.string().optional(),
+    args: import_zod28.z.array(import_zod28.z.any()).optional(),
+    error: import_zod28.z.string().optional()
   })
 });
 var LLM_TIMEOUT = 12e4;
 
 // src/evaluation/conversation.ts
-var import_zod28 = require("zod");
-var TextBlockSchema = import_zod28.z.object({
-  type: import_zod28.z.literal("text"),
-  text: import_zod28.z.string()
-});
-var ThinkingBlockSchema = import_zod28.z.object({
-  type: import_zod28.z.literal("thinking"),
-  thinking: import_zod28.z.string()
-});
-var ToolUseBlockSchema = import_zod28.z.object({
-  type: import_zod28.z.literal("tool_use"),
-  toolName: import_zod28.z.string(),
-  toolId: import_zod28.z.string(),
-  input: import_zod28.z.unknown()
-});
-var ToolResultBlockSchema = import_zod28.z.object({
-  type: import_zod28.z.literal("tool_result"),
-  toolUseId: import_zod28.z.string(),
-  content: import_zod28.z.string(),
-  isError: import_zod28.z.boolean().optional()
-});
-var ConversationBlockSchema = import_zod28.z.discriminatedUnion("type", [
+var import_zod29 = require("zod");
+var TextBlockSchema = import_zod29.z.object({
+  type: import_zod29.z.literal("text"),
+  text: import_zod29.z.string()
+});
+var ThinkingBlockSchema = import_zod29.z.object({
+  type: import_zod29.z.literal("thinking"),
+  thinking: import_zod29.z.string()
+});
+var ToolUseBlockSchema = import_zod29.z.object({
+  type: import_zod29.z.literal("tool_use"),
+  toolName: import_zod29.z.string(),
+  toolId: import_zod29.z.string(),
+  input: import_zod29.z.unknown()
+});
+var ToolResultBlockSchema = import_zod29.z.object({
+  type: import_zod29.z.literal("tool_result"),
+  toolUseId: import_zod29.z.string(),
+  content: import_zod29.z.string(),
+  isError: import_zod29.z.boolean().optional()
+});
+var ConversationBlockSchema = import_zod29.z.discriminatedUnion("type", [
   TextBlockSchema,
   ThinkingBlockSchema,
   ToolUseBlockSchema,
@@ -1382,18 +1449,18 @@ var ConversationMessageRoles = [
   "user",
   "system"
 ];
-var ConversationMessageSchema = import_zod28.z.object({
-  role: import_zod28.z.enum(ConversationMessageRoles),
-  content: import_zod28.z.array(ConversationBlockSchema),
-  timestamp: import_zod28.z.string()
+var ConversationMessageSchema = import_zod29.z.object({
+  role: import_zod29.z.enum(ConversationMessageRoles),
+  content: import_zod29.z.array(ConversationBlockSchema),
+  timestamp: import_zod29.z.string()
 });
-var ScenarioConversationSchema = import_zod28.z.object({
-  id: import_zod28.z.string(),
-  projectId: import_zod28.z.string(),
-  evalRunId: import_zod28.z.string(),
-  resultId: import_zod28.z.string(),
-  messages: import_zod28.z.array(ConversationMessageSchema),
-  createdAt: import_zod28.z.string()
+var ScenarioConversationSchema = import_zod29.z.object({
+  id: import_zod29.z.string(),
+  projectId: import_zod29.z.string(),
+  evalRunId: import_zod29.z.string(),
+  resultId: import_zod29.z.string(),
+  messages: import_zod29.z.array(ConversationMessageSchema),
+  createdAt: import_zod29.z.string()
 });
 
 // src/evaluation/eval-result.ts
@@ -1404,94 +1471,96 @@ var AssertionResultStatus = /* @__PURE__ */ ((AssertionResultStatus2) => {
   AssertionResultStatus2["ERROR"] = "error";
   return AssertionResultStatus2;
 })(AssertionResultStatus || {});
-var AssertionResultSchema = import_zod29.z.object({
-  id: import_zod29.z.string(),
-  assertionId: import_zod29.z.string(),
-  assertionType: import_zod29.z.string(),
-  assertionName: import_zod29.z.string(),
-  status: import_zod29.z.enum(AssertionResultStatus),
-  message: import_zod29.z.string().optional(),
-  expected: import_zod29.z.string().optional(),
-  actual: import_zod29.z.string().optional(),
-  duration: import_zod29.z.number().optional(),
-  details: import_zod29.z.record(import_zod29.z.string(), import_zod29.z.unknown()).optional(),
-  llmTraceSteps: import_zod29.z.array(LLMTraceStepSchema).optional()
-});
-var EvalRunResultSchema = import_zod29.z.object({
-  id: import_zod29.z.string(),
-  targetId: import_zod29.z.string(),
-  targetName: import_zod29.z.string().optional(),
+var AssertionResultSchema = import_zod30.z.object({
+  id: import_zod30.z.string(),
+  assertionId: import_zod30.z.string(),
+  assertionType: import_zod30.z.string(),
+  assertionName: import_zod30.z.string(),
+  status: import_zod30.z.enum(AssertionResultStatus),
+  message: import_zod30.z.string().optional(),
+  expected: import_zod30.z.string().optional(),
+  actual: import_zod30.z.string().optional(),
+  duration: import_zod30.z.number().optional(),
+  details: import_zod30.z.record(import_zod30.z.string(), import_zod30.z.unknown()).optional(),
+  llmTraceSteps: import_zod30.z.array(LLMTraceStepSchema).optional()
+});
+var EvalRunResultSchema = import_zod30.z.object({
+  id: import_zod30.z.string(),
+  targetId: import_zod30.z.string(),
+  targetName: import_zod30.z.string().optional(),
   /** SkillVersion ID used for this evaluation (for version tracking) */
-  skillVersionId: import_zod29.z.string().optional(),
+  skillVersionId: import_zod30.z.string().optional(),
   /** SkillVersion semver string (e.g., "1.0.0", "1.2.3") for display */
-  skillVersion: import_zod29.z.string().optional(),
-  scenarioId: import_zod29.z.string(),
-  scenarioName: import_zod29.z.string(),
+  skillVersion: import_zod30.z.string().optional(),
+  scenarioId: import_zod30.z.string(),
+  scenarioName: import_zod30.z.string(),
   modelConfig: ModelConfigSchema.optional(),
-  assertionResults: import_zod29.z.array(AssertionResultSchema),
+  assertionResults: import_zod30.z.array(AssertionResultSchema),
   metrics: EvalMetricsSchema.optional(),
-  passed: import_zod29.z.number(),
-  failed: import_zod29.z.number(),
-  passRate: import_zod29.z.number(),
-  duration: import_zod29.z.number(),
-  outputText: import_zod29.z.string().optional(),
-  files: import_zod29.z.array(ExpectedFileSchema).optional(),
-  fileDiffs: import_zod29.z.array(DiffContentSchema).optional(),
+  passed: import_zod30.z.number(),
+  failed: import_zod30.z.number(),
+  passRate: import_zod30.z.number(),
+  duration: import_zod30.z.number(),
+  outputText: import_zod30.z.string().optional(),
+  files: import_zod30.z.array(ExpectedFileSchema).optional(),
+  fileDiffs: import_zod30.z.array(DiffContentSchema).optional(),
   /** Full template files after execution with status indicators */
-  templateFiles: import_zod29.z.array(TemplateFileSchema).optional(),
-  startedAt: import_zod29.z.string().optional(),
-  completedAt: import_zod29.z.string().optional(),
+  templateFiles: import_zod30.z.array(TemplateFileSchema).optional(),
+  startedAt: import_zod30.z.string().optional(),
+  completedAt: import_zod30.z.string().optional(),
   llmTrace: LLMTraceSchema.optional(),
   /** Full conversation messages (only present in transit; stripped before DB storage) */
-  conversation: import_zod29.z.array(ConversationMessageSchema).optional()
-});
-var PromptResultSchema = import_zod29.z.object({
-  text: import_zod29.z.string(),
-  files: import_zod29.z.array(import_zod29.z.unknown()).optional(),
-  finishReason: import_zod29.z.string().optional(),
-  reasoning: import_zod29.z.string().optional(),
-  reasoningDetails: import_zod29.z.unknown().optional(),
-  toolCalls: import_zod29.z.array(import_zod29.z.unknown()).optional(),
-  toolResults: import_zod29.z.array(import_zod29.z.unknown()).optional(),
-  warnings: import_zod29.z.array(import_zod29.z.unknown()).optional(),
-  sources: import_zod29.z.array(import_zod29.z.unknown()).optional(),
-  steps: import_zod29.z.array(import_zod29.z.unknown()),
-  generationTimeMs: import_zod29.z.number(),
-  prompt: import_zod29.z.string(),
-  systemPrompt: import_zod29.z.string(),
-  usage: import_zod29.z.object({
-    totalTokens: import_zod29.z.number().optional(),
-    totalMicrocentsSpent: import_zod29.z.number().optional()
+  conversation: import_zod30.z.array(ConversationMessageSchema).optional(),
+  /** 0-based iteration index when a scenario is run multiple times within a single eval run */
+  iterationIndex: import_zod30.z.number().int().min(0).optional()
+});
+var PromptResultSchema = import_zod30.z.object({
+  text: import_zod30.z.string(),
+  files: import_zod30.z.array(import_zod30.z.unknown()).optional(),
+  finishReason: import_zod30.z.string().optional(),
+  reasoning: import_zod30.z.string().optional(),
+  reasoningDetails: import_zod30.z.unknown().optional(),
+  toolCalls: import_zod30.z.array(import_zod30.z.unknown()).optional(),
+  toolResults: import_zod30.z.array(import_zod30.z.unknown()).optional(),
+  warnings: import_zod30.z.array(import_zod30.z.unknown()).optional(),
+  sources: import_zod30.z.array(import_zod30.z.unknown()).optional(),
+  steps: import_zod30.z.array(import_zod30.z.unknown()),
+  generationTimeMs: import_zod30.z.number(),
+  prompt: import_zod30.z.string(),
+  systemPrompt: import_zod30.z.string(),
+  usage: import_zod30.z.object({
+    totalTokens: import_zod30.z.number().optional(),
+    totalMicrocentsSpent: import_zod30.z.number().optional()
   })
 });
-var EvaluationResultSchema = import_zod29.z.object({
-  id: import_zod29.z.string(),
-  runId: import_zod29.z.string(),
-  timestamp: import_zod29.z.number(),
+var EvaluationResultSchema = import_zod30.z.object({
+  id: import_zod30.z.string(),
+  runId: import_zod30.z.string(),
+  timestamp: import_zod30.z.number(),
   promptResult: PromptResultSchema,
-  testResults: import_zod29.z.array(import_zod29.z.unknown()),
-  tags: import_zod29.z.array(import_zod29.z.string()).optional(),
-  feedback: import_zod29.z.string().optional(),
-  score: import_zod29.z.number(),
-  suiteId: import_zod29.z.string().optional()
-});
-var LeanEvaluationResultSchema = import_zod29.z.object({
-  id: import_zod29.z.string(),
-  runId: import_zod29.z.string(),
-  timestamp: import_zod29.z.number(),
-  tags: import_zod29.z.array(import_zod29.z.string()).optional(),
-  scenarioId: import_zod29.z.string(),
-  scenarioVersion: import_zod29.z.number().optional(),
-  targetId: import_zod29.z.string(),
-  targetVersion: import_zod29.z.number().optional(),
-  suiteId: import_zod29.z.string().optional(),
-  score: import_zod29.z.number(),
-  time: import_zod29.z.number().optional(),
-  microcentsSpent: import_zod29.z.number().optional()
+  testResults: import_zod30.z.array(import_zod30.z.unknown()),
+  tags: import_zod30.z.array(import_zod30.z.string()).optional(),
+  feedback: import_zod30.z.string().optional(),
+  score: import_zod30.z.number(),
+  suiteId: import_zod30.z.string().optional()
+});
+var LeanEvaluationResultSchema = import_zod30.z.object({
+  id: import_zod30.z.string(),
+  runId: import_zod30.z.string(),
+  timestamp: import_zod30.z.number(),
+  tags: import_zod30.z.array(import_zod30.z.string()).optional(),
+  scenarioId: import_zod30.z.string(),
+  scenarioVersion: import_zod30.z.number().optional(),
+  targetId: import_zod30.z.string(),
+  targetVersion: import_zod30.z.number().optional(),
+  suiteId: import_zod30.z.string().optional(),
+  score: import_zod30.z.number(),
+  time: import_zod30.z.number().optional(),
+  microcentsSpent: import_zod30.z.number().optional()
 });
 
 // src/evaluation/eval-run-folder.ts
-var import_zod30 = require("zod");
+var import_zod31 = require("zod");
 var EvalRunFolderSchema = TenantEntitySchema.extend({});
 var CreateEvalRunFolderInputSchema = EvalRunFolderSchema.omit({
   id: true,
@@ -1505,26 +1574,26 @@ var UpdateEvalRunFolderInputSchema = EvalRunFolderSchema.omit({
   updatedAt: true,
   deleted: true
 }).partial();
-var EvalRunFolderMembershipSchema = import_zod30.z.object({
-  folderId: import_zod30.z.string(),
-  evalRunId: import_zod30.z.string(),
-  projectId: import_zod30.z.string(),
-  createdAt: import_zod30.z.string()
+var EvalRunFolderMembershipSchema = import_zod31.z.object({
+  folderId: import_zod31.z.string(),
+  evalRunId: import_zod31.z.string(),
+  projectId: import_zod31.z.string(),
+  createdAt: import_zod31.z.string()
 });
 
 // src/project/project.ts
-var import_zod31 = require("zod");
+var import_zod32 = require("zod");
 var ProjectSchema = BaseEntitySchema.extend({
-  appId: import_zod31.z.string().optional().describe("The ID of the app in Dev Center"),
-  scenarioTags: import_zod31.z.array(import_zod31.z.string()).optional().describe("Project-level tag vocabulary for scenarios"),
+  appId: import_zod32.z.string().optional().describe("The ID of the app in Dev Center"),
+  scenarioTags: import_zod32.z.array(import_zod32.z.string()).optional().describe("Project-level tag vocabulary for scenarios"),
   /** Per-project Wix auth token (write-only — never returned in GET responses). null = clear. */
-  wixAuthToken: import_zod31.z.string().nullable().optional().describe("Wix auth token for CLI/MCP authentication (encrypted at rest)"),
+  wixAuthToken: import_zod32.z.string().nullable().optional().describe("Wix auth token for CLI/MCP authentication (encrypted at rest)"),
   /** Per-project Base44 auth file content (write-only — never returned in GET responses). null = clear. */
-  base44AuthFile: import_zod31.z.string().nullable().optional().describe("Base64-encoded Base44 auth file content (encrypted at rest)"),
+  base44AuthFile: import_zod32.z.string().nullable().optional().describe("Base64-encoded Base44 auth file content (encrypted at rest)"),
   /** Resolved at runtime from the encrypted Wix auth token */
-  wixAuthEmail: import_zod31.z.string().optional().describe("Email associated with the Wix auth token (resolved at runtime)"),
+  wixAuthEmail: import_zod32.z.string().optional().describe("Email associated with the Wix auth token (resolved at runtime)"),
   /** Resolved at runtime from the encrypted Base44 auth file */
-  base44AuthEmail: import_zod31.z.string().optional().describe("Email from the Base44 auth file (resolved at runtime)")
+  base44AuthEmail: import_zod32.z.string().optional().describe("Email from the Base44 auth file (resolved at runtime)")
 });
 var CreateProjectInputSchema = ProjectSchema.omit({
   id: true,
@@ -1550,7 +1619,7 @@ var CreateTemplateInputSchema = TemplateSchema.omit({
 var UpdateTemplateInputSchema = CreateTemplateInputSchema.partial();
 
 // src/schedule/eval-schedule.ts
-var import_zod32 = require("zod");
+var import_zod33 = require("zod");
 var FrequencyType = /* @__PURE__ */ ((FrequencyType2) => {
   FrequencyType2["DAILY"] = "daily";
   FrequencyType2["WEEKDAY"] = "weekday";
@@ -1560,29 +1629,29 @@ var FrequencyType = /* @__PURE__ */ ((FrequencyType2) => {
 })(FrequencyType || {});
 var EvalScheduleSchema = TenantEntitySchema.extend({
   /** Whether the schedule is active */
-  enabled: import_zod32.z.boolean(),
+  enabled: import_zod33.z.boolean(),
   /** Test suite to run */
-  suiteId: import_zod32.z.string(),
+  suiteId: import_zod33.z.string(),
   /** Preset that provides agent + entities for this schedule */
-  presetId: import_zod32.z.string(),
+  presetId: import_zod33.z.string(),
   /** How often to run */
-  frequencyType: import_zod32.z.nativeEnum(FrequencyType),
+  frequencyType: import_zod33.z.nativeEnum(FrequencyType),
   /** Time of day in 24h format (HH:MM), hours 00-23, minutes 00-59 */
-  timeOfDay: import_zod32.z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/),
+  timeOfDay: import_zod33.z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/),
   /** Day of week (0=Sun, 6=Sat) for weekly schedules */
-  dayOfWeek: import_zod32.z.number().min(0).max(6).optional(),
+  dayOfWeek: import_zod33.z.number().min(0).max(6).optional(),
   /** Day of month (1-31) for monthly schedules */
-  dayOfMonth: import_zod32.z.number().min(1).max(31).optional(),
+  dayOfMonth: import_zod33.z.number().min(1).max(31).optional(),
   /** IANA timezone (e.g., 'America/New_York') */
-  timezone: import_zod32.z.string(),
+  timezone: import_zod33.z.string(),
   /** ID of the last eval run created by this schedule */
-  lastRunId: import_zod32.z.string().optional(),
+  lastRunId: import_zod33.z.string().optional(),
   /** Denormalized status of the last run */
-  lastRunStatus: import_zod32.z.string().optional(),
+  lastRunStatus: import_zod33.z.string().optional(),
   /** ISO timestamp of the last run */
-  lastRunAt: import_zod32.z.string().optional(),
+  lastRunAt: import_zod33.z.string().optional(),
   /** Next scheduled run time in UTC (pre-computed for efficient querying, set by backend) */
-  nextRunAt: import_zod32.z.string().optional()
+  nextRunAt: import_zod33.z.string().optional()
 });
 function isValidTimezone(tz) {
   try {
@@ -1595,14 +1664,14 @@ function isValidTimezone(tz) {
 function validateScheduleFields(data, ctx, options) {
   if (data.frequencyType === "weekly" /* WEEKLY */ && data.dayOfWeek == null) {
     ctx.addIssue({
-      code: import_zod32.z.ZodIssueCode.custom,
+      code: import_zod33.z.ZodIssueCode.custom,
       message: "dayOfWeek is required for weekly schedules",
       path: ["dayOfWeek"]
     });
   }
   if (data.frequencyType === "monthly" /* MONTHLY */ && data.dayOfMonth == null) {
     ctx.addIssue({
-      code: import_zod32.z.ZodIssueCode.custom,
+      code: import_zod33.z.ZodIssueCode.custom,
       message: "dayOfMonth is required for monthly schedules",
       path: ["dayOfMonth"]
     });
@@ -1610,7 +1679,7 @@ function validateScheduleFields(data, ctx, options) {
   const shouldValidateTz = options.partial ? data.timezone !== void 0 : true;
   if (shouldValidateTz && !isValidTimezone(data.timezone)) {
     ctx.addIssue({
-      code: import_zod32.z.ZodIssueCode.custom,
+      code: import_zod33.z.ZodIssueCode.custom,
       message: "Invalid IANA timezone",
       path: ["timezone"]
     });
@@ -1878,8 +1947,15 @@ function getSystemAssertion(id) {
   AssertionResultStatus,
   AssertionSchema,
   AssertionTypeSchema,
+  BATCH_IMPORT_LIMITS,
   BaseEntitySchema,
   BaseTestSchema,
+  BatchAssertionLinkSchema,
+  BatchImportPayloadSchema,
+  BatchImportResponseSchema,
+  BatchResultItemSchema,
+  BatchScenarioEntrySchema,
+  BatchSummarySchema,
   BuildCheckTestSchema,
   BuildPassedAssertionSchema,
   BuildPassedConfigSchema,
@@ -2022,11 +2098,13 @@ function getSystemAssertion(id) {
   UpdateTestScenarioInputSchema,
   UpdateTestSuiteInputSchema,
   VitestTestSchema,
+  classifyAssertionRef,
   formatTraceEventLine,
   getSystemAssertion,
   getSystemAssertions,
   isSystemAssertionId,
   isValidSkillFolderName,
+  normalizeBatchAssertionLink,
   normalizeModelId,
   parseTraceEventLine,
   validateAssertionConfig