@wix/evalforge-types 0.53.0 → 0.54.0

package/build/index.mjs

@@ -1415,85 +1415,34 @@ var TestSchema = z20.discriminatedUnion("type", [
   PlaywrightNLTestSchema
 ]);
 
-// src/scenario/assertions.ts
-import { z as z21 } from "zod";
-var SkillWasCalledAssertionSchema = z21.object({
-  type: z21.literal("skill_was_called"),
-  /** Names of the skills that must have been called (matched against trace Skill tool args) */
-  skillNames: z21.array(z21.string().min(1)).min(1)
-});
-var ToolCalledWithParamAssertionSchema = z21.object({
-  type: z21.literal("tool_called_with_param"),
-  /** Name of the tool that must have been called */
-  toolName: z21.string().min(1),
-  /** JSON string of key-value pairs for expected parameters (substring match) */
-  expectedParams: z21.string().min(1)
-});
-var BuildPassedAssertionSchema = z21.object({
-  type: z21.literal("build_passed"),
-  /** Command to run (default: "yarn build") */
-  command: z21.string().optional(),
-  /** Expected exit code (default: 0) */
-  expectedExitCode: z21.number().int().optional()
-});
-var CostAssertionSchema = z21.object({
-  type: z21.literal("cost"),
-  /** Maximum allowed cost in USD */
-  maxCostUsd: z21.number().positive()
-});
-var LlmJudgeAssertionSchema = z21.object({
-  type: z21.literal("llm_judge"),
-  /** Prompt template; placeholders: {{output}}, {{cwd}}, {{changedFiles}}, {{trace}} */
-  prompt: z21.string(),
-  /** Minimum score to pass (0-10, default 7) */
-  minScore: z21.number().int().min(0).max(10).optional(),
-  /** Model for the judge (e.g. claude-3-5-haiku) */
-  model: z21.string().optional(),
-  maxTokens: z21.number().int().optional(),
-  temperature: z21.number().min(0).max(1).optional()
-});
-var TimeAssertionSchema = z21.object({
-  type: z21.literal("time_limit"),
-  /** Maximum allowed duration in milliseconds */
-  maxDurationMs: z21.number().int().positive()
-});
-var AssertionSchema = z21.union([
-  SkillWasCalledAssertionSchema,
-  ToolCalledWithParamAssertionSchema,
-  BuildPassedAssertionSchema,
-  TimeAssertionSchema,
-  CostAssertionSchema,
-  LlmJudgeAssertionSchema
-]);
-
 // src/scenario/environment.ts
-import { z as z22 } from "zod";
-var LocalProjectConfigSchema = z22.object({
+import { z as z21 } from "zod";
+var LocalProjectConfigSchema = z21.object({
   /** Template ID to use for the local project */
-  templateId: z22.string().optional(),
+  templateId: z21.string().optional(),
   /** Files to create in the project */
-  files: z22.array(
-    z22.object({
-      path: z22.string().min(1),
-      content: z22.string().min(1)
+  files: z21.array(
+    z21.object({
+      path: z21.string().min(1),
+      content: z21.string().min(1)
     })
   ).optional()
 });
-var MetaSiteConfigSchema = z22.object({
-  configurations: z22.array(
-    z22.object({
-      name: z22.string().min(1),
-      apiCalls: z22.array(
-        z22.object({
-          url: z22.string().url(),
-          method: z22.enum(["POST", "PUT"]),
-          body: z22.string()
+var MetaSiteConfigSchema = z21.object({
+  configurations: z21.array(
+    z21.object({
+      name: z21.string().min(1),
+      apiCalls: z21.array(
+        z21.object({
+          url: z21.string().url(),
+          method: z21.enum(["POST", "PUT"]),
+          body: z21.string()
         })
       )
     })
   ).optional()
 });
-var EnvironmentSchema = z22.object({
+var EnvironmentSchema = z21.object({
   /** Local project configuration */
   localProject: LocalProjectConfigSchema.optional(),
   /** Meta site configuration */
@@ -1501,11 +1450,11 @@ var EnvironmentSchema = z22.object({
 });
 
 // src/scenario/test-scenario.ts
-import { z as z24 } from "zod";
+import { z as z23 } from "zod";
 
 // src/assertion/assertion.ts
-import { z as z23 } from "zod";
-var AssertionTypeSchema = z23.enum([
+import { z as z22 } from "zod";
+var AssertionTypeSchema = z22.enum([
   "skill_was_called",
   "tool_called_with_param",
   "build_passed",
@@ -1513,59 +1462,61 @@ var AssertionTypeSchema = z23.enum([
   "cost",
   "llm_judge"
 ]);
-var AssertionParameterTypeSchema = z23.enum([
+var AssertionParameterTypeSchema = z22.enum([
   "string",
   "number",
   "boolean"
 ]);
-var AssertionParameterSchema = z23.object({
+var AssertionParameterSchema = z22.object({
   /** Parameter name (used as key in params object) */
-  name: z23.string().min(1),
+  name: z22.string().min(1),
   /** Display label for the parameter */
-  label: z23.string().min(1),
+  label: z22.string().min(1),
   /** Parameter type */
   type: AssertionParameterTypeSchema,
   /** Whether this parameter is required */
-  required: z23.boolean(),
+  required: z22.boolean(),
   /** Default value (optional, used when not provided) */
-  defaultValue: z23.union([z23.string(), z23.number(), z23.boolean()]).optional(),
+  defaultValue: z22.union([z22.string(), z22.number(), z22.boolean()]).optional(),
   /** If true, parameter is hidden by default behind "Show advanced options" */
-  advanced: z23.boolean().optional()
+  advanced: z22.boolean().optional()
 });
-var ScenarioAssertionLinkSchema = z23.object({
+var ScenarioAssertionLinkSchema = z22.object({
   /** ID of the assertion (can be system assertion like 'system:skill_was_called' or custom assertion UUID) */
-  assertionId: z23.string(),
+  assertionId: z22.string(),
   /** Parameter values for this assertion in this scenario */
-  params: z23.record(
-    z23.string(),
-    z23.union([z23.string(), z23.number(), z23.boolean(), z23.null()])
+  params: z22.record(
+    z22.string(),
+    z22.union([z22.string(), z22.number(), z22.boolean(), z22.null()])
   ).optional()
 });
-var SkillWasCalledConfigSchema = z23.object({
+var SkillWasCalledConfigSchema = z22.object({
   /** Names of the skills that must have been called */
-  skillNames: z23.array(z23.string().min(1)).min(1)
+  skillNames: z22.array(z22.string().min(1)).min(1)
 });
-var CostConfigSchema = z23.strictObject({
+var CostConfigSchema = z22.strictObject({
   /** Maximum allowed cost in USD */
-  maxCostUsd: z23.number().positive()
+  maxCostUsd: z22.number().positive()
 });
-var ToolCalledWithParamConfigSchema = z23.strictObject({
+var ToolCalledWithParamConfigSchema = z22.strictObject({
   /** Name of the tool that must have been called */
-  toolName: z23.string().min(1),
+  toolName: z22.string().min(1),
   /** JSON string of key-value pairs for expected parameters (substring match) */
-  expectedParams: z23.string().min(1)
+  expectedParams: z22.string().min(1),
+  /** If true, the matching tool call must also have succeeded (step.success === true) */
+  requireSuccess: z22.boolean().optional()
 });
-var BuildPassedConfigSchema = z23.strictObject({
+var BuildPassedConfigSchema = z22.strictObject({
   /** Command to run (default: "yarn build") */
-  command: z23.string().optional(),
+  command: z22.string().optional(),
   /** Expected exit code (default: 0) */
-  expectedExitCode: z23.number().int().optional()
+  expectedExitCode: z22.number().int().optional()
 });
-var TimeConfigSchema = z23.strictObject({
+var TimeConfigSchema = z22.strictObject({
   /** Maximum allowed duration in milliseconds */
-  maxDurationMs: z23.number().int().positive()
+  maxDurationMs: z22.number().int().positive()
 });
-var LlmJudgeConfigSchema = z23.object({
+var LlmJudgeConfigSchema = z22.object({
   /**
    * Prompt template with placeholders:
    * - {{output}}: agent's final output
@@ -1576,19 +1527,45 @@ var LlmJudgeConfigSchema = z23.object({
    * - {{trace}}: step-by-step trace of tool calls
    * - Custom parameters defined in the parameters array
    */
-  prompt: z23.string().min(1),
+  prompt: z22.string().min(1),
   /** Minimum score to pass (0-10, default 7) */
-  minScore: z23.number().int().min(0).max(10).optional(),
+  minScore: z22.number().int().min(0).max(10).optional(),
   /** Model for the judge (e.g. claude-3-5-haiku-20241022) */
-  model: z23.string().optional(),
+  model: z22.string().optional(),
   /** Max output tokens */
-  maxTokens: z23.number().int().optional(),
+  maxTokens: z22.number().int().optional(),
   /** Temperature (0-1) */
-  temperature: z23.number().min(0).max(1).optional(),
+  temperature: z22.number().min(0).max(1).optional(),
   /** User-defined parameters for this assertion */
-  parameters: z23.array(AssertionParameterSchema).optional()
+  parameters: z22.array(AssertionParameterSchema).optional()
+});
+var SkillWasCalledAssertionSchema = SkillWasCalledConfigSchema.extend({
+  type: z22.literal("skill_was_called")
+});
+var ToolCalledWithParamAssertionSchema = ToolCalledWithParamConfigSchema.extend({
+  type: z22.literal("tool_called_with_param")
+});
+var BuildPassedAssertionSchema = BuildPassedConfigSchema.extend({
+  type: z22.literal("build_passed")
+});
+var CostAssertionSchema = CostConfigSchema.extend({
+  type: z22.literal("cost")
+});
+var LlmJudgeAssertionSchema = LlmJudgeConfigSchema.extend({
+  type: z22.literal("llm_judge")
 });
-var AssertionConfigSchema = z23.union([
+var TimeAssertionSchema = TimeConfigSchema.extend({
+  type: z22.literal("time_limit")
+});
+var AssertionSchema = z22.union([
+  SkillWasCalledAssertionSchema,
+  ToolCalledWithParamAssertionSchema,
+  BuildPassedAssertionSchema,
+  TimeAssertionSchema,
+  CostAssertionSchema,
+  LlmJudgeAssertionSchema
+]);
+var AssertionConfigSchema = z22.union([
   LlmJudgeConfigSchema,
   // requires prompt - check first
   SkillWasCalledConfigSchema,
@@ -1601,7 +1578,7 @@ var AssertionConfigSchema = z23.union([
   // requires maxCostUsd, uses strictObject
   BuildPassedConfigSchema,
   // all optional, uses strictObject to reject unknown keys
-  z23.object({})
+  z22.object({})
   // fallback empty config
 ]);
 var CustomAssertionSchema = TenantEntitySchema.extend({
@@ -1652,25 +1629,25 @@ function getLlmJudgeConfig(assertion) {
 }
 
 // src/scenario/test-scenario.ts
-var ExpectedFileSchema = z24.object({
+var ExpectedFileSchema = z23.object({
   /** Relative path where the file should be created */
-  path: z24.string(),
+  path: z23.string(),
   /** Optional expected content */
-  content: z24.string().optional()
+  content: z23.string().optional()
 });
 var TestScenarioSchema = TenantEntitySchema.extend({
   /** The prompt sent to the agent to trigger the task */
-  triggerPrompt: z24.string().min(10),
+  triggerPrompt: z23.string().min(10),
   /** ID of the template to use for this scenario (null = no template) */
-  templateId: z24.string().nullish(),
+  templateId: z23.string().nullish(),
   /** Inline assertions to evaluate for this scenario (legacy) */
-  assertions: z24.array(AssertionSchema).optional(),
+  assertions: z23.array(AssertionSchema).optional(),
   /** IDs of saved assertions to evaluate (from assertions table) - legacy, use assertionLinks */
-  assertionIds: z24.array(z24.string()).optional(),
+  assertionIds: z23.array(z23.string()).optional(),
   /** Linked assertions with per-scenario parameter values */
-  assertionLinks: z24.array(ScenarioAssertionLinkSchema).optional(),
+  assertionLinks: z23.array(ScenarioAssertionLinkSchema).optional(),
   /** Tags for categorisation and filtering */
-  tags: z24.array(z24.string()).optional()
+  tags: z23.array(z23.string()).optional()
 });
 var CreateTestScenarioInputSchema = TestScenarioSchema.omit({
   id: true,
@@ -1681,10 +1658,10 @@ var CreateTestScenarioInputSchema = TestScenarioSchema.omit({
 var UpdateTestScenarioInputSchema = CreateTestScenarioInputSchema.partial();
 
 // src/suite/test-suite.ts
-import { z as z25 } from "zod";
+import { z as z24 } from "zod";
 var TestSuiteSchema = TenantEntitySchema.extend({
   /** IDs of test scenarios in this suite */
-  scenarioIds: z25.array(z25.string())
+  scenarioIds: z24.array(z24.string())
 });
 var CreateTestSuiteInputSchema = TestSuiteSchema.omit({
   id: true,
@@ -1695,21 +1672,21 @@ var CreateTestSuiteInputSchema = TestSuiteSchema.omit({
 var UpdateTestSuiteInputSchema = CreateTestSuiteInputSchema.partial();
 
 // src/evaluation/metrics.ts
-import { z as z26 } from "zod";
-var TokenUsageSchema = z26.object({
-  prompt: z26.number(),
-  completion: z26.number(),
-  total: z26.number()
-});
-var EvalMetricsSchema = z26.object({
-  totalAssertions: z26.number(),
-  passed: z26.number(),
-  failed: z26.number(),
-  skipped: z26.number(),
-  errors: z26.number(),
-  passRate: z26.number(),
-  avgDuration: z26.number(),
-  totalDuration: z26.number()
+import { z as z25 } from "zod";
+var TokenUsageSchema = z25.object({
+  prompt: z25.number(),
+  completion: z25.number(),
+  total: z25.number()
+});
+var EvalMetricsSchema = z25.object({
+  totalAssertions: z25.number(),
+  passed: z25.number(),
+  failed: z25.number(),
+  skipped: z25.number(),
+  errors: z25.number(),
+  passRate: z25.number(),
+  avgDuration: z25.number(),
+  totalDuration: z25.number()
 });
 var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
   EvalStatus2["PENDING"] = "pending";
@@ -1719,7 +1696,7 @@ var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
   EvalStatus2["CANCELLED"] = "cancelled";
   return EvalStatus2;
 })(EvalStatus || {});
-var EvalStatusSchema = z26.enum(EvalStatus);
+var EvalStatusSchema = z25.enum(EvalStatus);
 var LLMStepType = /* @__PURE__ */ ((LLMStepType2) => {
   LLMStepType2["COMPLETION"] = "completion";
   LLMStepType2["TOOL_USE"] = "tool_use";
@@ -1727,52 +1704,52 @@ var LLMStepType = /* @__PURE__ */ ((LLMStepType2) => {
   LLMStepType2["THINKING"] = "thinking";
   return LLMStepType2;
 })(LLMStepType || {});
-var LLMTraceStepSchema = z26.object({
-  id: z26.string(),
-  stepNumber: z26.number(),
-  type: z26.enum(LLMStepType),
-  model: z26.string(),
-  provider: z26.string(),
-  startedAt: z26.string(),
-  durationMs: z26.number(),
+var LLMTraceStepSchema = z25.object({
+  id: z25.string(),
+  stepNumber: z25.number(),
+  type: z25.enum(LLMStepType),
+  model: z25.string(),
+  provider: z25.string(),
+  startedAt: z25.string(),
+  durationMs: z25.number(),
   tokenUsage: TokenUsageSchema,
-  costUsd: z26.number(),
-  toolName: z26.string().optional(),
-  toolArguments: z26.string().optional(),
-  inputPreview: z26.string().optional(),
-  outputPreview: z26.string().optional(),
-  success: z26.boolean(),
-  error: z26.string().optional()
-});
-var LLMBreakdownStatsSchema = z26.object({
-  count: z26.number(),
-  durationMs: z26.number(),
-  tokens: z26.number(),
-  costUsd: z26.number()
-});
-var LLMTraceSummarySchema = z26.object({
-  totalSteps: z26.number(),
-  totalDurationMs: z26.number(),
+  costUsd: z25.number(),
+  toolName: z25.string().optional(),
+  toolArguments: z25.string().optional(),
+  inputPreview: z25.string().optional(),
+  outputPreview: z25.string().optional(),
+  success: z25.boolean(),
+  error: z25.string().optional()
+});
+var LLMBreakdownStatsSchema = z25.object({
+  count: z25.number(),
+  durationMs: z25.number(),
+  tokens: z25.number(),
+  costUsd: z25.number()
+});
+var LLMTraceSummarySchema = z25.object({
+  totalSteps: z25.number(),
+  totalDurationMs: z25.number(),
   totalTokens: TokenUsageSchema,
-  totalCostUsd: z26.number(),
-  stepTypeBreakdown: z26.record(z26.string(), LLMBreakdownStatsSchema).optional(),
-  modelBreakdown: z26.record(z26.string(), LLMBreakdownStatsSchema),
-  modelsUsed: z26.array(z26.string())
-});
-var LLMTraceSchema = z26.object({
-  id: z26.string(),
-  steps: z26.array(LLMTraceStepSchema),
+  totalCostUsd: z25.number(),
+  stepTypeBreakdown: z25.record(z25.string(), LLMBreakdownStatsSchema).optional(),
+  modelBreakdown: z25.record(z25.string(), LLMBreakdownStatsSchema),
+  modelsUsed: z25.array(z25.string())
+});
+var LLMTraceSchema = z25.object({
+  id: z25.string(),
+  steps: z25.array(LLMTraceStepSchema),
   summary: LLMTraceSummarySchema
 });
 
 // src/evaluation/eval-result.ts
-import { z as z30 } from "zod";
+import { z as z29 } from "zod";
 
 // src/evaluation/eval-run.ts
-import { z as z28 } from "zod";
+import { z as z27 } from "zod";
 
 // src/evaluation/live-trace.ts
-import { z as z27 } from "zod";
+import { z as z26 } from "zod";
 var LiveTraceEventType = /* @__PURE__ */ ((LiveTraceEventType2) => {
   LiveTraceEventType2["THINKING"] = "thinking";
   LiveTraceEventType2["TOOL_USE"] = "tool_use";
@@ -1786,37 +1763,37 @@ var LiveTraceEventType = /* @__PURE__ */ ((LiveTraceEventType2) => {
   LiveTraceEventType2["USER"] = "user";
   return LiveTraceEventType2;
 })(LiveTraceEventType || {});
-var LiveTraceEventSchema = z27.object({
+var LiveTraceEventSchema = z26.object({
   /** The evaluation run ID */
-  evalRunId: z27.string(),
+  evalRunId: z26.string(),
   /** The scenario ID being executed */
-  scenarioId: z27.string(),
+  scenarioId: z26.string(),
   /** The scenario name for display */
-  scenarioName: z27.string(),
+  scenarioName: z26.string(),
   /** The target ID (skill, agent, etc.) */
-  targetId: z27.string(),
+  targetId: z26.string(),
   /** The target name for display */
-  targetName: z27.string(),
+  targetName: z26.string(),
   /** Step number in the current scenario execution */
-  stepNumber: z27.number(),
+  stepNumber: z26.number(),
   /** Type of trace event */
-  type: z27.enum(LiveTraceEventType),
+  type: z26.enum(LiveTraceEventType),
   /** Tool name if this is a tool_use event */
-  toolName: z27.string().optional(),
+  toolName: z26.string().optional(),
   /** Tool arguments preview (truncated JSON) */
-  toolArgs: z27.string().optional(),
+  toolArgs: z26.string().optional(),
   /** Output preview (truncated text) */
-  outputPreview: z27.string().optional(),
+  outputPreview: z26.string().optional(),
   /** File path for file operations */
-  filePath: z27.string().optional(),
+  filePath: z26.string().optional(),
   /** Elapsed time in milliseconds for progress events */
-  elapsedMs: z27.number().optional(),
+  elapsedMs: z26.number().optional(),
   /** Thinking/reasoning text from Claude */
-  thinking: z27.string().optional(),
+  thinking: z26.string().optional(),
   /** Timestamp when this event occurred */
-  timestamp: z27.string(),
+  timestamp: z26.string(),
   /** Whether this is the final event for this scenario */
-  isComplete: z27.boolean()
+  isComplete: z26.boolean()
 });
 var TRACE_EVENT_PREFIX = "TRACE_EVENT:";
 function parseTraceEventLine(line) {
@@ -1844,14 +1821,14 @@ var TriggerType = /* @__PURE__ */ ((TriggerType2) => {
   TriggerType2["MANUAL"] = "MANUAL";
   return TriggerType2;
 })(TriggerType || {});
-var TriggerMetadataSchema = z28.object({
-  version: z28.string().optional(),
-  resourceUpdated: z28.array(z28.string()).optional()
+var TriggerMetadataSchema = z27.object({
+  version: z27.string().optional(),
+  resourceUpdated: z27.array(z27.string()).optional()
 });
-var TriggerSchema = z28.object({
-  id: z28.string(),
+var TriggerSchema = z27.object({
+  id: z27.string(),
   metadata: TriggerMetadataSchema.optional(),
-  type: z28.enum(TriggerType)
+  type: z27.enum(TriggerType)
 });
 var FailureCategory = /* @__PURE__ */ ((FailureCategory2) => {
   FailureCategory2["MISSING_FILE"] = "missing_file";
@@ -1869,28 +1846,28 @@ var FailureSeverity = /* @__PURE__ */ ((FailureSeverity2) => {
   FailureSeverity2["LOW"] = "low";
   return FailureSeverity2;
 })(FailureSeverity || {});
-var DiffLineTypeSchema = z28.enum(["added", "removed", "unchanged"]);
-var DiffLineSchema = z28.object({
+var DiffLineTypeSchema = z27.enum(["added", "removed", "unchanged"]);
+var DiffLineSchema = z27.object({
   type: DiffLineTypeSchema,
-  content: z28.string(),
-  lineNumber: z28.number()
-});
-var DiffContentSchema = z28.object({
-  path: z28.string(),
-  expected: z28.string(),
-  actual: z28.string(),
-  diffLines: z28.array(DiffLineSchema),
-  renamedFrom: z28.string().optional()
-});
-var CommandExecutionSchema = z28.object({
-  command: z28.string(),
-  exitCode: z28.number(),
-  output: z28.string().optional(),
-  duration: z28.number()
-});
-var FileModificationSchema = z28.object({
-  path: z28.string(),
-  action: z28.enum(["created", "modified", "deleted"])
+  content: z27.string(),
+  lineNumber: z27.number()
+});
+var DiffContentSchema = z27.object({
+  path: z27.string(),
+  expected: z27.string(),
+  actual: z27.string(),
+  diffLines: z27.array(DiffLineSchema),
+  renamedFrom: z27.string().optional()
+});
+var CommandExecutionSchema = z27.object({
+  command: z27.string(),
+  exitCode: z27.number(),
+  output: z27.string().optional(),
+  duration: z27.number()
+});
+var FileModificationSchema = z27.object({
+  path: z27.string(),
+  action: z27.enum(["created", "modified", "deleted"])
 });
 var TemplateFileStatus = /* @__PURE__ */ ((TemplateFileStatus2) => {
   TemplateFileStatus2["NEW"] = "new";
@@ -1898,87 +1875,87 @@ var TemplateFileStatus = /* @__PURE__ */ ((TemplateFileStatus2) => {
   TemplateFileStatus2["UNCHANGED"] = "unchanged";
   return TemplateFileStatus2;
 })(TemplateFileStatus || {});
-var TemplateFileSchema = z28.object({
+var TemplateFileSchema = z27.object({
   /** Relative path within the template */
-  path: z28.string(),
+  path: z27.string(),
   /** Full file content after execution */
-  content: z28.string(),
+  content: z27.string(),
   /** File status (new, modified, unchanged) */
-  status: z28.enum(["new", "modified", "unchanged"])
-});
-var ApiCallSchema = z28.object({
-  endpoint: z28.string(),
-  tokensUsed: z28.number(),
-  duration: z28.number()
-});
-var ExecutionTraceSchema = z28.object({
-  commands: z28.array(CommandExecutionSchema),
-  filesModified: z28.array(FileModificationSchema),
-  apiCalls: z28.array(ApiCallSchema),
-  totalDuration: z28.number()
-});
-var FailureAnalysisSchema = z28.object({
-  category: z28.enum(FailureCategory),
-  severity: z28.enum(FailureSeverity),
-  summary: z28.string(),
-  details: z28.string(),
-  rootCause: z28.string(),
-  suggestedFix: z28.string(),
-  relatedAssertions: z28.array(z28.string()),
-  codeSnippet: z28.string().optional(),
-  similarIssues: z28.array(z28.string()).optional(),
-  patternId: z28.string().optional(),
+  status: z27.enum(["new", "modified", "unchanged"])
+});
+var ApiCallSchema = z27.object({
+  endpoint: z27.string(),
+  tokensUsed: z27.number(),
+  duration: z27.number()
+});
+var ExecutionTraceSchema = z27.object({
+  commands: z27.array(CommandExecutionSchema),
+  filesModified: z27.array(FileModificationSchema),
+  apiCalls: z27.array(ApiCallSchema),
+  totalDuration: z27.number()
+});
+var FailureAnalysisSchema = z27.object({
+  category: z27.enum(FailureCategory),
+  severity: z27.enum(FailureSeverity),
+  summary: z27.string(),
+  details: z27.string(),
+  rootCause: z27.string(),
+  suggestedFix: z27.string(),
+  relatedAssertions: z27.array(z27.string()),
+  codeSnippet: z27.string().optional(),
+  similarIssues: z27.array(z27.string()).optional(),
+  patternId: z27.string().optional(),
   // Extended fields for detailed debugging
   diff: DiffContentSchema.optional(),
   executionTrace: ExecutionTraceSchema.optional()
 });
 var EvalRunSchema = TenantEntitySchema.extend({
   /** Agent ID for this run */
-  agentId: z28.string().optional(),
+  agentId: z27.string().optional(),
   /** Preset ID that originated this run (optional) */
-  presetId: z28.string().optional(),
+  presetId: z27.string().optional(),
   /** Skill IDs for this run */
-  skillIds: z28.array(z28.string()).optional(),
+  skillIds: z27.array(z27.string()).optional(),
   /** Map of skillId to skillVersionId for this run */
-  skillVersions: z28.record(z28.string(), z28.string()).optional(),
+  skillVersions: z27.record(z27.string(), z27.string()).optional(),
   /** Scenario IDs to run (always present — resolved server-side from tags when needed) */
-  scenarioIds: z28.array(z28.string()),
+  scenarioIds: z27.array(z27.string()),
   /** Current status */
   status: EvalStatusSchema,
   /** Progress percentage (0-100) */
-  progress: z28.number(),
+  progress: z27.number(),
   /** Results for each scenario/target combination (lazy to break eval-result ↔ eval-run cycle) */
-  results: z28.array(z28.lazy(() => EvalRunResultSchema)),
+  results: z27.array(z27.lazy(() => EvalRunResultSchema)),
   /** Aggregated metrics across all results */
   aggregateMetrics: EvalMetricsSchema,
   /** Failure analyses */
-  failureAnalyses: z28.array(FailureAnalysisSchema).optional(),
+  failureAnalyses: z27.array(FailureAnalysisSchema).optional(),
   /** Aggregated LLM trace summary */
   llmTraceSummary: LLMTraceSummarySchema.optional(),
   /** What triggered this run */
   trigger: TriggerSchema.optional(),
   /** When the run started (set when evaluation is triggered) */
-  startedAt: z28.string().optional(),
+  startedAt: z27.string().optional(),
   /** When the run completed */
-  completedAt: z28.string().optional(),
+  completedAt: z27.string().optional(),
   /** Live trace events captured during execution (for playback on results page) */
-  liveTraceEvents: z28.array(LiveTraceEventSchema).optional(),
+  liveTraceEvents: z27.array(LiveTraceEventSchema).optional(),
   /** Remote job ID for tracking execution in Dev Machines */
-  jobId: z28.string().optional(),
+  jobId: z27.string().optional(),
   /** Remote job status from the Dev Machine API (PENDING, RUNNING, COMPLETED, FAILED, CANCELLED) */
-  jobStatus: z28.string().optional(),
+  jobStatus: z27.string().optional(),
   /** Remote job error message if the job failed */
-  jobError: z28.string().optional(),
+  jobError: z27.string().optional(),
   /** Timestamp of the last job status check */
-  jobStatusCheckedAt: z28.string().optional(),
+  jobStatusCheckedAt: z27.string().optional(),
   /** MCP server IDs to enable for this run (optional) */
-  mcpIds: z28.array(z28.string()).optional(),
+  mcpIds: z27.array(z27.string()).optional(),
   /** Sub-agent IDs to enable for this run (optional) */
-  subAgentIds: z28.array(z28.string()).optional(),
+  subAgentIds: z27.array(z27.string()).optional(),
   /** Rule IDs to enable for this run (optional) */
-  ruleIds: z28.array(z28.string()).optional(),
+  ruleIds: z27.array(z27.string()).optional(),
   /** Tags used to select scenarios for this run (for traceability) */
-  tags: z28.array(z28.string()).optional()
+  tags: z27.array(z27.string()).optional()
 });
 var CreateEvalRunInputSchema = EvalRunSchema.omit({
   id: true,
@@ -1993,60 +1970,60 @@ var CreateEvalRunInputSchema = EvalRunSchema.omit({
   scenarioIds: true
 }).extend({
   /** Optional on input — backend resolves from tags when not provided */
-  scenarioIds: z28.array(z28.string()).optional()
+  scenarioIds: z27.array(z27.string()).optional()
 }).refine(
   (data) => data.scenarioIds && data.scenarioIds.length > 0 || data.tags && data.tags.length > 0,
   { message: "Either scenarioIds or tags must be provided" }
 );
-var EvaluationProgressSchema = z28.object({
-  runId: z28.string(),
-  targetId: z28.string(),
-  totalScenarios: z28.number(),
-  completedScenarios: z28.number(),
-  scenarioProgress: z28.array(
-    z28.object({
-      scenarioId: z28.string(),
-      currentStep: z28.string(),
-      error: z28.string().optional()
+var EvaluationProgressSchema = z27.object({
+  runId: z27.string(),
+  targetId: z27.string(),
+  totalScenarios: z27.number(),
+  completedScenarios: z27.number(),
+  scenarioProgress: z27.array(
+    z27.object({
+      scenarioId: z27.string(),
+      currentStep: z27.string(),
+      error: z27.string().optional()
     })
   ),
-  createdAt: z28.number()
-});
-var EvaluationLogSchema = z28.object({
-  runId: z28.string(),
-  scenarioId: z28.string(),
-  log: z28.object({
-    level: z28.enum(["info", "error", "debug"]),
-    message: z28.string().optional(),
-    args: z28.array(z28.any()).optional(),
-    error: z28.string().optional()
+  createdAt: z27.number()
+});
+var EvaluationLogSchema = z27.object({
+  runId: z27.string(),
+  scenarioId: z27.string(),
+  log: z27.object({
+    level: z27.enum(["info", "error", "debug"]),
+    message: z27.string().optional(),
+    args: z27.array(z27.any()).optional(),
+    error: z27.string().optional()
   })
 });
 var LLM_TIMEOUT = 12e4;
 
 // src/evaluation/conversation.ts
-import { z as z29 } from "zod";
-var TextBlockSchema = z29.object({
-  type: z29.literal("text"),
-  text: z29.string()
-});
-var ThinkingBlockSchema = z29.object({
-  type: z29.literal("thinking"),
-  thinking: z29.string()
-});
-var ToolUseBlockSchema = z29.object({
-  type: z29.literal("tool_use"),
-  toolName: z29.string(),
-  toolId: z29.string(),
-  input: z29.unknown()
-});
-var ToolResultBlockSchema = z29.object({
-  type: z29.literal("tool_result"),
-  toolUseId: z29.string(),
-  content: z29.string(),
-  isError: z29.boolean().optional()
-});
-var ConversationBlockSchema = z29.discriminatedUnion("type", [
+import { z as z28 } from "zod";
+var TextBlockSchema = z28.object({
+  type: z28.literal("text"),
+  text: z28.string()
+});
+var ThinkingBlockSchema = z28.object({
+  type: z28.literal("thinking"),
+  thinking: z28.string()
+});
+var ToolUseBlockSchema = z28.object({
+  type: z28.literal("tool_use"),
+  toolName: z28.string(),
+  toolId: z28.string(),
+  input: z28.unknown()
+});
+var ToolResultBlockSchema = z28.object({
+  type: z28.literal("tool_result"),
+  toolUseId: z28.string(),
+  content: z28.string(),
+  isError: z28.boolean().optional()
+});
+var ConversationBlockSchema = z28.discriminatedUnion("type", [
   TextBlockSchema,
   ThinkingBlockSchema,
   ToolUseBlockSchema,
@@ -2057,18 +2034,18 @@ var ConversationMessageRoles = [
   "user",
   "system"
 ];
-var ConversationMessageSchema = z29.object({
-  role: z29.enum(ConversationMessageRoles),
-  content: z29.array(ConversationBlockSchema),
-  timestamp: z29.string()
+var ConversationMessageSchema = z28.object({
+  role: z28.enum(ConversationMessageRoles),
+  content: z28.array(ConversationBlockSchema),
+  timestamp: z28.string()
 });
-var ScenarioConversationSchema = z29.object({
-  id: z29.string(),
-  projectId: z29.string(),
-  evalRunId: z29.string(),
-  resultId: z29.string(),
-  messages: z29.array(ConversationMessageSchema),
-  createdAt: z29.string()
+var ScenarioConversationSchema = z28.object({
+  id: z28.string(),
+  projectId: z28.string(),
+  evalRunId: z28.string(),
+  resultId: z28.string(),
+  messages: z28.array(ConversationMessageSchema),
+  createdAt: z28.string()
 });
 
 // src/evaluation/eval-result.ts
@@ -2079,100 +2056,100 @@ var AssertionResultStatus = /* @__PURE__ */ ((AssertionResultStatus2) => {
   AssertionResultStatus2["ERROR"] = "error";
   return AssertionResultStatus2;
 })(AssertionResultStatus || {});
-var AssertionResultSchema = z30.object({
-  id: z30.string(),
-  assertionId: z30.string(),
-  assertionType: z30.string(),
-  assertionName: z30.string(),
-  status: z30.enum(AssertionResultStatus),
-  message: z30.string().optional(),
-  expected: z30.string().optional(),
-  actual: z30.string().optional(),
-  duration: z30.number().optional(),
-  details: z30.record(z30.string(), z30.unknown()).optional(),
-  llmTraceSteps: z30.array(LLMTraceStepSchema).optional()
-});
-var EvalRunResultSchema = z30.object({
-  id: z30.string(),
-  targetId: z30.string(),
-  targetName: z30.string().optional(),
+var AssertionResultSchema = z29.object({
+  id: z29.string(),
+  assertionId: z29.string(),
+  assertionType: z29.string(),
+  assertionName: z29.string(),
+  status: z29.enum(AssertionResultStatus),
+  message: z29.string().optional(),
+  expected: z29.string().optional(),
+  actual: z29.string().optional(),
+  duration: z29.number().optional(),
+  details: z29.record(z29.string(), z29.unknown()).optional(),
+  llmTraceSteps: z29.array(LLMTraceStepSchema).optional()
+});
+var EvalRunResultSchema = z29.object({
+  id: z29.string(),
+  targetId: z29.string(),
+  targetName: z29.string().optional(),
   /** SkillVersion ID used for this evaluation (for version tracking) */
-  skillVersionId: z30.string().optional(),
+  skillVersionId: z29.string().optional(),
   /** SkillVersion semver string (e.g., "1.0.0", "1.2.3") for display */
-  skillVersion: z30.string().optional(),
-  scenarioId: z30.string(),
-  scenarioName: z30.string(),
+  skillVersion: z29.string().optional(),
+  scenarioId: z29.string(),
+  scenarioName: z29.string(),
   modelConfig: ModelConfigSchema.optional(),
-  assertionResults: z30.array(AssertionResultSchema),
+  assertionResults: z29.array(AssertionResultSchema),
   metrics: EvalMetricsSchema.optional(),
-  passed: z30.number(),
-  failed: z30.number(),
-  passRate: z30.number(),
-  duration: z30.number(),
-  outputText: z30.string().optional(),
-  files: z30.array(ExpectedFileSchema).optional(),
-  fileDiffs: z30.array(DiffContentSchema).optional(),
+  passed: z29.number(),
+  failed: z29.number(),
+  passRate: z29.number(),
+  duration: z29.number(),
+  outputText: z29.string().optional(),
+  files: z29.array(ExpectedFileSchema).optional(),
+  fileDiffs: z29.array(DiffContentSchema).optional(),
   /** Full template files after execution with status indicators */
-  templateFiles: z30.array(TemplateFileSchema).optional(),
-  startedAt: z30.string().optional(),
-  completedAt: z30.string().optional(),
+  templateFiles: z29.array(TemplateFileSchema).optional(),
+  startedAt: z29.string().optional(),
+  completedAt: z29.string().optional(),
   llmTrace: LLMTraceSchema.optional(),
   /** Full conversation messages (only present in transit; stripped before DB storage) */
-  conversation: z30.array(ConversationMessageSchema).optional()
-});
-var PromptResultSchema = z30.object({
-  text: z30.string(),
-  files: z30.array(z30.unknown()).optional(),
-  finishReason: z30.string().optional(),
-  reasoning: z30.string().optional(),
-  reasoningDetails: z30.unknown().optional(),
-  toolCalls: z30.array(z30.unknown()).optional(),
-  toolResults: z30.array(z30.unknown()).optional(),
-  warnings: z30.array(z30.unknown()).optional(),
-  sources: z30.array(z30.unknown()).optional(),
-  steps: z30.array(z30.unknown()),
-  generationTimeMs: z30.number(),
-  prompt: z30.string(),
-  systemPrompt: z30.string(),
-  usage: z30.object({
-    totalTokens: z30.number().optional(),
-    totalMicrocentsSpent: z30.number().optional()
+  conversation: z29.array(ConversationMessageSchema).optional()
+});
+var PromptResultSchema = z29.object({
+  text: z29.string(),
+  files: z29.array(z29.unknown()).optional(),
+  finishReason: z29.string().optional(),
+  reasoning: z29.string().optional(),
+  reasoningDetails: z29.unknown().optional(),
+  toolCalls: z29.array(z29.unknown()).optional(),
+  toolResults: z29.array(z29.unknown()).optional(),
+  warnings: z29.array(z29.unknown()).optional(),
+  sources: z29.array(z29.unknown()).optional(),
+  steps: z29.array(z29.unknown()),
+  generationTimeMs: z29.number(),
+  prompt: z29.string(),
+  systemPrompt: z29.string(),
+  usage: z29.object({
+    totalTokens: z29.number().optional(),
+    totalMicrocentsSpent: z29.number().optional()
   })
 });
-var EvaluationResultSchema = z30.object({
-  id: z30.string(),
-  runId: z30.string(),
-  timestamp: z30.number(),
+var EvaluationResultSchema = z29.object({
+  id: z29.string(),
+  runId: z29.string(),
+  timestamp: z29.number(),
   promptResult: PromptResultSchema,
-  testResults: z30.array(z30.unknown()),
-  tags: z30.array(z30.string()).optional(),
-  feedback: z30.string().optional(),
-  score: z30.number(),
-  suiteId: z30.string().optional()
-});
-var LeanEvaluationResultSchema = z30.object({
-  id: z30.string(),
-  runId: z30.string(),
-  timestamp: z30.number(),
-  tags: z30.array(z30.string()).optional(),
-  scenarioId: z30.string(),
-  scenarioVersion: z30.number().optional(),
-  targetId: z30.string(),
-  targetVersion: z30.number().optional(),
-  suiteId: z30.string().optional(),
-  score: z30.number(),
-  time: z30.number().optional(),
-  microcentsSpent: z30.number().optional()
+  testResults: z29.array(z29.unknown()),
+  tags: z29.array(z29.string()).optional(),
+  feedback: z29.string().optional(),
+  score: z29.number(),
+  suiteId: z29.string().optional()
+});
+var LeanEvaluationResultSchema = z29.object({
+  id: z29.string(),
+  runId: z29.string(),
+  timestamp: z29.number(),
+  tags: z29.array(z29.string()).optional(),
+  scenarioId: z29.string(),
+  scenarioVersion: z29.number().optional(),
+  targetId: z29.string(),
+  targetVersion: z29.number().optional(),
+  suiteId: z29.string().optional(),
+  score: z29.number(),
+  time: z29.number().optional(),
+  microcentsSpent: z29.number().optional()
 });
 
 // src/project/project.ts
-import { z as z31 } from "zod";
+import { z as z30 } from "zod";
 var ProjectSchema = BaseEntitySchema.extend({
-  appId: z31.string().optional().describe("The ID of the app in Dev Center"),
-  appSecret: z31.string().optional().describe("The secret of the app in Dev Center"),
-  useWixAuth: z31.boolean().optional().describe("Enable Wix CLI/MCP auth for evaluations"),
-  useBase44Auth: z31.boolean().optional().describe("Enable Base44 auth for evaluations"),
-  scenarioTags: z31.array(z31.string()).optional().describe("Project-level tag vocabulary for scenarios")
+  appId: z30.string().optional().describe("The ID of the app in Dev Center"),
+  appSecret: z30.string().optional().describe("The secret of the app in Dev Center"),
+  useWixAuth: z30.boolean().optional().describe("Enable Wix CLI/MCP auth for evaluations"),
+  useBase44Auth: z30.boolean().optional().describe("Enable Base44 auth for evaluations"),
+  scenarioTags: z30.array(z30.string()).optional().describe("Project-level tag vocabulary for scenarios")
 });
 var CreateProjectInputSchema = ProjectSchema.omit({
   id: true,
@@ -2225,7 +2202,7 @@ var SYSTEM_ASSERTIONS = {
   [SYSTEM_ASSERTION_IDS.TOOL_CALLED_WITH_PARAM]: {
     id: SYSTEM_ASSERTION_IDS.TOOL_CALLED_WITH_PARAM,
     name: "Tool Called With Param",
-    description: "Check that a tool was called with expected parameters",
+    description: "Check that a tool was called with expected parameters (tool name is substring matched)",
     type: "tool_called_with_param",
     parameters: [
       {
@@ -2239,6 +2216,14 @@ var SYSTEM_ASSERTIONS = {
         label: "Expected Parameters (JSON, substring match)",
         type: "string",
         required: true
+      },
+      {
+        name: "requireSuccess",
+        label: "Require Successful Call",
+        type: "boolean",
+        required: false,
+        defaultValue: false,
+        advanced: true
       }
     ]
   },