@wix/evalforge-types 0.38.0 → 0.40.0

package/build/index.mjs

@@ -991,6 +991,17 @@ var RuleInputBaseSchema = RuleSchema.omit({
 var CreateRuleInputSchema = RuleInputBaseSchema;
 var UpdateRuleInputSchema = RuleInputBaseSchema.partial();
 
+// src/common/tool-names.ts
+var AVAILABLE_TOOL_NAMES = [
+  "Bash",
+  "Edit",
+  "Glob",
+  "Grep",
+  "Read",
+  "Skill",
+  "Write"
+];
+
 // src/target/target.ts
 var TargetSchema = TenantEntitySchema.extend({
   // Base for all testable entities
@@ -1329,6 +1340,13 @@ var SkillWasCalledAssertionSchema = z21.object({
   /** Names of the skills that must have been called (matched against trace Skill tool args) */
   skillNames: z21.array(z21.string().min(1)).min(1)
 });
+var ToolCalledWithParamAssertionSchema = z21.object({
+  type: z21.literal("tool_called_with_param"),
+  /** Name of the tool that must have been called */
+  toolName: z21.string().min(1),
+  /** JSON string of key-value pairs for expected parameters (substring match) */
+  expectedParams: z21.string().min(1)
+});
 var BuildPassedAssertionSchema = z21.object({
   type: z21.literal("build_passed"),
   /** Command to run (default: "yarn build") */
@@ -1345,8 +1363,6 @@ var LlmJudgeAssertionSchema = z21.object({
   type: z21.literal("llm_judge"),
   /** Prompt template; placeholders: {{output}}, {{cwd}}, {{changedFiles}}, {{trace}} */
   prompt: z21.string(),
-  /** Optional system prompt for the judge (default asks for JSON with score) */
-  systemPrompt: z21.string().optional(),
   /** Minimum score to pass (0-100, default 70) */
   minScore: z21.number().int().min(0).max(100).optional(),
   /** Model for the judge (e.g. claude-3-5-haiku) */
@@ -1361,6 +1377,7 @@ var TimeAssertionSchema = z21.object({
 });
 var AssertionSchema = z21.union([
   SkillWasCalledAssertionSchema,
+  ToolCalledWithParamAssertionSchema,
   BuildPassedAssertionSchema,
   TimeAssertionSchema,
   CostAssertionSchema,
@@ -1408,6 +1425,7 @@ import { z as z24 } from "zod";
 import { z as z23 } from "zod";
 var AssertionTypeSchema = z23.enum([
   "skill_was_called",
+  "tool_called_with_param",
   "build_passed",
   "time_limit",
   "cost",
@@ -1449,6 +1467,12 @@ var CostConfigSchema = z23.strictObject({
   /** Maximum allowed cost in USD */
   maxCostUsd: z23.number().positive()
 });
+var ToolCalledWithParamConfigSchema = z23.strictObject({
+  /** Name of the tool that must have been called */
+  toolName: z23.string().min(1),
+  /** JSON string of key-value pairs for expected parameters (substring match) */
+  expectedParams: z23.string().min(1)
+});
 var BuildPassedConfigSchema = z23.strictObject({
   /** Command to run (default: "yarn build") */
   command: z23.string().optional(),
@@ -1471,8 +1495,6 @@ var LlmJudgeConfigSchema = z23.object({
    * - Custom parameters defined in the parameters array
    */
   prompt: z23.string().min(1),
-  /** Optional system prompt for the judge */
-  systemPrompt: z23.string().optional(),
   /** Minimum score to pass (0-100, default 70) */
   minScore: z23.number().int().min(0).max(100).optional(),
   /** Model for the judge (e.g. claude-3-5-haiku-20241022) */
@@ -1489,6 +1511,8 @@ var AssertionConfigSchema = z23.union([
   // requires prompt - check first
   SkillWasCalledConfigSchema,
   // requires skillNames
+  ToolCalledWithParamConfigSchema,
+  // requires toolName + expectedParams, uses strictObject
   TimeConfigSchema,
   // requires maxDurationMs, uses strictObject
   CostConfigSchema,
@@ -1517,6 +1541,8 @@ function validateAssertionConfig(type, config) {
       return SkillWasCalledConfigSchema.safeParse(config).success;
     case "cost":
       return CostConfigSchema.safeParse(config).success;
+    case "tool_called_with_param":
+      return ToolCalledWithParamConfigSchema.safeParse(config).success;
     case "build_passed":
       return BuildPassedConfigSchema.safeParse(config).success;
     case "time_limit":
@@ -2025,6 +2051,7 @@ var UpdateTemplateInputSchema = CreateTemplateInputSchema.partial();
 // src/assertion/system-assertions.ts
 var SYSTEM_ASSERTION_IDS = {
   SKILL_WAS_CALLED: "system:skill_was_called",
+  TOOL_CALLED_WITH_PARAM: "system:tool_called_with_param",
   BUILD_PASSED: "system:build_passed",
   TIME_LIMIT: "system:time_limit",
   COST: "system:cost",
@@ -2048,6 +2075,26 @@ var SYSTEM_ASSERTIONS = {
       }
     ]
   },
+  [SYSTEM_ASSERTION_IDS.TOOL_CALLED_WITH_PARAM]: {
+    id: SYSTEM_ASSERTION_IDS.TOOL_CALLED_WITH_PARAM,
+    name: "Tool Called With Param",
+    description: "Check that a tool was called with expected parameters",
+    type: "tool_called_with_param",
+    parameters: [
+      {
+        name: "toolName",
+        label: "Tool Name",
+        type: "string",
+        required: true
+      },
+      {
+        name: "expectedParams",
+        label: "Expected Parameters (JSON, substring match)",
+        type: "string",
+        required: true
+      }
+    ]
+  },
   [SYSTEM_ASSERTION_IDS.BUILD_PASSED]: {
     id: SYSTEM_ASSERTION_IDS.BUILD_PASSED,
     name: "Build Passed",
@@ -2127,19 +2174,6 @@ var SYSTEM_ASSERTIONS = {
         required: true,
         defaultValue: "Verify the output meets the acceptance criteria."
       },
-      {
-        name: "systemPrompt",
-        label: "System Prompt (optional)",
-        type: "string",
-        required: false,
-        defaultValue: `You are judging a scenario run. Use these values:
-- {{output}}: the agent's final output
-- {{cwd}}: working directory
-- {{changedFiles}}: list of files changed (or "No files were changed")
-- {{trace}}: step-by-step trace (tool calls, completions) to check e.g. which tools were called and how many times
-
-Judge how well the output meets the acceptance criteria stated in the user prompt.`
-      },
       {
         name: "minScore",
         label: "Minimum Score (0-100)",
@@ -2166,6 +2200,7 @@ var export_ClaudeModel = import_types.ClaudeModel;
 export {
   AVAILABLE_MODEL_IDS,
   AVAILABLE_RUN_COMMANDS,
+  AVAILABLE_TOOL_NAMES,
   AgentRunCommand,
   AgentRunCommandSchema,
   AgentSchema,
@@ -2283,6 +2318,8 @@ export {
   TimeAssertionSchema,
   TimeConfigSchema,
   TokenUsageSchema,
+  ToolCalledWithParamAssertionSchema,
+  ToolCalledWithParamConfigSchema,
   ToolTestSchema,
   TriggerMetadataSchema,
   TriggerSchema,