@wix/evalforge-types 0.38.0 → 0.40.0

package/build/types/assertion/assertion.d.ts

@@ -2,6 +2,7 @@ import { z } from 'zod';
 /**
  * Assertion types:
  * - skill_was_called: Checks if a specific skill was invoked (deterministic, system-level)
+ * - tool_called_with_param: Checks if a tool was called with expected parameters (deterministic, system-level)
  * - build_passed: Runs a command and checks exit code (deterministic, system-level)
  * - time_limit: Checks that scenario completed within a duration threshold (deterministic, system-level)
  * - cost: Checks that scenario LLM cost stays within a USD threshold (deterministic, system-level)
@@ -9,6 +10,7 @@ import { z } from 'zod';
  */
 export declare const AssertionTypeSchema: z.ZodEnum<{
     skill_was_called: "skill_was_called";
+    tool_called_with_param: "tool_called_with_param";
     build_passed: "build_passed";
     cost: "cost";
     llm_judge: "llm_judge";
@@ -68,6 +70,15 @@ export declare const CostConfigSchema: z.ZodObject<{
     maxCostUsd: z.ZodNumber;
 }, z.core.$strict>;
 export type CostConfig = z.infer<typeof CostConfigSchema>;
+/** Configuration for tool_called_with_param assertion type.
+ * Uses strictObject to reject objects with unknown keys.
+ */
+export declare const ToolCalledWithParamConfigSchema: z.ZodObject<{
+    /** Name of the tool that must have been called */
+    toolName: z.ZodString;
+    /** JSON string of key-value pairs for expected parameters (substring match) */
+    expectedParams: z.ZodString;
+}, z.core.$strict>;
 /**
  * Configuration for build_passed assertion type.
  * Uses strictObject to reject objects with unknown keys (prevents matching LlmJudge configs).
@@ -94,7 +105,6 @@ export type TimeConfig = z.infer<typeof TimeConfigSchema>;
  */
 export declare const LlmJudgeConfigSchema: z.ZodObject<{
     prompt: z.ZodString;
-    systemPrompt: z.ZodOptional<z.ZodString>;
     minScore: z.ZodOptional<z.ZodNumber>;
     model: z.ZodOptional<z.ZodString>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -121,7 +131,6 @@ export type LlmJudgeConfig = z.infer<typeof LlmJudgeConfigSchema>;
  */
 export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
     prompt: z.ZodString;
-    systemPrompt: z.ZodOptional<z.ZodString>;
     minScore: z.ZodOptional<z.ZodNumber>;
     model: z.ZodOptional<z.ZodString>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -141,6 +150,11 @@ export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
 }, z.core.$strip>, z.ZodObject<{
     skillNames: z.ZodArray<z.ZodString>;
 }, z.core.$strip>, z.ZodObject<{
+    /** Name of the tool that must have been called */
+    toolName: z.ZodString;
+    /** JSON string of key-value pairs for expected parameters (substring match) */
+    expectedParams: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
     /** Maximum allowed duration in milliseconds */
     maxDurationMs: z.ZodNumber;
 }, z.core.$strict>, z.ZodObject<{
@@ -167,6 +181,7 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     projectId: z.ZodString;
     type: z.ZodEnum<{
         skill_was_called: "skill_was_called";
+        tool_called_with_param: "tool_called_with_param";
         build_passed: "build_passed";
         cost: "cost";
         llm_judge: "llm_judge";
@@ -174,7 +189,6 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     }>;
     config: z.ZodUnion<readonly [z.ZodObject<{
         prompt: z.ZodString;
-        systemPrompt: z.ZodOptional<z.ZodString>;
         minScore: z.ZodOptional<z.ZodNumber>;
         model: z.ZodOptional<z.ZodString>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -194,6 +208,11 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
     }, z.core.$strip>, z.ZodObject<{
+        /** Name of the tool that must have been called */
+        toolName: z.ZodString;
+        /** JSON string of key-value pairs for expected parameters (substring match) */
+        expectedParams: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
@@ -213,6 +232,7 @@ export type CustomAssertion = z.infer<typeof CustomAssertionSchema>;
 export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     type: z.ZodEnum<{
         skill_was_called: "skill_was_called";
+        tool_called_with_param: "tool_called_with_param";
         build_passed: "build_passed";
         cost: "cost";
         llm_judge: "llm_judge";
@@ -223,7 +243,6 @@ export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     projectId: z.ZodString;
     config: z.ZodUnion<readonly [z.ZodObject<{
         prompt: z.ZodString;
-        systemPrompt: z.ZodOptional<z.ZodString>;
         minScore: z.ZodOptional<z.ZodNumber>;
         model: z.ZodOptional<z.ZodString>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -243,6 +262,11 @@ export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
     }, z.core.$strip>, z.ZodObject<{
+        /** Name of the tool that must have been called */
+        toolName: z.ZodString;
+        /** JSON string of key-value pairs for expected parameters (substring match) */
+        expectedParams: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
@@ -262,6 +286,7 @@ export type CreateCustomAssertionInput = z.infer<typeof CreateCustomAssertionInp
 export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     type: z.ZodOptional<z.ZodEnum<{
         skill_was_called: "skill_was_called";
+        tool_called_with_param: "tool_called_with_param";
         build_passed: "build_passed";
         cost: "cost";
         llm_judge: "llm_judge";
@@ -272,7 +297,6 @@ export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     projectId: z.ZodOptional<z.ZodString>;
     config: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
         prompt: z.ZodString;
-        systemPrompt: z.ZodOptional<z.ZodString>;
         minScore: z.ZodOptional<z.ZodNumber>;
         model: z.ZodOptional<z.ZodString>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -292,6 +316,11 @@ export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
     }, z.core.$strip>, z.ZodObject<{
+        /** Name of the tool that must have been called */
+        toolName: z.ZodString;
+        /** JSON string of key-value pairs for expected parameters (substring match) */
+        expectedParams: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{

package/build/types/assertion/system-assertions.d.ts

@@ -20,6 +20,7 @@ export interface SystemAssertion {
  */
 export declare const SYSTEM_ASSERTION_IDS: {
     readonly SKILL_WAS_CALLED: "system:skill_was_called";
+    readonly TOOL_CALLED_WITH_PARAM: "system:tool_called_with_param";
     readonly BUILD_PASSED: "system:build_passed";
     readonly TIME_LIMIT: "system:time_limit";
     readonly COST: "system:cost";

package/build/types/common/index.d.ts

@@ -3,3 +3,4 @@ export * from './github-source.js';
 export * from './mcp.js';
 export * from './models.js';
 export * from './rule.js';
+export * from './tool-names.js';

package/build/types/common/tool-names.d.ts

@@ -0,0 +1 @@
+export declare const AVAILABLE_TOOL_NAMES: readonly ["Bash", "Edit", "Glob", "Grep", "Read", "Skill", "Write"];

package/build/types/scenario/assertions.d.ts

@@ -11,6 +11,18 @@ export declare const SkillWasCalledAssertionSchema: z.ZodObject<{
     skillNames: z.ZodArray<z.ZodString>;
 }, z.core.$strip>;
 export type SkillWasCalledAssertion = z.infer<typeof SkillWasCalledAssertionSchema>;
+/**
+ * Assertion: a specific tool must have been called with expected parameters.
+ * Checked by inspecting the LLM trace for tool calls with matching name and arguments.
+ * Each expected param value is matched as a substring against the actual argument value.
+ * All expected params must match on the same tool call for the assertion to pass.
+ */
+export declare const ToolCalledWithParamAssertionSchema: z.ZodObject<{
+    type: z.ZodLiteral<"tool_called_with_param">;
+    toolName: z.ZodString;
+    expectedParams: z.ZodString;
+}, z.core.$strip>;
+export type ToolCalledWithParamAssertion = z.infer<typeof ToolCalledWithParamAssertionSchema>;
 /**
  * Assertion: a build command must exit with the expected code (default 0).
  * Runs the command in the scenario working directory.
@@ -38,7 +50,6 @@ export type CostAssertion = z.infer<typeof CostAssertionSchema>;
 export declare const LlmJudgeAssertionSchema: z.ZodObject<{
     type: z.ZodLiteral<"llm_judge">;
     prompt: z.ZodString;
-    systemPrompt: z.ZodOptional<z.ZodString>;
     minScore: z.ZodOptional<z.ZodNumber>;
     model: z.ZodOptional<z.ZodString>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -62,6 +73,10 @@ export type TimeAssertion = z.infer<typeof TimeAssertionSchema>;
 export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
     type: z.ZodLiteral<"skill_was_called">;
     skillNames: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"tool_called_with_param">;
+    toolName: z.ZodString;
+    expectedParams: z.ZodString;
 }, z.core.$strip>, z.ZodObject<{
     type: z.ZodLiteral<"build_passed">;
     command: z.ZodOptional<z.ZodString>;
@@ -75,7 +90,6 @@ export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
 }, z.core.$strip>, z.ZodObject<{
     type: z.ZodLiteral<"llm_judge">;
     prompt: z.ZodString;
-    systemPrompt: z.ZodOptional<z.ZodString>;
     minScore: z.ZodOptional<z.ZodNumber>;
     model: z.ZodOptional<z.ZodString>;
     maxTokens: z.ZodOptional<z.ZodNumber>;

package/build/types/scenario/test-scenario.d.ts

@@ -28,6 +28,10 @@ export declare const TestScenarioSchema: z.ZodObject<{
     assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
         type: z.ZodLiteral<"skill_was_called">;
         skillNames: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"tool_called_with_param">;
+        toolName: z.ZodString;
+        expectedParams: z.ZodString;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"build_passed">;
         command: z.ZodOptional<z.ZodString>;
@@ -41,7 +45,6 @@ export declare const TestScenarioSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;
-        systemPrompt: z.ZodOptional<z.ZodString>;
         minScore: z.ZodOptional<z.ZodNumber>;
         model: z.ZodOptional<z.ZodString>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -66,6 +69,10 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
     assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
         type: z.ZodLiteral<"skill_was_called">;
         skillNames: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"tool_called_with_param">;
+        toolName: z.ZodString;
+        expectedParams: z.ZodString;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"build_passed">;
         command: z.ZodOptional<z.ZodString>;
@@ -79,7 +86,6 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;
-        systemPrompt: z.ZodOptional<z.ZodString>;
         minScore: z.ZodOptional<z.ZodNumber>;
         model: z.ZodOptional<z.ZodString>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -104,6 +110,10 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
     assertions: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
         type: z.ZodLiteral<"skill_was_called">;
         skillNames: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"tool_called_with_param">;
+        toolName: z.ZodString;
+        expectedParams: z.ZodString;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"build_passed">;
         command: z.ZodOptional<z.ZodString>;
@@ -117,7 +127,6 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;
-        systemPrompt: z.ZodOptional<z.ZodString>;
         minScore: z.ZodOptional<z.ZodNumber>;
         model: z.ZodOptional<z.ZodString>;
         maxTokens: z.ZodOptional<z.ZodNumber>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-types",
-  "version": "0.38.0",
+  "version": "0.40.0",
   "description": "Unified types for EvalForge agent evaluation system",
   "files": [
     "build"
@@ -47,5 +47,5 @@
       "artifactId": "evalforge-types"
     }
   },
-  "falconPackageHash": "dc3f87434a7b2a1b350369f78ca98d0fc64183d39b35b8a111054878"
+  "falconPackageHash": "b095aa7d94bbecb74ee45be17f04ba1501f2b9fdb2d57b9ad0c3b37f"
 }