@wix/evalforge-types 0.37.0 → 0.39.0

package/build/types/agent/adapter.d.ts

@@ -5,6 +5,7 @@ import type { ModelConfig } from '../common/models.js';
 import type { LLMTrace } from '../evaluation/metrics.js';
 import type { MCPEntity } from '../common/mcp.js';
 import type { SubAgent } from '../target/sub-agent.js';
+import type { Rule } from '../common/rule.js';
 /**
  * Trace context for live streaming of agent execution.
  * This is agent-agnostic and can be used by any adapter implementation.
@@ -57,6 +58,8 @@ export interface AgentExecutionContext {
     mcps?: MCPEntity[];
     /** Sub-agents to load (when present, written to .claude/agents/*.md) */
     subAgents?: SubAgent[];
+    /** Rules to write (CLAUDE.md, AGENTS.md, .cursor/rules/*.md based on ruleType) */
+    rules?: Rule[];
 }
 /**
  * Token usage statistics from agent execution.

package/build/types/assertion/assertion.d.ts

@@ -2,6 +2,7 @@ import { z } from 'zod';
 /**
  * Assertion types:
  * - skill_was_called: Checks if a specific skill was invoked (deterministic, system-level)
+ * - tool_called_with_param: Checks if a tool was called with expected parameters (deterministic, system-level)
  * - build_passed: Runs a command and checks exit code (deterministic, system-level)
  * - time_limit: Checks that scenario completed within a duration threshold (deterministic, system-level)
  * - cost: Checks that scenario LLM cost stays within a USD threshold (deterministic, system-level)
@@ -9,6 +10,7 @@ import { z } from 'zod';
  */
 export declare const AssertionTypeSchema: z.ZodEnum<{
     skill_was_called: "skill_was_called";
+    tool_called_with_param: "tool_called_with_param";
     build_passed: "build_passed";
     cost: "cost";
     llm_judge: "llm_judge";
@@ -68,6 +70,15 @@ export declare const CostConfigSchema: z.ZodObject<{
     maxCostUsd: z.ZodNumber;
 }, z.core.$strict>;
 export type CostConfig = z.infer<typeof CostConfigSchema>;
+/** Configuration for tool_called_with_param assertion type.
+ * Uses strictObject to reject objects with unknown keys.
+ */
+export declare const ToolCalledWithParamConfigSchema: z.ZodObject<{
+    /** Name of the tool that must have been called */
+    toolName: z.ZodString;
+    /** JSON string of key-value pairs for expected parameters (substring match) */
+    expectedParams: z.ZodString;
+}, z.core.$strict>;
 /**
  * Configuration for build_passed assertion type.
  * Uses strictObject to reject objects with unknown keys (prevents matching LlmJudge configs).
@@ -141,6 +152,11 @@ export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
 }, z.core.$strip>, z.ZodObject<{
     skillNames: z.ZodArray<z.ZodString>;
 }, z.core.$strip>, z.ZodObject<{
+    /** Name of the tool that must have been called */
+    toolName: z.ZodString;
+    /** JSON string of key-value pairs for expected parameters (substring match) */
+    expectedParams: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
     /** Maximum allowed duration in milliseconds */
     maxDurationMs: z.ZodNumber;
 }, z.core.$strict>, z.ZodObject<{
@@ -167,6 +183,7 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     projectId: z.ZodString;
     type: z.ZodEnum<{
         skill_was_called: "skill_was_called";
+        tool_called_with_param: "tool_called_with_param";
         build_passed: "build_passed";
         cost: "cost";
         llm_judge: "llm_judge";
@@ -194,6 +211,11 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
     }, z.core.$strip>, z.ZodObject<{
+        /** Name of the tool that must have been called */
+        toolName: z.ZodString;
+        /** JSON string of key-value pairs for expected parameters (substring match) */
+        expectedParams: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
@@ -213,6 +235,7 @@ export type CustomAssertion = z.infer<typeof CustomAssertionSchema>;
 export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     type: z.ZodEnum<{
         skill_was_called: "skill_was_called";
+        tool_called_with_param: "tool_called_with_param";
         build_passed: "build_passed";
         cost: "cost";
         llm_judge: "llm_judge";
@@ -243,6 +266,11 @@ export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
     }, z.core.$strip>, z.ZodObject<{
+        /** Name of the tool that must have been called */
+        toolName: z.ZodString;
+        /** JSON string of key-value pairs for expected parameters (substring match) */
+        expectedParams: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
@@ -262,6 +290,7 @@ export type CreateCustomAssertionInput = z.infer<typeof CreateCustomAssertionInp
 export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     type: z.ZodOptional<z.ZodEnum<{
         skill_was_called: "skill_was_called";
+        tool_called_with_param: "tool_called_with_param";
         build_passed: "build_passed";
         cost: "cost";
         llm_judge: "llm_judge";
@@ -292,6 +321,11 @@ export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
     }, z.core.$strip>, z.ZodObject<{
+        /** Name of the tool that must have been called */
+        toolName: z.ZodString;
+        /** JSON string of key-value pairs for expected parameters (substring match) */
+        expectedParams: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{

package/build/types/assertion/system-assertions.d.ts

@@ -20,6 +20,7 @@ export interface SystemAssertion {
  */
 export declare const SYSTEM_ASSERTION_IDS: {
     readonly SKILL_WAS_CALLED: "system:skill_was_called";
+    readonly TOOL_CALLED_WITH_PARAM: "system:tool_called_with_param";
     readonly BUILD_PASSED: "system:build_passed";
     readonly TIME_LIMIT: "system:time_limit";
     readonly COST: "system:cost";

package/build/types/common/index.d.ts

@@ -2,3 +2,5 @@ export * from './base-entity.js';
 export * from './github-source.js';
 export * from './mcp.js';
 export * from './models.js';
+export * from './rule.js';
+export * from './tool-names.js';

package/build/types/common/rule.d.ts

@@ -0,0 +1,47 @@
+import { z } from 'zod';
+export declare const RuleTypeSchema: z.ZodEnum<{
+    "claude-md": "claude-md";
+    "agents-md": "agents-md";
+    "cursor-rule": "cursor-rule";
+}>;
+export type RuleType = z.infer<typeof RuleTypeSchema>;
+export declare const RuleSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodString;
+    createdAt: z.ZodString;
+    updatedAt: z.ZodString;
+    deleted: z.ZodOptional<z.ZodBoolean>;
+    projectId: z.ZodString;
+    ruleType: z.ZodEnum<{
+        "claude-md": "claude-md";
+        "agents-md": "agents-md";
+        "cursor-rule": "cursor-rule";
+    }>;
+    content: z.ZodString;
+}, z.core.$strip>;
+export type Rule = z.infer<typeof RuleSchema>;
+export declare const CreateRuleInputSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodString;
+    projectId: z.ZodString;
+    ruleType: z.ZodEnum<{
+        "claude-md": "claude-md";
+        "agents-md": "agents-md";
+        "cursor-rule": "cursor-rule";
+    }>;
+    content: z.ZodString;
+}, z.core.$strip>;
+export type CreateRuleInput = z.infer<typeof CreateRuleInputSchema>;
+export declare const UpdateRuleInputSchema: z.ZodObject<{
+    name: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    projectId: z.ZodOptional<z.ZodString>;
+    ruleType: z.ZodOptional<z.ZodEnum<{
+        "claude-md": "claude-md";
+        "agents-md": "agents-md";
+        "cursor-rule": "cursor-rule";
+    }>>;
+    content: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type UpdateRuleInput = z.infer<typeof UpdateRuleInputSchema>;

package/build/types/common/tool-names.d.ts

@@ -0,0 +1 @@
+export declare const AVAILABLE_TOOL_NAMES: readonly ["Bash", "Edit", "Glob", "Grep", "Read", "Skill", "Write"];

package/build/types/evaluation/eval-run.d.ts

@@ -495,6 +495,7 @@ export declare const EvalRunSchema: z.ZodObject<{
     jobStatusCheckedAt: z.ZodOptional<z.ZodString>;
     mcpIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
     subAgentIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    ruleIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>;
 export type EvalRun = z.infer<typeof EvalRunSchema>;
 /**
@@ -612,6 +613,7 @@ export declare const CreateEvalRunInputSchema: z.ZodObject<{
     jobStatusCheckedAt: z.ZodOptional<z.ZodString>;
     mcpIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
     subAgentIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    ruleIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>;
 export type CreateEvalRunInput = z.infer<typeof CreateEvalRunInputSchema>;
 /**

package/build/types/scenario/assertions.d.ts

@@ -11,6 +11,18 @@ export declare const SkillWasCalledAssertionSchema: z.ZodObject<{
     skillNames: z.ZodArray<z.ZodString>;
 }, z.core.$strip>;
 export type SkillWasCalledAssertion = z.infer<typeof SkillWasCalledAssertionSchema>;
+/**
+ * Assertion: a specific tool must have been called with expected parameters.
+ * Checked by inspecting the LLM trace for tool calls with matching name and arguments.
+ * Each expected param value is matched as a substring against the actual argument value.
+ * All expected params must match on the same tool call for the assertion to pass.
+ */
+export declare const ToolCalledWithParamAssertionSchema: z.ZodObject<{
+    type: z.ZodLiteral<"tool_called_with_param">;
+    toolName: z.ZodString;
+    expectedParams: z.ZodString;
+}, z.core.$strip>;
+export type ToolCalledWithParamAssertion = z.infer<typeof ToolCalledWithParamAssertionSchema>;
 /**
  * Assertion: a build command must exit with the expected code (default 0).
  * Runs the command in the scenario working directory.
@@ -62,6 +74,10 @@ export type TimeAssertion = z.infer<typeof TimeAssertionSchema>;
 export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
     type: z.ZodLiteral<"skill_was_called">;
     skillNames: z.ZodArray<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"tool_called_with_param">;
+    toolName: z.ZodString;
+    expectedParams: z.ZodString;
 }, z.core.$strip>, z.ZodObject<{
     type: z.ZodLiteral<"build_passed">;
     command: z.ZodOptional<z.ZodString>;

package/build/types/scenario/test-scenario.d.ts

@@ -28,6 +28,10 @@ export declare const TestScenarioSchema: z.ZodObject<{
     assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
         type: z.ZodLiteral<"skill_was_called">;
         skillNames: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"tool_called_with_param">;
+        toolName: z.ZodString;
+        expectedParams: z.ZodString;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"build_passed">;
         command: z.ZodOptional<z.ZodString>;
@@ -66,6 +70,10 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
     assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
         type: z.ZodLiteral<"skill_was_called">;
         skillNames: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"tool_called_with_param">;
+        toolName: z.ZodString;
+        expectedParams: z.ZodString;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"build_passed">;
         command: z.ZodOptional<z.ZodString>;
@@ -104,6 +112,10 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
     assertions: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
         type: z.ZodLiteral<"skill_was_called">;
         skillNames: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"tool_called_with_param">;
+        toolName: z.ZodString;
+        expectedParams: z.ZodString;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"build_passed">;
         command: z.ZodOptional<z.ZodString>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-types",
-  "version": "0.37.0",
+  "version": "0.39.0",
   "description": "Unified types for EvalForge agent evaluation system",
   "files": [
     "build"
@@ -47,5 +47,5 @@
       "artifactId": "evalforge-types"
     }
   },
-  "falconPackageHash": "57c752f025e2a8ccb67557327757571462f50d8acadc4a28877770af"
+  "falconPackageHash": "a8c46ce4c7e60fa96cb1a06746bcd9fbfdd42cbf1f8be3200348eb2a"
 }