agentv 0.25.0 → 1.0.0

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   runCli
-} from "./chunk-ZVSFP6NK.js";
+} from "./chunk-RIJO5WBF.js";
 import "./chunk-UE4GLFVL.js";
 
 // src/cli.ts

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   app,
   runCli
-} from "./chunk-ZVSFP6NK.js";
+} from "./chunk-RIJO5WBF.js";
 import "./chunk-UE4GLFVL.js";
 export {
   app,

package/dist/templates/.claude/skills/agentv-eval-builder/SKILL.md

@@ -44,7 +44,7 @@ execution:
 ```
 
 **Contract:**
-- Input (stdin): JSON with `question`, `expected_outcome`, `reference_answer`, `candidate_answer`, `guideline_paths`, `input_files`, `input_segments`
+- Input (stdin): JSON with `question`, `expected_outcome`, `reference_answer`, `candidate_answer`, `guideline_files` (file paths), `input_files` (file paths, excludes guidelines), `input_messages`
 - Output (stdout): JSON with `score` (0.0-1.0), `hits`, `misses`, `reasoning`
 
 **Template:** See `references/custom-evaluators.md` for Python code evaluator template
@@ -61,8 +61,42 @@ execution:
       model: gpt-5-chat
 ```
 
-### Evaluator Chaining
-Evaluators run sequentially:
+### Tool Trajectory Evaluators
+Validate agent tool usage patterns (requires trace data from provider):
+
+```yaml
+execution:
+  evaluators:
+    - name: research_check
+      type: tool_trajectory
+      mode: any_order       # Options: any_order, in_order, exact
+      minimums:             # For any_order mode
+        knowledgeSearch: 2
+      expected:             # For in_order/exact modes
+        - tool: knowledgeSearch
+        - tool: documentRetrieve
+```
+
+See `references/tool-trajectory-evaluator.md` for modes and configuration.
+
+### Expected Tool Calls Evaluators
+Validate tool calls and inputs inline with conversation flow:
+
+```yaml
+expected_messages:
+  - role: assistant
+    tool_calls:
+      - tool: getMetrics
+        input: { server: "prod-1" }
+
+execution:
+  evaluators:
+    - name: input_check
+      type: expected_tool_calls
+```
+
+### Multiple Evaluators
+Define multiple evaluators to run sequentially. The final score is a weighted average of all results.
 
 ```yaml
 execution:
@@ -119,23 +153,6 @@ execution:
 
 See `references/composite-evaluator.md` for aggregation types and patterns.
 
-### Tool Trajectory Evaluator
-Validate agent tool usage from trace data:
-
-```yaml
-execution:
-  evaluators:
-    - name: workflow-check
-      type: tool_trajectory
-      mode: in_order  # or: any_order, exact
-      expected:
-        - tool: fetchData
-        - tool: processData
-        - tool: saveResults
-```
-
-See `references/tool-trajectory-evaluator.md` for modes and configuration.
-
 ## Example
 ```yaml
 $schema: agentv-eval-v2

package/dist/templates/.claude/skills/agentv-eval-builder/references/example-evals.md

@@ -78,13 +78,12 @@ evalcases:
     execution:
       evaluators:
         - name: json_format_validator
-          type: code
+          type: code_judge
           script: uv run validate_json.py
           cwd: ./evaluators
         - name: content_evaluator
           type: llm_judge
           prompt: ./judges/semantic_correctness.md
-          model: gpt-5-chat
     
     input_messages:
       - role: user
@@ -102,6 +101,99 @@ evalcases:
           }
 ```
 
+## Tool Trajectory Evaluation
+
+Validate that an agent uses specific tools during execution.
+
+```yaml
+$schema: agentv-eval-v2
+description: Tool usage validation
+target: mock_agent
+
+evalcases:
+  # Validate minimum tool usage (order doesn't matter)
+  - id: research-depth
+    expected_outcome: Agent researches thoroughly
+    input_messages:
+      - role: user
+        content: Research REST vs GraphQL
+    execution:
+      evaluators:
+        - name: research-check
+          type: tool_trajectory
+          mode: any_order
+          minimums:
+            knowledgeSearch: 2
+            documentRetrieve: 1
+
+  # Validate exact tool sequence
+  - id: auth-flow
+    expected_outcome: Agent follows auth sequence
+    input_messages:
+      - role: user
+        content: Authenticate user
+    execution:
+      evaluators:
+        - name: auth-sequence
+          type: tool_trajectory
+          mode: exact
+          expected:
+            - tool: checkCredentials
+            - tool: generateToken
+```
+
+## Expected Messages with Tool Calls
+
+Validate precise tool inputs inline with expected messages.
+
+```yaml
+$schema: agentv-eval-v2
+description: Tool input validation
+target: mock_agent
+
+evalcases:
+  - id: precise-inputs
+    expected_outcome: Agent calls tools with correct parameters
+    input_messages:
+      - role: user
+        content: Check CPU metrics for prod-1
+    expected_messages:
+      - role: assistant
+        content: Checking metrics...
+        tool_calls:
+          - tool: getCpuMetrics
+            input: { server: "prod-1" }
+    execution:
+      evaluators:
+        - name: input-validator
+          type: expected_tool_calls
+```
+
+## Static Trace Evaluation
+
+Evaluate pre-existing trace files without running an agent.
+
+```yaml
+$schema: agentv-eval-v2
+description: Static trace evaluation
+target: static_trace
+
+evalcases:
+  - id: validate-trace-file
+    expected_outcome: Trace contains required steps
+    input_messages:
+      - role: user
+        content: Analyze trace
+    execution:
+      evaluators:
+        - name: trace-check
+          type: tool_trajectory
+          mode: in_order
+          expected:
+            - tool: webSearch
+            - tool: readFile
+```
+
 ## Multi-Turn Conversation (Single Eval Case)
 
 ```yaml

package/dist/templates/.claude/skills/agentv-eval-builder/references/tool-trajectory-evaluator.md

@@ -76,7 +76,7 @@ execution:
 - Strict protocol validation
 - Regression testing specific behavior
 
-## Expected Messages Evaluator
+## Expected Tool Calls Evaluator
 
 For simpler cases, specify tool_calls inline in `expected_messages`:
 
@@ -84,11 +84,11 @@ For simpler cases, specify tool_calls inline in `expected_messages`:
 evalcases:
   - id: research-task
     expected_outcome: Agent searches and retrieves documents
-    
+
     input_messages:
       - role: user
         content: Research REST vs GraphQL differences
-    
+
     expected_messages:
       - role: assistant
         content: I'll research this topic.
@@ -96,11 +96,11 @@ evalcases:
           - tool: knowledgeSearch
           - tool: knowledgeSearch
           - tool: documentRetrieve
-    
+
     execution:
       evaluators:
         - name: tool-validator
-          type: expected_messages
+          type: expected_tool_calls
 ```
 
 ### With Input Matching
@@ -130,7 +130,7 @@ expected_messages:
 | `in_order` | (matched tools in sequence) / (expected tools count) |
 | `exact` | (correctly positioned tools) / (expected tools count) |
 
-### expected_messages Scoring
+### expected_tool_calls Scoring
 
 Sequential matching: `(matched tool_calls) / (expected tool_calls)`
 
@@ -215,7 +215,7 @@ evalcases:
     execution:
       evaluators:
         - name: pipeline-check
-          type: expected_messages
+          type: expected_tool_calls
 ```
 
 ## CLI Options for Traces
@@ -234,4 +234,4 @@ agentv eval evals/test.yaml --include-trace
 2. **Start with any_order** - Then tighten to `in_order` or `exact` as needed
 3. **Combine with other evaluators** - Use tool trajectory for execution, LLM judge for output quality
 4. **Test with --dump-traces** - Inspect actual traces to understand agent behavior
-5. **Use expected_messages for simple cases** - It's more readable for basic tool validation
+5. **Use expected_tool_calls for simple cases** - It's more readable for basic tool validation

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentv",
-  "version": "0.25.0",
+  "version": "1.0.0",
   "description": "CLI entry point for AgentV",
   "type": "module",
   "repository": {