agentv 0.23.0 → 0.26.0

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   runCli
-} from "./chunk-4T62HFF4.js";
+} from "./chunk-6ZM7WVSC.js";
 import "./chunk-UE4GLFVL.js";
 
 // src/cli.ts

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   app,
   runCli
-} from "./chunk-4T62HFF4.js";
+} from "./chunk-6ZM7WVSC.js";
 import "./chunk-UE4GLFVL.js";
 export {
   app,

package/dist/templates/.agentv/.env.template

@@ -1,16 +1,16 @@
-# Example environment configuration for AgentV
 # Copy this file to .env and fill in your credentials
 
-# Model Provider Selection (Optional - can be configured via targets.yaml)
-PROVIDER=azure
-
 # Azure OpenAI Configuration
-# These are the default environment variable names used in the provided targets.yaml
 AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/
-AZURE_OPENAI_API_KEY=your-api-key-here
-AZURE_DEPLOYMENT_NAME=gpt-4o
+AZURE_OPENAI_API_KEY=your-openai-api-key-here
+AZURE_DEPLOYMENT_NAME=gpt-5-chat
+AZURE_OPENAI_API_VERSION=2024-12-01-preview
+
+# Google Gemini
+GOOGLE_GENERATIVE_AI_API_KEY=your-gemini-api-key-here
+GEMINI_MODEL_NAME=gemini-2.5-flash
 
-# Anthropic Configuration (if using Anthropic provider)
+# Anthropic
 ANTHROPIC_API_KEY=your-anthropic-api-key-here
 
 # VS Code Workspace Paths for Execution Targets
@@ -19,5 +19,5 @@ ANTHROPIC_API_KEY=your-anthropic-api-key-here
 PROJECTX_WORKSPACE_PATH=C:/Users/your-username/OneDrive - Company Pty Ltd/sample.code-workspace
 
 # CLI provider sample (used by the local_cli target)
-PROJECT_ROOT=D:/GitHub/your-username/agentv/docs/examples/simple
-LOCAL_AGENT_TOKEN=your-cli-token
+CLI_EVALS_DIR=./docs/examples/simple/evals/local-cli
+LOCAL_AGENT_TOKEN=dummytoken

package/dist/templates/.agentv/targets.yaml

@@ -10,6 +10,7 @@ targets:
     endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
     api_key: ${{ AZURE_OPENAI_API_KEY }}
     model: ${{ AZURE_DEPLOYMENT_NAME }}
+    # version: ${{ AZURE_OPENAI_API_VERSION }}  # Optional: uncomment to override default (2024-12-01-preview)
 
   - name: vscode
     provider: vscode
@@ -49,13 +50,19 @@ targets:
     endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
     api_key: ${{ AZURE_OPENAI_API_KEY }}
     model: ${{ AZURE_DEPLOYMENT_NAME }}
+    version: ${{ AZURE_OPENAI_API_VERSION }}
+
+  - name: gemini_base
+    provider: gemini
+    api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }}
+    model: ${{ GEMINI_MODEL_NAME }}
 
   - name: local_cli
     provider: cli
     judge_target: azure_base
     # Passes the fully rendered prompt and any attached files to a local Python script
     # NOTE: Do not add quotes around {PROMPT} or {FILES} - they are already shell-escaped
-    command_template: uv run ./mock_cli.py --prompt {PROMPT} {FILES}
+    command_template: uv run ./mock_cli.py --prompt {PROMPT} {FILES} --output {OUTPUT_FILE}
     # Format for each file in {FILES}. {path} and {basename} are automatically shell-escaped, so no quotes needed
     files_format: --file {path}
     # Optional working directory resolved from .env

package/dist/templates/.claude/skills/agentv-eval-builder/SKILL.md

@@ -10,10 +10,17 @@ description: Create and maintain AgentV YAML evaluation files for testing AI age
 - Format: YAML with structured content arrays
 - Examples: `references/example-evals.md`
 
+## Feature Reference
+- Rubrics: `references/rubric-evaluator.md` - Structured criteria-based evaluation
+- Composite Evaluators: `references/composite-evaluator.md` - Combine multiple evaluators
+- Tool Trajectory: `references/tool-trajectory-evaluator.md` - Validate agent tool usage
+- Custom Evaluators: `references/custom-evaluators.md` - Code and LLM judge templates
+
 ## Structure Requirements
-- Root level: `description` (optional), `execution` (optional), `evalcases` (required)
-- Eval case fields: `id` (required), `expected_outcome` (required), `input_messages` (required), `expected_messages` (required)
-- Optional fields: `conversation_id`, `note`, `execution`
+- Root level: `description` (optional), `target` (optional), `execution` (optional), `evalcases` (required)
+- Eval case fields: `id` (required), `expected_outcome` (required), `input_messages` (required)
+- Optional fields: `expected_messages`, `conversation_id`, `rubrics`, `execution`
+- `expected_messages` is optional - omit for outcome-only evaluation where the LLM judge evaluates based on `expected_outcome` criteria alone
 - Message fields: `role` (required), `content` (required)
 - Message roles: `system`, `user`, `assistant`, `tool`
 - Content types: `text` (inline), `file` (relative or absolute path)
@@ -31,13 +38,13 @@ Scripts that validate output programmatically:
 execution:
   evaluators:
     - name: json_format_validator
-      type: code
+      type: code_judge
       script: uv run validate_output.py
       cwd: ../../evaluators/scripts
 ```
 
 **Contract:**
-- Input (stdin): JSON with `question`, `expected_outcome`, `reference_answer`, `candidate_answer`, `guideline_paths`, `input_files`, `input_messages`
+- Input (stdin): JSON with `question`, `expected_outcome`, `reference_answer`, `candidate_answer`, `guideline_files` (file paths), `input_files` (file paths, excludes guidelines), `input_messages`
 - Output (stdout): JSON with `score` (0.0-1.0), `hits`, `misses`, `reasoning`
 
 **Template:** See `references/custom-evaluators.md` for Python code evaluator template
@@ -61,12 +68,74 @@ Evaluators run sequentially:
 execution:
   evaluators:
     - name: format_check      # Runs first
-      type: code
+      type: code_judge
       script: uv run validate_json.py
     - name: content_check     # Runs second
       type: llm_judge
 ```
 
+### Rubric Evaluator
+Inline rubrics for structured criteria-based evaluation:
+
+```yaml
+evalcases:
+  - id: explanation-task
+    expected_outcome: Clear explanation of quicksort
+    input_messages:
+      - role: user
+        content: Explain quicksort
+    rubrics:
+      - Mentions divide-and-conquer approach
+      - Explains the partition step
+      - id: complexity
+        description: States time complexity correctly
+        weight: 2.0
+        required: true
+```
+
+See `references/rubric-evaluator.md` for detailed rubric configuration.
+
+### Composite Evaluator
+Combine multiple evaluators with aggregation:
+
+```yaml
+execution:
+  evaluators:
+    - name: release_gate
+      type: composite
+      evaluators:
+        - name: safety
+          type: llm_judge
+          prompt: ./prompts/safety.md
+        - name: quality
+          type: llm_judge
+          prompt: ./prompts/quality.md
+      aggregator:
+        type: weighted_average
+        weights:
+          safety: 0.3
+          quality: 0.7
+```
+
+See `references/composite-evaluator.md` for aggregation types and patterns.
+
+### Tool Trajectory Evaluator
+Validate agent tool usage from trace data:
+
+```yaml
+execution:
+  evaluators:
+    - name: workflow-check
+      type: tool_trajectory
+      mode: in_order  # or: any_order, exact
+      expected:
+        - tool: fetchData
+        - tool: processData
+        - tool: saveResults
+```
+
+See `references/tool-trajectory-evaluator.md` for modes and configuration.
+
 ## Example
 ```yaml
 $schema: agentv-eval-v2

package/dist/templates/.claude/skills/agentv-eval-builder/references/composite-evaluator.md

@@ -0,0 +1,215 @@
+# Composite Evaluator Guide
+
+Composite evaluators combine multiple evaluators and aggregate their results. This enables sophisticated evaluation patterns like safety gates, weighted scoring, and conflict resolution.
+
+## Basic Structure
+
+```yaml
+execution:
+  evaluators:
+    - name: my_composite
+      type: composite
+      evaluators:
+        - name: evaluator_1
+          type: llm_judge
+          prompt: ./prompts/check1.md
+        - name: evaluator_2
+          type: code_judge
+          script: uv run check2.py
+      aggregator:
+        type: weighted_average
+        weights:
+          evaluator_1: 0.6
+          evaluator_2: 0.4
+```
+
+## Aggregator Types
+
+### 1. Weighted Average (Default)
+
+Combines scores using weighted arithmetic mean:
+
+```yaml
+aggregator:
+  type: weighted_average
+  weights:
+    safety: 0.3      # 30% weight
+    quality: 0.7     # 70% weight
+```
+
+If weights are omitted, all evaluators have equal weight (1.0).
+
+**Score calculation:**
+```
+final_score = Σ(score_i × weight_i) / Σ(weight_i)
+```
+
+### 2. Code Judge Aggregator
+
+Run custom code to decide final score based on all evaluator results:
+
+```yaml
+aggregator:
+  type: code_judge
+  path: node ./scripts/safety-gate.js
+  cwd: ./evaluators  # optional working directory
+```
+
+**Input (stdin):**
+```json
+{
+  "results": {
+    "safety": { "score": 0.9, "hits": [...], "misses": [...] },
+    "quality": { "score": 0.85, "hits": [...], "misses": [...] }
+  }
+}
+```
+
+**Output (stdout):**
+```json
+{
+  "score": 0.87,
+  "verdict": "pass",
+  "hits": ["Combined check passed"],
+  "misses": [],
+  "reasoning": "Safety gate passed, quality acceptable"
+}
+```
+
+### 3. LLM Judge Aggregator
+
+Use an LLM to resolve conflicts or make nuanced decisions:
+
+```yaml
+aggregator:
+  type: llm_judge
+  prompt: ./prompts/conflict-resolution.md
+```
+
+The `{{EVALUATOR_RESULTS_JSON}}` variable is replaced with the JSON results from all child evaluators.
+
+## Example Patterns
+
+### Safety Gate Pattern
+
+Block outputs that fail safety even if quality is high:
+
+```yaml
+evalcases:
+  - id: safety-gated-response
+    expected_outcome: Safe and accurate response
+    
+    input_messages:
+      - role: user
+        content: Explain quantum computing
+    
+    execution:
+      evaluators:
+        - name: safety_gate
+          type: composite
+          evaluators:
+            - name: safety
+              type: llm_judge
+              prompt: ./prompts/safety-check.md
+            - name: quality
+              type: llm_judge
+              prompt: ./prompts/quality-check.md
+          aggregator:
+            type: code_judge
+            path: ./scripts/safety-gate.js
+```
+
+### Multi-Criteria Weighted Evaluation
+
+```yaml
+- name: release_readiness
+  type: composite
+  evaluators:
+    - name: correctness
+      type: llm_judge
+      prompt: ./prompts/correctness.md
+    - name: style
+      type: code_judge
+      script: uv run style_checker.py
+    - name: security
+      type: llm_judge
+      prompt: ./prompts/security.md
+  aggregator:
+    type: weighted_average
+    weights:
+      correctness: 0.5
+      style: 0.2
+      security: 0.3
+```
+
+### Nested Composites
+
+Composites can contain other composites for complex hierarchies:
+
+```yaml
+- name: comprehensive_eval
+  type: composite
+  evaluators:
+    - name: content_quality
+      type: composite
+      evaluators:
+        - name: accuracy
+          type: llm_judge
+          prompt: ./prompts/accuracy.md
+        - name: clarity
+          type: llm_judge
+          prompt: ./prompts/clarity.md
+      aggregator:
+        type: weighted_average
+        weights:
+          accuracy: 0.6
+          clarity: 0.4
+    - name: safety
+      type: llm_judge
+      prompt: ./prompts/safety.md
+  aggregator:
+    type: weighted_average
+    weights:
+      content_quality: 0.7
+      safety: 0.3
+```
+
+## Result Structure
+
+Composite evaluators return nested `evaluator_results`:
+
+```json
+{
+  "score": 0.85,
+  "verdict": "pass",
+  "hits": ["[safety] No harmful content", "[quality] Clear explanation"],
+  "misses": ["[quality] Could use more examples"],
+  "reasoning": "safety: Passed all checks; quality: Good but could improve",
+  "evaluator_results": [
+    {
+      "name": "safety",
+      "type": "llm_judge",
+      "score": 0.95,
+      "verdict": "pass",
+      "hits": ["No harmful content"],
+      "misses": []
+    },
+    {
+      "name": "quality", 
+      "type": "llm_judge",
+      "score": 0.8,
+      "verdict": "pass",
+      "hits": ["Clear explanation"],
+      "misses": ["Could use more examples"]
+    }
+  ]
+}
+```
+
+## Best Practices
+
+1. **Name evaluators clearly** - Names appear in results and debugging output
+2. **Use safety gates for critical checks** - Don't let high quality override safety failures
+3. **Balance weights thoughtfully** - Consider which aspects matter most for your use case
+4. **Keep nesting shallow** - Deep nesting makes debugging harder
+5. **Test aggregators independently** - Verify your custom aggregation logic with unit tests

package/dist/templates/.claude/skills/agentv-eval-builder/references/rubric-evaluator.md

@@ -0,0 +1,139 @@
+# Rubric Evaluator Guide
+
+Rubrics provide structured evaluation through lists of criteria that define what makes a good response. Rubrics are checked by an LLM judge and scored based on weights and requirements.
+
+## Basic Usage
+
+### Simple String Rubrics
+
+Define rubrics as simple strings - each becomes a required criterion with weight 1.0:
+
+```yaml
+$schema: agentv-eval-v2
+
+evalcases:
+  - id: quicksort-explanation
+    expected_outcome: Explain how quicksort works
+    
+    input_messages:
+      - role: user
+        content: Explain how the quicksort algorithm works
+    
+    rubrics:
+      - Mentions divide-and-conquer approach
+      - Explains the partition step
+      - States time complexity correctly
+```
+
+### Detailed Rubric Objects
+
+Use objects for fine-grained control over weights and requirements:
+
+```yaml
+evalcases:
+  - id: technical-guide
+    expected_outcome: Write a comprehensive HTTP status codes guide
+    
+    input_messages:
+      - role: user
+        content: Write a guide explaining HTTP status codes
+    
+    rubrics:
+      - id: structure
+        description: Has clear headings and organization
+        weight: 1.0
+        required: true
+        
+      - id: success-codes
+        description: Covers 2xx success codes with examples
+        weight: 2.0
+        required: true
+        
+      - id: client-errors
+        description: Explains 4xx client error codes
+        weight: 2.0
+        required: true
+        
+      - id: server-errors
+        description: Explains 5xx server error codes
+        weight: 1.5
+        required: false
+        
+      - id: practical-examples
+        description: Includes practical use case examples
+        weight: 1.0
+        required: false
+```
+
+## Rubric Object Fields
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `id` | string | auto-generated | Unique identifier for the rubric |
+| `description` | string | required | The criterion being evaluated |
+| `weight` | number | 1.0 | Relative importance (higher = more impact on score) |
+| `required` | boolean | true | If true, failing this rubric forces verdict to 'fail' |
+
+## Scoring and Verdicts
+
+**Score Calculation:**
+```
+score = (sum of satisfied weights) / (total weights)
+```
+
+**Verdict Rules:**
+- `pass`: Score ≥ 0.8 AND all required rubrics satisfied
+- `borderline`: Score ≥ 0.6 AND all required rubrics satisfied  
+- `fail`: Score < 0.6 OR any required rubric failed
+
+## Combining Rubrics with Other Evaluators
+
+Rubrics can be combined with code evaluators for comprehensive validation:
+
+```yaml
+evalcases:
+  - id: email-validator
+    expected_outcome: Python function to validate email addresses
+    
+    input_messages:
+      - role: user
+        content: Write a Python function to validate email addresses
+    
+    # Semantic evaluation via rubrics
+    rubrics:
+      - Uses regular expressions for validation
+      - Includes type hints
+      - Has docstring documentation
+      - Handles edge cases (None, empty string)
+    
+    execution:
+      evaluators:
+        # Rubric evaluator is auto-added from inline rubrics field
+        
+        # Additional code evaluator for syntax checking
+        - name: python_syntax
+          type: code_judge
+          script: uv run python -m py_compile
+```
+
+## Generate Rubrics from Expected Outcome
+
+Use the CLI to auto-generate rubrics from `expected_outcome`:
+
+```bash
+# Generate rubrics for eval cases that don't have them
+agentv generate rubrics evals/my-eval.yaml
+
+# Use a specific LLM target for generation
+agentv generate rubrics evals/my-eval.yaml --target azure_base
+```
+
+This analyzes each `expected_outcome` and creates appropriate rubric items.
+
+## Best Practices
+
+1. **Use required sparingly** - Only mark rubrics as `required: true` for critical criteria
+2. **Balance weights** - Use higher weights (2.0+) for core requirements, lower (0.5) for nice-to-haves
+3. **Be specific** - "Includes error handling" is better than "Good code quality"
+4. **Keep rubrics atomic** - Each rubric should test one thing
+5. **Consider partial credit** - Non-required rubrics allow partial scores