agentv 0.23.0 → 0.25.0

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   runCli
-} from "./chunk-4T62HFF4.js";
+} from "./chunk-ZVSFP6NK.js";
 import "./chunk-UE4GLFVL.js";
 
 // src/cli.ts

package/dist/cli.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\r\nimport { runCli } from './index.js';\r\n\r\nrunCli().catch((error) => {\r\n  console.error(error);\r\n  process.exit(1);\r\n});\r\n"],"mappings":";;;;;;;AAGA,OAAO,EAAE,MAAM,CAAC,UAAU;AACxB,UAAQ,MAAM,KAAK;AACnB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
+{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { runCli } from './index.js';\n\nrunCli().catch((error) => {\n  console.error(error);\n  process.exit(1);\n});\n"],"mappings":";;;;;;;AAGA,OAAO,EAAE,MAAM,CAAC,UAAU;AACxB,UAAQ,MAAM,KAAK;AACnB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   app,
   runCli
-} from "./chunk-4T62HFF4.js";
+} from "./chunk-ZVSFP6NK.js";
 import "./chunk-UE4GLFVL.js";
 export {
   app,

package/dist/templates/.agentv/.env.template

@@ -1,16 +1,16 @@
-# Example environment configuration for AgentV
 # Copy this file to .env and fill in your credentials
 
-# Model Provider Selection (Optional - can be configured via targets.yaml)
-PROVIDER=azure
-
 # Azure OpenAI Configuration
-# These are the default environment variable names used in the provided targets.yaml
 AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/
-AZURE_OPENAI_API_KEY=your-api-key-here
-AZURE_DEPLOYMENT_NAME=gpt-4o
+AZURE_OPENAI_API_KEY=your-openai-api-key-here
+AZURE_DEPLOYMENT_NAME=gpt-5-chat
+AZURE_OPENAI_API_VERSION=2024-12-01-preview
+
+# Google Gemini
+GOOGLE_GENERATIVE_AI_API_KEY=your-gemini-api-key-here
+GEMINI_MODEL_NAME=gemini-2.5-flash
 
-# Anthropic Configuration (if using Anthropic provider)
+# Anthropic
 ANTHROPIC_API_KEY=your-anthropic-api-key-here
 
 # VS Code Workspace Paths for Execution Targets
@@ -19,5 +19,5 @@ ANTHROPIC_API_KEY=your-anthropic-api-key-here
 PROJECTX_WORKSPACE_PATH=C:/Users/your-username/OneDrive - Company Pty Ltd/sample.code-workspace
 
 # CLI provider sample (used by the local_cli target)
-PROJECT_ROOT=D:/GitHub/your-username/agentv/docs/examples/simple
-LOCAL_AGENT_TOKEN=your-cli-token
+CLI_EVALS_DIR=./docs/examples/simple/evals/local-cli
+LOCAL_AGENT_TOKEN=dummytoken

package/dist/templates/.agentv/targets.yaml

@@ -10,6 +10,7 @@ targets:
     endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
     api_key: ${{ AZURE_OPENAI_API_KEY }}
     model: ${{ AZURE_DEPLOYMENT_NAME }}
+    # version: ${{ AZURE_OPENAI_API_VERSION }}  # Optional: uncomment to override default (2024-12-01-preview)
 
   - name: vscode
     provider: vscode
@@ -49,13 +50,19 @@ targets:
     endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
     api_key: ${{ AZURE_OPENAI_API_KEY }}
     model: ${{ AZURE_DEPLOYMENT_NAME }}
+    version: ${{ AZURE_OPENAI_API_VERSION }}
+
+  - name: gemini_base
+    provider: gemini
+    api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }}
+    model: ${{ GEMINI_MODEL_NAME }}
 
   - name: local_cli
     provider: cli
     judge_target: azure_base
     # Passes the fully rendered prompt and any attached files to a local Python script
     # NOTE: Do not add quotes around {PROMPT} or {FILES} - they are already shell-escaped
-    command_template: uv run ./mock_cli.py --prompt {PROMPT} {FILES}
+    command_template: uv run ./mock_cli.py --prompt {PROMPT} {FILES} --output {OUTPUT_FILE}
     # Format for each file in {FILES}. {path} and {basename} are automatically shell-escaped, so no quotes needed
     files_format: --file {path}
     # Optional working directory resolved from .env

package/dist/templates/.claude/skills/agentv-eval-builder/SKILL.md

@@ -10,10 +10,17 @@ description: Create and maintain AgentV YAML evaluation files for testing AI age
 - Format: YAML with structured content arrays
 - Examples: `references/example-evals.md`
 
+## Feature Reference
+- Rubrics: `references/rubric-evaluator.md` - Structured criteria-based evaluation
+- Composite Evaluators: `references/composite-evaluator.md` - Combine multiple evaluators
+- Tool Trajectory: `references/tool-trajectory-evaluator.md` - Validate agent tool usage
+- Custom Evaluators: `references/custom-evaluators.md` - Code and LLM judge templates
+
 ## Structure Requirements
-- Root level: `description` (optional), `execution` (optional), `evalcases` (required)
-- Eval case fields: `id` (required), `expected_outcome` (required), `input_messages` (required), `expected_messages` (required)
-- Optional fields: `conversation_id`, `note`, `execution`
+- Root level: `description` (optional), `target` (optional), `execution` (optional), `evalcases` (required)
+- Eval case fields: `id` (required), `expected_outcome` (required), `input_messages` (required)
+- Optional fields: `expected_messages`, `conversation_id`, `rubrics`, `execution`
+- `expected_messages` is optional - omit for outcome-only evaluation where the LLM judge evaluates based on `expected_outcome` criteria alone
 - Message fields: `role` (required), `content` (required)
 - Message roles: `system`, `user`, `assistant`, `tool`
 - Content types: `text` (inline), `file` (relative or absolute path)
@@ -31,13 +38,13 @@ Scripts that validate output programmatically:
 execution:
   evaluators:
     - name: json_format_validator
-      type: code
+      type: code_judge
       script: uv run validate_output.py
       cwd: ../../evaluators/scripts
 ```
 
 **Contract:**
-- Input (stdin): JSON with `question`, `expected_outcome`, `reference_answer`, `candidate_answer`, `guideline_paths`, `input_files`, `input_messages`
+- Input (stdin): JSON with `question`, `expected_outcome`, `reference_answer`, `candidate_answer`, `guideline_paths`, `input_files`, `input_segments`
 - Output (stdout): JSON with `score` (0.0-1.0), `hits`, `misses`, `reasoning`
 
 **Template:** See `references/custom-evaluators.md` for Python code evaluator template
@@ -61,12 +68,74 @@ Evaluators run sequentially:
 execution:
   evaluators:
     - name: format_check      # Runs first
-      type: code
+      type: code_judge
       script: uv run validate_json.py
     - name: content_check     # Runs second
       type: llm_judge
 ```
 
+### Rubric Evaluator
+Inline rubrics for structured criteria-based evaluation:
+
+```yaml
+evalcases:
+  - id: explanation-task
+    expected_outcome: Clear explanation of quicksort
+    input_messages:
+      - role: user
+        content: Explain quicksort
+    rubrics:
+      - Mentions divide-and-conquer approach
+      - Explains the partition step
+      - id: complexity
+        description: States time complexity correctly
+        weight: 2.0
+        required: true
+```
+
+See `references/rubric-evaluator.md` for detailed rubric configuration.
+
+### Composite Evaluator
+Combine multiple evaluators with aggregation:
+
+```yaml
+execution:
+  evaluators:
+    - name: release_gate
+      type: composite
+      evaluators:
+        - name: safety
+          type: llm_judge
+          prompt: ./prompts/safety.md
+        - name: quality
+          type: llm_judge
+          prompt: ./prompts/quality.md
+      aggregator:
+        type: weighted_average
+        weights:
+          safety: 0.3
+          quality: 0.7
+```
+
+See `references/composite-evaluator.md` for aggregation types and patterns.
+
+### Tool Trajectory Evaluator
+Validate agent tool usage from trace data:
+
+```yaml
+execution:
+  evaluators:
+    - name: workflow-check
+      type: tool_trajectory
+      mode: in_order  # or: any_order, exact
+      expected:
+        - tool: fetchData
+        - tool: processData
+        - tool: saveResults
+```
+
+See `references/tool-trajectory-evaluator.md` for modes and configuration.
+
 ## Example
 ```yaml
 $schema: agentv-eval-v2

package/dist/templates/.claude/skills/agentv-eval-builder/references/composite-evaluator.md

@@ -0,0 +1,215 @@
+# Composite Evaluator Guide
+
+Composite evaluators combine multiple evaluators and aggregate their results. This enables sophisticated evaluation patterns like safety gates, weighted scoring, and conflict resolution.
+
+## Basic Structure
+
+```yaml
+execution:
+  evaluators:
+    - name: my_composite
+      type: composite
+      evaluators:
+        - name: evaluator_1
+          type: llm_judge
+          prompt: ./prompts/check1.md
+        - name: evaluator_2
+          type: code_judge
+          script: uv run check2.py
+      aggregator:
+        type: weighted_average
+        weights:
+          evaluator_1: 0.6
+          evaluator_2: 0.4
+```
+
+## Aggregator Types
+
+### 1. Weighted Average (Default)
+
+Combines scores using weighted arithmetic mean:
+
+```yaml
+aggregator:
+  type: weighted_average
+  weights:
+    safety: 0.3      # 30% weight
+    quality: 0.7     # 70% weight
+```
+
+If weights are omitted, all evaluators have equal weight (1.0).
+
+**Score calculation:**
+```
+final_score = Σ(score_i × weight_i) / Σ(weight_i)
+```
+
+### 2. Code Judge Aggregator
+
+Run custom code to decide final score based on all evaluator results:
+
+```yaml
+aggregator:
+  type: code_judge
+  path: node ./scripts/safety-gate.js
+  cwd: ./evaluators  # optional working directory
+```
+
+**Input (stdin):**
+```json
+{
+  "results": {
+    "safety": { "score": 0.9, "hits": [...], "misses": [...] },
+    "quality": { "score": 0.85, "hits": [...], "misses": [...] }
+  }
+}
+```
+
+**Output (stdout):**
+```json
+{
+  "score": 0.87,
+  "verdict": "pass",
+  "hits": ["Combined check passed"],
+  "misses": [],
+  "reasoning": "Safety gate passed, quality acceptable"
+}
+```
+
+### 3. LLM Judge Aggregator
+
+Use an LLM to resolve conflicts or make nuanced decisions:
+
+```yaml
+aggregator:
+  type: llm_judge
+  prompt: ./prompts/conflict-resolution.md
+```
+
+The `{{EVALUATOR_RESULTS_JSON}}` variable is replaced with the JSON results from all child evaluators.
+
+## Example Patterns
+
+### Safety Gate Pattern
+
+Block outputs that fail safety even if quality is high:
+
+```yaml
+evalcases:
+  - id: safety-gated-response
+    expected_outcome: Safe and accurate response
+    
+    input_messages:
+      - role: user
+        content: Explain quantum computing
+    
+    execution:
+      evaluators:
+        - name: safety_gate
+          type: composite
+          evaluators:
+            - name: safety
+              type: llm_judge
+              prompt: ./prompts/safety-check.md
+            - name: quality
+              type: llm_judge
+              prompt: ./prompts/quality-check.md
+          aggregator:
+            type: code_judge
+            path: ./scripts/safety-gate.js
+```
+
+### Multi-Criteria Weighted Evaluation
+
+```yaml
+- name: release_readiness
+  type: composite
+  evaluators:
+    - name: correctness
+      type: llm_judge
+      prompt: ./prompts/correctness.md
+    - name: style
+      type: code_judge
+      script: uv run style_checker.py
+    - name: security
+      type: llm_judge
+      prompt: ./prompts/security.md
+  aggregator:
+    type: weighted_average
+    weights:
+      correctness: 0.5
+      style: 0.2
+      security: 0.3
+```
+
+### Nested Composites
+
+Composites can contain other composites for complex hierarchies:
+
+```yaml
+- name: comprehensive_eval
+  type: composite
+  evaluators:
+    - name: content_quality
+      type: composite
+      evaluators:
+        - name: accuracy
+          type: llm_judge
+          prompt: ./prompts/accuracy.md
+        - name: clarity
+          type: llm_judge
+          prompt: ./prompts/clarity.md
+      aggregator:
+        type: weighted_average
+        weights:
+          accuracy: 0.6
+          clarity: 0.4
+    - name: safety
+      type: llm_judge
+      prompt: ./prompts/safety.md
+  aggregator:
+    type: weighted_average
+    weights:
+      content_quality: 0.7
+      safety: 0.3
+```
+
+## Result Structure
+
+Composite evaluators return nested `evaluator_results`:
+
+```json
+{
+  "score": 0.85,
+  "verdict": "pass",
+  "hits": ["[safety] No harmful content", "[quality] Clear explanation"],
+  "misses": ["[quality] Could use more examples"],
+  "reasoning": "safety: Passed all checks; quality: Good but could improve",
+  "evaluator_results": [
+    {
+      "name": "safety",
+      "type": "llm_judge",
+      "score": 0.95,
+      "verdict": "pass",
+      "hits": ["No harmful content"],
+      "misses": []
+    },
+    {
+      "name": "quality", 
+      "type": "llm_judge",
+      "score": 0.8,
+      "verdict": "pass",
+      "hits": ["Clear explanation"],
+      "misses": ["Could use more examples"]
+    }
+  ]
+}
+```
+
+## Best Practices
+
+1. **Name evaluators clearly** - Names appear in results and debugging output
+2. **Use safety gates for critical checks** - Don't let high quality override safety failures
+3. **Balance weights thoughtfully** - Consider which aspects matter most for your use case
+4. **Keep nesting shallow** - Deep nesting makes debugging harder
+5. **Test aggregators independently** - Verify your custom aggregation logic with unit tests