agentv 1.6.1 → 2.0.2

package/dist/cli.js

@@ -1,11 +1,13 @@
 #!/usr/bin/env node
 import {
   runCli
-} from "./chunk-HU4B6ODF.js";
+} from "./chunk-5AJ7DFUO.js";
 import "./chunk-UE4GLFVL.js";
 
 // src/cli.ts
-runCli().catch((error) => {
+runCli().then(() => {
+  process.exit(0);
+}).catch((error) => {
   console.error(error);
   process.exit(1);
 });

package/dist/cli.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { runCli } from './index.js';\n\nrunCli().catch((error) => {\n  console.error(error);\n  process.exit(1);\n});\n"],"mappings":";;;;;;;AAGA,OAAO,EAAE,MAAM,CAAC,UAAU;AACxB,UAAQ,MAAM,KAAK;AACnB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
+{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { runCli } from './index.js';\n\nrunCli()\n  .then(() => {\n    process.exit(0);\n  })\n  .catch((error) => {\n    console.error(error);\n    process.exit(1);\n  });\n"],"mappings":";;;;;;;AAGA,OAAO,EACJ,KAAK,MAAM;AACV,UAAQ,KAAK,CAAC;AAChB,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,UAAQ,MAAM,KAAK;AACnB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   app,
   runCli
-} from "./chunk-HU4B6ODF.js";
+} from "./chunk-5AJ7DFUO.js";
 import "./chunk-UE4GLFVL.js";
 export {
   app,

package/dist/templates/.agentv/.env.template

@@ -1,23 +1,23 @@
-# Copy this file to .env and fill in your credentials
-
-# Azure OpenAI Configuration
-AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/
-AZURE_OPENAI_API_KEY=your-openai-api-key-here
-AZURE_DEPLOYMENT_NAME=gpt-5-chat
-AZURE_OPENAI_API_VERSION=2024-12-01-preview
-
-# Google Gemini
-GOOGLE_GENERATIVE_AI_API_KEY=your-gemini-api-key-here
-GEMINI_MODEL_NAME=gemini-2.5-flash
-
-# Anthropic
-ANTHROPIC_API_KEY=your-anthropic-api-key-here
-
-# VS Code Workspace Paths for Execution Targets
-# Note: Using forward slashes is recommended for paths in .env files 
-# to avoid issues with escape characters.
-PROJECTX_WORKSPACE_PATH=C:/Users/your-username/OneDrive - Company Pty Ltd/sample.code-workspace
-
-# CLI provider sample (used by the local_cli target)
-CLI_EVALS_DIR=./docs/examples/simple/evals/local-cli
-LOCAL_AGENT_TOKEN=dummytoken
+# Copy this file to .env and fill in your credentials
+
+# Azure OpenAI Configuration
+AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/
+AZURE_OPENAI_API_KEY=your-openai-api-key-here
+AZURE_DEPLOYMENT_NAME=gpt-5-chat
+AZURE_OPENAI_API_VERSION=2024-12-01-preview
+
+# Google Gemini
+GOOGLE_GENERATIVE_AI_API_KEY=your-gemini-api-key-here
+GEMINI_MODEL_NAME=gemini-2.5-flash
+
+# Anthropic
+ANTHROPIC_API_KEY=your-anthropic-api-key-here
+
+# VS Code Workspace Paths for Execution Targets
+# Note: Using forward slashes is recommended for paths in .env files 
+# to avoid issues with escape characters.
+PROJECTX_WORKSPACE_PATH=C:/Users/your-username/OneDrive - Company Pty Ltd/sample.code-workspace
+
+# CLI provider sample (used by the local_cli target)
+CLI_EVALS_DIR=./docs/examples/simple/evals/local-cli
+LOCAL_AGENT_TOKEN=dummytoken

package/dist/templates/.agentv/config.yaml

@@ -1,15 +1,15 @@
-$schema: agentv-config-v2
-
-# Customize which files are treated as guidelines vs regular file content
-
-# Custom guideline patterns:
-guideline_patterns:
-  - "**/*.instructions.md"
-  - "**/*.prompt.md"
-  - "**/SKILL.md"
-  
-# Notes:
-# - Patterns use standard glob syntax (via micromatch library)
-# - Paths are normalized to forward slashes for cross-platform compatibility
-# - Only files matching these patterns are loaded as guidelines
-# - All other files referenced in eval cases are treated as regular file content
+$schema: agentv-config-v2
+
+# Customize which files are treated as guidelines vs regular file content
+
+# Custom guideline patterns:
+guideline_patterns:
+  - "**/*.instructions.md"
+  - "**/*.prompt.md"
+  - "**/SKILL.md"
+  
+# Notes:
+# - Patterns use standard glob syntax (via micromatch library)
+# - Paths are normalized to forward slashes for cross-platform compatibility
+# - Only files matching these patterns are loaded as guidelines
+# - All other files referenced in eval cases are treated as regular file content

package/dist/templates/.agentv/targets.yaml

@@ -31,6 +31,22 @@ targets:
     log_dir: ${{ CODEX_LOG_DIR }}              # Optional: where Codex CLI stream logs are stored (defaults to ./.agentv/logs/codex)
     log_format: json                    # Optional: 'summary' (default) or 'json' for raw event logs
 
+  # Claude Code - Anthropic's autonomous coding CLI
+  - name: claude-code
+    provider: claude-code
+    judge_target: azure_base
+    # Uses the Claude Code CLI (defaults to `claude` on PATH)
+    # executable: ${{ CLAUDE_CODE_CLI_PATH }}  # Optional: override executable path
+    # model: claude-sonnet-4-20250514          # Optional: override model
+    # args:                                    # Optional additional CLI arguments
+    #   - --allowedTools
+    #   - Read,Write,Edit,Bash
+    timeout_seconds: 180
+    # cwd: ${{ CLAUDE_CODE_WORKSPACE_DIR }}    # Optional: working directory (defaults to process.cwd())
+    # log_dir: ${{ CLAUDE_CODE_LOG_DIR }}      # Optional: where stream logs are stored (defaults to ./.agentv/logs/claude-code)
+    log_format: json                           # Optional: 'summary' (default) or 'json' for raw event logs
+    # system_prompt: optional override (default instructs agent to include code in response)
+
   - name: vscode_projectx
     provider: vscode
     workspace_template: ${{ PROJECTX_WORKSPACE_PATH }}

package/dist/templates/.claude/skills/agentv-eval-builder/SKILL.md

@@ -14,6 +14,7 @@ description: Create and maintain AgentV YAML evaluation files for testing AI age
 - Rubrics: `references/rubric-evaluator.md` - Structured criteria-based evaluation
 - Composite Evaluators: `references/composite-evaluator.md` - Combine multiple evaluators
 - Tool Trajectory: `references/tool-trajectory-evaluator.md` - Validate agent tool usage
+- Structured Data + Metrics: `references/structured-data-evaluators.md` - `field_accuracy`, `latency`, `cost`
 - Custom Evaluators: `references/custom-evaluators.md` - Code and LLM judge templates
 - Batch CLI: `references/batch-cli-evaluator.md` - Evaluate batch runner output (JSONL)
 - Compare: `references/compare-command.md` - Compare evaluation results between runs
@@ -49,7 +50,9 @@ execution:
 - Input (stdin): JSON with `question`, `expected_outcome`, `reference_answer`, `candidate_answer`, `guideline_files` (file paths), `input_files` (file paths, excludes guidelines), `input_messages`
 - Output (stdout): JSON with `score` (0.0-1.0), `hits`, `misses`, `reasoning`
 
-**Template:** See `references/custom-evaluators.md` for Python code evaluator template
+**TypeScript evaluators:** Keep `.ts` source files and run them via Node-compatible loaders such as `npx --yes tsx` so global `agentv` installs stay portable. See `references/custom-evaluators.md` for complete templates and command examples.
+
+**Template:** See `references/custom-evaluators.md` for Python and TypeScript templates
 
 ### LLM Judges
 Language models evaluate response quality:

package/dist/templates/.claude/skills/agentv-eval-builder/references/batch-cli-evaluator.md

@@ -17,6 +17,18 @@ Batch CLI evaluation is used when:
 3. **AgentV** parses JSONL, routes each record to its matching evalcase by `id`
 4. **Per-case evaluator** validates the output for each evalcase independently
 
+## Error Handling
+
+### Missing IDs / Per-item Failures
+
+In batch mode, a runner can succeed overall while still failing to produce an output record for a specific evalcase `id`.
+
+- When an output record is missing, AgentV treats this as a per-item error.
+- The evaluation result will include an `error` field describing the issue.
+- The CLI progress display will show that evalcase as failed (❌) while still continuing to evaluate other cases.
+
+This behavior is intentional so one bad/missing record does not discard the entire batch.
+
 ## Eval File Structure
 
 ```yaml

package/dist/templates/.claude/skills/agentv-eval-builder/references/compare-command.md

@@ -5,7 +5,7 @@ Compare evaluation results between two runs to measure performance differences.
 ## Usage
 
 ```bash
-agentv compare <baseline.jsonl> <candidate.jsonl> [--threshold <value>]
+agentv compare <baseline.jsonl> <candidate.jsonl> [options]
 ```
 
 ## Arguments
@@ -15,6 +15,8 @@ agentv compare <baseline.jsonl> <candidate.jsonl> [--threshold <value>]
 | `result1` | Path to baseline JSONL result file |
 | `result2` | Path to candidate JSONL result file |
 | `--threshold`, `-t` | Score delta threshold for win/loss classification (default: 0.1) |
+| `--format`, `-f` | Output format: `table` (default) or `json` |
+| `--json` | Shorthand for `--format=json` |
 
 ## How It Works
 
@@ -25,10 +27,30 @@ agentv compare <baseline.jsonl> <candidate.jsonl> [--threshold <value>]
    - `win`: delta >= threshold (candidate better)
    - `loss`: delta <= -threshold (baseline better)
    - `tie`: |delta| < threshold (no significant difference)
-5. **Output Summary**: JSON with matched results, unmatched counts, and statistics
+5. **Output Summary**: Human-readable table (default) or JSON
 
 ## Output Format
 
+### Table Format (default)
+
+```
+Comparing: baseline.jsonl → candidate.jsonl
+
+  Eval ID        Baseline  Candidate     Delta  Result
+  ─────────────  ────────  ─────────  ────────  ────────
+  safety-check       0.70       0.90     +0.20  ✓ win
+  accuracy-test      0.85       0.80     -0.05  = tie
+  latency-eval       0.90       0.75     -0.15  ✗ loss
+
+Summary: 1 win, 1 loss, 1 tie | Mean Δ: +0.000 | Status: neutral
+```
+
+Colors are used to highlight wins (green), losses (red), and ties (gray). Colors are automatically disabled when output is piped or `NO_COLOR` is set.
+
+### JSON Format (`--json`)
+
+Output uses snake_case for Python ecosystem compatibility:
+
 ```json
 {
   "matched": [
@@ -50,7 +72,7 @@ agentv compare <baseline.jsonl> <candidate.jsonl> [--threshold <value>]
     "wins": 1,
     "losses": 0,
     "ties": 0,
-    "meanDelta": 0.2
+    "mean_delta": 0.2
   }
 }
 ```

package/dist/templates/.claude/skills/agentv-eval-builder/references/composite-evaluator.md

@@ -1,215 +1,215 @@
-# Composite Evaluator Guide
-
-Composite evaluators combine multiple evaluators and aggregate their results. This enables sophisticated evaluation patterns like safety gates, weighted scoring, and conflict resolution.
-
-## Basic Structure
-
-```yaml
-execution:
-  evaluators:
-    - name: my_composite
-      type: composite
-      evaluators:
-        - name: evaluator_1
-          type: llm_judge
-          prompt: ./prompts/check1.md
-        - name: evaluator_2
-          type: code_judge
-          script: uv run check2.py
-      aggregator:
-        type: weighted_average
-        weights:
-          evaluator_1: 0.6
-          evaluator_2: 0.4
-```
-
-## Aggregator Types
-
-### 1. Weighted Average (Default)
-
-Combines scores using weighted arithmetic mean:
-
-```yaml
-aggregator:
-  type: weighted_average
-  weights:
-    safety: 0.3      # 30% weight
-    quality: 0.7     # 70% weight
-```
-
-If weights are omitted, all evaluators have equal weight (1.0).
-
-**Score calculation:**
-```
-final_score = Σ(score_i × weight_i) / Σ(weight_i)
-```
-
-### 2. Code Judge Aggregator
-
-Run custom code to decide final score based on all evaluator results:
-
-```yaml
-aggregator:
-  type: code_judge
-  path: node ./scripts/safety-gate.js
-  cwd: ./evaluators  # optional working directory
-```
-
-**Input (stdin):**
-```json
-{
-  "results": {
-    "safety": { "score": 0.9, "hits": [...], "misses": [...] },
-    "quality": { "score": 0.85, "hits": [...], "misses": [...] }
-  }
-}
-```
-
-**Output (stdout):**
-```json
-{
-  "score": 0.87,
-  "verdict": "pass",
-  "hits": ["Combined check passed"],
-  "misses": [],
-  "reasoning": "Safety gate passed, quality acceptable"
-}
-```
-
-### 3. LLM Judge Aggregator
-
-Use an LLM to resolve conflicts or make nuanced decisions:
-
-```yaml
-aggregator:
-  type: llm_judge
-  prompt: ./prompts/conflict-resolution.md
-```
-
-The `{{EVALUATOR_RESULTS_JSON}}` variable is replaced with the JSON results from all child evaluators.
-
-## Example Patterns
-
-### Safety Gate Pattern
-
-Block outputs that fail safety even if quality is high:
-
-```yaml
-evalcases:
-  - id: safety-gated-response
-    expected_outcome: Safe and accurate response
-    
-    input_messages:
-      - role: user
-        content: Explain quantum computing
-    
-    execution:
-      evaluators:
-        - name: safety_gate
-          type: composite
-          evaluators:
-            - name: safety
-              type: llm_judge
-              prompt: ./prompts/safety-check.md
-            - name: quality
-              type: llm_judge
-              prompt: ./prompts/quality-check.md
-          aggregator:
-            type: code_judge
-            path: ./scripts/safety-gate.js
-```
-
-### Multi-Criteria Weighted Evaluation
-
-```yaml
-- name: release_readiness
-  type: composite
-  evaluators:
-    - name: correctness
-      type: llm_judge
-      prompt: ./prompts/correctness.md
-    - name: style
-      type: code_judge
-      script: uv run style_checker.py
-    - name: security
-      type: llm_judge
-      prompt: ./prompts/security.md
-  aggregator:
-    type: weighted_average
-    weights:
-      correctness: 0.5
-      style: 0.2
-      security: 0.3
-```
-
-### Nested Composites
-
-Composites can contain other composites for complex hierarchies:
-
-```yaml
-- name: comprehensive_eval
-  type: composite
-  evaluators:
-    - name: content_quality
-      type: composite
-      evaluators:
-        - name: accuracy
-          type: llm_judge
-          prompt: ./prompts/accuracy.md
-        - name: clarity
-          type: llm_judge
-          prompt: ./prompts/clarity.md
-      aggregator:
-        type: weighted_average
-        weights:
-          accuracy: 0.6
-          clarity: 0.4
-    - name: safety
-      type: llm_judge
-      prompt: ./prompts/safety.md
-  aggregator:
-    type: weighted_average
-    weights:
-      content_quality: 0.7
-      safety: 0.3
-```
-
-## Result Structure
-
-Composite evaluators return nested `evaluator_results`:
-
-```json
-{
-  "score": 0.85,
-  "verdict": "pass",
-  "hits": ["[safety] No harmful content", "[quality] Clear explanation"],
-  "misses": ["[quality] Could use more examples"],
-  "reasoning": "safety: Passed all checks; quality: Good but could improve",
-  "evaluator_results": [
-    {
-      "name": "safety",
-      "type": "llm_judge",
-      "score": 0.95,
-      "verdict": "pass",
-      "hits": ["No harmful content"],
-      "misses": []
-    },
-    {
-      "name": "quality", 
-      "type": "llm_judge",
-      "score": 0.8,
-      "verdict": "pass",
-      "hits": ["Clear explanation"],
-      "misses": ["Could use more examples"]
-    }
-  ]
-}
-```
-
-## Best Practices
-
-1. **Name evaluators clearly** - Names appear in results and debugging output
-2. **Use safety gates for critical checks** - Don't let high quality override safety failures
-3. **Balance weights thoughtfully** - Consider which aspects matter most for your use case
-4. **Keep nesting shallow** - Deep nesting makes debugging harder
-5. **Test aggregators independently** - Verify your custom aggregation logic with unit tests
+# Composite Evaluator Guide
+
+Composite evaluators combine multiple evaluators and aggregate their results. This enables sophisticated evaluation patterns like safety gates, weighted scoring, and conflict resolution.
+
+## Basic Structure
+
+```yaml
+execution:
+  evaluators:
+    - name: my_composite
+      type: composite
+      evaluators:
+        - name: evaluator_1
+          type: llm_judge
+          prompt: ./prompts/check1.md
+        - name: evaluator_2
+          type: code_judge
+          script: uv run check2.py
+      aggregator:
+        type: weighted_average
+        weights:
+          evaluator_1: 0.6
+          evaluator_2: 0.4
+```
+
+## Aggregator Types
+
+### 1. Weighted Average (Default)
+
+Combines scores using weighted arithmetic mean:
+
+```yaml
+aggregator:
+  type: weighted_average
+  weights:
+    safety: 0.3      # 30% weight
+    quality: 0.7     # 70% weight
+```
+
+If weights are omitted, all evaluators have equal weight (1.0).
+
+**Score calculation:**
+```
+final_score = Σ(score_i × weight_i) / Σ(weight_i)
+```
+
+### 2. Code Judge Aggregator
+
+Run custom code to decide final score based on all evaluator results:
+
+```yaml
+aggregator:
+  type: code_judge
+  path: node ./scripts/safety-gate.js
+  cwd: ./evaluators  # optional working directory
+```
+
+**Input (stdin):**
+```json
+{
+  "results": {
+    "safety": { "score": 0.9, "hits": [...], "misses": [...] },
+    "quality": { "score": 0.85, "hits": [...], "misses": [...] }
+  }
+}
+```
+
+**Output (stdout):**
+```json
+{
+  "score": 0.87,
+  "verdict": "pass",
+  "hits": ["Combined check passed"],
+  "misses": [],
+  "reasoning": "Safety gate passed, quality acceptable"
+}
+```
+
+### 3. LLM Judge Aggregator
+
+Use an LLM to resolve conflicts or make nuanced decisions:
+
+```yaml
+aggregator:
+  type: llm_judge
+  prompt: ./prompts/conflict-resolution.md
+```
+
+The `{{EVALUATOR_RESULTS_JSON}}` variable is replaced with the JSON results from all child evaluators.
+
+## Example Patterns
+
+### Safety Gate Pattern
+
+Block outputs that fail safety even if quality is high:
+
+```yaml
+evalcases:
+  - id: safety-gated-response
+    expected_outcome: Safe and accurate response
+    
+    input_messages:
+      - role: user
+        content: Explain quantum computing
+    
+    execution:
+      evaluators:
+        - name: safety_gate
+          type: composite
+          evaluators:
+            - name: safety
+              type: llm_judge
+              prompt: ./prompts/safety-check.md
+            - name: quality
+              type: llm_judge
+              prompt: ./prompts/quality-check.md
+          aggregator:
+            type: code_judge
+            path: ./scripts/safety-gate.js
+```
+
+### Multi-Criteria Weighted Evaluation
+
+```yaml
+- name: release_readiness
+  type: composite
+  evaluators:
+    - name: correctness
+      type: llm_judge
+      prompt: ./prompts/correctness.md
+    - name: style
+      type: code_judge
+      script: uv run style_checker.py
+    - name: security
+      type: llm_judge
+      prompt: ./prompts/security.md
+  aggregator:
+    type: weighted_average
+    weights:
+      correctness: 0.5
+      style: 0.2
+      security: 0.3
+```
+
+### Nested Composites
+
+Composites can contain other composites for complex hierarchies:
+
+```yaml
+- name: comprehensive_eval
+  type: composite
+  evaluators:
+    - name: content_quality
+      type: composite
+      evaluators:
+        - name: accuracy
+          type: llm_judge
+          prompt: ./prompts/accuracy.md
+        - name: clarity
+          type: llm_judge
+          prompt: ./prompts/clarity.md
+      aggregator:
+        type: weighted_average
+        weights:
+          accuracy: 0.6
+          clarity: 0.4
+    - name: safety
+      type: llm_judge
+      prompt: ./prompts/safety.md
+  aggregator:
+    type: weighted_average
+    weights:
+      content_quality: 0.7
+      safety: 0.3
+```
+
+## Result Structure
+
+Composite evaluators return nested `evaluator_results`:
+
+```json
+{
+  "score": 0.85,
+  "verdict": "pass",
+  "hits": ["[safety] No harmful content", "[quality] Clear explanation"],
+  "misses": ["[quality] Could use more examples"],
+  "reasoning": "safety: Passed all checks; quality: Good but could improve",
+  "evaluator_results": [
+    {
+      "name": "safety",
+      "type": "llm_judge",
+      "score": 0.95,
+      "verdict": "pass",
+      "hits": ["No harmful content"],
+      "misses": []
+    },
+    {
+      "name": "quality", 
+      "type": "llm_judge",
+      "score": 0.8,
+      "verdict": "pass",
+      "hits": ["Clear explanation"],
+      "misses": ["Could use more examples"]
+    }
+  ]
+}
+```
+
+## Best Practices
+
+1. **Name evaluators clearly** - Names appear in results and debugging output
+2. **Use safety gates for critical checks** - Don't let high quality override safety failures
+3. **Balance weights thoughtfully** - Consider which aspects matter most for your use case
+4. **Keep nesting shallow** - Deep nesting makes debugging harder
+5. **Test aggregators independently** - Verify your custom aggregation logic with unit tests