agentv 3.9.2 → 3.10.1

package/dist/templates/.claude/skills/agentv-eval-builder/references/batch-cli-evaluator.md

@@ -0,0 +1,316 @@
+# Batch CLI Evaluation Guide
+
+Guide for evaluating batch CLI output where a single runner processes all evalcases at once and outputs JSONL.
+
+## Overview
+
+Batch CLI evaluation is used when:
+- An external tool processes multiple inputs in a single invocation (e.g., AML screening, bulk classification)
+- The runner reads the eval YAML directly to extract all evalcases
+- Output is JSONL with records keyed by evalcase `id`
+- Each evalcase has its own evaluator to validate its corresponding output record
+
+## Execution Flow
+
+1. **AgentV** invokes the batch runner once, passing `--eval <yaml-path>` and `--output <jsonl-path>`
+2. **Batch runner** reads the eval YAML, extracts all evalcases, processes them, writes JSONL output keyed by `id`
+3. **AgentV** parses JSONL, routes each record to its matching evalcase by `id`
+4. **Per-case evaluator** validates the output for each evalcase independently
+
+## Eval File Structure
+
+```yaml
+description: Batch CLI demo using structured input_messages
+execution:
+  target: batch_cli
+
+evalcases:
+  - id: case-001
+    expected_outcome: |-
+      Batch runner returns JSON with decision=CLEAR.
+
+    expected_messages:
+      - role: assistant
+        content:
+          decision: CLEAR  # Structured expected output
+
+    input_messages:
+      - role: system
+        content: You are a batch processor.
+      - role: user
+        content:  # Structured input (runner extracts this)
+          request:
+            type: screening_check
+            jurisdiction: AU
+          row:
+            id: case-001
+            name: Example A
+            amount: 5000
+
+    execution:
+      evaluators:
+        - name: decision-check
+          type: code_judge
+          script: bun run ./scripts/check-output.ts
+          cwd: .
+
+  - id: case-002
+    expected_outcome: |-
+      Batch runner returns JSON with decision=REVIEW.
+
+    expected_messages:
+      - role: assistant
+        content:
+          decision: REVIEW
+
+    input_messages:
+      - role: system
+        content: You are a batch processor.
+      - role: user
+        content:
+          request:
+            type: screening_check
+            jurisdiction: AU
+          row:
+            id: case-002
+            name: Example B
+            amount: 25000
+
+    execution:
+      evaluators:
+        - name: decision-check
+          type: code_judge
+          script: bun run ./scripts/check-output.ts
+          cwd: .
+```
+
+## Batch Runner Implementation
+
+The batch runner reads the eval YAML directly and processes all evalcases in one invocation.
+
+### Runner Contract
+
+**Input:** The runner receives the eval file path via `--eval` flag:
+```bash
+bun run batch-runner.ts --eval ./my-eval.yaml --output ./results.jsonl
+```
+
+**Output:** JSONL file where each line is a JSON object with:
+```json
+{"id": "case-001", "text": "{\"decision\": \"CLEAR\", ...}"}
+{"id": "case-002", "text": "{\"decision\": \"REVIEW\", ...}"}
+```
+
+The `id` field must match the evalcase `id` for AgentV to route output to the correct evaluator.
+
+### Output with Tool Trajectory Support
+
+To enable `tool_trajectory` evaluation, include `output_messages` with `tool_calls`:
+
+```json
+{
+  "id": "case-001",
+  "text": "{\"decision\": \"CLEAR\", ...}",
+  "output_messages": [
+    {
+      "role": "assistant",
+      "tool_calls": [
+        {
+          "tool": "screening_check",
+          "input": { "origin_country": "NZ", "amount": 5000 },
+          "output": { "decision": "CLEAR", "reasons": [] }
+        }
+      ]
+    },
+    {
+      "role": "assistant",
+      "content": { "decision": "CLEAR" }
+    }
+  ]
+}
+```
+
+AgentV extracts tool calls directly from `output_messages[].tool_calls[]` for `tool_trajectory` evaluators. This is the recommended format for batch runners that make tool calls.
+
+### Example Runner (TypeScript)
+
+```typescript
+import fs from 'node:fs/promises';
+import { parse } from 'yaml';
+
+type EvalCase = {
+  id: string;
+  input_messages: Array<{ role: string; content: unknown }>;
+};
+
+async function main() {
+  const args = process.argv.slice(2);
+  const evalPath = getFlag(args, '--eval');
+  const outPath = getFlag(args, '--output');
+
+  // Read and parse eval YAML
+  const yamlText = await fs.readFile(evalPath, 'utf8');
+  const parsed = parse(yamlText);
+  const evalcases = parsed.evalcases as EvalCase[];
+
+  // Process each evalcase
+  const results: Array<{ id: string; text: string }> = [];
+  for (const evalcase of evalcases) {
+    const userContent = findUserContent(evalcase.input_messages);
+    const decision = processInput(userContent); // Your logic here
+
+    results.push({
+      id: evalcase.id,
+      text: JSON.stringify({ decision, ...otherFields }),
+    });
+  }
+
+  // Write JSONL output
+  const jsonl = results.map((r) => JSON.stringify(r)).join('\n') + '\n';
+  await fs.writeFile(outPath, jsonl, 'utf8');
+}
+
+function getFlag(args: string[], name: string): string {
+  const idx = args.indexOf(name);
+  return args[idx + 1];
+}
+
+function findUserContent(messages: Array<{ role: string; content: unknown }>) {
+  return messages.find((m) => m.role === 'user')?.content;
+}
+```
+
+## Evaluator Implementation
+
+Each evalcase has its own evaluator that validates the output. The evaluator receives the standard code_judge input.
+
+### Evaluator Contract
+
+**Input (stdin):** Standard AgentV code_judge format:
+```json
+{
+  "candidate_answer": "{\"id\":\"case-001\",\"decision\":\"CLEAR\",...}",
+  "expected_messages": [{"role": "assistant", "content": {"decision": "CLEAR"}}],
+  "input_messages": [...],
+  ...
+}
+```
+
+**Output (stdout):** Standard evaluator result:
+```json
+{
+  "score": 1.0,
+  "hits": ["decision matches: CLEAR"],
+  "misses": [],
+  "reasoning": "Batch runner decision matches expected."
+}
+```
+
+### Example Evaluator (TypeScript)
+
+```typescript
+import fs from 'node:fs';
+
+type EvalInput = {
+  candidate_answer?: string;
+  expected_messages?: Array<{ role: string; content: unknown }>;
+};
+
+function main() {
+  const stdin = fs.readFileSync(0, 'utf8');
+  const input = JSON.parse(stdin) as EvalInput;
+
+  // Extract expected value from expected_messages
+  const expectedDecision = findExpectedDecision(input.expected_messages);
+
+  // Parse candidate answer (output from batch runner)
+  let candidateDecision: string | undefined;
+  try {
+    const parsed = JSON.parse(input.candidate_answer ?? '');
+    candidateDecision = parsed.decision;
+  } catch {
+    candidateDecision = undefined;
+  }
+
+  // Compare
+  const hits: string[] = [];
+  const misses: string[] = [];
+
+  if (expectedDecision === candidateDecision) {
+    hits.push(`decision matches: ${expectedDecision}`);
+  } else {
+    misses.push(`mismatch: expected=${expectedDecision} actual=${candidateDecision}`);
+  }
+
+  const score = misses.length === 0 ? 1 : 0;
+
+  process.stdout.write(JSON.stringify({
+    score,
+    hits,
+    misses,
+    reasoning: score === 1
+      ? 'Batch runner output matches expected.'
+      : 'Batch runner output did not match expected.',
+  }));
+}
+
+function findExpectedDecision(messages?: Array<{ role: string; content: unknown }>) {
+  if (!messages) return undefined;
+  for (const msg of messages) {
+    if (typeof msg.content === 'object' && msg.content !== null) {
+      return (msg.content as Record<string, unknown>).decision as string;
+    }
+  }
+  return undefined;
+}
+
+main();
+```
+
+## Structured Content in expected_messages
+
+For batch evaluation, use structured objects in `expected_messages.content` to define expected output fields:
+
+```yaml
+expected_messages:
+  - role: assistant
+    content:
+      decision: CLEAR
+      confidence: high
+      reasons: []
+```
+
+The evaluator then extracts these fields and compares against the parsed candidate output.
+
+## Best Practices
+
+1. **Use unique evalcase IDs** - The batch runner and AgentV use `id` to route outputs
+2. **Structured input_messages** - Put structured data in `user.content` for the runner to extract
+3. **Structured expected_messages** - Define expected output as objects for easy validation
+4. **Deterministic runners** - Batch runners should produce consistent output for testing
+5. **Healthcheck support** - Add `--healthcheck` flag for runner validation:
+   ```typescript
+   if (args.includes('--healthcheck')) {
+     console.log('batch-runner: healthy');
+     return;
+   }
+   ```
+
+## Target Configuration
+
+Configure the batch CLI provider in your target:
+
+```yaml
+# In agentv-targets.yaml or eval file
+targets:
+  batch_cli:
+    provider: cli
+    commandTemplate: bun run ./scripts/batch-runner.ts --eval {EVAL_FILE} --output {OUTPUT_FILE}
+    provider_batching: true
+```
+
+Key settings:
+- `provider: cli` - Use CLI provider
+- `provider_batching: true` - Run once for all evalcases
+- `{EVAL_FILE}` - Placeholder for eval file path
+- `{OUTPUT_FILE}` - Placeholder for JSONL output path

package/dist/templates/.claude/skills/agentv-eval-builder/references/compare-command.md

@@ -0,0 +1,137 @@
+# Compare Command
+
+Compare evaluation results between two runs to measure performance differences.
+
+## Usage
+
+```bash
+agentv compare <baseline.jsonl> <candidate.jsonl> [options]
+```
+
+## Arguments
+
+| Argument | Description |
+|----------|-------------|
+| `result1` | Path to baseline JSONL result file |
+| `result2` | Path to candidate JSONL result file |
+| `--threshold`, `-t` | Score delta threshold for win/loss classification (default: 0.1) |
+| `--format`, `-f` | Output format: `table` (default) or `json` |
+| `--json` | Shorthand for `--format=json` |
+
+## How It Works
+
+1. **Load Results**: Reads both JSONL files containing evaluation results
+2. **Match by eval_id**: Pairs results with matching `eval_id` fields
+3. **Compute Deltas**: Calculates `delta = score2 - score1` for each pair
+4. **Classify Outcomes**:
+   - `win`: delta >= threshold (candidate better)
+   - `loss`: delta <= -threshold (baseline better)
+   - `tie`: |delta| < threshold (no significant difference)
+5. **Output Summary**: Human-readable table (default) or JSON
+
+## Output Format
+
+### Table Format (default)
+
+```
+Comparing: baseline.jsonl → candidate.jsonl
+
+  Eval ID        Baseline  Candidate     Delta  Result
+  ─────────────  ────────  ─────────  ────────  ────────
+  safety-check       0.70       0.90     +0.20  ✓ win
+  accuracy-test      0.85       0.80     -0.05  = tie
+  latency-eval       0.90       0.75     -0.15  ✗ loss
+
+Summary: 1 win, 1 loss, 1 tie | Mean Δ: +0.000 | Status: neutral
+```
+
+Colors are used to highlight wins (green), losses (red), and ties (gray). Colors are automatically disabled when output is piped or `NO_COLOR` is set.
+
+### JSON Format (`--json`)
+
+Output uses snake_case for Python ecosystem compatibility:
+
+```json
+{
+  "matched": [
+    {
+      "eval_id": "case-1",
+      "score1": 0.7,
+      "score2": 0.9,
+      "delta": 0.2,
+      "outcome": "win"
+    }
+  ],
+  "unmatched": {
+    "file1": 0,
+    "file2": 0
+  },
+  "summary": {
+    "total": 2,
+    "matched": 1,
+    "wins": 1,
+    "losses": 0,
+    "ties": 0,
+    "mean_delta": 0.2
+  }
+}
+```
+
+## Exit Codes
+
+| Code | Meaning |
+|------|---------|
+| `0` | Candidate is equal or better (meanDelta >= 0) |
+| `1` | Baseline is better (regression detected) |
+
+## Workflow Examples
+
+### Model Comparison
+
+Compare different model versions:
+
+```bash
+# Run baseline evaluation
+agentv eval evals/*.yaml --target gpt-4 --out baseline.jsonl
+
+# Run candidate evaluation
+agentv eval evals/*.yaml --target gpt-4o --out candidate.jsonl
+
+# Compare results
+agentv compare baseline.jsonl candidate.jsonl
+```
+
+### Prompt Optimization
+
+Compare before/after prompt changes:
+
+```bash
+# Run with original prompt
+agentv eval evals/*.yaml --out before.jsonl
+
+# Modify prompt, then run again
+agentv eval evals/*.yaml --out after.jsonl
+
+# Compare with strict threshold
+agentv compare before.jsonl after.jsonl --threshold 0.05
+```
+
+### CI Quality Gate
+
+Fail CI if candidate regresses:
+
+```bash
+#!/bin/bash
+agentv compare baseline.jsonl candidate.jsonl
+if [ $? -eq 1 ]; then
+  echo "Regression detected! Candidate performs worse than baseline."
+  exit 1
+fi
+echo "Candidate is equal or better than baseline."
+```
+
+## Tips
+
+- **Threshold Selection**: Default 0.1 means 10% difference required. Use stricter thresholds (0.05) for critical evaluations.
+- **Unmatched Results**: Check `unmatched` counts to identify eval cases that only exist in one file.
+- **Multiple Comparisons**: Compare against multiple baselines by running the command multiple times.

package/dist/templates/.claude/skills/agentv-eval-builder/references/composite-evaluator.md

@@ -0,0 +1,215 @@
+# Composite Evaluator Guide
+
+Composite evaluators combine multiple evaluators and aggregate their results. This enables sophisticated evaluation patterns like safety gates, weighted scoring, and conflict resolution.
+
+## Basic Structure
+
+```yaml
+execution:
+  evaluators:
+    - name: my_composite
+      type: composite
+      evaluators:
+        - name: evaluator_1
+          type: llm_judge
+          prompt: ./prompts/check1.md
+        - name: evaluator_2
+          type: code_judge
+          script: uv run check2.py
+      aggregator:
+        type: weighted_average
+        weights:
+          evaluator_1: 0.6
+          evaluator_2: 0.4
+```
+
+## Aggregator Types
+
+### 1. Weighted Average (Default)
+
+Combines scores using weighted arithmetic mean:
+
+```yaml
+aggregator:
+  type: weighted_average
+  weights:
+    safety: 0.3      # 30% weight
+    quality: 0.7     # 70% weight
+```
+
+If weights are omitted, all evaluators have equal weight (1.0).
+
+**Score calculation:**
+```
+final_score = Σ(score_i × weight_i) / Σ(weight_i)
+```
+
+### 2. Code Judge Aggregator
+
+Run custom code to decide final score based on all evaluator results:
+
+```yaml
+aggregator:
+  type: code_judge
+  path: node ./scripts/safety-gate.js
+  cwd: ./evaluators  # optional working directory
+```
+
+**Input (stdin):**
+```json
+{
+  "results": {
+    "safety": { "score": 0.9, "hits": [...], "misses": [...] },
+    "quality": { "score": 0.85, "hits": [...], "misses": [...] }
+  }
+}
+```
+
+**Output (stdout):**
+```json
+{
+  "score": 0.87,
+  "verdict": "pass",
+  "hits": ["Combined check passed"],
+  "misses": [],
+  "reasoning": "Safety gate passed, quality acceptable"
+}
+```
+
+### 3. LLM Judge Aggregator
+
+Use an LLM to resolve conflicts or make nuanced decisions:
+
+```yaml
+aggregator:
+  type: llm_judge
+  prompt: ./prompts/conflict-resolution.md
+```
+
+The `{{EVALUATOR_RESULTS_JSON}}` variable is replaced with the JSON results from all child evaluators.
+
+## Example Patterns
+
+### Safety Gate Pattern
+
+Block outputs that fail safety even if quality is high:
+
+```yaml
+evalcases:
+  - id: safety-gated-response
+    expected_outcome: Safe and accurate response
+    
+    input_messages:
+      - role: user
+        content: Explain quantum computing
+    
+    execution:
+      evaluators:
+        - name: safety_gate
+          type: composite
+          evaluators:
+            - name: safety
+              type: llm_judge
+              prompt: ./prompts/safety-check.md
+            - name: quality
+              type: llm_judge
+              prompt: ./prompts/quality-check.md
+          aggregator:
+            type: code_judge
+            path: ./scripts/safety-gate.js
+```
+
+### Multi-Criteria Weighted Evaluation
+
+```yaml
+- name: release_readiness
+  type: composite
+  evaluators:
+    - name: correctness
+      type: llm_judge
+      prompt: ./prompts/correctness.md
+    - name: style
+      type: code_judge
+      script: uv run style_checker.py
+    - name: security
+      type: llm_judge
+      prompt: ./prompts/security.md
+  aggregator:
+    type: weighted_average
+    weights:
+      correctness: 0.5
+      style: 0.2
+      security: 0.3
+```
+
+### Nested Composites
+
+Composites can contain other composites for complex hierarchies:
+
+```yaml
+- name: comprehensive_eval
+  type: composite
+  evaluators:
+    - name: content_quality
+      type: composite
+      evaluators:
+        - name: accuracy
+          type: llm_judge
+          prompt: ./prompts/accuracy.md
+        - name: clarity
+          type: llm_judge
+          prompt: ./prompts/clarity.md
+      aggregator:
+        type: weighted_average
+        weights:
+          accuracy: 0.6
+          clarity: 0.4
+    - name: safety
+      type: llm_judge
+      prompt: ./prompts/safety.md
+  aggregator:
+    type: weighted_average
+    weights:
+      content_quality: 0.7
+      safety: 0.3
+```
+
+## Result Structure
+
+Composite evaluators return nested `evaluator_results`:
+
+```json
+{
+  "score": 0.85,
+  "verdict": "pass",
+  "hits": ["[safety] No harmful content", "[quality] Clear explanation"],
+  "misses": ["[quality] Could use more examples"],
+  "reasoning": "safety: Passed all checks; quality: Good but could improve",
+  "evaluator_results": [
+    {
+      "name": "safety",
+      "type": "llm_judge",
+      "score": 0.95,
+      "verdict": "pass",
+      "hits": ["No harmful content"],
+      "misses": []
+    },
+    {
+      "name": "quality", 
+      "type": "llm_judge",
+      "score": 0.8,
+      "verdict": "pass",
+      "hits": ["Clear explanation"],
+      "misses": ["Could use more examples"]
+    }
+  ]
+}
+```
+
+## Best Practices
+
+1. **Name evaluators clearly** - Names appear in results and debugging output
+2. **Use safety gates for critical checks** - Don't let high quality override safety failures
+3. **Balance weights thoughtfully** - Consider which aspects matter most for your use case
+4. **Keep nesting shallow** - Deep nesting makes debugging harder
+5. **Test aggregators independently** - Verify your custom aggregation logic with unit tests

package/dist/templates/.claude/skills/agentv-eval-builder/references/config-schema.json

@@ -0,0 +1,27 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "AgentV Config Schema",
+  "description": "Schema for .agentv/config.yaml configuration files",
+  "type": "object",
+  "properties": {
+    "$schema": {
+      "type": "string",
+      "description": "Schema identifier",
+      "enum": ["agentv-config-v2"]
+    },
+    "guideline_patterns": {
+      "type": "array",
+      "description": "Glob patterns for identifying guideline files (instructions, prompts). Files matching these patterns are treated as guidelines, while non-matching files are treated as regular file content.",
+      "items": {
+        "type": "string",
+        "description": "Glob pattern (e.g., '**/*.instructions.md', '**/prompts/**')"
+      },
+      "examples": [
+        ["**/*.instructions.md", "**/instructions/**", "**/*.prompt.md", "**/prompts/**"],
+        ["**/*.guide.md", "**/guidelines/**", "docs/AGENTS.md"]
+      ]
+    }
+  },
+  "required": ["$schema"],
+  "additionalProperties": false
+}