agentv 0.11.0 → 0.13.0

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   runCli
-} from "./chunk-7CJK3EYC.js";
+} from "./chunk-WMO5PVPX.js";
 
 // src/cli.ts
 void runCli();

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   createProgram,
   runCli
-} from "./chunk-7CJK3EYC.js";
+} from "./chunk-WMO5PVPX.js";
 export {
   createProgram,
   runCli

package/dist/templates/.agentv/.env.template

@@ -0,0 +1,23 @@
+# Example environment configuration for AgentV
+# Copy this file to .env and fill in your credentials
+
+# Model Provider Selection (Optional - can be configured via targets.yaml)
+PROVIDER=azure
+
+# Azure OpenAI Configuration
+# These are the default environment variable names used in the provided targets.yaml
+AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/
+AZURE_OPENAI_API_KEY=your-api-key-here
+AZURE_DEPLOYMENT_NAME=gpt-4o
+
+# Anthropic Configuration (if using Anthropic provider)
+ANTHROPIC_API_KEY=your-anthropic-api-key-here
+
+# VS Code Workspace Paths for Execution Targets
+# Note: Using forward slashes is recommended for paths in .env files 
+# to avoid issues with escape characters.
+PROJECTX_WORKSPACE_PATH=C:/Users/your-username/OneDrive - Company Pty Ltd/sample.code-workspace
+
+# CLI provider sample (used by the local_cli target)
+PROJECT_ROOT=D:/GitHub/your-username/agentv/docs/examples/simple
+LOCAL_AGENT_TOKEN=your-cli-token

package/dist/templates/{github/prompts/eval-build.prompt.md → .claude/skills/agentv-eval-builder/SKILL.md}

@@ -1,10 +1,14 @@
 ---
-description: 'Apply when writing evals in YAML format'
+name: eval-builder
+description: Create and maintain AgentV YAML evaluation files for testing AI agent performance. Use this skill when creating new eval files, adding eval cases, or configuring custom evaluators (code validators or LLM judges) for agent testing workflows.
 ---
 
+# Eval Builder
+
 ## Schema Reference
-- Schema: `@../contexts/eval-schema.json` (JSON Schema for validation and tooling)
+- Schema: `references/eval-schema.json` (JSON Schema for validation and tooling)
 - Format: YAML with structured content arrays
+- Examples: `references/example-evals.md`
 
 ## Structure Requirements
 - Root level: `$schema` (required: "agentv-eval-v2"), `description` (optional), `target` (optional), `evalcases` (required)
@@ -14,7 +18,54 @@ description: 'Apply when writing evals in YAML format'
 - Message roles: `system`, `user`, `assistant`, `tool`
 - Content types: `text` (inline), `file` (relative or absolute path)
 - Attachments (type: `file`) should default to the `user` role
-- File paths must start with "/" for absolute paths (e.g., "/prompts/file.md")
+- File paths: Relative (from eval file dir) or absolute with "/" prefix (from repo root)
+
+## Custom Evaluators
+
+Configure multiple evaluators per eval case via `execution.evaluators` array.
+
+### Code Evaluators
+Scripts that validate output programmatically:
+
+```yaml
+execution:
+  evaluators:
+    - name: json_format_validator
+      type: code
+      script: uv run validate_output.py
+      cwd: ../../evaluators/scripts
+```
+
+**Contract:**
+- Input (stdin): JSON with `task`, `outcome`, `expected`, `output`, `system_message`, etc.
+- Output (stdout): JSON with `score` (0.0-1.0), `hits`, `misses`, `reasoning`
+
+**Template:** See `references/custom-evaluators.md` for Python code evaluator template
+
+### LLM Judges
+Language models evaluate response quality:
+
+```yaml
+execution:
+  evaluators:
+    - name: content_evaluator
+      type: llm_judge
+      prompt: /evaluators/prompts/correctness.md
+      model: gpt-5-chat
+```
+
+### Evaluator Chaining
+Evaluators run sequentially:
+
+```yaml
+execution:
+  evaluators:
+    - name: format_check      # Runs first
+      type: code
+      script: uv run validate_json.py
+    - name: content_check     # Runs second
+      type: llm_judge
+```
 
 ## Example
 ```yaml
@@ -40,7 +91,6 @@ evalcases:
               def add(a, b):
                   return a + b
               ```
-          # File paths can be relative or absolute
           - type: file
             value: /prompts/python.instructions.md
     
@@ -62,7 +112,8 @@ evalcases:
       evaluators:
         - name: keyword_check
           type: code
-          script: /evaluators/scripts/check_keywords.py
+          script: uv run check_keywords.py
+          cwd: /evaluators/scripts
         - name: semantic_judge
           type: llm_judge
           prompt: /evaluators/prompts/correctness.md
@@ -98,4 +149,4 @@ evalcases:
               unique.sort(reverse=True)
               return unique[1]
           ```
-```
+```

package/dist/templates/.claude/skills/agentv-eval-builder/references/custom-evaluators.md

@@ -0,0 +1,399 @@
+# Custom Evaluators Guide
+
+Guide for writing custom code evaluators and LLM judges for AgentV eval files.
+
+## Code Evaluator Contract
+
+Code evaluators receive input via stdin and write output to stdout, both as JSON.
+
+### Input Format (via stdin)
+
+```json
+{
+  "task": "string describing the task",
+  "outcome": "expected outcome description",
+  "expected": "expected output string",
+  "output": "generated code/text from the agent",
+  "system_message": "system message if any",
+  "guideline_paths": ["path1", "path2"],
+  "attachments": ["file1", "file2"],
+  "user_segments": [{"type": "text", "value": "..."}]
+}
+```
+
+### Output Format (to stdout)
+
+```json
+{
+  "score": 0.85,
+  "hits": ["successful check 1", "successful check 2"],
+  "misses": ["failed check 1"],
+  "reasoning": "Brief explanation of the score"
+}
+```
+
+**Field Requirements:**
+- `score`: Float between 0.0 and 1.0 (required)
+- `hits`: Array of strings describing what passed (optional but recommended)
+- `misses`: Array of strings describing what failed (optional but recommended)
+- `reasoning`: String explaining the score (optional but recommended)
+
+## Python Code Evaluator Template
+
+```python
+#!/usr/bin/env python3
+"""
+Example code evaluator for AgentV
+
+This evaluator checks for specific keywords in the output.
+Replace validation logic as needed.
+"""
+
+import json
+import sys
+from typing import Any
+
+
+def evaluate(input_data: dict[str, Any]) -> dict[str, Any]:
+    """
+    Evaluate the agent output.
+    
+    Args:
+        input_data: Full input context from AgentV
+    
+    Returns:
+        Evaluation result with score, hits, misses, reasoning
+    """
+    # Extract only the fields you need
+    # Most evaluators only need 'output' - avoid using unnecessary fields
+    output = input_data.get("output", "")
+    
+    # Your validation logic here
+    hits = []
+    misses = []
+    
+    # Example: Check for keywords
+    required_keywords = ["async", "await"]
+    for keyword in required_keywords:
+        if keyword in output:
+            hits.append(f"Contains required keyword: {keyword}")
+        else:
+            misses.append(f"Missing required keyword: {keyword}")
+    
+    # Calculate score
+    if not required_keywords:
+        score = 1.0
+    else:
+        score = len(hits) / len(required_keywords)
+    
+    # Build result
+    return {
+        "score": score,
+        "hits": hits,
+        "misses": misses,
+        "reasoning": f"Found {len(hits)}/{len(required_keywords)} required keywords"
+    }
+
+
+def main():
+    """Main entry point for AgentV code evaluator."""
+    try:
+        # Read input from stdin
+        input_data = json.loads(sys.stdin.read())
+        
+        # Run evaluation
+        result = evaluate(input_data)
+        
+        # Write result to stdout
+        print(json.dumps(result, indent=2))
+        
+    except Exception as e:
+        # Error handling: return zero score with error message
+        error_result = {
+            "score": 0.0,
+            "hits": [],
+            "misses": [f"Evaluator error: {str(e)}"],
+            "reasoning": f"Evaluator error: {str(e)}"
+        }
+        print(json.dumps(error_result, indent=2))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
+```
+
+## JSON Format Validator Example
+
+A common pattern is validating JSON output structure:
+
+```python
+#!/usr/bin/env python3
+"""
+JSON Format Validator for AgentV
+Validates that output is valid JSON with required keys.
+"""
+
+import json
+import sys
+from typing import Any
+
+
+def validate_json_format(output: str, required_keys: list[str]) -> dict[str, Any]:
+    """
+    Validate that output is valid JSON with required keys.
+    
+    Args:
+        output: The candidate output to validate
+        required_keys: List of required top-level keys
+    
+    Returns:
+        Evaluation result dict
+    """
+    # Try to parse as JSON
+    try:
+        parsed = json.loads(output.strip())
+    except json.JSONDecodeError as e:
+        return {
+            "score": 0.0,
+            "hits": [],
+            "misses": ["Not valid JSON"],
+            "reasoning": f"Output is not valid JSON. Parse error: {str(e)}"
+        }
+    
+    # Check if it's a dict
+    if not isinstance(parsed, dict):
+        return {
+            "score": 0.0,
+            "hits": [],
+            "misses": ["JSON is not an object/dict"],
+            "reasoning": f"Output is valid JSON but not an object. Got: {type(parsed).__name__}"
+        }
+    
+    # Check for required keys
+    missing_keys = [key for key in required_keys if key not in parsed]
+    present_keys = [key for key in required_keys if key in parsed]
+    
+    if missing_keys:
+        return {
+            "score": 0.0,
+            "hits": [f"Has key: {key}" for key in present_keys],
+            "misses": [f"Missing key: {key}" for key in missing_keys],
+            "reasoning": f"Valid JSON but missing required keys: {', '.join(missing_keys)}"
+        }
+    
+    # All checks passed
+    return {
+        "score": 1.0,
+        "hits": [f"Valid JSON with all required keys: {', '.join(required_keys)}"],
+        "misses": [],
+        "reasoning": f"Valid JSON with all required keys: {', '.join(required_keys)}"
+    }
+
+
+def main():
+    """Main entry point."""
+    try:
+        input_data = json.loads(sys.stdin.read())
+        output = input_data.get("output", "")
+        
+        # Define required keys (customize as needed)
+        required_keys = ["criticalityRating", "reasoning"]
+        
+        result = validate_json_format(output, required_keys)
+        print(json.dumps(result, indent=2))
+        
+    except Exception as e:
+        error_result = {
+            "score": 0.0,
+            "hits": [],
+            "misses": [f"Evaluator error: {str(e)}"],
+            "reasoning": f"Evaluator error: {str(e)}"
+        }
+        print(json.dumps(error_result, indent=2))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
+```
+
+## LLM Judge Prompt Template
+
+LLM judges use markdown prompts to guide evaluation:
+
+```markdown
+# Code Quality Judge
+
+Evaluate the candidate code for quality, correctness, and best practices.
+
+## Evaluation Criteria
+
+Rate the code on:
+1. **Correctness** - Does it solve the problem?
+2. **Style** - Does it follow best practices?
+3. **Completeness** - Are edge cases handled?
+4. **Documentation** - Are there helpful comments/docstrings?
+
+## Scoring Guidelines
+
+- **0.9-1.0:** Excellent - Correct, clean, well-documented
+- **0.7-0.8:** Good - Correct with minor style issues
+- **0.5-0.6:** Adequate - Works but has quality issues
+- **0.3-0.4:** Poor - Has bugs or major style problems
+- **0.0-0.2:** Unacceptable - Does not work or completely wrong
+
+## Output Format
+
+Respond with valid JSON:
+
+```json
+{
+  "score": 0.85,
+  "hits": [
+    "Correctly implements the algorithm",
+    "Good error handling"
+  ],
+  "misses": [
+    "Missing type hints",
+    "No docstring"
+  ],
+  "reasoning": "Code is correct and handles errors well, but lacks documentation."
+}
+```
+```
+
+## Best Practices
+
+### For Code Evaluators
+
+1. **Focus on relevant fields** - Most evaluators only need the `output` field
+2. **Avoid false positives** - Don't check fields like `task` or `expected` unless you specifically need context
+3. **Be deterministic** - Same input should always produce same output
+4. **Handle errors gracefully** - Return a valid result even when evaluation fails
+5. **Provide helpful feedback** - Use `hits` and `misses` to explain the score
+
+### For LLM Judges
+
+1. **Clear criteria** - Define what you're evaluating
+2. **Specific guidelines** - Provide scoring rubrics
+3. **JSON output** - Enforce structured output format
+4. **Examples** - Show what good/bad looks like
+5. **Concise prompts** - Keep instructions focused
+
+### Common Pitfalls to Avoid
+
+**❌ Checking unnecessary fields:**
+```python
+# BAD: Checking 'task' or 'expected' when you only need to validate format
+if "async" in input_data.get("task", ""):
+    # This creates false positives
+```
+
+**✅ Focus on output:**
+```python
+# GOOD: Only check the actual output
+output = input_data.get("output", "")
+if "async" in output:
+    # This is what you actually want to validate
+```
+
+**❌ Brittle string matching:**
+```python
+# BAD: Exact match is too strict
+if output == "The answer is 42":
+    score = 1.0
+```
+
+**✅ Flexible validation:**
+```python
+# GOOD: Check for semantic correctness
+if "42" in output and "answer" in output.lower():
+    score = 1.0
+```
+
+## Running Code Evaluators
+
+### In Eval Files
+
+```yaml
+execution:
+  evaluators:
+    - name: my_validator
+      type: code
+      script: uv run my_validator.py
+      cwd: ./evaluators
+```
+
+### Command Line Testing
+
+Test your evaluator locally:
+
+```bash
+# Create test input
+echo '{
+  "output": "test output here",
+  "task": "test task"
+}' | uv run my_validator.py
+
+# Should output:
+# {
+#   "score": 0.8,
+#   "hits": ["check 1 passed"],
+#   "misses": ["check 2 failed"],
+#   "reasoning": "..."
+# }
+```
+
+## Advanced Patterns
+
+### Combining Multiple Checks
+
+```python
+def evaluate(input_data: dict[str, Any]) -> dict[str, Any]:
+    output = input_data.get("output", "")
+    
+    checks = [
+        ("has_async", "async" in output, "Contains async keyword"),
+        ("has_await", "await" in output, "Contains await keyword"),
+        ("has_try", "try:" in output, "Has error handling"),
+    ]
+    
+    hits = [msg for _, passed, msg in checks if passed]
+    misses = [msg for _, passed, msg in checks if not passed]
+    score = len(hits) / len(checks)
+    
+    return {
+        "score": score,
+        "hits": hits,
+        "misses": misses,
+        "reasoning": f"Passed {len(hits)}/{len(checks)} checks"
+    }
+```
+
+### Weighted Scoring
+
+```python
+def evaluate(input_data: dict[str, Any]) -> dict[str, Any]:
+    output = input_data.get("output", "")
+    
+    # Define checks with weights
+    checks = [
+        ("correctness", is_correct(output), 0.5),
+        ("style", has_good_style(output), 0.3),
+        ("docs", has_docs(output), 0.2),
+    ]
+    
+    hits = [name for name, passed, _ in checks if passed]
+    misses = [name for name, passed, _ in checks if not passed]
+    
+    # Weighted score
+    score = sum(weight for _, passed, weight in checks if passed)
+    
+    return {
+        "score": score,
+        "hits": hits,
+        "misses": misses,
+        "reasoning": f"Weighted score: {score:.2f}"
+    }
+```