agentv 3.10.2 → 3.10.3

package/dist/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json

@@ -1,278 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "AgentV Eval Schema",
-  "description": "Schema for YAML evaluation files with conversation flows, multiple evaluators, and execution configuration",
-  "type": "object",
-  "properties": {
-    "description": {
-      "type": "string",
-      "description": "Description of what this eval suite covers"
-    },
-    "target": {
-      "type": "string",
-      "description": "(Deprecated: use execution.target instead) Default target configuration name. Can be overridden per eval case."
-    },
-    "execution": {
-      "type": "object",
-      "description": "Default execution configuration for all eval cases (can be overridden per case)",
-      "properties": {
-        "target": {
-          "type": "string",
-          "description": "Default target configuration name (e.g., default, azure_base, vscode_projectx). Can be overridden per eval case."
-        },
-        "evaluators": {
-          "type": "array",
-          "description": "Default evaluators for all eval cases (code-based and LLM judges)",
-          "items": {
-            "type": "object",
-            "properties": {
-              "name": {
-                "type": "string",
-                "description": "Evaluator name/identifier"
-              },
-              "type": {
-                "type": "string",
-                "enum": [
-                  "code",
-                  "llm_judge",
-                  "composite",
-                  "tool_trajectory",
-                  "field_accuracy",
-                  "latency",
-                  "cost",
-                  "token_usage"
-                ],
-                "description": "Evaluator type: 'code' for scripts/regex/keywords, 'llm_judge' for LLM-based evaluation"
-              },
-              "script": {
-                "type": "string",
-                "description": "Path to evaluator script (for type: code)"
-              },
-              "prompt": {
-                "type": "string",
-                "description": "Path to judge prompt file (for type: llm_judge)"
-              }
-            },
-            "required": ["name", "type"],
-            "additionalProperties": true
-          }
-        }
-      },
-      "additionalProperties": true
-    },
-    "evalcases": {
-      "type": "array",
-      "description": "Array of evaluation cases",
-      "minItems": 1,
-      "items": {
-        "type": "object",
-        "properties": {
-          "id": {
-            "type": "string",
-            "description": "Unique identifier for the eval case"
-          },
-          "conversation_id": {
-            "type": "string",
-            "description": "Optional conversation identifier for threading multiple eval cases together"
-          },
-          "expected_outcome": {
-            "type": "string",
-            "description": "Description of what the AI should accomplish in this eval"
-          },
-          "note": {
-            "type": "string",
-            "description": "Optional note or additional context for the eval case. Use this to document test-specific considerations, known limitations, or rationale for expected behavior."
-          },
-          "input_messages": {
-            "type": "array",
-            "description": "Input messages for the conversation",
-            "minItems": 1,
-            "items": {
-              "type": "object",
-              "properties": {
-                "role": {
-                  "type": "string",
-                  "enum": ["system", "user", "assistant", "tool"],
-                  "description": "Message role"
-                },
-                "content": {
-                  "oneOf": [
-                    {
-                      "type": "string",
-                      "description": "Simple text content"
-                    },
-                    {
-                      "type": "array",
-                      "description": "Mixed content items (text and file references)",
-                      "items": {
-                        "type": "object",
-                        "properties": {
-                          "type": {
-                            "type": "string",
-                            "enum": ["text", "file"],
-                            "description": "Content type: 'text' for inline content, 'file' for file references"
-                          },
-                          "value": {
-                            "type": "string",
-                            "description": "Text content or file path. Relative paths (e.g., ../prompts/file.md) are resolved from eval file directory. Absolute paths (e.g., /docs/examples/prompts/file.md) are resolved from repo root."
-                          }
-                        },
-                        "required": ["type", "value"],
-                        "additionalProperties": false
-                      }
-                    }
-                  ]
-                }
-              },
-              "required": ["role", "content"],
-              "additionalProperties": false
-            }
-          },
-          "input": {
-            "description": "Alias for input_messages with shorthand support. String expands to single user message, array of messages passes through.",
-            "oneOf": [
-              {
-                "type": "string",
-                "description": "Shorthand: single user message content"
-              },
-              {
-                "type": "array",
-                "description": "Array of messages (same format as input_messages)",
-                "items": {
-                  "type": "object",
-                  "properties": {
-                    "role": {
-                      "type": "string",
-                      "enum": ["system", "user", "assistant", "tool"]
-                    },
-                    "content": {
-                      "oneOf": [{ "type": "string" }, { "type": "array" }]
-                    }
-                  },
-                  "required": ["role", "content"]
-                }
-              }
-            ]
-          },
-          "expected_messages": {
-            "type": "array",
-            "description": "Expected response messages. Canonical form — use this or expected_output (alias). The content of the last entry is derived as the template variable 'reference_answer' for evaluator prompts.",
-            "minItems": 1,
-            "items": {
-              "type": "object",
-              "properties": {
-                "role": {
-                  "type": "string",
-                  "enum": ["system", "user", "assistant", "tool"],
-                  "description": "Message role"
-                },
-                "content": {
-                  "oneOf": [
-                    {
-                      "type": "string",
-                      "description": "Simple text content"
-                    },
-                    {
-                      "type": "array",
-                      "description": "Mixed content items",
-                      "items": {
-                        "type": "object",
-                        "properties": {
-                          "type": {
-                            "type": "string",
-                            "enum": ["text", "file"]
-                          },
-                          "value": {
-                            "type": "string"
-                          }
-                        },
-                        "required": ["type", "value"],
-                        "additionalProperties": false
-                      }
-                    }
-                  ]
-                }
-              },
-              "required": ["role", "content"],
-              "additionalProperties": false
-            }
-          },
-          "expected_output": {
-            "description": "Alias for expected_messages with shorthand support. String expands to single assistant message, object wraps as assistant message content. Resolves to expected_messages internally — the content of the last resolved entry becomes the template variable 'reference_answer'.",
-            "oneOf": [
-              {
-                "type": "string",
-                "description": "Shorthand: single assistant message content"
-              },
-              {
-                "type": "object",
-                "description": "Shorthand: structured content wraps as assistant message"
-              },
-              {
-                "type": "array",
-                "description": "Array of messages (same format as expected_messages)",
-                "items": {
-                  "type": "object",
-                  "properties": {
-                    "role": {
-                      "type": "string",
-                      "enum": ["system", "user", "assistant", "tool"]
-                    },
-                    "content": {
-                      "oneOf": [{ "type": "string" }, { "type": "object" }, { "type": "array" }]
-                    }
-                  },
-                  "required": ["role", "content"]
-                }
-              }
-            ]
-          },
-          "execution": {
-            "type": "object",
-            "description": "Per-case execution configuration",
-            "properties": {
-              "target": {
-                "type": "string",
-                "description": "Override target for this specific eval case"
-              },
-              "evaluators": {
-                "type": "array",
-                "description": "Multiple evaluators (code-based and LLM judges)",
-                "items": {
-                  "type": "object",
-                  "properties": {
-                    "name": {
-                      "type": "string",
-                      "description": "Evaluator name/identifier"
-                    },
-                    "type": {
-                      "type": "string",
-                      "enum": ["code", "llm_judge"],
-                      "description": "Evaluator type: 'code' for scripts/regex/keywords, 'llm_judge' for LLM-based evaluation"
-                    },
-                    "script": {
-                      "type": "string",
-                      "description": "Path to evaluator script (for type: code)"
-                    },
-                    "prompt": {
-                      "type": "string",
-                      "description": "Path to judge prompt file (for type: llm_judge)"
-                    }
-                  },
-                  "required": ["name", "type"],
-                  "additionalProperties": true
-                }
-              }
-            },
-            "additionalProperties": true
-          }
-        },
-        "required": ["id", "expected_outcome"],
-        "anyOf": [{ "required": ["input_messages"] }, { "required": ["input"] }],
-        "additionalProperties": true
-      }
-    }
-  },
-  "required": ["evalcases"],
-  "additionalProperties": false
-}

package/dist/templates/.claude/skills/agentv-eval-builder/references/example-evals.md

@@ -1,333 +0,0 @@
-# Example Eval Files
-
-This document contains complete examples of well-structured eval files demonstrating various AgentV patterns and best practices.
-
-## Basic Example: Simple Q&A Eval
-
-```yaml
-description: Basic arithmetic evaluation
-execution:
-  target: default
-
-evalcases:
-  - id: simple-addition
-    expected_outcome: Correctly calculates 2+2
-    
-    input_messages:
-      - role: user
-        content: What is 2 + 2?
-    
-    expected_messages:
-      - role: assistant
-        content: "4"
-```
-
-## Code Review with File References
-
-```yaml
-description: Code review with guidelines
-execution:
-  target: azure_base
-
-evalcases:
-  - id: code-review-basic
-    expected_outcome: Assistant provides helpful code analysis with security considerations
-    
-    input_messages:
-      - role: system
-        content: You are an expert code reviewer.
-      - role: user
-        content:
-          - type: text
-            value: |-
-              Review this function for security issues:
-              
-              ```python
-              def get_user(user_id):
-                  query = f"SELECT * FROM users WHERE id = {user_id}"
-                  return db.execute(query)
-              ```
-          - type: file
-            value: /prompts/security-guidelines.md
-    
-    expected_messages:
-      - role: assistant
-        content: |-
-          This code has a critical SQL injection vulnerability. The user_id is directly 
-          interpolated into the query string without sanitization.
-          
-          Recommended fix:
-          ```python
-          def get_user(user_id):
-              query = "SELECT * FROM users WHERE id = ?"
-              return db.execute(query, (user_id,))
-          ```
-```
-
-## Multi-Evaluator Configuration
-
-```yaml
-description: JSON generation with validation
-execution:
-  target: default
-
-evalcases:
-  - id: json-generation-with-validation
-    expected_outcome: Generates valid JSON with required fields
-    
-    execution:
-      evaluators:
-        - name: json_format_validator
-          type: code_judge
-          script: uv run validate_json.py
-          cwd: ./evaluators
-        - name: content_evaluator
-          type: llm_judge
-          prompt: ./judges/semantic_correctness.md
-    
-    input_messages:
-      - role: user
-        content: |-
-          Generate a JSON object for a user with name "Alice", 
-          email "alice@example.com", and role "admin".
-    
-    expected_messages:
-      - role: assistant
-        content: |-
-          {
-            "name": "Alice",
-            "email": "alice@example.com",
-            "role": "admin"
-          }
-```
-
-## Tool Trajectory Evaluation
-
-Validate that an agent uses specific tools during execution.
-
-```yaml
-description: Tool usage validation
-execution:
-  target: mock_agent
-
-evalcases:
-  # Validate minimum tool usage (order doesn't matter)
-  - id: research-depth
-    expected_outcome: Agent researches thoroughly
-    input_messages:
-      - role: user
-        content: Research REST vs GraphQL
-    execution:
-      evaluators:
-        - name: research-check
-          type: tool_trajectory
-          mode: any_order
-          minimums:
-            knowledgeSearch: 2
-            documentRetrieve: 1
-
-  # Validate exact tool sequence
-  - id: auth-flow
-    expected_outcome: Agent follows auth sequence
-    input_messages:
-      - role: user
-        content: Authenticate user
-    execution:
-      evaluators:
-        - name: auth-sequence
-          type: tool_trajectory
-          mode: exact
-          expected:
-            - tool: checkCredentials
-            - tool: generateToken
-```
-
-## Static Trace Evaluation
-
-Evaluate pre-existing trace files without running an agent.
-
-```yaml
-description: Static trace evaluation
-execution:
-  target: static_trace
-
-evalcases:
-  - id: validate-trace-file
-    expected_outcome: Trace contains required steps
-    input_messages:
-      - role: user
-        content: Analyze trace
-    execution:
-      evaluators:
-        - name: trace-check
-          type: tool_trajectory
-          mode: in_order
-          expected:
-            - tool: webSearch
-            - tool: readFile
-```
-
-## Multi-Turn Conversation (Single Eval Case)
-
-```yaml
-description: Multi-turn debugging session with clarifying questions
-execution:
-  target: default
-
-evalcases:
-  - id: debug-with-clarification
-    expected_outcome: |-
-      Assistant conducts a multi-turn debugging session, asking clarification
-      questions when needed, correctly diagnosing the bug, and proposing a clear
-      fix with rationale.
-    
-    input_messages:
-      - role: system
-        content: You are an expert debugging assistant who reasons step by step, asks clarifying questions, and explains fixes clearly.
-      - role: user
-        content: |-
-          I'm getting an off-by-one error in this function, but I can't see why:
-
-          ```python
-          def get_items(items):
-              result = []
-              for i in range(len(items) - 1):
-                  result.append(items[i])
-              return result
-          ```
-
-          Sometimes the last element is missing. Can you help debug this?
-      - role: assistant
-        content: |-
-          I can help debug this. Before I propose a fix, could you tell me:
-          - What output you expect for an example input list
-          - What output you actually get
-      - role: user
-        content: |-
-          For `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`.
-    
-    expected_messages:
-      - role: assistant
-        content: |-
-          You have an off-by-one error in your loop bounds.
-          You're iterating with `range(len(items) - 1)`, which stops before the last index.
-          To include all items, you can either:
-          - Use `range(len(items))`, or
-          - Iterate directly over the list: `for item in items:`
-          
-          Here's a corrected version:
-
-          ```python
-          def get_items(items):
-              result = []
-              for item in items:
-                  result.append(item)
-              return result
-          ```
-```
-
-## Batch CLI Evaluation
-
-Evaluate external batch runners that process all evalcases in one invocation.
-
-```yaml
-description: Batch CLI demo (AML screening)
-execution:
-  target: batch_cli
-
-evalcases:
-  - id: aml-001
-    expected_outcome: |-
-      Batch runner returns JSON with decision=CLEAR.
-
-    expected_messages:
-      - role: assistant
-        content:
-          decision: CLEAR
-
-    input_messages:
-      - role: system
-        content: You are a deterministic AML screening batch checker.
-      - role: user
-        content:
-          request:
-            type: aml_screening_check
-            jurisdiction: AU
-            effective_date: 2025-01-01
-          row:
-            id: aml-001
-            customer_name: Example Customer A
-            origin_country: NZ
-            destination_country: AU
-            transaction_type: INTERNATIONAL_TRANSFER
-            amount: 5000
-            currency: USD
-
-    execution:
-      evaluators:
-        - name: decision-check
-          type: code_judge
-          script: bun run ./scripts/check-batch-cli-output.ts
-          cwd: .
-
-  - id: aml-002
-    expected_outcome: |-
-      Batch runner returns JSON with decision=REVIEW.
-
-    expected_messages:
-      - role: assistant
-        content:
-          decision: REVIEW
-
-    input_messages:
-      - role: system
-        content: You are a deterministic AML screening batch checker.
-      - role: user
-        content:
-          request:
-            type: aml_screening_check
-            jurisdiction: AU
-            effective_date: 2025-01-01
-          row:
-            id: aml-002
-            customer_name: Example Customer B
-            origin_country: IR
-            destination_country: AU
-            transaction_type: INTERNATIONAL_TRANSFER
-            amount: 2000
-            currency: USD
-
-    execution:
-      evaluators:
-        - name: decision-check
-          type: code_judge
-          script: bun run ./scripts/check-batch-cli-output.ts
-          cwd: .
-```
-
-### Batch CLI Pattern Notes
-- **execution.target: batch_cli** - Configure CLI provider with `provider_batching: true`
-- **Batch runner** - Reads eval YAML via `--eval` flag, outputs JSONL keyed by `id`
-- **Structured input** - Put data in `user.content` as objects for runner to extract
-- **Structured expected** - Use `expected_messages.content` with object fields
-- **Per-case evaluators** - Each evalcase has its own evaluator to validate output
-
-## Notes on Examples
-
-### File Path Conventions
-- **Absolute paths** (start with `/`): Resolved from repository root
-  - Example: `/prompts/guidelines.md` → `<repo_root>/prompts/guidelines.md`
-- **Relative paths** (start with `./` or `../`): Resolved from eval file directory
-  - Example: `../../prompts/file.md` → Two directories up, then into prompts/
-
-### expected_outcome Writing Tips
-- Be specific about what success looks like
-- Mention key elements that must be present
-- For classification tasks, specify the expected category
-- For reasoning tasks, describe the thought process expected
-
-### Expected Messages
-- Show the pattern, not rigid templates
-- Allow for natural language variation
-- Focus on semantic correctness over exact matching
-- Evaluators will handle the actual validation

package/dist/templates/.claude/skills/agentv-eval-builder/references/rubric-evaluator.md

@@ -1,79 +0,0 @@
-# Rubric Evaluator
-
-## Field Reference
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `id` | string | auto-generated | Unique identifier |
-| `expected_outcome` | string | required* | Criterion being evaluated (*optional if `score_ranges` used) |
-| `weight` | number | 1.0 | Relative importance |
-| `required` | boolean | true | Failing forces verdict to 'fail' (checklist mode) |
-| `required_min_score` | integer | - | Minimum 0-10 score to pass (score-range mode) |
-| `score_ranges` | map or array | - | Score range definitions for analytic scoring |
-
-`description` is a backward-compatible alias for `expected_outcome`.
-
-## Checklist Mode
-
-```yaml
-rubrics:
-  - Mentions divide-and-conquer approach
-  - id: complexity
-    expected_outcome: States time complexity correctly
-    weight: 2.0
-    required: true
-  - id: examples
-    expected_outcome: Includes code examples
-    weight: 1.0
-    required: false
-```
-
-## Score-Range Mode
-
-Shorthand map format (recommended):
-
-```yaml
-rubrics:
-  - id: correctness
-    weight: 2.0
-    required_min_score: 7
-    score_ranges:
-      0: Critical bugs
-      3: Minor bugs
-      6: Correct with minor issues
-      9: Fully correct
-```
-
-Map keys are lower bounds (0-10). Each range extends from its key to (next key - 1), with the last extending to 10. Must start at 0.
-
-Array format is also accepted:
-
-```yaml
-    score_ranges:
-      - score_range: [0, 2]
-        expected_outcome: Critical bugs
-      - score_range: [3, 5]
-        expected_outcome: Minor bugs
-      - score_range: [6, 8]
-        expected_outcome: Correct with minor issues
-      - score_range: [9, 10]
-        expected_outcome: Fully correct
-```
-
-Ranges must be integers 0-10, non-overlapping, covering all values 0-10.
-
-## Scoring
-
-**Checklist:** `score = sum(satisfied weights) / sum(all weights)`
-
-**Score-range:** `score = weighted_average(raw_score / 10)` per criterion
-
-## Verdicts
-
-| Verdict | Condition |
-|---------|-----------|
-| `pass` | score >= 0.8 AND all gating criteria satisfied |
-| `borderline` | score >= 0.6 AND all gating criteria satisfied |
-| `fail` | score < 0.6 OR any gating criterion failed |
-
-Gating: checklist uses `required: true`, score-range uses `required_min_score: N`.