agentv 2.6.0 → 2.7.1-next.3

package/dist/templates/.agents/skills/agentv-eval-builder/references/eval-schema.json

@@ -0,0 +1,251 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "AgentV Eval Schema",
+  "description": "Schema for YAML evaluation files with conversation flows, multiple evaluators, and execution configuration",
+  "type": "object",
+  "properties": {
+    "description": {
+      "type": "string",
+      "description": "Description of what this eval suite covers"
+    },
+    "target": {
+      "type": "string",
+      "description": "(Deprecated: use execution.target instead) Default target configuration name. Can be overridden per test."
+    },
+    "execution": {
+      "type": "object",
+      "description": "Default execution configuration for all tests (can be overridden per test)",
+      "properties": {
+        "target": {
+          "type": "string",
+          "description": "Default target configuration name (e.g., default, azure_base, vscode_projectx). Can be overridden per test."
+        },
+        "evaluators": {
+          "type": "array",
+          "description": "Default evaluators appended to every test's evaluators (unless skip_defaults is set per test)",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string",
+                "description": "Evaluator name/identifier"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "code",
+                  "llm_judge",
+                  "composite",
+                  "tool_trajectory",
+                  "field_accuracy",
+                  "latency",
+                  "cost",
+                  "token_usage"
+                ],
+                "description": "Evaluator type: 'code' for scripts/regex/keywords, 'llm_judge' for LLM-based evaluation"
+              },
+              "script": {
+                "type": "string",
+                "description": "Path to evaluator script (for type: code)"
+              },
+              "prompt": {
+                "type": "string",
+                "description": "Path to judge prompt file (for type: llm_judge)"
+              }
+            },
+            "required": ["name", "type"],
+            "additionalProperties": true
+          }
+        }
+      },
+      "additionalProperties": true
+    },
+    "tests": {
+      "type": "array",
+      "description": "Array of evaluation tests",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string",
+            "description": "Unique identifier for the test"
+          },
+          "conversation_id": {
+            "type": "string",
+            "description": "Optional conversation identifier for threading multiple tests together"
+          },
+          "criteria": {
+            "type": "string",
+            "description": "Description of what the AI should accomplish in this eval"
+          },
+          "note": {
+            "type": "string",
+            "description": "Optional note or additional context for the test. Use this to document test-specific considerations, known limitations, or rationale for expected behavior."
+          },
+          "input": {
+            "description": "Input messages for the conversation. String expands to single user message, array of messages passes through.",
+            "oneOf": [
+              {
+                "type": "string",
+                "description": "Shorthand: single user message content"
+              },
+              {
+                "type": "array",
+                "description": "Array of messages",
+                "minItems": 1,
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "role": {
+                      "type": "string",
+                      "enum": ["system", "user", "assistant", "tool"],
+                      "description": "Message role"
+                    },
+                    "content": {
+                      "oneOf": [
+                        {
+                          "type": "string",
+                          "description": "Simple text content"
+                        },
+                        {
+                          "type": "array",
+                          "description": "Mixed content items (text and file references)",
+                          "items": {
+                            "type": "object",
+                            "properties": {
+                              "type": {
+                                "type": "string",
+                                "enum": ["text", "file"],
+                                "description": "Content type: 'text' for inline content, 'file' for file references"
+                              },
+                              "value": {
+                                "type": "string",
+                                "description": "Text content or file path. Relative paths (e.g., ../prompts/file.md) are resolved from eval file directory. Absolute paths (e.g., /docs/examples/prompts/file.md) are resolved from repo root."
+                              }
+                            },
+                            "required": ["type", "value"],
+                            "additionalProperties": false
+                          }
+                        }
+                      ]
+                    }
+                  },
+                  "required": ["role", "content"],
+                  "additionalProperties": false
+                }
+              }
+            ]
+          },
+          "expected_output": {
+            "description": "Expected response messages. String expands to single assistant message, object wraps as assistant message content. The content of the last resolved entry becomes the template variable 'reference_answer'.",
+            "oneOf": [
+              {
+                "type": "string",
+                "description": "Shorthand: single assistant message content"
+              },
+              {
+                "type": "object",
+                "description": "Shorthand: structured content wraps as assistant message"
+              },
+              {
+                "type": "array",
+                "description": "Array of messages",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "role": {
+                      "type": "string",
+                      "enum": ["system", "user", "assistant", "tool"],
+                      "description": "Message role"
+                    },
+                    "content": {
+                      "oneOf": [
+                        {
+                          "type": "string",
+                          "description": "Simple text content"
+                        },
+                        {
+                          "type": "object",
+                          "description": "Structured content object"
+                        },
+                        {
+                          "type": "array",
+                          "description": "Mixed content items",
+                          "items": {
+                            "type": "object",
+                            "properties": {
+                              "type": {
+                                "type": "string",
+                                "enum": ["text", "file"]
+                              },
+                              "value": {
+                                "type": "string"
+                              }
+                            },
+                            "required": ["type", "value"],
+                            "additionalProperties": false
+                          }
+                        }
+                      ]
+                    }
+                  },
+                  "required": ["role", "content"],
+                  "additionalProperties": false
+                }
+              }
+            ]
+          },
+          "execution": {
+            "type": "object",
+            "description": "Per-case execution configuration",
+            "properties": {
+              "target": {
+                "type": "string",
+                "description": "Override target for this specific test"
+              },
+              "skip_defaults": {
+                "type": "boolean",
+                "description": "When true, root-level execution.evaluators are not appended to this test's evaluators"
+              },
+              "evaluators": {
+                "type": "array",
+                "description": "Per-test evaluators (root-level evaluators are appended unless skip_defaults is true)",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "name": {
+                      "type": "string",
+                      "description": "Evaluator name/identifier"
+                    },
+                    "type": {
+                      "type": "string",
+                      "enum": ["code", "llm_judge"],
+                      "description": "Evaluator type: 'code' for scripts/regex/keywords, 'llm_judge' for LLM-based evaluation"
+                    },
+                    "script": {
+                      "type": "string",
+                      "description": "Path to evaluator script (for type: code)"
+                    },
+                    "prompt": {
+                      "type": "string",
+                      "description": "Path to judge prompt file (for type: llm_judge)"
+                    }
+                  },
+                  "required": ["name", "type"],
+                  "additionalProperties": true
+                }
+              }
+            },
+            "additionalProperties": true
+          }
+        },
+        "required": ["id", "criteria"],
+        "anyOf": [{ "required": ["input"] }],
+        "additionalProperties": true
+      }
+    }
+  },
+  "required": ["tests"],
+  "additionalProperties": false
+}

package/dist/templates/.agents/skills/agentv-eval-builder/references/rubric-evaluator.md

@@ -0,0 +1,77 @@
+# Rubric Evaluator
+
+## Field Reference
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `id` | string | auto-generated | Unique identifier |
+| `outcome` | string | required* | Criterion being evaluated (*optional if `score_ranges` used) |
+| `weight` | number | 1.0 | Relative importance |
+| `required` | boolean | true | Failing forces verdict to 'fail' (checklist mode) |
+| `required_min_score` | integer | - | Minimum 0-10 score to pass (score-range mode) |
+| `score_ranges` | map or array | - | Score range definitions for analytic scoring |
+
+## Checklist Mode
+
+```yaml
+rubrics:
+  - Mentions divide-and-conquer approach
+  - id: complexity
+    outcome: States time complexity correctly
+    weight: 2.0
+    required: true
+  - id: examples
+    outcome: Includes code examples
+    weight: 1.0
+    required: false
+```
+
+## Score-Range Mode
+
+Shorthand map format (recommended):
+
+```yaml
+rubrics:
+  - id: correctness
+    weight: 2.0
+    required_min_score: 7
+    score_ranges:
+      0: Critical bugs
+      3: Minor bugs
+      6: Correct with minor issues
+      9: Fully correct
+```
+
+Map keys are lower bounds (0-10). Each range extends from its key to (next key - 1), with the last extending to 10. Must start at 0.
+
+Array format is also accepted:
+
+```yaml
+    score_ranges:
+      - score_range: [0, 2]
+        outcome: Critical bugs
+      - score_range: [3, 5]
+        outcome: Minor bugs
+      - score_range: [6, 8]
+        outcome: Correct with minor issues
+      - score_range: [9, 10]
+        outcome: Fully correct
+```
+
+Ranges must be integers 0-10, non-overlapping, covering all values 0-10.
+
+## Scoring
+
+**Checklist:** `score = sum(satisfied weights) / sum(all weights)`
+
+**Score-range:** `score = weighted_average(raw_score / 10)` per criterion
+
+## Verdicts
+
+| Verdict | Condition |
+|---------|-----------|
+| `pass` | score >= 0.8 AND all gating criteria satisfied |
+| `borderline` | score >= 0.6 AND all gating criteria satisfied |
+| `fail` | score < 0.6 OR any gating criterion failed |
+
+Gating: checklist uses `required: true`, score-range uses `required_min_score: N`.

package/dist/templates/.agents/skills/agentv-eval-orchestrator/SKILL.md

@@ -0,0 +1,50 @@
+---
+name: agentv-eval-orchestrator
+description: Run AgentV evaluations without API keys by orchestrating eval subcommands. Use this skill when asked to run evals, evaluate an agent, or test prompt quality using agentv.
+---
+
+# AgentV Eval Orchestrator
+
+Run AgentV evaluations by acting as the LLM yourself — no API keys needed.
+
+## Quick Start
+
+```bash
+agentv prompt eval <eval-file.yaml>
+```
+
+This outputs a complete orchestration prompt with step-by-step instructions and all test IDs. Follow its instructions.
+
+## Workflow
+
+For each test, run these three steps:
+
+### 1. Get Task Input
+
+```bash
+agentv prompt eval input <path> --test-id <id>
+```
+
+Returns JSON with `input`, `guideline_paths`, and `criteria`. File references in messages use absolute paths — read them from the filesystem.
+
+### 2. Execute the Task
+
+You ARE the candidate LLM. Read `input` from step 1, read any referenced files, and answer the task. Save your response to a temp file.
+
+**Important**: Do not leak `criteria` into your answer — it's for your reference when judging, not part of the task.
+
+### 3. Judge the Result
+
+```bash
+agentv prompt eval judge <path> --test-id <id> --answer-file /tmp/eval_<id>.txt
+```
+
+Returns JSON with an `evaluators` array. Each evaluator has a `status`:
+
+- **`"completed"`** — Deterministic score is final. Read `result.score` (0.0–1.0).
+- **`"prompt_ready"`** — LLM grading required. Send `prompt.system_prompt` as system and `prompt.user_prompt` as user to yourself. Parse the JSON response to get `score`, `hits`, `misses`.
+
+## When to use this vs `agentv eval`
+
+- **`agentv eval`** — You have API keys configured. Runs everything end-to-end automatically.
+- **`agentv prompt`** — No API keys. You orchestrate: get input, answer the task yourself, judge the result.

package/dist/templates/.agents/skills/agentv-prompt-optimizer/SKILL.md

@@ -0,0 +1,78 @@
+---
+name: agentv-prompt-optimizer
+description: Iteratively optimize prompt files against AgentV evaluation datasets by analyzing failures and refining instructions.
+---
+
+# AgentV Prompt Optimizer
+
+## Input Variables
+- `eval-path`: Path or glob pattern to the AgentV evaluation file(s) to optimize against
+- `optimization-log-path` (optional): Path where optimization progress should be logged
+
+## Workflow
+
+1.  **Initialize**
+    - Verify `<eval-path>` (file or glob) targets the correct system.
+    - **Identify Prompt Files**:
+        - Infer prompt files from the eval file content (look for `file:` references in `input` that match these patterns).
+        - Recursively check referenced prompt files for *other* prompt references (dependencies).
+        - If multiple prompts are found, consider ALL of them as candidates for optimization.
+    - **Identify Optimization Log**:
+        - If `<optimization-log-path>` is provided, use it.
+        - If not, create a new one in the parent directory of the eval files: `optimization-[timestamp].md`.
+    - Read content of the identified prompt file.
+
+2.  **Optimization Loop** (Max 10 iterations)
+    - **Execute (The Generator)**: Run `agentv eval <eval-path>`.
+        - *Targeted Run*: If iterating on specific stubborn failures, use `--test-id <test_id>` to run only the relevant tests.
+    - **Analyze (The Reflector)**:
+        - Locate the results file path from the console output (e.g., `.agentv/results/eval_...jsonl`).
+        - **Orchestrate Subagent**: Use `runSubagent` to analyze the results.
+            - **Task**: Read the results file, calculate pass rate, and perform root cause analysis.
+            - **Output**: Return a structured analysis including:
+                - **Score**: Current pass rate.
+                - **Root Cause**: Why failures occurred (e.g., "Ambiguous definition", "Hallucination").
+                - **Insight**: Key learning or pattern identified from the failures.
+                - **Strategy**: High-level plan to fix the prompt (e.g., "Clarify section X", "Add negative constraint").
+    - **Decide**:
+        - If **100% pass**: STOP and report success.
+        - If **Score decreased**: Revert last change, try different approach.
+        - If **No improvement** (2x): STOP and report stagnation.
+    - **Refine (The Curator)**:
+        - **Orchestrate Subagent**: Use `runSubagent` to apply the fix.
+            - **Task**: Read the relevant prompt file(s), apply the **Strategy** from the Reflector, and generate the log entry.
+            - **Output**: The **Log Entry** describing the specific operation performed.
+                  ```markdown
+                  ### Iteration [N]
+                  - **Operation**: [ADD / UPDATE / DELETE]
+                  - **Target**: [Section Name]
+                  - **Change**: [Specific text added/modified]
+                  - **Trigger**: [Specific failing test case or error pattern]
+                  - **Rationale**: [From Reflector: Root Cause]
+                  - **Score**: [From Reflector: Current Pass Rate]
+                  - **Insight**: [From Reflector: Key Learning]
+                  ```
+        - **Strategy**: Treat the prompt as a structured set of rules. Execute atomic operations:
+            - **ADD**: Insert a new rule if a constraint was missed.
+            - **UPDATE**: Refine an existing rule to be clearer or more general.
+                - *Clarify*: Make ambiguous instructions specific.
+                - *Generalize*: Refactor specific fixes into high-level principles (First Principles).
+            - **DELETE**: Remove obsolete, redundant, or harmful rules.
+                - *Prune*: If a general rule covers specific cases, delete the specific ones.
+            - **Negative Constraint**: If hallucinating, explicitly state what NOT to do. Prefer generalized prohibitions over specific forbidden tokens where possible.
+            - **Safety Check**: Ensure new rules don't contradict existing ones (unless intended).
+        - **Constraint**: Avoid rewriting large sections. Make surgical, additive changes to preserve existing behavior.
+    - **Log Result**:
+        - Append the **Log Entry** returned by the Curator to the optimization log file.
+
+3.  **Completion**
+    - Report final score.
+    - Summarize key changes made to the prompt.
+    - **Finalize Optimization Log**: Add a summary header to the optimization log file indicating the session completion and final score.
+
+## Guidelines
+- **Generalization First**: Prefer broad, principle-based guidelines over specific examples or "hotfixes". Only use specific rules if generalized instructions fail to achieve the desired score.
+- **Simplicity ("Less is More")**: Avoid overfitting to the test set. If a specific rule doesn't significantly improve the score compared to a general one, choose the general one.
+- **Structure**: Maintain existing Markdown headers/sections.
+- **Progressive Disclosure**: If the prompt grows too large (>200 lines), consider moving specialized logic into a separate file or skill.
+- **Quality Criteria**: Ensure the prompt defines a clear persona, specific task, and measurable success criteria.

package/dist/templates/.agentv/.env.example

@@ -1,23 +1,23 @@
-# Copy this file to .env and fill in your credentials
-
-# Azure OpenAI Configuration
-AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/
-AZURE_OPENAI_API_KEY=your-openai-api-key-here
-AZURE_DEPLOYMENT_NAME=gpt-5-chat
-AZURE_OPENAI_API_VERSION=2024-12-01-preview
-
-# Google Gemini
-GOOGLE_GENERATIVE_AI_API_KEY=your-gemini-api-key-here
-GEMINI_MODEL_NAME=gemini-2.5-flash
-
-# Anthropic
-ANTHROPIC_API_KEY=your-anthropic-api-key-here
-
-# VS Code Workspace Paths for Execution Targets
-# Note: Using forward slashes is recommended for paths in .env files 
-# to avoid issues with escape characters.
-PROJECTX_WORKSPACE_PATH=C:/Users/your-username/OneDrive - Company Pty Ltd/sample.code-workspace
-
-# CLI provider sample (used by the local_cli target)
-CLI_EVALS_DIR=./docs/examples/simple/evals/local-cli
-LOCAL_AGENT_TOKEN=dummytoken
+# Copy this file to .env and fill in your credentials
+
+# Azure OpenAI Configuration
+AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/
+AZURE_OPENAI_API_KEY=your-openai-api-key-here
+AZURE_DEPLOYMENT_NAME=gpt-5-chat
+AZURE_OPENAI_API_VERSION=2024-12-01-preview
+
+# Google Gemini
+GOOGLE_GENERATIVE_AI_API_KEY=your-gemini-api-key-here
+GEMINI_MODEL_NAME=gemini-2.5-flash
+
+# Anthropic
+ANTHROPIC_API_KEY=your-anthropic-api-key-here
+
+# VS Code Workspace Paths for Execution Targets
+# Note: Using forward slashes is recommended for paths in .env files 
+# to avoid issues with escape characters.
+PROJECTX_WORKSPACE_PATH=C:/Users/your-username/OneDrive - Company Pty Ltd/sample.code-workspace
+
+# CLI provider sample (used by the local_cli target)
+CLI_EVALS_DIR=./docs/examples/simple/evals/local-cli
+LOCAL_AGENT_TOKEN=dummytoken

package/dist/templates/.agentv/config.yaml

@@ -1,15 +1,15 @@
-$schema: agentv-config-v2
-
-# Customize which files are treated as guidelines vs regular file content
-
-# Custom guideline patterns:
-guideline_patterns:
-  - "**/*.instructions.md"
-  - "**/*.prompt.md"
-  - "**/SKILL.md"
-  
-# Notes:
-# - Patterns use standard glob syntax (via micromatch library)
-# - Paths are normalized to forward slashes for cross-platform compatibility
-# - Only files matching these patterns are loaded as guidelines
-# - All other files referenced in eval cases are treated as regular file content
+$schema: agentv-config-v2
+
+# Customize which files are treated as guidelines vs regular file content
+
+# Custom guideline patterns:
+guideline_patterns:
+  - "**/*.instructions.md"
+  - "**/*.prompt.md"
+  - "**/SKILL.md"
+  
+# Notes:
+# - Patterns use standard glob syntax (via micromatch library)
+# - Paths are normalized to forward slashes for cross-platform compatibility
+# - Only files matching these patterns are loaded as guidelines
+# - All other files referenced in eval cases are treated as regular file content

package/dist/templates/.claude/skills/agentv-eval-builder/SKILL.md

@@ -14,9 +14,9 @@ description: Example eval
 execution:
   target: default
 
-evalcases:
+cases:
   - id: greeting
-    expected_outcome: Friendly greeting
+    criteria: Friendly greeting
     input: "Say hello"
     expected_output: "Hello! How can I help you?"
     rubrics:
@@ -26,7 +26,7 @@ evalcases:
 
 ## Eval File Structure
 
-**Required:** `evalcases` (array)
+**Required:** `cases` (array)
 **Optional:** `description`, `execution`, `dataset`
 
 **Eval case fields:**
@@ -34,7 +34,7 @@ evalcases:
 | Field | Required | Description |
 |-------|----------|-------------|
 | `id` | yes | Unique identifier |
-| `expected_outcome` | yes | What the response should accomplish |
+| `criteria` | yes | What the response should accomplish |
 | `input` / `input_messages` | yes | Input to the agent |
 | `expected_output` / `expected_messages` | no | Gold-standard reference answer |
 | `rubrics` | no | Inline evaluation criteria |
@@ -65,7 +65,9 @@ Configure via `execution.evaluators` array. Multiple evaluators produce a weight
   target: {}              # optional: enable LLM target proxy (max_calls: 50)
 ```
 Contract: stdin JSON -> stdout JSON `{score, hits, misses, reasoning}`
-See `references/custom-evaluators.md` for templates.
+Input includes: `question`, `criteria`, `candidate_answer`, `reference_answer`, `output_messages`, `trace_summary`, `file_changes`, `workspace_path`, `config`
+When `workspace_template` is configured, `workspace_path` is the absolute path to the workspace dir (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the workspace).
+See docs at https://agentv.dev/evaluators/code-judges/
 
 ### llm_judge
 ```yaml
@@ -76,7 +78,7 @@ See `references/custom-evaluators.md` for templates.
   config:                       # passed to script templates as context.config
     strictness: high
 ```
-Variables: `{{question}}`, `{{expected_outcome}}`, `{{candidate_answer}}`, `{{reference_answer}}`, `{{input_messages}}`, `{{expected_messages}}`, `{{output_messages}}`
+Variables: `{{question}}`, `{{criteria}}`, `{{candidate_answer}}`, `{{reference_answer}}`, `{{input_messages}}`, `{{expected_messages}}`, `{{output_messages}}`, `{{file_changes}}`
 - Markdown templates: use `{{variable}}` syntax
 - TypeScript templates: use `definePromptTemplate(fn)` from `@agentv/eval`, receives context object with all variables + `config`
 
@@ -144,12 +146,27 @@ Compares `output_messages` fields against `expected_messages` fields.
   max_total_tokens: 4000
 ```
 
+### execution_metrics
+```yaml
+- name: efficiency
+  type: execution_metrics
+  max_tool_calls: 10        # Maximum tool invocations
+  max_llm_calls: 5          # Maximum LLM calls (assistant messages)
+  max_tokens: 5000          # Maximum total tokens (input + output)
+  max_cost_usd: 0.05        # Maximum cost in USD
+  max_duration_ms: 30000    # Maximum execution duration
+  target_exploration_ratio: 0.6   # Target ratio of read-only tool calls
+  exploration_tolerance: 0.2      # Tolerance for ratio check (default: 0.2)
+```
+Declarative threshold-based checks on execution metrics. Only specified thresholds are checked.
+Score is proportional: `hits / (hits + misses)`. Missing data counts as a miss.
+
 ### rubric (inline)
 ```yaml
 rubrics:
   - Simple string criterion
   - id: weighted
-    expected_outcome: Detailed criterion
+    criteria: Detailed criterion
     weight: 2.0
     required: true
 ```
@@ -158,17 +175,25 @@ See `references/rubric-evaluator.md` for score-range mode and scoring formula.
 ## CLI Commands
 
 ```bash
-# Run evaluation
-bun agentv eval <file.yaml> [--eval-id <id>] [--target <name>] [--dry-run]
+# Run evaluation (requires API keys)
+agentv eval <file.yaml> [--eval-id <id>] [--target <name>] [--dry-run]
+
+# Run with trace persistence (writes to .agentv/traces/)
+agentv eval <file.yaml> --trace
+
+# Agent-orchestrated evals (no API keys needed)
+agentv eval prompt <file.yaml>                                      # orchestration overview
+agentv eval prompt input <file.yaml> --eval-id <id>                 # task input JSON (file paths, not embedded content)
+agentv eval prompt judge <file.yaml> --eval-id <id> --answer-file f # judge prompts / code judge results
 
 # Validate eval file
-bun agentv validate <file.yaml>
+agentv validate <file.yaml>
 
 # Compare results between runs
-bun agentv compare <results1.jsonl> <results2.jsonl>
+agentv compare <results1.jsonl> <results2.jsonl>
 
-# Generate rubrics from expected_outcome
-bun agentv generate rubrics <file.yaml> [--target <name>]
+# Generate rubrics from criteria
+agentv generate rubrics <file.yaml> [--target <name>]
 ```
 
 ## Schemas