agentv 3.10.2 → 3.11.0

package/dist/templates/.claude/skills/agentv-eval-builder/references/custom-evaluators.md

@@ -1,115 +0,0 @@
-# Custom Evaluators
-
-## Wire Format
-
-### Input (stdin JSON)
-
-```json
-{
-  "question": "string",
-  "expected_outcome": "string",
-  "reference_answer": "string",
-  "candidate_answer": "string",
-  "guideline_files": ["path"],
-  "input_files": ["path"],
-  "input_messages": [{"role": "user", "content": "..."}],
-  "expected_messages": [{"role": "assistant", "content": "..."}],
-  "output_messages": [{"role": "assistant", "content": "..."}],
-  "trace_summary": {
-    "event_count": 5,
-    "tool_names": ["fetch"],
-    "tool_calls_by_name": {"fetch": 1},
-    "error_count": 0,
-    "token_usage": {"input": 1000, "output": 500},
-    "cost_usd": 0.0015,
-    "duration_ms": 3500
-  }
-}
-```
-
-### Output (stdout JSON)
-
-```json
-{
-  "score": 0.85,
-  "hits": ["passed check"],
-  "misses": ["failed check"],
-  "reasoning": "explanation"
-}
-```
-
-`score` (0.0-1.0) required. `hits`, `misses`, `reasoning` optional.
-
-## SDK Functions
-
-```typescript
-import { defineCodeJudge, createTargetClient, definePromptTemplate } from '@agentv/eval';
-```
-
-- `defineCodeJudge(fn)` - Wraps evaluation function with stdin/stdout handling
-- `createTargetClient()` - Returns LLM proxy client (when `target: {}` configured)
-  - `.invoke({question, systemPrompt})` - Single LLM call
-  - `.invokeBatch(requests)` - Batch LLM calls
-- `definePromptTemplate(fn)` - Wraps prompt generation function
-  - Context fields: `question`, `candidateAnswer`, `referenceAnswer`, `expectedOutcome`, `expectedMessages`, `outputMessages`, `config`, `traceSummary`
-
-## Python Example
-
-```python
-#!/usr/bin/env python3
-import json, sys
-
-def evaluate(data: dict) -> dict:
-    candidate = data.get("candidate_answer", "")
-    hits, misses = [], []
-    for kw in ["async", "await"]:
-        (hits if kw in candidate else misses).append(f"Keyword '{kw}'")
-    return {
-        "score": len(hits) / max(len(hits) + len(misses), 1),
-        "hits": hits, "misses": misses
-    }
-
-if __name__ == "__main__":
-    try:
-        print(json.dumps(evaluate(json.loads(sys.stdin.read()))))
-    except Exception as e:
-        print(json.dumps({"score": 0, "misses": [str(e)]}))
-        sys.exit(1)
-```
-
-## TypeScript Example
-
-```typescript
-#!/usr/bin/env bun
-import { defineCodeJudge } from '@agentv/eval';
-
-export default defineCodeJudge(({ candidateAnswer, expectedOutcome }) => {
-  const hits: string[] = [];
-  const misses: string[] = [];
-  if (candidateAnswer.includes(expectedOutcome)) {
-    hits.push('Matches expected outcome');
-  } else {
-    misses.push('Does not match expected outcome');
-  }
-  return {
-    score: hits.length / Math.max(hits.length + misses.length, 1),
-    hits, misses,
-  };
-});
-```
-
-## Template Variables
-
-Derived from eval case fields (users never author these directly):
-
-| Variable | Source |
-|----------|--------|
-| `question` | First user message in `input_messages` |
-| `expected_outcome` | Eval case `expected_outcome` field |
-| `reference_answer` | Last entry in `expected_messages` |
-| `candidate_answer` | Last entry in `output_messages` (runtime) |
-| `input_messages` | Full resolved input array (JSON) |
-| `expected_messages` | Full resolved expected array (JSON) |
-| `output_messages` | Full provider output array (JSON) |
-
-Markdown templates use `{{variable}}` syntax. TypeScript templates receive context object.

package/dist/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json

@@ -1,278 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "AgentV Eval Schema",
-  "description": "Schema for YAML evaluation files with conversation flows, multiple evaluators, and execution configuration",
-  "type": "object",
-  "properties": {
-    "description": {
-      "type": "string",
-      "description": "Description of what this eval suite covers"
-    },
-    "target": {
-      "type": "string",
-      "description": "(Deprecated: use execution.target instead) Default target configuration name. Can be overridden per eval case."
-    },
-    "execution": {
-      "type": "object",
-      "description": "Default execution configuration for all eval cases (can be overridden per case)",
-      "properties": {
-        "target": {
-          "type": "string",
-          "description": "Default target configuration name (e.g., default, azure_base, vscode_projectx). Can be overridden per eval case."
-        },
-        "evaluators": {
-          "type": "array",
-          "description": "Default evaluators for all eval cases (code-based and LLM judges)",
-          "items": {
-            "type": "object",
-            "properties": {
-              "name": {
-                "type": "string",
-                "description": "Evaluator name/identifier"
-              },
-              "type": {
-                "type": "string",
-                "enum": [
-                  "code",
-                  "llm_judge",
-                  "composite",
-                  "tool_trajectory",
-                  "field_accuracy",
-                  "latency",
-                  "cost",
-                  "token_usage"
-                ],
-                "description": "Evaluator type: 'code' for scripts/regex/keywords, 'llm_judge' for LLM-based evaluation"
-              },
-              "script": {
-                "type": "string",
-                "description": "Path to evaluator script (for type: code)"
-              },
-              "prompt": {
-                "type": "string",
-                "description": "Path to judge prompt file (for type: llm_judge)"
-              }
-            },
-            "required": ["name", "type"],
-            "additionalProperties": true
-          }
-        }
-      },
-      "additionalProperties": true
-    },
-    "evalcases": {
-      "type": "array",
-      "description": "Array of evaluation cases",
-      "minItems": 1,
-      "items": {
-        "type": "object",
-        "properties": {
-          "id": {
-            "type": "string",
-            "description": "Unique identifier for the eval case"
-          },
-          "conversation_id": {
-            "type": "string",
-            "description": "Optional conversation identifier for threading multiple eval cases together"
-          },
-          "expected_outcome": {
-            "type": "string",
-            "description": "Description of what the AI should accomplish in this eval"
-          },
-          "note": {
-            "type": "string",
-            "description": "Optional note or additional context for the eval case. Use this to document test-specific considerations, known limitations, or rationale for expected behavior."
-          },
-          "input_messages": {
-            "type": "array",
-            "description": "Input messages for the conversation",
-            "minItems": 1,
-            "items": {
-              "type": "object",
-              "properties": {
-                "role": {
-                  "type": "string",
-                  "enum": ["system", "user", "assistant", "tool"],
-                  "description": "Message role"
-                },
-                "content": {
-                  "oneOf": [
-                    {
-                      "type": "string",
-                      "description": "Simple text content"
-                    },
-                    {
-                      "type": "array",
-                      "description": "Mixed content items (text and file references)",
-                      "items": {
-                        "type": "object",
-                        "properties": {
-                          "type": {
-                            "type": "string",
-                            "enum": ["text", "file"],
-                            "description": "Content type: 'text' for inline content, 'file' for file references"
-                          },
-                          "value": {
-                            "type": "string",
-                            "description": "Text content or file path. Relative paths (e.g., ../prompts/file.md) are resolved from eval file directory. Absolute paths (e.g., /docs/examples/prompts/file.md) are resolved from repo root."
-                          }
-                        },
-                        "required": ["type", "value"],
-                        "additionalProperties": false
-                      }
-                    }
-                  ]
-                }
-              },
-              "required": ["role", "content"],
-              "additionalProperties": false
-            }
-          },
-          "input": {
-            "description": "Alias for input_messages with shorthand support. String expands to single user message, array of messages passes through.",
-            "oneOf": [
-              {
-                "type": "string",
-                "description": "Shorthand: single user message content"
-              },
-              {
-                "type": "array",
-                "description": "Array of messages (same format as input_messages)",
-                "items": {
-                  "type": "object",
-                  "properties": {
-                    "role": {
-                      "type": "string",
-                      "enum": ["system", "user", "assistant", "tool"]
-                    },
-                    "content": {
-                      "oneOf": [{ "type": "string" }, { "type": "array" }]
-                    }
-                  },
-                  "required": ["role", "content"]
-                }
-              }
-            ]
-          },
-          "expected_messages": {
-            "type": "array",
-            "description": "Expected response messages. Canonical form — use this or expected_output (alias). The content of the last entry is derived as the template variable 'reference_answer' for evaluator prompts.",
-            "minItems": 1,
-            "items": {
-              "type": "object",
-              "properties": {
-                "role": {
-                  "type": "string",
-                  "enum": ["system", "user", "assistant", "tool"],
-                  "description": "Message role"
-                },
-                "content": {
-                  "oneOf": [
-                    {
-                      "type": "string",
-                      "description": "Simple text content"
-                    },
-                    {
-                      "type": "array",
-                      "description": "Mixed content items",
-                      "items": {
-                        "type": "object",
-                        "properties": {
-                          "type": {
-                            "type": "string",
-                            "enum": ["text", "file"]
-                          },
-                          "value": {
-                            "type": "string"
-                          }
-                        },
-                        "required": ["type", "value"],
-                        "additionalProperties": false
-                      }
-                    }
-                  ]
-                }
-              },
-              "required": ["role", "content"],
-              "additionalProperties": false
-            }
-          },
-          "expected_output": {
-            "description": "Alias for expected_messages with shorthand support. String expands to single assistant message, object wraps as assistant message content. Resolves to expected_messages internally — the content of the last resolved entry becomes the template variable 'reference_answer'.",
-            "oneOf": [
-              {
-                "type": "string",
-                "description": "Shorthand: single assistant message content"
-              },
-              {
-                "type": "object",
-                "description": "Shorthand: structured content wraps as assistant message"
-              },
-              {
-                "type": "array",
-                "description": "Array of messages (same format as expected_messages)",
-                "items": {
-                  "type": "object",
-                  "properties": {
-                    "role": {
-                      "type": "string",
-                      "enum": ["system", "user", "assistant", "tool"]
-                    },
-                    "content": {
-                      "oneOf": [{ "type": "string" }, { "type": "object" }, { "type": "array" }]
-                    }
-                  },
-                  "required": ["role", "content"]
-                }
-              }
-            ]
-          },
-          "execution": {
-            "type": "object",
-            "description": "Per-case execution configuration",
-            "properties": {
-              "target": {
-                "type": "string",
-                "description": "Override target for this specific eval case"
-              },
-              "evaluators": {
-                "type": "array",
-                "description": "Multiple evaluators (code-based and LLM judges)",
-                "items": {
-                  "type": "object",
-                  "properties": {
-                    "name": {
-                      "type": "string",
-                      "description": "Evaluator name/identifier"
-                    },
-                    "type": {
-                      "type": "string",
-                      "enum": ["code", "llm_judge"],
-                      "description": "Evaluator type: 'code' for scripts/regex/keywords, 'llm_judge' for LLM-based evaluation"
-                    },
-                    "script": {
-                      "type": "string",
-                      "description": "Path to evaluator script (for type: code)"
-                    },
-                    "prompt": {
-                      "type": "string",
-                      "description": "Path to judge prompt file (for type: llm_judge)"
-                    }
-                  },
-                  "required": ["name", "type"],
-                  "additionalProperties": true
-                }
-              }
-            },
-            "additionalProperties": true
-          }
-        },
-        "required": ["id", "expected_outcome"],
-        "anyOf": [{ "required": ["input_messages"] }, { "required": ["input"] }],
-        "additionalProperties": true
-      }
-    }
-  },
-  "required": ["evalcases"],
-  "additionalProperties": false
-}