agentv 4.26.1 → 4.27.0

package/dist/skills/agentv-eval-writer/references/config-schema.json

@@ -0,0 +1,63 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "AgentV Config Schema",
+  "description": "Schema for .agentv/config.yaml configuration files",
+  "type": "object",
+  "properties": {
+    "$schema": {
+      "type": "string",
+      "description": "Schema identifier",
+      "enum": ["agentv-config-v2"]
+    },
+    "required_version": {
+      "type": "string",
+      "description": "Minimum AgentV version required to run this project's evals. Uses semver range syntax (e.g., '>=2.11.0', '^2.11.0'). When the installed version is below the range, AgentV warns and prompts to update.",
+      "examples": [">=2.11.0", "^2.12.0", ">=2.11.0 <3.0.0"]
+    },
+    "eval_patterns": {
+      "type": "array",
+      "description": "Glob patterns for discovering eval files during interactive mode (`agentv eval` with no args). Defaults to ['**/evals/**/dataset*.yaml', '**/evals/**/eval.yaml'] if not specified.",
+      "items": {
+        "type": "string",
+        "description": "Glob pattern (e.g., '**/evals/**/dataset*.yaml', '**/evals/**/eval.yaml')"
+      },
+      "examples": [["**/evals/**/dataset*.yaml", "**/evals/**/eval.yaml"], ["**/evals/**/*.yaml"]]
+    },
+    "execution": {
+      "type": "object",
+      "description": "Default execution options. CLI flags take precedence over these values.",
+      "properties": {
+        "verbose": {
+          "type": "boolean",
+          "description": "Enable verbose logging (equivalent to --verbose)",
+          "default": false
+        },
+        "keep_workspaces": {
+          "type": "boolean",
+          "description": "Always keep temp workspaces after eval (equivalent to --keep-workspaces)",
+          "default": false
+        },
+        "otel_file": {
+          "type": "string",
+          "description": "Write OTLP JSON trace to this path (equivalent to --otel-file). Supports {timestamp} placeholder.",
+          "examples": [".agentv/results/otel-{timestamp}.json"]
+        }
+      },
+      "additionalProperties": false
+    },
+    "hooks": {
+      "type": "object",
+      "description": "Lifecycle hooks that run at specific points during agentv execution.",
+      "properties": {
+        "before_session": {
+          "type": "string",
+          "description": "Shell command to run once at agentv startup, before any command executes. stdout is parsed for env var exports (KEY=value or export KEY=\"value\") and injected into process.env. Keys already set in the environment are not overwritten. stderr is forwarded to the user. Non-zero exit aborts with an error.",
+          "examples": ["bun scripts/load-secrets.ts", "eval $(aws ssm get-parameters-by-path ...)"]
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "required": ["$schema"],
+  "additionalProperties": false
+}

package/dist/skills/agentv-eval-writer/references/custom-evaluators.md

@@ -0,0 +1,119 @@
+# Custom Graders
+
+## Wire Format
+
+### Input (stdin JSON)
+
+```json
+{
+  "question": "string",
+  "criteria": "string",
+  "reference_answer": "string",
+  "answer": "string",
+  "input_files": ["path"],
+  "input": [{"role": "user", "content": "..."}],
+  "expected_output": [{"role": "assistant", "content": "..."}],
+  "output": [{"role": "assistant", "content": "..."}],
+  "trace": {
+    "event_count": 5,
+    "tool_calls": {"fetch": 1},
+    "error_count": 0,
+    "llm_call_count": 2
+  },
+  "token_usage": {"input": 1000, "output": 500},
+  "cost_usd": 0.0015,
+  "duration_ms": 3500,
+  "start_time": "2026-02-13T10:00:00.000Z",
+  "end_time": "2026-02-13T10:00:03.500Z"
+}
+```
+
+### Output (stdout JSON)
+
+```json
+{
+  "score": 0.85,
+  "assertions": [
+    { "text": "passed check", "passed": true },
+    { "text": "failed check", "passed": false }
+  ],
+  "reasoning": "explanation"
+}
+```
+
+`score` (0.0-1.0) required. `assertions`, `reasoning` optional.
+
+## SDK Functions
+
+```typescript
+import { defineCodeGrader, createTargetClient, definePromptTemplate } from '@agentv/eval';
+```
+
+- `defineCodeGrader(fn)` - Wraps evaluation function with stdin/stdout handling
+- `createTargetClient()` - Returns LLM proxy client (when `target: {}` configured)
+  - `.invoke({question, systemPrompt})` - Single LLM call
+  - `.invokeBatch(requests)` - Batch LLM calls
+- `definePromptTemplate(fn)` - Wraps prompt generation function
+  - Context fields: `question`, `answer`, `referenceAnswer`, `criteria`, `expectedOutput`, `output`, `config`, `trace`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`
+
+## Python Example
+
+```python
+#!/usr/bin/env python3
+import json, sys
+
+def evaluate(data: dict) -> dict:
+    candidate = data.get("answer", "")
+    assertions = []
+    for kw in ["async", "await"]:
+        assertions.append({"text": f"Keyword '{kw}'", "passed": kw in candidate})
+    passed = sum(1 for a in assertions if a["passed"])
+    return {
+        "score": passed / max(len(assertions), 1),
+        "assertions": assertions,
+    }
+
+if __name__ == "__main__":
+    try:
+        print(json.dumps(evaluate(json.loads(sys.stdin.read()))))
+    except Exception as e:
+        print(json.dumps({"score": 0, "assertions": [{"text": str(e), "passed": False}]}))
+        sys.exit(1)
+```
+
+## TypeScript Example
+
+```typescript
+#!/usr/bin/env bun
+import { defineCodeGrader } from '@agentv/eval';
+
+export default defineCodeGrader(({ answer, criteria }) => {
+  const assertions: Array<{ text: string; passed: boolean }> = [];
+  if (answer.includes(criteria)) {
+    assertions.push({ text: 'Matches expected outcome', passed: true });
+  } else {
+    assertions.push({ text: 'Does not match expected outcome', passed: false });
+  }
+  const passed = assertions.filter(a => a.passed).length;
+  return {
+    score: passed / Math.max(assertions.length, 1),
+    assertions,
+  };
+});
+```
+
+## Template Variables
+
+Derived from test fields (users never author these directly):
+
+| Variable | Source |
+|----------|--------|
+| `question` | First user message in `input` |
+| `criteria` | Test `criteria` field |
+| `reference_answer` | Last entry in `expected_output` |
+| `answer` | Last entry in `output` (runtime) |
+| `input` | Full resolved input array (JSON) |
+| `expected_output` | Full resolved expected array (JSON) |
+| `output` | Full provider output array (JSON) |
+
+Markdown templates use `{{variable}}` syntax. TypeScript templates receive context object.