agentv 5.0.0-next.1 → 5.2.0-next.1

package/dist/skills/agentv-eval-writer/references/{config-schema.json → config.schema.json}

@@ -1,7 +1,7 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "AgentV Config Schema",
-  "description": "Schema for .agentv/config.yaml configuration files",
+  "description": "Schema for .agentv/config.yaml and .agentv/config.local.yaml configuration files",
   "type": "object",
   "properties": {
     "$schema": {
@@ -37,6 +37,11 @@
           "description": "Always keep temp workspaces after eval (equivalent to --keep-workspaces)",
           "default": false
         },
+        "workspace_path": {
+          "type": "string",
+          "description": "Machine-local existing workspace directory for eval runs (equivalent to --workspace-path). Only honored in .agentv/config.local.yaml; do not commit local paths.",
+          "examples": ["/home/user/workspaces/my-eval"]
+        },
         "otel_file": {
           "type": "string",
           "description": "Write OTLP JSON trace to this path (equivalent to --otel-file). Supports {timestamp} placeholder.",
@@ -145,10 +150,6 @@
             "auto_push": {
               "type": "boolean",
               "description": "Push result commits best-effort after each completed run."
-            },
-            "require_push": {
-              "type": "boolean",
-              "description": "Fail the command if a configured push fails after writing local artifacts."
             }
           },
           "additionalProperties": false

package/dist/skills/agentv-eval-writer/references/custom-evaluators.md

@@ -1,4 +1,8 @@
-# Custom Graders
+# Custom Graders and Assertions
+
+Use **custom assertions** for reusable assertion types discovered from `.agentv/assertions/` via `defineAssertion()`. Use **script graders** for command-backed or LLM-backed scoring components referenced with `type: script` and `command:`.
+
+Promptfoo normally calls eval custom logic assertions and uses fixed assertion types such as `javascript`, `python`, `ruby`, and `webhook`. AgentV follows that assertion terminology for `defineAssertion()` and extends it with arbitrary discovered assertion type names.
 
 ## Wire Format
 
@@ -6,7 +10,6 @@
 
 ```json
 {
-  "criteria": "string",
   "input_files": ["path"],
   "input": [{"role": "user", "content": "..."}],
   "expected_output": [{"role": "assistant", "content": "..."}],
@@ -54,34 +57,34 @@
 ```typescript
 import {
   createTargetClient,
-  defineCodeGrader,
+  defineScriptGrader,
   defineEval,
   definePromptTemplate,
   graders,
 } from '@agentv/sdk';
 ```
 
-- `defineCodeGrader(fn)` - Wraps evaluation function with stdin/stdout handling
+- `defineScriptGrader(fn)` - Wraps evaluation function with stdin/stdout handling
 - `defineEval(definition)` - Defines a YAML-aligned `.eval.ts` suite
-- `graders` - Helper catalog that returns ordinary AgentV `assertions` entries
+- `graders` - Helper catalog that returns ordinary AgentV `assert` entries
 - `createTargetClient()` - Returns LLM proxy client (when `target: {}` configured)
   - `.invoke({question, systemPrompt})` - Single LLM call
   - `.invokeBatch(requests)` - Batch LLM calls
 - `definePromptTemplate(fn)` - Wraps prompt generation function
   - Raw stdin uses `snake_case`; SDK handlers receive `camelCase`
-  - Context fields: `input`, `expectedOutput`, `output`, `messages`, `criteria`, `config`, `trace`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`
+  - Context fields: `input`, `expectedOutput`, `output`, `messages`, `config`, `trace`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`
 
 For Python, the repo-local helper example in `examples/features/sdk-python/` keeps canonical `snake_case` fields and rejects deprecated wire aliases like `output_text`, `input_text`, and `reference_answer`. It is not a separate Python runner or a promised published package; generated evals still run through the AgentV CLI.
 
 ## YAML-Aligned Helper Example
 
-Use helper factories for reusable Braintrust/DeepEval-inspired checks, but keep the result as AgentV `assertions`:
+Use helper factories for reusable Braintrust/DeepEval-inspired checks, but keep the result as AgentV `assert` entries:
 
 ```typescript
 import { defineEval, graders } from '@agentv/sdk';
 
 function ragFaithfulness() {
-  return graders.llmGrader({
+  return graders.llmRubric(undefined, {
     name: 'rag-faithfulness',
     target: 'grader-target',
     prompt: 'Grade whether the answer is supported by the retrieved context.',
@@ -94,7 +97,7 @@ export default defineEval({
     {
       id: 'grounded-answer',
       input: 'Answer using the retrieved context.',
-      assertions: [
+      assert: [
         graders.contains('source', { name: 'mentions-source' }),
         ragFaithfulness(),
       ],
@@ -106,12 +109,12 @@ export default defineEval({
 The helper lowers to ordinary YAML:
 
 ```yaml
-assertions:
+assert:
   - name: mentions-source
     type: contains
     value: source
   - name: rag-faithfulness
-    type: llm-grader
+    type: llm-rubric
     target: grader-target
     prompt: Grade whether the answer is supported by the retrieved context.
 ```
@@ -120,7 +123,7 @@ assertions:
 
 ```python
 #!/usr/bin/env python3
-from agentv_py.grader import Assertion, CodeGraderResult, define_code_grader
+from agentv_py.grader import Assertion, ScriptGraderResult, define_script_grader
 
 
 def evaluate(context):
@@ -129,26 +132,29 @@ def evaluate(context):
     for kw in ["async", "await"]:
         assertions.append(Assertion(text=f"Keyword '{kw}'", passed=kw in candidate))
     passed = sum(1 for item in assertions if item.passed)
-    return CodeGraderResult(
+    return ScriptGraderResult(
         score=passed / max(len(assertions), 1),
         assertions=assertions,
     )
 
 
 if __name__ == "__main__":
-    define_code_grader(evaluate)
+    define_script_grader(evaluate)
 ```
 
 ## TypeScript Example
 
 ```typescript
 #!/usr/bin/env bun
-import { defineCodeGrader } from '@agentv/sdk';
+import { defineScriptGrader } from '@agentv/sdk';
 
-export default defineCodeGrader(({ output, criteria }) => {
+export default defineScriptGrader(({ output, expectedOutput }) => {
   const candidate = output ?? '';
+  const expected = expectedOutput
+    ?.map((message) => (typeof message.content === 'string' ? message.content : ''))
+    .join('\n') ?? '';
   const assertions: Array<{ text: string; passed: boolean }> = [];
-  if (candidate.includes(criteria)) {
+  if (expected.length > 0 && candidate.includes(expected)) {
     assertions.push({ text: 'Matches expected outcome', passed: true });
   } else {
     assertions.push({ text: 'Does not match expected outcome', passed: false });
@@ -167,7 +173,6 @@ Derived from test fields (users never author these directly):
 
 | Variable | Source |
 |----------|--------|
-| `criteria` | Test `criteria` field |
 | `input` | Full resolved input array (JSON) |
 | `expected_output` | Full resolved expected array (JSON) |
 | `output` | Final answer / scored result string |