agentv 4.42.4 → 4.43.0-next.1

package/dist/skills/agentv-eval-writer/SKILL.md

@@ -18,7 +18,7 @@ Comprehensive docs: https://agentv.dev
 
 Treat YAML as the canonical portable model. Prefer authoring `.eval.yaml` / `EVAL.yaml` first, then use TypeScript helpers, Python scripts, or executable graders only when they lower to the same fields or when the evaluation logic must actually run code.
 
-Use `@agentv/sdk` for TypeScript helper imports. Do not use `@agentv/eval` for new evals, examples, scaffolds, or skill guidance; it is only a deprecated compatibility shim for existing consumers during migration.
+Use `@agentv/sdk` for TypeScript helper imports. Do not use `@agentv/eval` for new evals, examples, scaffolds, or skill guidance; it was a deprecated compatibility package and has been removed from this repository.
 
 ## Evaluation Types
 
@@ -119,7 +119,7 @@ tests:
 |-------|----------|-------------|
 | `id` | yes | Unique identifier |
 | `criteria` | yes | What the response should accomplish |
-| `input` | yes | Input to the agent (string shorthand or full message array) |
+| `input` | yes | Input to the agent (string/object shorthand or full message array) |
 | `expected_output` | no | Gold-standard reference answer (string shorthand or full message array) |
 | `assertions` | no | Graders: deterministic checks, rubrics, and LLM/code graders |
 | `execution` | no | Per-case execution overrides |
@@ -128,7 +128,9 @@ tests:
 | `conversation_id` | no | Thread grouping |
 
 **Shorthand forms:**
-- `input` (string) expands to `[{role: "user", content: "..."}]`
+- `input` (string, including YAML block scalars) expands to `[{role: "user", content: "..."}]`
+- `input` (object without a top-level `role`) expands to `[{role: "user", content: {...}}]`
+- top-level `role` is reserved for message objects; use `{role, content}` when you mean a message, or nest payload role data under another key
 - `expected_output` (string/object) expands to `[{role: "assistant", content: ...}]`
 - Use these canonical field names on disk; keep the wire format `snake_case`
 
@@ -158,14 +160,12 @@ requires:
 
 ## Suite-level Input
 
-Prepend shared input messages to every test (like suite-level `assertions`). Avoids repeating the same prompt file in each test:
+Prepend shared input messages to every test (like suite-level `assertions`). Avoids repeating the same prompt or instruction in each test:
 
 ```yaml
-input:
-  - role: user
-    content:
-      - type: file
-        value: ./system-prompt.md
+input: |
+  Read AGENTS.md before answering.
+  Explain tradeoffs clearly.
 
 tests: ./cases.yaml
 
@@ -176,7 +176,7 @@ tests: ./cases.yaml
 ```
 
 Effective input: `[...suite input, ...test input]`. Skipped when `execution.skip_defaults: true`.
-Accepts same formats as test `input` (string or message array).
+Accepts the same formats as test `input`: string/block scalar, structured object without a top-level `role`, single message object, or full message array. Use the full message array only when you need multiple messages or file/image content blocks.
 
 ## Tests as String Path
 
@@ -365,6 +365,13 @@ Contract: stdin JSON -> stdout JSON `{score, assertions: [{text, passed, evidenc
 Raw stdin uses snake_case and includes: `criteria`, `input`, `expected_output`, `output` (final answer string), `messages`, `trace`, `trace_summary`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config`
 SDK handlers receive the same payload in camelCase: `expectedOutput`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`, `fileChanges`, `workspacePath`.
 When a workspace is configured, `workspace_path` is the absolute path to the workspace dir (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the workspace).
+For deterministic workspace checks that fit normal Vitest `expect(...)` tests, prefer a plain verifier file and the built-in adapter:
+```yaml
+- name: welcome_banner
+  type: code-grader
+  command: [agentv, eval, graders/welcome-banner.test.ts]
+```
+AgentV infers the Vitest adapter for `*.test.ts`, `*.spec.ts`, and Vercel-style `EVAL.ts` files. Use the explicit `agentv eval vitest` subcommand only when you need adapter flags such as `--cwd`, `--in-workspace`, or `--vitest-command`.
 See docs at https://agentv.dev/graders/code-graders/
 
 ### llm-grader
@@ -551,16 +558,16 @@ agentv eval assert <grader-name> --agent-output "..." --agent-input "..."
 agentv import claude --session-id <uuid>
 
 # Re-run only execution errors from a previous run
-agentv eval <file.yaml> --retry-errors .agentv/results/runs/<timestamp>/index.jsonl
+agentv eval <file.yaml> --retry-errors .agentv/results/default/<timestamp>/index.jsonl
 
 # Validate eval file
 agentv validate <file.yaml>
 
 # Compare results — N-way matrix from a canonical run manifest
-agentv compare .agentv/results/runs/<timestamp>/index.jsonl
-agentv compare .agentv/results/runs/<timestamp>/index.jsonl --baseline <target>                   # CI regression gate
-agentv compare .agentv/results/runs/<timestamp>/index.jsonl --baseline <target> --candidate <target>  # pairwise
-agentv compare .agentv/results/runs/<baseline-timestamp>/index.jsonl .agentv/results/runs/<candidate-timestamp>/index.jsonl
+agentv compare .agentv/results/default/<timestamp>/index.jsonl
+agentv compare .agentv/results/default/<timestamp>/index.jsonl --baseline <target>                   # CI regression gate
+agentv compare .agentv/results/default/<timestamp>/index.jsonl --baseline <target> --candidate <target>  # pairwise
+agentv compare .agentv/results/default/<baseline-timestamp>/index.jsonl .agentv/results/default/<candidate-timestamp>/index.jsonl
 
 # Author assertions directly in the eval file
 # Prefer simple assertions when they fit the criteria; use deterministic or LLM-based graders when needed
@@ -644,7 +651,7 @@ export default defineCodeGrader(({ output, trace }) => {
 });
 ```
 
-`defineAssertion()` files go in `.agentv/assertions/` and are referenced by filename as `type: <name>`. `defineCodeGrader()` scripts are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`.
+`defineAssertion()` files go in `.agentv/assertions/` and are referenced by filename as `type: <name>`. `defineCodeGrader()` scripts are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`. Plain Vitest workspace verifier files can use `command: [agentv, eval, graders/check.test.ts]`.
 
 ### Convention-Based Discovery
 

package/dist/skills/agentv-eval-writer/references/config-schema.json

@@ -45,6 +45,16 @@
       },
       "additionalProperties": false
     },
+    "results": {
+      "$ref": "#/definitions/resultsConfig"
+    },
+    "projects": {
+      "type": "array",
+      "description": "Global Dashboard project registry entries. Only valid in $AGENTV_HOME/config.yaml.",
+      "items": {
+        "$ref": "#/definitions/projectEntry"
+      }
+    },
     "hooks": {
       "type": "object",
       "description": "Lifecycle hooks that run at specific points during agentv execution.",
@@ -58,6 +68,110 @@
       "additionalProperties": false
     }
   },
+  "definitions": {
+    "sourceRepoBlock": {
+      "type": "object",
+      "description": "Source project repository block. Use url for the Git endpoint, path for the local checkout path, and branch for the checkout branch.",
+      "properties": {
+        "url": {
+          "type": "string",
+          "description": "Source Git remote URL.",
+          "examples": ["https://github.com/example/repo.git", "git@github.com:example/repo.git"]
+        },
+        "path": {
+          "type": "string",
+          "description": "Local source checkout path.",
+          "examples": ["/srv/agentv/my-evals"]
+        },
+        "branch": {
+          "type": "string",
+          "description": "Source checkout branch.",
+          "examples": ["main"]
+        }
+      },
+      "additionalProperties": false
+    },
+    "resultsRepoBlock": {
+      "type": "object",
+      "description": "Results repository block. Use remote for the Git endpoint URL, path for an existing local results checkout or managed clone path, and branch for the storage branch.",
+      "properties": {
+        "remote": {
+          "type": "string",
+          "description": "Results Git remote endpoint URL used for fetching and pushing.",
+          "examples": [
+            "https://github.com/example/results.git",
+            "git@github.com:example/results.git"
+          ]
+        },
+        "path": {
+          "type": "string",
+          "description": "Existing local results checkout path when remote is omitted, or managed results clone path when remote is set.",
+          "examples": [".", "/srv/agentv/results", "~/data/agentv-results"]
+        },
+        "branch": {
+          "type": "string",
+          "description": "Results storage branch.",
+          "examples": ["agentv/results/v1"]
+        }
+      },
+      "additionalProperties": false
+    },
+    "resultsConfig": {
+      "type": "object",
+      "description": "Git-backed results repo configuration. Canonical form uses a nested repo block.",
+      "properties": {
+        "repo": {
+          "$ref": "#/definitions/resultsRepoBlock"
+        },
+        "sync": {
+          "type": "object",
+          "properties": {
+            "auto_push": {
+              "type": "boolean",
+              "description": "Push result commits best-effort after each completed run."
+            },
+            "require_push": {
+              "type": "boolean",
+              "description": "Fail the command if a configured push fails after writing local artifacts."
+            }
+          },
+          "additionalProperties": false
+        },
+        "branch_prefix": {
+          "type": "string",
+          "description": "Prefix for temporary result branch names; this is not the storage branch."
+        }
+      },
+      "required": ["repo"],
+      "additionalProperties": false
+    },
+    "projectEntry": {
+      "type": "object",
+      "description": "Dashboard project registry entry.",
+      "properties": {
+        "id": {
+          "type": "string"
+        },
+        "name": {
+          "type": "string"
+        },
+        "repo": {
+          "$ref": "#/definitions/sourceRepoBlock"
+        },
+        "results": {
+          "$ref": "#/definitions/resultsConfig"
+        },
+        "added_at": {
+          "type": "string"
+        },
+        "last_opened_at": {
+          "type": "string"
+        }
+      },
+      "required": ["id", "name", "repo"],
+      "additionalProperties": false
+    }
+  },
   "required": ["$schema"],
   "additionalProperties": false
 }

package/dist/skills/agentv-eval-writer/references/eval-schema.json

@@ -49,6 +49,55 @@
             {
               "type": "string"
             },
+            {
+              "type": "object",
+              "properties": {
+                "role": {
+                  "type": "string",
+                  "enum": ["system", "user", "assistant", "tool"]
+                },
+                "content": {
+                  "anyOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object",
+                      "properties": {},
+                      "additionalProperties": {}
+                    },
+                    {
+                      "type": "array",
+                      "items": {
+                        "type": "object",
+                        "properties": {
+                          "type": {
+                            "type": "string",
+                            "enum": ["text", "file", "image"]
+                          },
+                          "value": {
+                            "type": "string"
+                          }
+                        },
+                        "required": ["type", "value"],
+                        "additionalProperties": false
+                      }
+                    }
+                  ]
+                }
+              },
+              "required": ["role", "content"],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "role": {
+                  "not": {}
+                }
+              },
+              "additionalProperties": {}
+            },
             {
               "type": "array",
               "items": {
@@ -63,6 +112,11 @@
                       {
                         "type": "string"
                       },
+                      {
+                        "type": "object",
+                        "properties": {},
+                        "additionalProperties": {}
+                      },
                       {
                         "type": "array",
                         "items": {
@@ -119,6 +173,55 @@
                       {
                         "type": "string"
                       },
+                      {
+                        "type": "object",
+                        "properties": {
+                          "role": {
+                            "type": "string",
+                            "enum": ["system", "user", "assistant", "tool"]
+                          },
+                          "content": {
+                            "anyOf": [
+                              {
+                                "type": "string"
+                              },
+                              {
+                                "type": "object",
+                                "properties": {},
+                                "additionalProperties": {}
+                              },
+                              {
+                                "type": "array",
+                                "items": {
+                                  "type": "object",
+                                  "properties": {
+                                    "type": {
+                                      "type": "string",
+                                      "enum": ["text", "file", "image"]
+                                    },
+                                    "value": {
+                                      "type": "string"
+                                    }
+                                  },
+                                  "required": ["type", "value"],
+                                  "additionalProperties": false
+                                }
+                              }
+                            ]
+                          }
+                        },
+                        "required": ["role", "content"],
+                        "additionalProperties": false
+                      },
+                      {
+                        "type": "object",
+                        "properties": {
+                          "role": {
+                            "not": {}
+                          }
+                        },
+                        "additionalProperties": {}
+                      },
                       {
                         "type": "array",
                         "items": {
@@ -133,6 +236,11 @@
                                 {
                                   "type": "string"
                                 },
+                                {
+                                  "type": "object",
+                                  "properties": {},
+                                  "additionalProperties": {}
+                                },
                                 {
                                   "type": "array",
                                   "items": {
@@ -189,6 +297,11 @@
                                 {
                                   "type": "string"
                                 },
+                                {
+                                  "type": "object",
+                                  "properties": {},
+                                  "additionalProperties": {}
+                                },
                                 {
                                   "type": "array",
                                   "items": {
@@ -5378,6 +5491,11 @@
                                 {
                                   "type": "string"
                                 },
+                                {
+                                  "type": "object",
+                                  "properties": {},
+                                  "additionalProperties": {}
+                                },
                                 {
                                   "type": "array",
                                   "items": {
@@ -5409,6 +5527,11 @@
                                 {
                                   "type": "string"
                                 },
+                                {
+                                  "type": "object",
+                                  "properties": {},
+                                  "additionalProperties": {}
+                                },
                                 {
                                   "type": "array",
                                   "items": {
@@ -6642,6 +6765,55 @@
                       {
                         "type": "string"
                       },
+                      {
+                        "type": "object",
+                        "properties": {
+                          "role": {
+                            "type": "string",
+                            "enum": ["system", "user", "assistant", "tool"]
+                          },
+                          "content": {
+                            "anyOf": [
+                              {
+                                "type": "string"
+                              },
+                              {
+                                "type": "object",
+                                "properties": {},
+                                "additionalProperties": {}
+                              },
+                              {
+                                "type": "array",
+                                "items": {
+                                  "type": "object",
+                                  "properties": {
+                                    "type": {
+                                      "type": "string",
+                                      "enum": ["text", "file", "image"]
+                                    },
+                                    "value": {
+                                      "type": "string"
+                                    }
+                                  },
+                                  "required": ["type", "value"],
+                                  "additionalProperties": false
+                                }
+                              }
+                            ]
+                          }
+                        },
+                        "required": ["role", "content"],
+                        "additionalProperties": false
+                      },
+                      {
+                        "type": "object",
+                        "properties": {
+                          "role": {
+                            "not": {}
+                          }
+                        },
+                        "additionalProperties": {}
+                      },
                       {
                         "type": "array",
                         "items": {
@@ -6656,6 +6828,11 @@
                                 {
                                   "type": "string"
                                 },
+                                {
+                                  "type": "object",
+                                  "properties": {},
+                                  "additionalProperties": {}
+                                },
                                 {
                                   "type": "array",
                                   "items": {
@@ -6712,6 +6889,11 @@
                                 {
                                   "type": "string"
                                 },
+                                {
+                                  "type": "object",
+                                  "properties": {},
+                                  "additionalProperties": {}
+                                },
                                 {
                                   "type": "array",
                                   "items": {
@@ -11901,6 +12083,11 @@
                                 {
                                   "type": "string"
                                 },
+                                {
+                                  "type": "object",
+                                  "properties": {},
+                                  "additionalProperties": {}
+                                },
                                 {
                                   "type": "array",
                                   "items": {
@@ -11932,6 +12119,11 @@
                                 {
                                   "type": "string"
                                 },
+                                {
+                                  "type": "object",
+                                  "properties": {},
+                                  "additionalProperties": {}
+                                },
                                 {
                                   "type": "array",
                                   "items": {

package/dist/{ts-eval-loader-PYEU2PMP-YLXIQTEQ.js → ts-eval-loader-HQDMXQQV-LBW3N3W6.js}

@@ -2,7 +2,7 @@ import { createRequire } from 'node:module'; const require = createRequire(impor
 import {
   loadTsEvalFile,
   loadTsEvalSuite
-} from "./chunk-ZPFM52HV.js";
+} from "./chunk-RFJM2UBD.js";
 import "./chunk-NPVGBFF6.js";
 import "./chunk-M7BUKBAF.js";
 import "./chunk-5H446C7X.js";
@@ -10,4 +10,4 @@ export {
   loadTsEvalFile,
   loadTsEvalSuite
 };
-//# sourceMappingURL=ts-eval-loader-PYEU2PMP-YLXIQTEQ.js.map
+//# sourceMappingURL=ts-eval-loader-HQDMXQQV-LBW3N3W6.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentv",
-  "version": "4.42.4",
+  "version": "4.43.0-next.1",
   "description": "CLI entry point for AgentV",
   "type": "module",
   "repository": {
@@ -17,19 +17,19 @@
   "files": ["dist", "README.md"],
   "scripts": {
     "dev": "bun src/cli.ts",
-    "build": "tsup && bun run copy-readme",
+    "build": "(cd ../../packages/sdk && bun run build) && tsup && bun run copy-readme",
     "copy-readme": "bun -e \"import { cpSync } from 'fs'; cpSync('../../README.md', 'README.md')\"",
     "prepublishOnly": "node -e \"if(process.env.ALLOW_PUBLISH!=='1'){console.error('ERROR: Use bun run publish:next, then bun run promote:latest');process.exit(1)}\"",
-    "typecheck": "tsc --noEmit",
+    "typecheck": "(cd ../../packages/sdk && bun run build) && tsc --noEmit",
     "lint": "biome check .",
     "format": "biome format --write .",
     "fix": "biome check --write .",
-    "test": "bun test",
+    "test": "(cd ../../packages/sdk && bun run build) && bun test",
     "test:watch": "bun test --watch"
   },
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.2.49",
-    "@github/copilot-sdk": "^0.1.25",
+    "@github/copilot-sdk": "^1.0.3",
     "@hono/node-server": "^1.19.11",
     "@inquirer/prompts": "^8.2.1",
     "@earendil-works/pi-ai": "^0.74.0",
@@ -53,6 +53,7 @@
   },
   "devDependencies": {
     "@agentv/core": "workspace:*",
+    "@agentv/sdk": "workspace:*",
     "@types/semver": "^7.7.1",
     "execa": "^9.3.0"
   }