agentv 5.3.1-next.1 → 5.3.2-next.1

package/dist/skills/agentv-eval-writer/references/python-helpers.md

@@ -18,7 +18,7 @@ Use it when the user wants Python-based custom graders or wants to emit AgentV Y
   - `load_grader_input()`
   - `run_script_grader(handler)`
   - `define_script_grader(handler)`
-  - `TargetClient.from_env()`
+  - `ProviderClient.from_env()`
 - `agentv_py.evals`
   - `EvalDefinition`
   - `EvalTest`
@@ -57,7 +57,7 @@ def rag_faithfulness():
     return {
         "name": "rag-faithfulness",
         "type": "llm-rubric",
-        "target": "grader-target",
+        "provider": "grader-provider",
         "prompt": "Grade whether the answer is supported by the retrieved context.",
     }
 

package/dist/skills/agentv-eval-writer/references/rubric-evaluator.md

@@ -1,12 +1,12 @@
 # Rubric Graders
 
-Rubrics are defined as `assert` entries with plain strings or `type: llm-rubric`. They support binary checklist grading and score-range analytic grading.
+Rubrics are defined as `assert` entries with plain strings, `type: llm-rubric`, or `type: agent-rubric`. They support binary checklist grading and score-range analytic grading. Use `agent-rubric` only when the grader provider is agent-capable and should inspect workspace evidence.
 
 ## Field Reference
 
 | Field | Type | Default | Description |
 |-------|------|---------|-------------|
-| `type` | string | required | Use `llm-rubric` for a structured rubric; plain strings in `assert` use the same grader path |
+| `type` | string | required | Use `llm-rubric` for a structured rubric; use `agent-rubric` for Promptfoo-compatible agent-backed rubric checks; plain strings in `assert` use the same non-agent rubric path |
 | `value` | array | required | List of criterion strings or objects |
 | `required` | boolean or number | - | Gate: `true` requires score >= 0.8; a number (0–1) sets a custom threshold |
 
@@ -117,6 +117,23 @@ Array format is also accepted:
 
 Ranges must be integers 0-10, non-overlapping, covering all values 0-10.
 
+## Agent Rubric Mode
+
+`agent-rubric` accepts the same `value`, `provider`, `max_steps`, `required`, and
+`min_score` fields as `llm-rubric`, but the resolved grader provider must be
+agent-capable:
+
+```yaml
+assert:
+  - type: agent-rubric
+    provider: codex-grader
+    value: Inspect the workspace and verify the claimed files exist.
+```
+
+AgentV asks the grader agent to write a verdict JSON file shaped like
+`{"pass": boolean, "score": number, "reason": string}`. The score must be a
+finite 0-1 value; invalid verdict files fail closed.
+
 ## Scoring
 
 **Checklist:** `score = sum(satisfied weights) / sum(all weights)`

package/dist/templates/.agentv/providers.yaml

@@ -0,0 +1,42 @@
+providers:
+  - id: azure
+    label: default
+    endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}"
+    api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
+    model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
+  - id: codex
+    label: codex
+    cwd: "{{ env.CODEX_WORKSPACE_DIR }}"
+    log_dir: "{{ env.CODEX_LOG_DIR }}"
+    stream_log: raw
+  - id: claude
+    label: claude
+    stream_log: raw
+  - id: azure
+    label: azure-llm
+    endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}"
+    api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
+    model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
+    version: "{{ env.AZURE_OPENAI_API_VERSION }}"
+  - id: gemini
+    label: gemini-llm
+    api_key: "{{ env.GOOGLE_GENERATIVE_AI_API_KEY }}"
+    model: "{{ env.GEMINI_MODEL_NAME }}"
+  - id: cli
+    label: local_cli
+    command: uv run ./mock_cli.py --prompt {PROMPT} {FILES} --output {OUTPUT_FILE}
+    files_format: --file {path}
+    cwd: "{{ env.CLI_EVALS_DIR }}"
+    healthcheck:
+      command: uv run ./mock_cli.py --healthcheck
+  - id: openrouter
+    label: mimo
+    api_key: "{{ env.OPENROUTER_API_KEY }}"
+    model: xiaomi/mimo-v2.5-pro
+  - id: openrouter
+    label: mimo-flash
+    api_key: "{{ env.OPENROUTER_API_KEY }}"
+    model: xiaomi/mimo-v2-flash
+defaults:
+  provider: default
+  grader: azure-llm

package/dist/templates/.env.example

@@ -18,5 +18,5 @@ GEMINI_MODEL_NAME=gemini-3-flash-preview
 # Anthropic
 ANTHROPIC_API_KEY=your-anthropic-api-key-here
 
-# CLI provider sample (used by the local_cli target)
-CLI_EVALS_DIR=./docs/examples/simple/evals/local-cli
+# CLI provider sample (used by the local_cli provider)
+CLI_EVALS_DIR=./docs/examples/simple/evals/local-cli

package/dist/{ts-eval-loader-2RFVZHCT-7CZ3DCDD.js → ts-eval-loader-3G5GEAEC-6F52JLPP.js}

@@ -4,10 +4,10 @@ import {
   loadTsEvalFile,
   loadTsEvalSuite,
   typeScriptEvalConfigGlob
-} from "./chunk-ELCJ23K4.js";
+} from "./chunk-6262OKXM.js";
+import "./chunk-M7BUKBAF.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
-import "./chunk-M7BUKBAF.js";
 import "./chunk-QMRVH5ZP.js";
 export {
   isTypeScriptEvalConfigFileName,
@@ -15,4 +15,4 @@ export {
   loadTsEvalSuite,
   typeScriptEvalConfigGlob
 };
-//# sourceMappingURL=ts-eval-loader-2RFVZHCT-7CZ3DCDD.js.map
+//# sourceMappingURL=ts-eval-loader-3G5GEAEC-6F52JLPP.js.map

package/dist/ts-eval-loader-3G5GEAEC-6F52JLPP.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentv",
-  "version": "5.3.1-next.1",
+  "version": "5.3.2-next.1",
   "description": "CLI entry point for AgentV",
   "type": "module",
   "repository": {
@@ -14,17 +14,42 @@
   "bin": {
     "agentv": "./dist/cli.js"
   },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./sdk": {
+      "types": "./dist/sdk.d.ts",
+      "import": "./dist/sdk.js"
+    },
+    "./provider": {
+      "types": "./dist/provider.d.ts",
+      "import": "./dist/provider.js"
+    },
+    "./contracts": {
+      "types": "./dist/contracts.d.ts",
+      "import": "./dist/contracts.js"
+    },
+    "./config": {
+      "types": "./dist/config.d.ts",
+      "import": "./dist/config.js"
+    },
+    "./package.json": "./package.json"
+  },
   "files": ["dist", "README.md"],
   "scripts": {
     "dev": "bun src/cli.ts",
-    "build": "(cd ../../packages/sdk && bun run build) && tsup && bun run copy-readme",
+    "build": "tsup && bun run copy-readme",
     "copy-readme": "bun -e \"import { cpSync } from 'fs'; cpSync('../../README.md', 'README.md')\"",
     "prepublishOnly": "node -e \"if(process.env.ALLOW_PUBLISH!=='1'){console.error('ERROR: Use bun run publish:next, then bun run promote:latest');process.exit(1)}\"",
-    "typecheck": "(cd ../../packages/sdk && bun run build) && tsc --noEmit",
+    "typecheck": "tsc --noEmit",
     "lint": "biome check .",
     "format": "biome format --write .",
     "fix": "biome check --write .",
-    "test": "(cd ../../packages/sdk && bun run build) && bun test",
+    "test": "bun test",
     "test:watch": "bun test --watch"
   },
   "dependencies": {