agentv 5.3.1-next.1 → 5.3.3

package/dist/sdk.js

@@ -0,0 +1,145 @@
+import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
+import {
+  ProviderInvocationError,
+  ProviderNotAvailableError,
+  codeGrader,
+  containsGrader,
+  createProviderClient,
+  createWorkspace,
+  defineAssertion,
+  defineCodeGrader,
+  defineEval,
+  definePromptTemplate,
+  defineScriptGrader,
+  defineWorkspaceGrader,
+  equalsGrader,
+  exactGrader,
+  graders,
+  isJsonGrader,
+  jsonGrader,
+  llmRubricGrader,
+  normalizeWorkspaceGraderResult,
+  regexGrader,
+  runWorkspaceGrader,
+  scriptGrader,
+  serializeEvalYaml,
+  toEvalYamlObject
+} from "./chunk-MMDLXYBX.js";
+import {
+  CodeGraderInputSchema,
+  CodeGraderResultSchema,
+  ContentFileSchema,
+  ContentImageSchema,
+  ContentSchema,
+  ContentTextSchema,
+  MessageSchema,
+  PromptTemplateInputSchema,
+  ScriptGraderCheckSchema,
+  ScriptGraderInputSchema,
+  ScriptGraderResultSchema,
+  TRACE_EVENT_TYPES,
+  TRACE_REDACTION_LEVELS,
+  TRACE_SOURCE_KINDS,
+  TRACE_TOOL_STATUSES,
+  TokenUsageSchema,
+  ToolCallSchema,
+  TraceArtifactSchema,
+  TraceBranchSchema,
+  TraceErrorSchema,
+  TraceEventSchema,
+  TraceMessageSchema,
+  TraceModelSchema,
+  TraceRawEvidenceSchema,
+  TraceRedactionStateSchema,
+  TraceSchema,
+  TraceSessionSchema,
+  TraceSourceRefSchema,
+  TraceSourceSchema,
+  TraceSummarySchema,
+  TraceToolSchema,
+  defineVitestWorkspaceGrader,
+  runCodeGrader,
+  runScriptGrader,
+  runVitestWorkspaceGrader,
+  vitestReportToCodeGraderResult,
+  vitestReportToScriptGraderResult
+} from "./chunk-AQ5BIAXF.js";
+import "./chunk-TEEXVJWM.js";
+import {
+  defineConfig
+} from "./chunk-JGSRUJZQ.js";
+import {
+  evaluate,
+  external_exports
+} from "./chunk-6262OKXM.js";
+import "./chunk-M7BUKBAF.js";
+import "./chunk-7BGERE6L.js";
+import "./chunk-PEUTJS7B.js";
+import "./chunk-QMRVH5ZP.js";
+export {
+  CodeGraderInputSchema,
+  CodeGraderResultSchema,
+  ContentFileSchema,
+  ContentImageSchema,
+  ContentSchema,
+  ContentTextSchema,
+  MessageSchema,
+  PromptTemplateInputSchema,
+  ProviderInvocationError,
+  ProviderNotAvailableError,
+  ScriptGraderCheckSchema,
+  ScriptGraderInputSchema,
+  ScriptGraderResultSchema,
+  TRACE_EVENT_TYPES,
+  TRACE_REDACTION_LEVELS,
+  TRACE_SOURCE_KINDS,
+  TRACE_TOOL_STATUSES,
+  TokenUsageSchema,
+  ToolCallSchema,
+  TraceArtifactSchema,
+  TraceBranchSchema,
+  TraceErrorSchema,
+  TraceEventSchema,
+  TraceMessageSchema,
+  TraceModelSchema,
+  TraceRawEvidenceSchema,
+  TraceRedactionStateSchema,
+  TraceSchema,
+  TraceSessionSchema,
+  TraceSourceRefSchema,
+  TraceSourceSchema,
+  TraceSummarySchema,
+  TraceToolSchema,
+  codeGrader,
+  containsGrader,
+  createProviderClient,
+  createWorkspace,
+  defineAssertion,
+  defineCodeGrader,
+  defineConfig,
+  defineEval,
+  definePromptTemplate,
+  defineScriptGrader,
+  defineVitestWorkspaceGrader,
+  defineWorkspaceGrader,
+  equalsGrader,
+  evaluate,
+  exactGrader,
+  graders,
+  isJsonGrader,
+  jsonGrader,
+  llmRubricGrader,
+  normalizeWorkspaceGraderResult,
+  regexGrader,
+  runCodeGrader,
+  runScriptGrader,
+  runVitestWorkspaceGrader,
+  runWorkspaceGrader,
+  scriptGrader,
+  serializeEvalYaml,
+  toEvalYamlObject,
+  vitestReportToCodeGraderResult,
+  vitestReportToScriptGraderResult,
+  external_exports as z
+};
+//# sourceMappingURL=sdk.js.map

package/dist/sdk.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/skills/agentv-eval-migrations/references/breaking-changes.md

@@ -74,7 +74,7 @@ v4.42.4 docs and schema used `assertions` for suite-level and per-test graders:
 ```yaml
 assertions:
   - name: correctness
-    type: llm-grader
+    type: llm-rubric
     prompt: ./graders/correctness.md
 
 tests:
@@ -539,11 +539,13 @@ experiment:
 
 ### Current Shape
 
-Current schema accepts top-level `experiment` only as a non-empty string run
-grouping label. Current docs also support promptfoo-shaped `tags.experiment`.
+Current schema rejects top-level `experiment` in authored eval YAML. Use the
+promptfoo-shaped `tags.experiment` key when the label belongs in the eval file,
+or pass CLI `--experiment` when the label is run-time context.
 
 ```yaml
-experiment: with-skills
+tags:
+  experiment: with-skills
 target: codex
 timeout_seconds: 600
 evaluate_options:
@@ -552,14 +554,6 @@ evaluate_options:
     strategy: pass_any
 ```
 
-or:
-
-```yaml
-tags:
-  experiment: with-skills
-target: codex
-```
-
 ### Migration Steps
 
 - If `experiment` is an object, move runtime fields out:
@@ -568,7 +562,8 @@ target: codex
   - budget -> `evaluate_options.budget_usd`
   - timeout -> top-level `timeout_seconds`
   - threshold -> top-level `threshold`
-- Keep only a string label in `experiment`, or use `tags.experiment`.
+- Move string experiment labels to `tags.experiment`, or supply them at run time
+  with CLI `--experiment`.
 
 ### Verification
 
@@ -577,6 +572,8 @@ bun apps/cli/src/cli.ts validate path/to/eval.eval.yaml
 rg -n "^experiment:" path/to/evals
 ```
 
+Any match under authored eval YAML should be removed or migrated.
+
 ### Compatibility Notes
 
 Do not describe a v4.42.4 eval as if `experiment:` was already the main runtime
@@ -895,8 +892,10 @@ targets:
 - Remove `use_target`; current authored target definitions must resolve to
   concrete provider objects.
 - Keep supported AgentV target extensions such as `grader_target`,
-  `fallback_targets`, `workers`, and `batch_requests` as top-level fields on
-  target objects.
+  `fallback_targets`, and `workers` as top-level fields on target objects.
+- Remove runner-level request batching from migrated configs. CLI providers are
+  invoked once per eval case; throughput batching belongs inside provider
+  adapters or CLIs without eval YAML batch configuration.
 
 ### Verification
 
@@ -961,7 +960,7 @@ assert:
 - `type: g-eval` -> `type: llm-rubric`.
 - `type: code-grader`, `code-judge`, `code_grader`, or `code_judge` ->
   `type: script`.
-- `type: llm_judge` or `llm_grader` -> `type: llm-grader`.
+- `type: llm_judge` or `llm_grader` -> `type: llm-rubric`.
 - Convert multi-word snake_case deterministic types to kebab-case:
   `is_json` -> `is-json`, `contains_all` -> `contains-all`,
   `starts_with` -> `starts-with`, and so on.
@@ -990,9 +989,9 @@ v4.42.4 LLM grader docs allowed both:
 
 ```yaml
 assertions:
-  - type: llm-grader
+  - type: llm-rubric
     prompt: ./graders/correctness.md
-  - type: llm-grader
+  - type: llm-rubric
     prompt: file://graders/correctness.md
 ```
 

package/dist/skills/agentv-eval-writer/SKILL.md

@@ -20,14 +20,14 @@ Promptfoo parity matrix: https://agentv.dev/docs/reference/promptfoo-parity/
 Treat YAML as the canonical portable model. Prefer authoring `.eval.yaml` / `EVAL.yaml` first, then use TypeScript helpers, Python scripts, or executable graders only when they lower to the same fields or when the evaluation logic must actually run code.
 
 Eval files define what is tested and how it runs: prompts, datasets, assertions,
-task fixtures, top-level `target`, and suite run controls. Use field-local file
+task fixtures, top-level `providers`, and suite run controls. Use field-local file
 refs such as `tests: file://...`, `prompts: file://...`, `default_test:
 file://...`, and `environment: file://...`. String-valued `tests` and string
 entries inside `tests[]` are raw-case refs for direct paths, directories, and
 globs. Run several full eval suites directly with CLI multi-file selection and
 tags. Use scoped `run:` on individual tests only for `threshold`, `repeat`,
-`timeout_seconds`, and legacy `budget_usd`; keep target selection at top-level
-`target` or CLI `--target`, put suite budget caps under
+`timeout_seconds`, and legacy `budget_usd`; keep provider selection at top-level
+`providers` or CLI `--provider`, put suite budget caps under
 `evaluate_options.budget_usd`, authored concurrency under
 `evaluate_options.max_concurrency`, suite repeat policy under
 `evaluate_options.repeat`, coding-agent testbed setup under `environment`,
@@ -39,7 +39,7 @@ Use `@agentv/sdk` for TypeScript helper imports. Do not use `@agentv/eval` for n
 ## Authoring Checklist
 
 - Put grading criteria in `assert`, not in test-level `criteria`. Plain assertion strings become an `llm-rubric` grader.
-- Prefer plain assertion strings for semantic checks when the default rubric grader can judge them. Use `type: llm-rubric` for structured criteria, custom prompts, custom grader targets, or assertion-level transforms, and `type: script` when grading must execute code.
+- Prefer plain assertion strings for semantic checks when the default rubric grader can judge them. Use `type: llm-rubric` for structured criteria, custom prompts, custom grader providers, or assertion-level transforms. Use `type: agent-rubric` when the grader itself must be an agent-capable provider that can inspect the workspace. Use `type: script` when grading must execute code.
 - Put reference answers in `tests[].vars.expected_output` or `default_test.vars.expected_output`, and consume them with an explicit assertion such as `type: llm-rubric` with `value: "Matches the reference answer: {{ expected_output }}"`. Do not write criteria, scoring instructions, or "the agent should..." rubric prose as the reference answer.
 - For historical or repo-state evals, materialize the repo through a pinned `environment` setup recipe. Mentioning a SHA only in prompt prose is not enough because the agent needs an actual checkout to inspect.
 
@@ -122,7 +122,8 @@ tests:
 
 ```yaml
 description: Example eval
-target: default
+providers:
+  - default
 
 prompts:
   - "{{ prompt }}"
@@ -142,7 +143,7 @@ tests:
 ## Eval File Structure
 
 **Required:** `tests` (array or string raw-case path) or `scenarios`
-**Optional:** `name`, `description`, `experiment`, `version`, `author`, `tags`, `license`, `requires`, `target`, `targets`, `prompts`, `default_test`, `timeout_seconds`, `evaluate_options`, `threshold`, `suite`, `environment`, `env`, `extensions`, `assert`
+**Optional:** `name`, `description`, `version`, `author`, `tags`, `license`, `requires`, `providers`, `prompts`, `default_test`, `timeout_seconds`, `evaluate_options`, `threshold`, `suite`, `environment`, `env`, `extensions`, `assert`
 
 **Test fields:**
 
@@ -151,12 +152,22 @@ tests:
 | `id` | yes | Unique identifier |
 | `vars` | yes when the prompt needs row data | Prompt-template variables for this row |
 | `vars.expected_output` | no | Conventional reference-answer var consumed by explicit graders |
-| `assert` | yes | Graders: deterministic checks, `llm-rubric` checks, script graders, or plain string rubric criteria |
-| `execution` | no | Per-case grader/default overrides such as `skip_defaults`; target selection belongs in top-level `target` or CLI `--target` |
+| `assert` | yes | Graders: deterministic checks, `llm-rubric` / `agent-rubric` checks, script graders, or plain string rubric criteria |
+| `execution` | no | Per-case grader/default overrides such as `skip_defaults`; provider selection belongs in top-level `providers` or CLI `--provider` |
 | `environment` | no | Per-case coding-agent testbed config (overrides suite-level) |
 | `metadata` | no | Arbitrary key-value pairs passed to setup/teardown scripts |
 | `conversation_id` | no | Thread grouping |
 
+**Provider declarations:** AgentV accepts Promptfoo-shaped provider entries
+where they map cleanly: strings such as `openai:gpt-4.1-mini`, object form with
+`id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`,
+and provider maps such as `{ "openai:gpt-4": { label, config } }`. In object
+form, `id` is the backend/spec and `label` is the stable AgentV selection and
+result identity. AgentV-only `runtime`, provider-local `environment`, and
+provider `hooks` must be explicit extensions; direct Promptfoo runs do not
+execute them, and export must lower supported cases or reject unsupported ones
+clearly.
+
 ## Prompt Templates and Vars
 
 Use top-level `prompts` plus `tests[].vars` for the Promptfoo-compatible canonical
@@ -167,7 +178,8 @@ repeat samples.
 
 ```yaml
 description: Prompt matrix example
-target: default
+providers:
+  - default
 
 prompts:
   - id: support-chat
@@ -387,7 +399,7 @@ tests:
 
 When `assert` is defined, **only the declared graders run**. For
 semantic checks, add plain rubric strings. If you need a custom LLM prompt or
-grader target, declare `llm-rubric` explicitly:
+grader provider, declare `llm-rubric` explicitly:
 
 ```yaml
 prompts:
@@ -539,7 +551,7 @@ Configure via the `assert` array. Multiple graders produce a weighted average sc
   type: script
   command: [uv, run, validate.py]
   cwd: ./scripts          # optional working directory
-  target: {}              # optional: enable LLM target proxy (max_calls: 50)
+  provider: {}            # optional: enable LLM provider proxy (max_calls: 50)
 ```
 Contract: stdin JSON -> stdout JSON `{pass, score, reason, checks?: [{text, pass, score?, reason}]}`
 Raw stdin uses snake_case and includes: `input`, `expected_output`, `output` (final answer string), `messages`, `trace`, `trace_summary`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config`
@@ -560,7 +572,7 @@ See the Script Graders docs for the full stdin/stdout contract.
 - name: quality
   type: llm-rubric
   prompt: ./prompts/eval.md     # markdown template or command config
-  target: grader_gpt_5_mini     # optional: override the grader target for this grader
+  provider: grader_gpt_5_mini   # optional: override the grader provider for this grader
   model: gpt-5-chat            # optional model override
   config:                       # passed to prompt templates as context.config
     strictness: high
@@ -568,7 +580,7 @@ See the Script Graders docs for the full stdin/stdout contract.
 Variables: `{{criteria}}`, `{{input}}`, `{{expected_output}}`, `{{output}}`, `{{metadata}}`, `{{metadata_json}}`, `{{rubrics}}`, `{{rubrics_json}}`, `{{file_changes}}`, `{{tool_calls}}`
 - Markdown templates: use `{{variable}}` syntax
 - TypeScript templates: use `definePromptTemplate(fn)` from `@agentv/sdk`, receives context object with all variables + `config`
-- Use `target:` to run different `llm-rubric` graders against different named LLM targets in the same eval (useful for grader panels / ensembles)
+- Use `provider:` to run different `llm-rubric` graders against different named LLM providers in the same eval (useful for grader panels / ensembles)
 
 ### assert-set
 ```yaml
@@ -813,7 +825,7 @@ import { graders } from '@agentv/sdk';
 export function ragFaithfulness() {
   return graders.llmRubric(undefined, {
     name: 'rag-faithfulness',
-    target: 'grader-target',
+    provider: 'grader-provider',
     prompt: 'Grade whether the answer is supported by the retrieved context.',
   });
 }
@@ -862,7 +874,7 @@ export default defineScriptGrader(({ output, trace }) => {
 });
 ```
 
-Use `defineScriptGrader()` when the custom component is a command-backed grader with explicit score control, check arrays, workspace commands, or LLM calls through a grader target. `defineScriptGrader()` scripts are referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. Plain Vitest workspace verifier files can use `command: [agentv, eval, graders/check.test.ts]`.
+Use `defineScriptGrader()` when the custom component is a command-backed grader with explicit score control, check arrays, workspace commands, or LLM calls through a grader provider. `defineScriptGrader()` scripts are referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. Plain Vitest workspace verifier files can use `command: [agentv, eval, graders/check.test.ts]`.
 
 ### Convention-Based Discovery
 

package/dist/skills/agentv-eval-writer/references/custom-evaluators.md

@@ -56,7 +56,7 @@ Promptfoo normally calls eval custom logic assertions and uses fixed assertion t
 
 ```typescript
 import {
-  createTargetClient,
+  createProviderClient,
   defineScriptGrader,
   defineEval,
   type EvalConfig,
@@ -69,7 +69,7 @@ import {
 - `EvalConfig` - Public TypeScript eval authoring type for default-exported `*.eval.ts` and `*.eval.mts` files
 - `defineEval(definition)` - Optional thin helper over the same `EvalConfig` shape
 - `graders` - Helper catalog that returns ordinary AgentV `assert` entries
-- `createTargetClient()` - Returns LLM proxy client (when `target: {}` configured)
+- `createProviderClient()` - Returns LLM proxy client (when `provider: {}` configured)
   - `.invoke({question, systemPrompt})` - Single LLM call
   - `.invokeBatch(requests)` - Batch LLM calls
 - `definePromptTemplate(fn)` - Wraps prompt generation function
@@ -88,7 +88,7 @@ import { graders, type EvalConfig } from '@agentv/sdk';
 function ragFaithfulness() {
   return graders.llmRubric(undefined, {
     name: 'rag-faithfulness',
-    target: 'grader-target',
+    provider: 'grader-provider',
     prompt: 'Grade whether the answer is supported by the retrieved context.',
   });
 }
@@ -120,7 +120,7 @@ assert:
     value: source
   - name: rag-faithfulness
     type: llm-rubric
-    target: grader-target
+    provider: grader-provider
     prompt: Grade whether the answer is supported by the retrieved context.
 ```
 
@@ -181,6 +181,6 @@ Derived from test fields (users never author these directly):
 | `input` | Full resolved input array (JSON) |
 | `expected_output` | Full resolved expected array (JSON) |
 | `output` | Final answer / scored result string |
-| `messages` | Transcript messages from target execution |
+| `messages` | Transcript messages from provider execution |
 
 Markdown templates use `{{variable}}` syntax. TypeScript templates receive context object.