agentv 5.3.1-next.1 → 5.3.3

package/README.md

@@ -1,6 +1,8 @@
 # AgentV
 
-Test AI targets on real repo tasks and measure what actually works.
+> **Deprecated:** AgentV has been replaced by [oh-my-promptfoo](https://github.com/allagentsdev/oh-my-promptfoo) and [Promptfoo](https://github.com/promptfoo/promptfoo). Use oh-my-promptfoo for workspace setup, and migrate your YAML configs to `promptfooconfig.yaml` for Promptfoo.
+
+Test AI providers on real repo tasks and measure what actually works.
 
 ## Why?
 
@@ -10,18 +12,18 @@ Test AI targets on real repo tasks and measure what actually works.
 - **Version-controlled** — evals, judges, and results all live in Git
 - **Hybrid graders** — deterministic code checks + LLM-based subjective scoring
 - **CI/CD native** — exit codes, JSONL output, threshold flags for pipeline gating
-- **Any target** — run against agents, model providers, gateways, replay targets, CLI wrappers, transcript providers, and future app or service wrappers
+- **Any provider** — run against agents, model providers, gateways, replay providers, CLI wrappers, transcript providers, and future app or service wrappers
 
 ## Core Concepts
 
 - **Eval suite / tests** are the task corpus: the prompts, cases, datasets, and reusable field-local files you want to evaluate.
 - **Category** is derived from where the eval lives, such as folder path and file name. Use paths to organize the corpus instead of repeating category labels in every eval.
 - **Environment / fixtures / graders** are task-owned context: host or Docker setup, repos, setup scripts, files, fixtures, deterministic checks, and LLM grading prompts.
-- **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target` by configured target `id` or with an eval-local target object.
-- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and target/model names out of that tag.
-- **Evaluate options** configure eval run behavior such as `max_concurrency`, repeat policy, and budgets.
+- **Provider** is the configured system under test: an agent, model provider, gateway, replay provider, CLI wrapper, transcript provider, or future app/service wrapper. Each provider entry uses `id` for the backend/spec and optional `label` for the stable AgentV selection and result identity.
+- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and provider/model names out of that tag.
+- **Evaluate options** configure eval run behavior such as `max_concurrency`, repeat sample count, and budgets.
 - **Default test** configures inherited per-test defaults such as score `threshold`.
-- **Run** is one concrete execution of a tagged eval against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend.
+- **Run** is one concrete execution of a tagged eval against a resolved provider that writes portable artifacts for readers such as Dashboard, compare, and trend.
 
 ## Quick start
 
@@ -31,12 +33,12 @@ npm install -g agentv
 agentv init
 ```
 
-**2. Configure targets and graders** in `.agentv/config.yaml` — point to the system under test and the reusable grader. Provider settings live under `config`, and target `id` is the selection name used by evals and CLI flags:
+**2. Configure providers and graders** in `.agentv/providers.yaml` — point to the system under test and the reusable grader. Provider `id` names the backend/spec; `label` is the stable selection name used by evals and CLI flags:
 
 ```yaml
-targets:
-  - id: local-openai
-    provider: openai
+providers:
+  - id: openai
+    label: local-openai
     runtime: host
     config:
       api_format: chat
@@ -44,9 +46,9 @@ targets:
       api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
       model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}"
 
-graders:
-  - id: local-openai-grader
-    provider: openai
+  - id: openai
+    label: local-openai-grader
+    runtime: host
     config:
       api_format: chat
       base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
@@ -54,7 +56,7 @@ graders:
       model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}"
 
 defaults:
-  target: local-openai
+  provider: local-openai
   grader: local-openai-grader
 ```
 
@@ -82,7 +84,8 @@ options:
 description: Code generation quality
 tags:
   experiment: with-skills
-target: local-openai
+providers:
+  - local-openai
 evaluate_options:
   max_concurrency: 2
 
@@ -112,29 +115,27 @@ tests:
 Plain assertion strings are short-form rubric criteria: AgentV groups them into
 `llm-rubric` and writes grader detail to `grading.json.component_results` for
 the Dashboard. Use explicit `type: llm-rubric` when you need weights, required
-flags, `score_ranges`, a custom grader prompt, a grader target, or output
+flags, `score_ranges`, a custom grader prompt, a grader provider, or output
 transforms; use string `value` for free-form rubric checks. Executable graders
 use `type: script`.
 
-The target can be an eval-local object when this eval needs target settings of its own:
+The provider can be an eval-local object when this eval needs provider settings of its own:
 
 ```yaml
-description: Code generation quality with eval-local target settings
+description: Code generation quality with eval-local provider settings
 tags:
   experiment: with-skills
-target:
-  id: local-mini
-  provider: openai
-  runtime: host
-  config:
-    api_format: chat
-    base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
-    api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
-    model: gpt-5.4-mini
+providers:
+  - id: openai
+    label: local-mini
+    runtime: host
+    config:
+      api_format: chat
+      base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
+      api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
+      model: gpt-5.4-mini
 evaluate_options:
-  repeat:
-    count: 2
-    strategy: pass_any
+  repeat: 2
 
 default_test:
   threshold: 0.85
@@ -148,7 +149,7 @@ tests:
       input: Write FizzBuzz in Python
 ```
 
-`target: local-openai` resolves the configured target id from `.agentv/config.yaml` and uses its provider, model, hooks, and provider settings. The object form above defines a full eval-local target and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved target metadata.
+`providers: [local-openai]` resolves the configured provider label from `.agentv/providers.yaml` and uses its backend, model, hooks, and provider settings. The object form above defines a full eval-local provider and must include enough provider configuration to run. AgentV records the resolved provider information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved provider metadata.
 
 Use `default_test.threshold` for the inherited per-test pass cutoff. `default_test` can also point at a shared file:
 
@@ -179,7 +180,7 @@ agentv results compare .agentv/results/<baseline-run-id>/.internal/index.jsonl .
 
 ## Results
 
-Each run writes a portable bundle directly under `.agentv/results/<run_id>/`. In this example, `tags.experiment: with-skills` names the condition being measured and `target: local-openai` selects the system under test from `.agentv/config.yaml`; both are recorded as metadata, not path segments. The `.internal/index.jsonl` file is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and target configuration used for the run.
+Each run writes a portable bundle directly under `.agentv/results/<run_id>/`. In this example, `tags.experiment: with-skills` names the condition being measured and `providers: [local-openai]` selects the system under test from `.agentv/providers.yaml`; both are recorded as metadata, not path segments. The `.internal/index.jsonl` file is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and provider configuration used for the run.
 
 ```bash
 agentv eval evals/my-eval.eval.yaml
@@ -192,11 +193,11 @@ Run bundle layout:
 .agentv/results/
 ├── 2026-06-30T08-30-00-000Z/     # <run_id> — one committed run bundle
 │   ├── summary.json              # run rollup: metadata, pass rate, counts, cost
-│   ├── fizzbuzz--a1b2c3d4/       # <result_dir> for one test/target row
+│   ├── fizzbuzz--a1b2c3d4/       # <result_dir> for one test/provider row
 │   │   ├── summary.json          # optional per-case rollup across samples
 │   │   ├── test/                 # generated test bundle: frozen inputs for reproducibility
 │   │   │   ├── EVAL.yaml         #   resolved eval spec
-│   │   │   ├── targets.yaml      #   resolved target config
+│   │   │   ├── providers.yaml    #   resolved provider config
 │   │   │   └── graders/          #   grader files used
 │   │   └── sample-1/             # one materialized sample
 │   │       ├── result.json       # compact sample manifest
@@ -216,7 +217,7 @@ Run bundle layout:
 Use `evaluate()` when your application owns the run:
 
 ```typescript
-import { evaluate } from '@agentv/sdk';
+import { evaluate } from 'agentv';
 
 const { results, summary } = await evaluate({
   experiment: 'with-skills',
@@ -243,20 +244,27 @@ console.log(`${summary.passed}/${summary.total} passed`);
 Use `*.eval.ts` when you want AgentV to run a TypeScript eval config:
 
 ```typescript
-import type { EvalConfig } from '@agentv/sdk';
+import type { EvalConfig } from 'agentv';
 
 const config: EvalConfig = {
   description: 'Code generation quality',
   tags: { experiment: 'with-skills' },
-  target: {
-    extends: 'copilot-sdk',
-    model: 'claude-sonnet-4.6',
-  },
-  repeat: {
-    count: 3,
-    strategy: 'pass_any',
-    earlyExit: false,
+  providers: [
+    {
+      id: 'copilot-sdk',
+      label: 'copilot',
+      config: { model: 'claude-sonnet-4.6' },
+    },
+    {
+      id: 'openai:gpt-5-mini',
+      label: 'grader',
+    },
+  ],
+  defaults: {
+    provider: 'copilot',
+    grader: 'grader',
   },
+  repeat: 3,
   threshold: 0.8,
   prompts: ['{{ input }}'],
   environment: {

package/dist/{artifact-writer-7NBCOAYC.js → artifact-writer-KJEOROKQ.js}

@@ -4,8 +4,8 @@ import {
   buildResultIndexArtifact,
   writeArtifactsFromResults,
   writePerTestArtifacts
-} from "./chunk-RKE7SSET.js";
-import "./chunk-LXBI3SPX.js";
+} from "./chunk-WIAJ7JAV.js";
+import "./chunk-JGSRUJZQ.js";
 import {
   RESULT_INDEX_FILENAME,
   RUN_CONFIG_FILENAME,
@@ -23,10 +23,10 @@ import {
   readRunConfigArtifact,
   writeArtifacts,
   writeInitialRunSummaryArtifact
-} from "./chunk-ELCJ23K4.js";
+} from "./chunk-6262OKXM.js";
+import "./chunk-M7BUKBAF.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
-import "./chunk-M7BUKBAF.js";
 import "./chunk-QMRVH5ZP.js";
 export {
   RESULT_INDEX_FILENAME,
@@ -50,4 +50,4 @@ export {
   writeInitialRunSummaryArtifact,
   writePerTestArtifacts
 };
-//# sourceMappingURL=artifact-writer-7NBCOAYC.js.map
+//# sourceMappingURL=artifact-writer-KJEOROKQ.js.map