npm - agentv - Versions diffs - 5.1.0-next.1 → 5.2.0-next.1 - Mend

agentv 5.1.0-next.1 → 5.2.0-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (38) hide show

package/README.md +16 -12
package/dist/{artifact-writer-IORG7CW5.js → artifact-writer-SN6UNL5G.js} +5 -5
package/dist/{chunk-ZGPCX334.js → chunk-3S6A2RKR.js} +2803 -959
package/dist/chunk-3S6A2RKR.js.map +1 -0
package/dist/{chunk-GIRUNQGP.js → chunk-3X357HS4.js} +205 -153
package/dist/chunk-3X357HS4.js.map +1 -0
package/dist/{chunk-CBBBRTQ3.js → chunk-46K2OET3.js} +64 -3
package/dist/chunk-46K2OET3.js.map +1 -0
package/dist/{chunk-GJWPQXHN.js → chunk-ALVZQUZP.js} +104 -188
package/dist/chunk-ALVZQUZP.js.map +1 -0
package/dist/{chunk-AM2IPHKS.js → chunk-T4AD3H3Y.js} +236 -154
package/dist/chunk-T4AD3H3Y.js.map +1 -0
package/dist/cli.js +6 -6
package/dist/dashboard/assets/{index-BSHoWrdr.js → index-DNgf3qJ2.js} +1 -1
package/dist/dashboard/assets/{index-zvUKAkKn.css → index-D_bokML8.css} +1 -1
package/dist/dashboard/assets/index-r_jSJmlw.js +121 -0
package/dist/dashboard/index.html +2 -2
package/dist/{dist-2D2GBHZV.js → dist-DEPJOMCA.js} +6 -8
package/dist/index.js +6 -6
package/dist/{interactive-V4SEHLNY.js → interactive-36ZNMQB2.js} +6 -6
package/dist/skills/agentv-bench/SKILL.md +16 -12
package/dist/skills/agentv-bench/references/eval-yaml-spec.md +15 -10
package/dist/skills/agentv-bench/references/migrating-from-skill-creator.md +1 -1
package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +26 -27
package/dist/skills/agentv-eval-writer/SKILL.md +86 -71
package/dist/skills/agentv-eval-writer/references/eval.schema.json +772 -1367
package/dist/{ts-eval-loader-SSPJVEHD-AL2A3V5Z.js → ts-eval-loader-OMG2JBHP-OP5RR7A3.js} +3 -2
package/package.json +8 -2
package/dist/chunk-AM2IPHKS.js.map +0 -1
package/dist/chunk-CBBBRTQ3.js.map +0 -1
package/dist/chunk-GIRUNQGP.js.map +0 -1
package/dist/chunk-GJWPQXHN.js.map +0 -1
package/dist/chunk-ZGPCX334.js.map +0 -1
package/dist/dashboard/assets/index-BaLLxmVO.js +0 -121
/package/dist/{artifact-writer-IORG7CW5.js.map → artifact-writer-SN6UNL5G.js.map} +0 -0
/package/dist/{dist-2D2GBHZV.js.map → dist-DEPJOMCA.js.map} +0 -0
/package/dist/{interactive-V4SEHLNY.js.map → interactive-36ZNMQB2.js.map} +0 -0
/package/dist/{ts-eval-loader-SSPJVEHD-AL2A3V5Z.js.map → ts-eval-loader-OMG2JBHP-OP5RR7A3.js.map} +0 -0

package/dist/dashboard/index.html CHANGED Viewed

@@ -4,8 +4,8 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>AgentV</title>
-    <script type="module" crossorigin src="/assets/index-BaLLxmVO.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-zvUKAkKn.css">
+    <script type="module" crossorigin src="/assets/index-r_jSJmlw.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-D_bokML8.css">
   </head>
   <body class="bg-gray-950 text-gray-100">
     <div id="root"></div>

package/dist/{dist-2D2GBHZV.js → dist-DEPJOMCA.js} RENAMED Viewed

@@ -44,7 +44,6 @@ import {
   resolveResultsRepoRunsDir,
   resolveResultsRepoUrl,
   runBeforeSessionHook,
-  scanRepoDeps,
   setupWipWorktree,
   stageResultsArtifacts,
   syncProject,
@@ -55,10 +54,7 @@ import {
   transpileEvalYaml,
   transpileEvalYamlFile,
   trimBaselineResult
-} from "./chunk-GJWPQXHN.js";
-import {
-  DockerWorkspaceProvider
-} from "./chunk-PEUTJS7B.js";
+} from "./chunk-ALVZQUZP.js";
 import {
   AGENTV_CONFIG_FILE_NAME,
   AGENTV_CONFIG_YML_FILE_NAME,
@@ -377,8 +373,11 @@ import {
   writeArtifactsFromResults,
   writeInitialRunSummaryArtifact,
   writePerTestArtifacts
-} from "./chunk-ZGPCX334.js";
+} from "./chunk-3S6A2RKR.js";
 import "./chunk-7BGERE6L.js";
+import {
+  DockerWorkspaceProvider
+} from "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";
 import "./chunk-QMRVH5ZP.js";
 export {
@@ -700,7 +699,6 @@ export {
   sameReplayEvalPath,
   sanitizeExternalTraceMetadata,
   saveProjectRegistry,
-  scanRepoDeps,
   scoreRangeEvaluationSchema,
   scoreToVerdict,
   serializeEvaluationResultWire,
@@ -756,4 +754,4 @@ export {
   writeInitialRunSummaryArtifact,
   writePerTestArtifacts
 };
-//# sourceMappingURL=dist-2D2GBHZV.js.map
+//# sourceMappingURL=dist-DEPJOMCA.js.map

package/dist/index.js CHANGED Viewed

@@ -6,13 +6,13 @@ import {
   runCli,
   shouldRunBeforeSessionHook,
   usesDeprecatedStudioAlias
-} from "./chunk-GIRUNQGP.js";
-import "./chunk-AM2IPHKS.js";
-import "./chunk-CBBBRTQ3.js";
-import "./chunk-GJWPQXHN.js";
-import "./chunk-PEUTJS7B.js";
-import "./chunk-ZGPCX334.js";
+} from "./chunk-3X357HS4.js";
+import "./chunk-T4AD3H3Y.js";
+import "./chunk-46K2OET3.js";
+import "./chunk-ALVZQUZP.js";
+import "./chunk-3S6A2RKR.js";
 import "./chunk-7BGERE6L.js";
+import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";
 import "./chunk-QMRVH5ZP.js";
 export {

package/dist/{interactive-V4SEHLNY.js → interactive-36ZNMQB2.js} RENAMED Viewed

@@ -8,16 +8,16 @@ import {
   getCategories,
   resolveExistingRunPrimaryPath,
   runEvalCommand
-} from "./chunk-AM2IPHKS.js";
-import "./chunk-CBBBRTQ3.js";
-import "./chunk-GJWPQXHN.js";
-import "./chunk-PEUTJS7B.js";
+} from "./chunk-T4AD3H3Y.js";
+import "./chunk-46K2OET3.js";
+import "./chunk-ALVZQUZP.js";
 import {
   getAgentvConfigDir,
   listTargetNames,
   readTargetDefinitions
-} from "./chunk-ZGPCX334.js";
+} from "./chunk-3S6A2RKR.js";
 import "./chunk-7BGERE6L.js";
+import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";
 import "./chunk-QMRVH5ZP.js";
@@ -344,4 +344,4 @@ ${ANSI_DIM}Retrying execution errors...${ANSI_RESET}
 export {
   launchInteractiveWizard
 };
-//# sourceMappingURL=interactive-V4SEHLNY.js.map
+//# sourceMappingURL=interactive-36ZNMQB2.js.map

package/dist/skills/agentv-bench/SKILL.md CHANGED Viewed

@@ -65,24 +65,28 @@ Before running or optimizing, understand what you're working with.
 AgentV supports two evaluation formats:
-**EVAL.yaml** (native, full features) — supports workspaces, script graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, multi-provider targets. Use this for agent evaluation.
+**EVAL.yaml** (native, full features) — supports environment recipes, script graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, and multi-provider targets. Use this for agent evaluation.
 ```yaml
 # example.eval.yaml
+prompts:
+  - "{{ task }}"
+environment:
+  type: host
+  workdir: ./workspace-template
+  setup:
+    command: ["bash", "-lc", "bun install && bun run build"]
+    cwd: "."
 tests:
   - id: basic-code-review
-    input: "Review this TypeScript file for bugs and suggest improvements"
-    criteria: "Identifies the null pointer bug on line 12 and suggests a fix"
+    vars:
+      task: "Review this TypeScript file for bugs and suggest improvements"
     assert:
       - type: contains
         value: "null"
       - Review identifies the null pointer bug and suggests a concrete fix
-workspace:
-  template: ./workspace-template
-  hooks:
-    before_each:
-      reset: fast
 ```
 Multi-skill evaluation is handled naturally via input messages — describe the task in the test input, and the agent uses whatever skills it needs.
@@ -255,11 +259,11 @@ All artifacts use established schemas — see `references/schemas.md` for the fu
 Write artifacts to `.agentv/artifacts/` or the iteration directory.
-### Workspace features (EVAL.yaml only)
+### Environment features (EVAL.yaml only)
-- **Workspace scope** — clone repos, run setup/teardown hooks (before_all, before_each, after_each, after_all), and choose `suite` or `attempt` lifetime
+- **Environment recipes** — prepare host or Docker testbeds with `type`, `workdir`, and argv-only `setup.command`
 - **Static local override** — use `--workspace-path` only when an existing local directory should be reused as-is
-- **Multi-repo** — clone multiple repos with sparse checkout and shallow clone support
+- **Multi-repo** — materialize repos from setup scripts when a task needs several checkouts
 - **File change tracking** — grade by diffing workspace files before/after agent execution
 ---

package/dist/skills/agentv-bench/references/eval-yaml-spec.md CHANGED Viewed

@@ -9,32 +9,37 @@ The grader agent uses this to evaluate assertions without the CLI.
 - `name` (string, optional) — eval name
 - `description` (string, optional) — description
-- `execution` (object, optional) — `target`, `model`, etc.
-- `workspace` (object, optional) — workspace config (template, repos, hooks)
-- `input` (string | object | Message | Message[], optional) — suite-level input prepended to each test. String/block shorthand expands to a user message.
+- `target` or `targets` (string | object | object[], optional) — system under test selection
+- `environment` (object | `file://...`, optional) — AgentV coding-agent testbed recipe
+- `env` (object, optional) — provider/eval environment variable overrides and template inputs
+- `extensions` (array, optional) — lifecycle hooks such as `beforeAll`, `beforeEach`, `afterEach`, `afterAll`
+- `prompts` (array, optional) — Promptfoo-compatible prompt matrix entries
 - `tests` (array, required) — test cases
 ### Per-test fields
 - `id` (string, required) — unique test identifier
-- `input` (string | object | Message | Message[], required) — task input. String shorthand expands to `[{role: user, content: "..."}]`; object shorthand preserves structured user content when the object has no top-level `role`. Top-level `role` is reserved for message objects.
+- `vars` (object, required when prompts need row data) — prompt-template variables for this row
 - `expected_output` (string | Message[], optional) — passive reference answer. String shorthand expands to `[{role: assistant, content: "..."}]`. It is available to declared graders, but does not add an implicit grader when `assertions` is present.
 - `criteria` (string, optional) — human-readable success criteria
-- `assertions` (array, optional) — grader assertions
+- `assert` (array, optional) — grader assertions
+- `environment` (object | `file://...`, optional) — per-case testbed override
 - `conversation_id` (string, optional) — groups related tests
 - `execution` (object, optional) — per-test execution override
-If `assertions` already state the grading contract, omit `criteria` instead of
+If `assert` already states the grading contract, omit `criteria` instead of
 duplicating the same rubric. Prefer plain assertion strings for semantic checks
 when the default LLM rubric grader can judge them; use multiple named
 `type: llm-rubric` blocks only for custom prompts, custom grader targets, or
 intentional grader panels. Write `expected_output` as a golden/reference answer,
 not as criteria or scoring instructions.
-For historical or repo-state evals, materialize the repository under
-`workspace.repos[]` and pin `commit` to the commit under test. A SHA in prompt
-prose or metadata is context only; it does not give the agent an actual
-checkout.
+For historical or repo-state evals, materialize the repository through
+`environment.setup.command` and pass the repo/ref as argv inputs. A SHA in
+prompt prose or metadata is context only; it does not give the agent an actual
+checkout. `setup.command` is a non-empty string array. Put the executable at
+`command[0]`, put CLI arguments in the remaining array entries, and use
+`timeout_ms` for setup timeout.
 ## 2. Assertion Types and Grading Recipes

package/dist/skills/agentv-bench/references/migrating-from-skill-creator.md CHANGED Viewed

@@ -64,7 +64,7 @@ When evals.json becomes limiting, convert to EVAL.yaml for the full feature set:
 # Convert evals.json to EVAL.yaml
 agentv convert evals.json
-# Edit the generated YAML to add workspace config, script graders, etc.
+# Edit the generated YAML to add environment setup, script graders, etc.
 # Then run with the full lifecycle
 agentv eval eval.yaml
 ```

package/dist/skills/agentv-eval-migrations/references/breaking-changes.md CHANGED Viewed

@@ -40,19 +40,19 @@ For a v4.42.4-era eval:
    `execution.max_concurrency` to `evaluate_options.max_concurrency`, or leave
    it to `--workers` / project config if it is operator policy.
 10. Remove top-level `execution`; current eval YAML rejects it.
-11. Replace `workspace.isolation: shared|per_test` with
-    `workspace.scope: suite|attempt`.
+11. Move authored coding-agent testbed setup from public `workspace` fields to
+    `environment`.
 12. Remove `workspace.mode` and `workspace.path` from committed eval YAML.
     Use `--workspace-path` or `.agentv/config.local.yaml` for local static
     directories.
-13. Replace workspace hook `script:` with `command:`.
-14. For executable setup, prefer top-level `extensions`; keep
-    `workspace.hooks.after_each.reset` for reset policy.
+13. Move lifecycle hooks to top-level `extensions`.
+14. Put reset policy and portable testbed setup under `environment`; keep
+    provider environment overrides under top-level `env`.
 15. Replace authored `preprocessors` and deprecated Promptfoo `postprocess`
     with `transform` at `default_test.options`, `tests[].options`, or the
     assertion that needs the shaped output.
-16. Keep raw cases under `tests` or `imports.tests`; import full eval suites
-    with `imports.suites`.
+16. Keep raw cases under `tests` / `tests: file://...`; run full eval suites
+    directly with CLI multi-file selection and tags.
 17. Validate with `bun apps/cli/src/cli.ts validate <eval-file>`.
 ## Assertions Renamed To `assert`
@@ -266,7 +266,7 @@ tests:
   contains `{type: file, value: "{{ file_path }}"}` and a text block. Store file
   paths in `default_test.vars` or `tests[].vars`.
 - Keep `input` only in external raw-case files imported through
-  `tests: file://...` or `imports.tests` when preserving existing raw datasets.
+  `tests: file://...` when preserving existing raw datasets.
   Do not copy that compatibility shape back into normal eval YAML.
 ### Verification
@@ -945,19 +945,18 @@ The external file contained raw case rows.
 ### Current Shape
-Current eval YAML still accepts inline `tests` and `tests: ./cases.yaml`, but
-adds explicit imports for composition:
+Current eval YAML accepts inline `tests`, `tests: ./cases.yaml`, and field-local
+file refs for raw case data. Run multiple full eval suites directly with CLI
+selection and tags:
 ```yaml
-imports:
-  suites:
-    - path: ../suites/refunds.eval.yaml
-  tests:
-    - path: ../cases/refund-smoke.cases.yaml
+prompts: file://../prompts/refund.yaml
+default_test: file://../defaults/refund.yaml
 tests:
+  - file://../cases/refund-smoke.cases.yaml
   - id: local-edge-case
-    input: Can a final-sale item be refunded after damage in transit?
+    vars:
+      question: Can a final-sale item be refunded after damage in transit?
     assert:
       - Explains the final-sale exception
 ```
@@ -966,21 +965,21 @@ tests:
 - Keep `tests: ./cases.yaml` when the file is a raw case array, JSONL, CSV,
   directory, glob, or script-backed dataset.
-- Use `imports.tests` when importing raw rows into the parent suite context.
-- Use `imports.suites` when importing full child eval suites that own their
-  own `workspace`, input, assertions, and task environment.
-- Do not define a parent `workspace` in a wrapper eval that imports child
-  suites through `imports.suites`; child suites own their environments.
-- Replace legacy `tests[].include` entries with `imports.suites` or
-  `imports.tests` where possible.
-- Use `run:` on import entries only for scoped overrides:
-  `threshold`, `repeat`, `timeout_seconds`, and `budget_usd`.
+- Use `tests: file://...` or string entries inside `tests` for raw rows that run
+  in the parent suite context.
+- Run full eval suites directly with CLI multi-file selection and tags. Do not
+  add wrapper-suite import semantics.
+- Use `prompts: file://...`, `default_test: file://...`, and
+  `environment: file://...` to share reusable config locally at the field that
+  consumes it.
+- Use `run:` on individual tests only for scoped overrides: `threshold`,
+  `repeat`, `timeout_seconds`, and `budget_usd`.
 ### Verification
 ```bash
 bun apps/cli/src/cli.ts validate path/to/eval.eval.yaml
-rg -n "include:|imports:|tests:" path/to/evals
+rg -n "include:|tests:" path/to/evals
 ```
 ### Compatibility Notes

package/dist/skills/agentv-eval-writer/SKILL.md CHANGED Viewed

@@ -20,20 +20,19 @@ Promptfoo parity matrix: https://agentv.dev/docs/reference/promptfoo-parity/
 Treat YAML as the canonical portable model. Prefer authoring `.eval.yaml` / `EVAL.yaml` first, then use TypeScript helpers, Python scripts, or executable graders only when they lower to the same fields or when the evaluation logic must actually run code.
 Eval files define what is tested and how it runs: prompts, datasets, assertions,
-task fixtures, top-level `target`, and suite run controls. Use `imports.suites`
-for full child suites that preserve their workspace, shared input, `assert`,
-fixtures, and graders. Use `imports.tests` for raw case rows that should run in
-the parent file's context. Inline `tests` are also parent-owned raw cases.
-String-valued `tests` and string entries inside `tests[]` are raw-case import
-shorthand for direct paths, directories, and globs. Legacy `tests[].include`
-entries still load with a migration warning, but new evals should use
-`imports.suites` or `imports.tests`. Use scoped `run:` on import entries or
-individual tests only for `threshold`, `repeat`, `timeout_seconds`, and
-legacy `budget_usd`; keep target selection at top-level `target` or CLI `--target`,
-put suite budget caps under `evaluate_options.budget_usd`, authored concurrency
-under `evaluate_options.max_concurrency`, suite repeat policy under
-`evaluate_options.repeat`,
-and keep setup and workspace mutation under `workspace`.
+task fixtures, top-level `target`, and suite run controls. Use field-local file
+refs such as `tests: file://...`, `prompts: file://...`, `default_test:
+file://...`, and `environment: file://...`. String-valued `tests` and string
+entries inside `tests[]` are raw-case refs for direct paths, directories, and
+globs. Run several full eval suites directly with CLI multi-file selection and
+tags. Use scoped `run:` on individual tests only for `threshold`, `repeat`,
+`timeout_seconds`, and legacy `budget_usd`; keep target selection at top-level
+`target` or CLI `--target`, put suite budget caps under
+`evaluate_options.budget_usd`, authored concurrency under
+`evaluate_options.max_concurrency`, suite repeat policy under
+`evaluate_options.repeat`, coding-agent testbed setup under `environment`,
+provider environment overrides under `env`, and lifecycle hooks under
+`extensions`.
 Use `@agentv/sdk` for TypeScript helper imports. Do not use `@agentv/eval` for new evals, examples, scaffolds, or skill guidance; it was a deprecated compatibility package and has been removed from this repository.
@@ -42,7 +41,7 @@ Use `@agentv/sdk` for TypeScript helper imports. Do not use `@agentv/eval` for n
 - Put grading criteria in `assert`, not in test-level `criteria`. Plain assertion strings become an `llm-rubric` grader.
 - Prefer plain assertion strings for semantic checks when the default rubric grader can judge them. Use `type: llm-rubric` for structured criteria, custom prompts, custom grader targets, or assertion-level transforms, and `type: script` when grading must execute code.
 - Write `expected_output` as a golden/reference answer the target could have produced. Do not write criteria, scoring instructions, or "the agent should..." rubric prose there.
-- For historical or repo-state evals, materialize the repo under `workspace.repos[]` pinned to the commit under test. Mentioning a SHA only in prompt prose is not enough because the agent needs an actual checkout to inspect.
+- For historical or repo-state evals, materialize the repo through a pinned `environment` setup recipe. Mentioning a SHA only in prompt prose is not enough because the agent needs an actual checkout to inspect.
 ## Evaluation Types
@@ -62,7 +61,7 @@ agentv convert evals.json
 agentv eval evals.json
 ```
-The converter maps `prompt` → `input`, `expected_output` → `expected_output`, and Agent Skills `assertions` → AgentV `assert` (`llm-rubric` checks), and resolves `files[]` paths. The generated YAML includes TODO comments for AgentV features to add (workspace setup, script graders, rubrics, required gates).
+The converter maps `prompt` → `input`, `expected_output` → `expected_output`, and Agent Skills `assertions` → AgentV `assert` (`llm-rubric` checks), and resolves `files[]` paths. The generated YAML includes TODO comments for AgentV features to add (environment setup, script graders, rubrics, required gates).
 After converting, enhance the YAML with AgentV-specific capabilities shown below.
@@ -132,8 +131,8 @@ tests:
 ## Eval File Structure
-**Required:** `tests` (array or string raw-case path) or `imports`
-**Optional:** `name`, `description`, `experiment`, `version`, `author`, `tags`, `license`, `requires`, `target`, `targets`, `prompts`, `default_test`, `timeout_seconds`, `evaluate_options`, `threshold`, `suite`, `workspace`, `assert`
+**Required:** `tests` (array or string raw-case path) or `scenarios`
+**Optional:** `name`, `description`, `experiment`, `version`, `author`, `tags`, `license`, `requires`, `target`, `targets`, `prompts`, `default_test`, `timeout_seconds`, `evaluate_options`, `threshold`, `suite`, `environment`, `env`, `extensions`, `assert`
 **Test fields:**
@@ -144,7 +143,7 @@ tests:
 | `expected_output` | no | Gold-standard reference answer (string shorthand or full message array) |
 | `assert` | yes | Graders: deterministic checks, `llm-rubric` checks, script graders, or plain string rubric criteria |
 | `execution` | no | Per-case grader/default overrides such as `skip_defaults`; target selection belongs in top-level `target` or CLI `--target` |
-| `workspace` | no | Per-case workspace config (overrides suite-level) |
+| `environment` | no | Per-case coding-agent testbed config (overrides suite-level) |
 | `metadata` | no | Arbitrary key-value pairs passed to setup/teardown scripts |
 | `conversation_id` | no | Thread grouping |
@@ -215,7 +214,7 @@ then render those vars from the prompt template next to the input.
 **JSONL format:** One test per line as JSON. Optional `.yaml` sidecar for shared defaults. See `examples/features/basic-jsonl/`.
-**Environment variables:** Use `{{ env.VAR }}` templates in authored config. Missing vars resolve to empty string. Works in eval files, external case files, and workspace configs. `.env` files are loaded automatically.
+**Environment variables:** Use `{{ env.VAR }}` templates in authored config. Missing vars resolve to empty string. Works in eval files, external case files, and environment configs. `.env` files are loaded automatically.
 ## Output Transforms
@@ -334,19 +333,29 @@ For repo-state evals, combine a pinned checkout, a golden answer, and assertion
 shorthand:
 ```yaml
-workspace:
-  repos:
-    - path: ./agentv
-      repo: https://github.com/EntityProcess/agentv.git
-      commit: 5e3c8f46d80fe66b1a75659e4fd94e38a7e09215
+environment:
+  type: host
+  workdir: ./agentv
+  setup:
+    command:
+      - bash
+      - ./scripts/materialize-repo.sh
+      - ./agentv
+      - https://github.com/EntityProcess/agentv.git
+      - 5e3c8f46d80fe66b1a75659e4fd94e38a7e09215
+    cwd: "."
+prompts:
+  - "{{ task }}"
 tests:
   - id: verification-learning-capture
-    input: |
-      The eval harness has prepared ./agentv at the commit before the
-      verification guidance was added.
+    vars:
+      task: |
+        The eval harness has prepared ./agentv at the commit before the
+        verification guidance was added.
-      Decide what durable repo change should be made and explain why.
+        Decide what durable repo change should be made and explain why.
     expected_output: |
       The durable repo change is to update .agents/verification.md with the
       reusable verification workflow lessons. AGENTS.md already routes this
@@ -431,25 +440,20 @@ assert:
 If a required grader scores below its threshold, the overall verdict is forced to `fail`.
-## Workspace Setup/Teardown
+## Environment Setup/Teardown
 Run scripts before/after each test. Define at suite level or override per case:
 ```yaml
-workspace:
-  template: ./workspace-templates/my-project
-  repos:
-    - path: ./repo
-      repo: sympy/sympy
-      commit: "abc123"
-  hooks:
-    before_all:
-      command: ["bun", "run", "setup.ts"]
-      timeout_ms: 120000
-    after_each:
-      reset: fast
-    after_all:
-      command: ["bun", "run", "teardown.ts"]
+environment:
+  type: host
+  workdir: ./repo
+  setup:
+    command: ["bun", "run", "setup.ts", "sympy/sympy", "abc123"]
+    cwd: "."
+    timeout_ms: 120000
+extensions:
+  - file://scripts/teardown.mjs:afterAll
 tests:
   - id: case-1
@@ -459,42 +463,53 @@ tests:
       source_commit: "abc123"
 ```
-**Lifecycle:** template copy → repo materialization → workspace before_all → target before_all → git baseline → before_each hooks → agent → file changes → after_each hooks → after_all hooks → cleanup
-**Merge:** Case-level fields replace suite-level fields.
-**Commands receive stdin JSON:** `{workspace_path, test_id, eval_run_id, case_input, case_metadata}`
+**Lifecycle:** environment setup → lifecycle extensions → target setup → agent → grading → teardown extensions → cleanup
+**Merge:** Case-level environment fields replace suite-level fields.
+**Setup commands:** `setup.command` is an argv command. `command[0]` is the executable and `command[1...]` are CLI arguments. Use `["bash", "-lc", "..."]` when shell behavior is required.
+**Setup stdin:** AgentV sends environment metadata JSON on stdin for setup commands.
 **Setup failure:** aborts case. **Teardown failure:** non-fatal (warning).
 For SWE-bench-style evals, put operational checkout state under
-`workspace.repos[].commit`; treat `metadata.source_commit` as informational
-only. A SHA in the prompt or metadata without a matching workspace repo pin is
+`environment.setup.command` CLI arguments; treat `metadata.source_commit` as informational only.
+A SHA in the prompt or metadata without a matching environment setup recipe is
 not an operational checkout.
-### Repository Lifecycle
+### Environment Lifecycle
-Materialize repos into the eval workspace automatically. Repo entries declare identity and checkout pins only; AgentV resolves acquisition from configured `repo_resolvers`, then registered projects, `git_cache.mirrors`, its mirror cache, and remote clone. `repo_resolvers` and `git_cache.mirrors` may be defined in `$AGENTV_HOME/config.yaml`, the project's committed `.agentv/config.yaml`, or a gitignored `.agentv/config.override.yaml` (highest precedence) — use the override for machine-specific local bindings without editing tracked or user-global config. Shared repo workspaces use fresh temp materialization by default:
+Describe coding-agent testbeds with `environment`. Reusable recipes should live
+in field-local files and be loaded with `environment: file://...`:
 ```yaml
-workspace:
-  repos:
-    - path: ./repo
-      repo: https://github.com/org/repo.git
-      commit: main
-      ancestor: 1       # parent commit
-  hooks:
-    after_each:
-      reset: fast          # none | fast | strict
-  scope: suite             # suite | attempt
+environment: file://.agentv/environments/repo.yaml
 ```
-- `repo`: full clone URL or GitHub `org/name` shorthand
-- `commit`: branch, tag, or SHA to check out
-- `ancestor`: walk N commits back from the checked-out ref
-- `sparse`: sparse checkout paths array
-- Harness-managed repo workspaces use temp materialization by default; use `workspace.scope: suite | attempt` for portable lifetime
-- Existing local workspace directories are machine-local bindings; use `--workspace-path` or `.agentv/config.local.yaml` with `execution.workspace_path`
-- `hooks.enabled`: boolean (default `true`); set `false` to skip all lifecycle hooks
-- `agentv workspace deps <eval-paths>` scans eval files and outputs a JSON manifest of required git repos (useful for CI pre-cloning)
+```yaml
+# .agentv/environments/repo.yaml
+type: host
+workdir: ./repo
+setup:
+  command: ["bash", "./scripts/materialize-repo.sh", "https://github.com/org/repo.git", "main", "1"]
+  cwd: "."
+  timeout_ms: 120000
+```
-See https://agentv.dev/targets/configuration/#repository-lifecycle
+```yaml
+# .agentv/environments/docker-repo.yaml
+type: docker
+context: ./environment
+dockerfile: Dockerfile
+workdir: /app
+setup:
+  command: ["bash", "-lc", "bun install && bun run build"]
+  cwd: "."
+  timeout_ms: 120000
+```
+- `type`: `host` or `docker`
+- `workdir`: path the target and graders should use
+- `setup`: argv command, optional `cwd`, and optional `timeout_ms` for repository/testbed materialization
+- Top-level `env`: provider/eval environment overrides
+- `environment.env`: recipe-scoped process environment, distinct from top-level `env`
+- `extensions`: lifecycle hooks such as `beforeAll`, `beforeEach`, `afterEach`, and `afterAll`
 ## Grader Types
@@ -511,7 +526,7 @@ Configure via the `assert` array. Multiple graders produce a weighted average sc
 Contract: stdin JSON -> stdout JSON `{score, assertions: [{text, passed, evidence?}], reasoning}`
 Raw stdin uses snake_case and includes: `input`, `expected_output`, `output` (final answer string), `messages`, `trace`, `trace_summary`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config`
 SDK handlers receive the same payload in camelCase: `expectedOutput`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`, `fileChanges`, `workspacePath`.
-When a workspace is configured, `workspace_path` is the absolute path to the workspace dir (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the workspace).
+When an environment prepares a workspace directory, `workspace_path` is the absolute path to that directory (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the prepared workdir).
 For deterministic workspace checks that fit normal Vitest `expect(...)` tests, prefer a plain verifier file and the built-in adapter:
 ```yaml
 - name: welcome_banner