agentv 5.0.0-next.1 → 5.1.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +234 -52
- package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js → agentv-provider-AYXH7WLW-6YZKZ7PS.js} +2 -2
- package/dist/{artifact-writer-QD52FC56.js → artifact-writer-IORG7CW5.js} +15 -8
- package/dist/{chunk-IJEZFZJR.js → chunk-7BGERE6L.js} +2 -2
- package/dist/chunk-7BGERE6L.js.map +1 -0
- package/dist/{chunk-IBI7A5RP.js → chunk-AM2IPHKS.js} +3355 -2244
- package/dist/chunk-AM2IPHKS.js.map +1 -0
- package/dist/{chunk-6JVM4GJI.js → chunk-CBBBRTQ3.js} +126 -80
- package/dist/chunk-CBBBRTQ3.js.map +1 -0
- package/dist/{chunk-6Q56CEXV.js → chunk-GIRUNQGP.js} +8171 -9308
- package/dist/chunk-GIRUNQGP.js.map +1 -0
- package/dist/{chunk-UG4JHZYM.js → chunk-GJWPQXHN.js} +673 -1346
- package/dist/chunk-GJWPQXHN.js.map +1 -0
- package/dist/{chunk-RCIEHFQ2.js → chunk-PEUTJS7B.js} +3 -3
- package/dist/{chunk-5H446C7X.js → chunk-QMRVH5ZP.js} +2 -8
- package/dist/{chunk-GZJMODLW.js → chunk-ZGPCX334.js} +45022 -31528
- package/dist/chunk-ZGPCX334.js.map +1 -0
- package/dist/cli.js +8 -9
- package/dist/cli.js.map +1 -1
- package/dist/dashboard/assets/{index-D5_8DE_B.js → index-BSHoWrdr.js} +1 -1
- package/dist/dashboard/assets/index-BaLLxmVO.js +121 -0
- package/dist/dashboard/assets/index-zvUKAkKn.css +1 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/{dist-KJ6PELQY.js → dist-2D2GBHZV.js} +72 -56
- package/dist/{docker-workspace-ZJ2ESVBI-6UDETQII.js → docker-workspace-NNK4Z6FR-QRTIG6TM.js} +3 -3
- package/dist/{exec-MYPAJJP7-C5HD5WG5.js → exec-4HFFRQ3B-AXDMJNLJ.js} +3 -3
- package/dist/index.js +8 -9
- package/dist/{interactive-2WMVAASS.js → interactive-V4SEHLNY.js} +13 -29
- package/dist/interactive-V4SEHLNY.js.map +1 -0
- package/dist/skills/agentv-bench/SKILL.md +24 -24
- package/dist/skills/agentv-bench/agents/analyzer.md +6 -6
- package/dist/skills/agentv-bench/agents/comparator.md +4 -5
- package/dist/skills/agentv-bench/agents/grader.md +4 -4
- package/dist/skills/agentv-bench/references/autoresearch.md +3 -3
- package/dist/skills/agentv-bench/references/description-optimization.md +2 -2
- package/dist/skills/agentv-bench/references/environment-adaptation.md +10 -10
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +42 -26
- package/dist/skills/agentv-bench/references/migrating-from-skill-creator.md +4 -4
- package/dist/skills/agentv-bench/references/subagent-pipeline.md +13 -12
- package/dist/skills/agentv-eval-migrations/SKILL.md +33 -0
- package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +1013 -0
- package/dist/skills/agentv-eval-review/SKILL.md +6 -3
- package/dist/skills/agentv-eval-review/scripts/lint_eval.py +1 -1
- package/dist/skills/agentv-eval-writer/SKILL.md +328 -209
- package/dist/skills/agentv-eval-writer/references/{config-schema.json → config.schema.json} +6 -5
- package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +23 -18
- package/dist/skills/agentv-eval-writer/references/eval.schema.json +10447 -0
- package/dist/skills/agentv-eval-writer/references/python-helpers.md +8 -8
- package/dist/skills/agentv-eval-writer/references/rubric-evaluator.md +21 -22
- package/dist/skills/agentv-trace-analyst/SKILL.md +4 -4
- package/dist/templates/.agentv/targets.yaml +41 -40
- package/dist/{ts-eval-loader-RE43OAGX-AMRCUKBZ.js → ts-eval-loader-SSPJVEHD-AL2A3V5Z.js} +4 -4
- package/package.json +13 -4
- package/dist/chunk-6JVM4GJI.js.map +0 -1
- package/dist/chunk-6Q56CEXV.js.map +0 -1
- package/dist/chunk-76FOHROU.js +0 -96
- package/dist/chunk-76FOHROU.js.map +0 -1
- package/dist/chunk-GZJMODLW.js.map +0 -1
- package/dist/chunk-HSHTTN6C.js +0 -565
- package/dist/chunk-HSHTTN6C.js.map +0 -1
- package/dist/chunk-IBI7A5RP.js.map +0 -1
- package/dist/chunk-IJEZFZJR.js.map +0 -1
- package/dist/chunk-LRULMAAA.js +0 -1711
- package/dist/chunk-LRULMAAA.js.map +0 -1
- package/dist/chunk-R3CXYNTN.js +0 -1238
- package/dist/chunk-R3CXYNTN.js.map +0 -1
- package/dist/chunk-UG4JHZYM.js.map +0 -1
- package/dist/chunk-VQ2ZO7XJ.js +0 -2098
- package/dist/chunk-VQ2ZO7XJ.js.map +0 -1
- package/dist/chunk-XALGXSKB.js +0 -21
- package/dist/chunk-XALGXSKB.js.map +0 -1
- package/dist/dashboard/assets/index-BVMrdN8a.js +0 -119
- package/dist/dashboard/assets/index-DZWXfWsv.css +0 -1
- package/dist/esm-NNMQYIKT.js +0 -32
- package/dist/esm-R77SNOF5.js +0 -65
- package/dist/esm-RVQPUGWH.js +0 -1207
- package/dist/esm-RVQPUGWH.js.map +0 -1
- package/dist/esm-TTEMJEVV.js +0 -933
- package/dist/esm-TTEMJEVV.js.map +0 -1
- package/dist/esm-ZADQ4XQH-QUFGGZL4.js +0 -924
- package/dist/esm-ZADQ4XQH-QUFGGZL4.js.map +0 -1
- package/dist/exec-MYPAJJP7-C5HD5WG5.js.map +0 -1
- package/dist/getMachineId-bsd-HSK5LZMG.js +0 -41
- package/dist/getMachineId-bsd-HSK5LZMG.js.map +0 -1
- package/dist/getMachineId-darwin-4DP6CCJV.js +0 -41
- package/dist/getMachineId-darwin-4DP6CCJV.js.map +0 -1
- package/dist/getMachineId-linux-44LJ5UJB.js +0 -33
- package/dist/getMachineId-linux-44LJ5UJB.js.map +0 -1
- package/dist/getMachineId-unsupported-NVK6IATM.js +0 -24
- package/dist/getMachineId-unsupported-NVK6IATM.js.map +0 -1
- package/dist/getMachineId-win-YZ36S7VA.js +0 -43
- package/dist/getMachineId-win-YZ36S7VA.js.map +0 -1
- package/dist/interactive-2WMVAASS.js.map +0 -1
- package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js +0 -9
- package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js.map +0 -1
- package/dist/skills/agentv-eval-writer/references/eval-schema.json +0 -17295
- package/dist/skills/agentv-eval-writer/references/experiment-schema.json +0 -278
- package/dist/src-76ISS3LH.js +0 -1733
- package/dist/src-76ISS3LH.js.map +0 -1
- package/dist/ts-eval-loader-RE43OAGX-AMRCUKBZ.js.map +0 -1
- /package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js.map → agentv-provider-AYXH7WLW-6YZKZ7PS.js.map} +0 -0
- /package/dist/{artifact-writer-QD52FC56.js.map → artifact-writer-IORG7CW5.js.map} +0 -0
- /package/dist/{chunk-RCIEHFQ2.js.map → chunk-PEUTJS7B.js.map} +0 -0
- /package/dist/{chunk-5H446C7X.js.map → chunk-QMRVH5ZP.js.map} +0 -0
- /package/dist/{dist-KJ6PELQY.js.map → dist-2D2GBHZV.js.map} +0 -0
- /package/dist/{docker-workspace-ZJ2ESVBI-6UDETQII.js.map → docker-workspace-NNK4Z6FR-QRTIG6TM.js.map} +0 -0
- /package/dist/{esm-NNMQYIKT.js.map → exec-4HFFRQ3B-AXDMJNLJ.js.map} +0 -0
- /package/dist/{esm-R77SNOF5.js.map → ts-eval-loader-SSPJVEHD-AL2A3V5Z.js.map} +0 -0
package/README.md
CHANGED
|
@@ -1,36 +1,8 @@
|
|
|
1
1
|
# AgentV
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Test AI targets on real repo tasks and measure what actually works.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
npm install -g agentv
|
|
7
|
-
agentv init
|
|
8
|
-
agentv eval evals/example.yaml
|
|
9
|
-
```
|
|
10
|
-
|
|
11
|
-
That's it. Results in seconds, not minutes.
|
|
12
|
-
|
|
13
|
-
## What it does
|
|
14
|
-
|
|
15
|
-
AgentV runs evaluation cases against your AI agents and scores them with deterministic code graders + customizable LLM graders. Everything lives in Git — YAML eval files, markdown judge prompts, JSONL results.
|
|
16
|
-
|
|
17
|
-
```yaml
|
|
18
|
-
# evals/math.yaml
|
|
19
|
-
description: Math problem solving
|
|
20
|
-
tests:
|
|
21
|
-
- id: addition
|
|
22
|
-
input: What is 15 + 27?
|
|
23
|
-
expected_output: "42"
|
|
24
|
-
assertions:
|
|
25
|
-
- type: contains
|
|
26
|
-
value: "42"
|
|
27
|
-
```
|
|
28
|
-
|
|
29
|
-
```bash
|
|
30
|
-
agentv eval evals/math.yaml
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
## Why AgentV?
|
|
5
|
+
## Why?
|
|
34
6
|
|
|
35
7
|
- **Local-first** — runs on your machine, no cloud accounts or API keys for eval infrastructure
|
|
36
8
|
- **Repo-backed workspaces** — reuse real repos, setup scripts, and existing harnesses instead of rebuilding synthetic tasks
|
|
@@ -38,7 +10,18 @@ agentv eval evals/math.yaml
|
|
|
38
10
|
- **Version-controlled** — evals, judges, and results all live in Git
|
|
39
11
|
- **Hybrid graders** — deterministic code checks + LLM-based subjective scoring
|
|
40
12
|
- **CI/CD native** — exit codes, JSONL output, threshold flags for pipeline gating
|
|
41
|
-
- **Any
|
|
13
|
+
- **Any target** — run against agents, model providers, gateways, replay targets, CLI wrappers, transcript providers, and future app or service wrappers
|
|
14
|
+
|
|
15
|
+
## Core Concepts
|
|
16
|
+
|
|
17
|
+
- **Eval suite / imports / tests** are the task corpus: the prompts, cases, datasets, and imported benchmarks you want to evaluate.
|
|
18
|
+
- **Category** is derived from where the eval lives, such as folder path and file name. Use paths to organize the corpus instead of repeating category labels in every eval.
|
|
19
|
+
- **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts.
|
|
20
|
+
- **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target` by configured target `id` or with an eval-local target object.
|
|
21
|
+
- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and target/model names out of that tag.
|
|
22
|
+
- **Evaluate options** configure eval run behavior such as `max_concurrency`, repeat policy, and budgets.
|
|
23
|
+
- **Default test** configures inherited per-test defaults such as score `threshold`.
|
|
24
|
+
- **Run** is one concrete execution of a tagged eval against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend.
|
|
42
25
|
|
|
43
26
|
## Quick start
|
|
44
27
|
|
|
@@ -48,54 +31,208 @@ npm install -g agentv
|
|
|
48
31
|
agentv init
|
|
49
32
|
```
|
|
50
33
|
|
|
51
|
-
**2. Configure targets** in `.agentv/
|
|
34
|
+
**2. Configure targets and graders** in `.agentv/config.yaml` — point to the system under test and the reusable grader. Provider settings live under `config`, and target `id` is the selection name used by evals and CLI flags:
|
|
35
|
+
|
|
36
|
+
```yaml
|
|
37
|
+
targets:
|
|
38
|
+
- id: local-openai
|
|
39
|
+
provider: openai
|
|
40
|
+
runtime: host
|
|
41
|
+
config:
|
|
42
|
+
api_format: chat
|
|
43
|
+
base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
|
|
44
|
+
api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
|
|
45
|
+
model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
|
|
46
|
+
|
|
47
|
+
graders:
|
|
48
|
+
- id: local-openai-grader
|
|
49
|
+
provider: openai
|
|
50
|
+
config:
|
|
51
|
+
api_format: chat
|
|
52
|
+
base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
|
|
53
|
+
api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
|
|
54
|
+
model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
|
|
55
|
+
|
|
56
|
+
defaults:
|
|
57
|
+
target: local-openai
|
|
58
|
+
grader: local-openai-grader
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
**3. Create shared test defaults** in `evals/default-test.yaml`. This is a partial test config that AgentV applies to each test:
|
|
62
|
+
|
|
63
|
+
```yaml
|
|
64
|
+
threshold: 0.8
|
|
65
|
+
options:
|
|
66
|
+
rubric_prompt: |
|
|
67
|
+
You are an expert grader. Evaluate the candidate answer against each rubric item.
|
|
68
|
+
Award credit only when the answer directly supports the criterion.
|
|
69
|
+
|
|
70
|
+
[[ ## question ## ]]
|
|
71
|
+
{{ input }}
|
|
72
|
+
|
|
73
|
+
[[ ## rubric ## ]]
|
|
74
|
+
{{ rubrics }}
|
|
75
|
+
|
|
76
|
+
[[ ## answer ## ]]
|
|
77
|
+
{{ output }}
|
|
78
|
+
```
|
|
52
79
|
|
|
53
|
-
**
|
|
80
|
+
**4. Create an eval** in `evals/my-eval.eval.yaml`:
|
|
54
81
|
```yaml
|
|
55
82
|
description: Code generation quality
|
|
83
|
+
tags:
|
|
84
|
+
experiment: with-skills
|
|
85
|
+
target: local-openai
|
|
86
|
+
evaluate_options:
|
|
87
|
+
max_concurrency: 2
|
|
88
|
+
|
|
89
|
+
default_test: file://./default-test.yaml
|
|
90
|
+
|
|
91
|
+
prompts:
|
|
92
|
+
- "{{ input }}"
|
|
93
|
+
|
|
56
94
|
tests:
|
|
57
95
|
- id: fizzbuzz
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
96
|
+
vars:
|
|
97
|
+
input: Write FizzBuzz in Python. Use lowercase output strings "fizz", "buzz", and "fizzbuzz". Return only one Python code block.
|
|
98
|
+
assert:
|
|
61
99
|
- type: contains
|
|
62
100
|
value: "fizz"
|
|
63
|
-
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
101
|
+
- Implements correct FizzBuzz logic for multiples of 3, 5, and 15
|
|
102
|
+
- type: script
|
|
103
|
+
command: ["python3", "../validators/check_syntax.py"]
|
|
104
|
+
- type: llm-rubric
|
|
105
|
+
value:
|
|
106
|
+
- outcome: Solution is simple and idiomatic Python
|
|
107
|
+
weight: 0.5
|
|
108
|
+
- outcome: Handles the 3, 5, and 15 branches correctly
|
|
109
|
+
weight: 1.5
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Plain assertion strings are short-form rubric criteria: AgentV groups them into
|
|
113
|
+
`llm-rubric` and writes each criterion to `grading.json.assertion_results` for the
|
|
114
|
+
Dashboard. Use explicit `type: llm-rubric` when you need weights, required flags, or
|
|
115
|
+
`score_ranges`, or when you need a custom grader prompt, grader target, or
|
|
116
|
+
output transforms; use string `value` for free-form rubric checks. Executable
|
|
117
|
+
graders use `type: script`.
|
|
118
|
+
|
|
119
|
+
The target can be an eval-local object when this eval needs target settings of its own:
|
|
120
|
+
|
|
121
|
+
```yaml
|
|
122
|
+
description: Code generation quality with eval-local target settings
|
|
123
|
+
tags:
|
|
124
|
+
experiment: with-skills
|
|
125
|
+
target:
|
|
126
|
+
id: local-mini
|
|
127
|
+
provider: openai
|
|
128
|
+
runtime: host
|
|
129
|
+
config:
|
|
130
|
+
api_format: chat
|
|
131
|
+
base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
|
|
132
|
+
api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
|
|
133
|
+
model: gpt-5.4-mini
|
|
134
|
+
evaluate_options:
|
|
135
|
+
repeat:
|
|
136
|
+
count: 2
|
|
137
|
+
strategy: pass_any
|
|
138
|
+
|
|
139
|
+
default_test:
|
|
140
|
+
threshold: 0.85
|
|
141
|
+
|
|
142
|
+
prompts:
|
|
143
|
+
- "{{ input }}"
|
|
144
|
+
|
|
145
|
+
tests:
|
|
146
|
+
- id: fizzbuzz
|
|
147
|
+
vars:
|
|
148
|
+
input: Write FizzBuzz in Python
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
`target: local-openai` resolves the configured target id from `.agentv/config.yaml` and uses its provider, model, hooks, and provider settings. The object form above defines a full eval-local target and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved target metadata.
|
|
152
|
+
|
|
153
|
+
Use `default_test.threshold` for the inherited per-test pass cutoff. `default_test` can also point at a shared file:
|
|
154
|
+
|
|
155
|
+
```yaml
|
|
156
|
+
default_test: file://{{ env.AGENTV_REPO_ROOT }}/.agentv/default-test.yaml
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
AgentV makes `AGENTV_REPO_ROOT` available during eval/config interpolation. Projects that prefer a short name can define their own reference in `.agentv/config.yaml`; `global-default` below is just an example key:
|
|
160
|
+
|
|
161
|
+
```yaml
|
|
162
|
+
refs:
|
|
163
|
+
global-default: file://{{ env.AGENTV_REPO_ROOT }}/.agentv/default-test.yaml
|
|
67
164
|
```
|
|
68
165
|
|
|
69
|
-
|
|
166
|
+
Then eval files in that project can use `default_test: ref://global-default`.
|
|
167
|
+
|
|
168
|
+
The checked-in version of this quickstart lives in [`examples/features/readme-quickstart/`](examples/features/readme-quickstart/).
|
|
169
|
+
|
|
170
|
+
**5. Run it:**
|
|
70
171
|
```bash
|
|
71
|
-
agentv eval evals/my-eval.yaml
|
|
172
|
+
agentv eval evals/my-eval.eval.yaml
|
|
72
173
|
```
|
|
73
174
|
|
|
74
|
-
**
|
|
175
|
+
**6. Compare two runs** (pass two run indexes — e.g. before and after a change):
|
|
75
176
|
```bash
|
|
76
|
-
agentv compare .agentv/results/
|
|
177
|
+
agentv results compare .agentv/results/<baseline-run-id>/.internal/index.jsonl .agentv/results/<candidate-run-id>/.internal/index.jsonl
|
|
77
178
|
```
|
|
78
179
|
|
|
79
|
-
##
|
|
180
|
+
## Results
|
|
181
|
+
|
|
182
|
+
Each run writes a portable bundle directly under `.agentv/results/<run_id>/`. In this example, `tags.experiment: with-skills` names the condition being measured and `target: local-openai` selects the system under test from `.agentv/config.yaml`; both are recorded as metadata, not path segments. The `.internal/index.jsonl` file is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and target configuration used for the run.
|
|
80
183
|
|
|
81
184
|
```bash
|
|
82
|
-
agentv eval evals/my-eval.yaml
|
|
83
|
-
cat
|
|
185
|
+
agentv eval evals/my-eval.eval.yaml
|
|
186
|
+
cat .agentv/results/<run_id>/.internal/index.jsonl
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Run bundle layout:
|
|
190
|
+
|
|
191
|
+
```
|
|
192
|
+
.agentv/results/
|
|
193
|
+
├── 2026-06-30T08-30-00-000Z/ # <run_id> — one committed run bundle
|
|
194
|
+
│ ├── summary.json # run rollup: metadata, pass rate, counts, cost
|
|
195
|
+
│ ├── fizzbuzz--a1b2c3d4/ # <result_dir> for one test/target row
|
|
196
|
+
│ │ ├── summary.json # optional per-case rollup across attempts
|
|
197
|
+
│ │ ├── test/ # generated test bundle: frozen inputs for reproducibility
|
|
198
|
+
│ │ │ ├── EVAL.yaml # resolved eval spec
|
|
199
|
+
│ │ │ ├── targets.yaml # resolved target config
|
|
200
|
+
│ │ │ └── graders/ # grader files used
|
|
201
|
+
│ │ └── sample-1/ # one materialized sample
|
|
202
|
+
│ │ ├── result.json # compact attempt manifest
|
|
203
|
+
│ │ ├── grading.json # assertion_results and grader evidence
|
|
204
|
+
│ │ ├── metrics.json # tool calls, transcript stats, behavior metrics
|
|
205
|
+
│ │ ├── transcript.json # normalized agent transcript
|
|
206
|
+
│ │ ├── transcript-raw.jsonl # raw agent output (debugging)
|
|
207
|
+
│ │ └── outputs/ # captured stdout and grader outputs
|
|
208
|
+
│ └── .internal/
|
|
209
|
+
│ └── index.jsonl # row index for scripts/CI and `agentv results compare`
|
|
210
|
+
├── .indexes/ # reserved local/rebuildable indexes
|
|
211
|
+
└── .cache/ # reserved local cache
|
|
84
212
|
```
|
|
85
213
|
|
|
86
214
|
## TypeScript SDK
|
|
87
215
|
|
|
88
|
-
Use
|
|
216
|
+
Use `evaluate()` when your application owns the run:
|
|
89
217
|
|
|
90
218
|
```typescript
|
|
91
219
|
import { evaluate } from '@agentv/sdk';
|
|
92
220
|
|
|
93
221
|
const { results, summary } = await evaluate({
|
|
222
|
+
experiment: 'with-skills',
|
|
223
|
+
task: async (input) => runMyAppTarget(input),
|
|
224
|
+
threshold: 0.8,
|
|
225
|
+
prompts: ['{{ input }}'],
|
|
94
226
|
tests: [
|
|
95
227
|
{
|
|
96
|
-
id: '
|
|
97
|
-
input: '
|
|
98
|
-
|
|
228
|
+
id: 'fizzbuzz',
|
|
229
|
+
vars: { input: 'Write FizzBuzz in Python' },
|
|
230
|
+
assert: [
|
|
231
|
+
{ type: 'contains', value: 'fizz' },
|
|
232
|
+
'Implements correct FizzBuzz logic for multiples of 3, 5, and 15',
|
|
233
|
+
{ type: 'script', command: ['python3', './validators/check_syntax.py'] },
|
|
234
|
+
{ type: 'llm-rubric', value: ['Solution is simple and idiomatic Python'] },
|
|
235
|
+
],
|
|
99
236
|
},
|
|
100
237
|
],
|
|
101
238
|
});
|
|
@@ -103,12 +240,57 @@ const { results, summary } = await evaluate({
|
|
|
103
240
|
console.log(`${summary.passed}/${summary.total} passed`);
|
|
104
241
|
```
|
|
105
242
|
|
|
243
|
+
Use `defineEval()` when you want AgentV to run the TypeScript eval file:
|
|
244
|
+
|
|
245
|
+
```typescript
|
|
246
|
+
import { defineEval } from '@agentv/sdk';
|
|
247
|
+
|
|
248
|
+
export default defineEval({
|
|
249
|
+
description: 'Code generation quality',
|
|
250
|
+
tags: { experiment: 'with-skills' },
|
|
251
|
+
target: {
|
|
252
|
+
extends: 'copilot-sdk',
|
|
253
|
+
model: 'claude-sonnet-4.6',
|
|
254
|
+
},
|
|
255
|
+
repeat: {
|
|
256
|
+
count: 3,
|
|
257
|
+
strategy: 'pass_any',
|
|
258
|
+
earlyExit: false,
|
|
259
|
+
},
|
|
260
|
+
threshold: 0.8,
|
|
261
|
+
prompts: ['{{ input }}'],
|
|
262
|
+
workspace: {
|
|
263
|
+
scope: 'attempt',
|
|
264
|
+
repos: [
|
|
265
|
+
{
|
|
266
|
+
path: './fixture',
|
|
267
|
+
repo: 'EntityProcess/agentv-contract-fixture',
|
|
268
|
+
commit: '21a34daed7ebcfe36cbed053607622a55e5e94cb',
|
|
269
|
+
},
|
|
270
|
+
],
|
|
271
|
+
},
|
|
272
|
+
tests: [
|
|
273
|
+
{
|
|
274
|
+
id: 'fizzbuzz',
|
|
275
|
+
vars: { input: 'Write FizzBuzz in Python' },
|
|
276
|
+
assert: [
|
|
277
|
+
{ type: 'contains', value: 'fizz' },
|
|
278
|
+
'Implements correct FizzBuzz logic for multiples of 3, 5, and 15',
|
|
279
|
+
{ type: 'script', command: ['python3', './validators/check_syntax.py'] },
|
|
280
|
+
{ type: 'llm-rubric', value: ['Solution is simple and idiomatic Python'] },
|
|
281
|
+
],
|
|
282
|
+
},
|
|
283
|
+
],
|
|
284
|
+
});
|
|
285
|
+
```
|
|
286
|
+
|
|
106
287
|
## Documentation
|
|
107
288
|
|
|
108
289
|
Full docs at [agentv.dev/docs](https://agentv.dev/docs/getting-started/introduction/).
|
|
109
290
|
|
|
110
291
|
- [Eval files](https://agentv.dev/docs/evaluation/eval-files/) — format and structure
|
|
111
|
-
- [Custom
|
|
292
|
+
- [Custom assertions](https://agentv.dev/docs/graders/custom-assertions/) — reusable assertion types
|
|
293
|
+
- [Script graders](https://agentv.dev/docs/graders/script-graders/) — command-backed graders in any language
|
|
112
294
|
- [Rubrics](https://agentv.dev/docs/evaluation/rubrics/) — structured criteria scoring
|
|
113
295
|
- [Targets](https://agentv.dev/docs/targets/configuration/) — configure agents and providers
|
|
114
296
|
- [Compare results](https://agentv.dev/docs/tools/compare/) — A/B testing and regression detection
|
|
@@ -2,8 +2,8 @@ import { createRequire } from 'node:module'; const require = createRequire(impor
|
|
|
2
2
|
import {
|
|
3
3
|
AgentvProvider
|
|
4
4
|
} from "./chunk-M7BUKBAF.js";
|
|
5
|
-
import "./chunk-
|
|
5
|
+
import "./chunk-QMRVH5ZP.js";
|
|
6
6
|
export {
|
|
7
7
|
AgentvProvider
|
|
8
8
|
};
|
|
9
|
-
//# sourceMappingURL=agentv-provider-AYXH7WLW-
|
|
9
|
+
//# sourceMappingURL=agentv-provider-AYXH7WLW-6YZKZ7PS.js.map
|
|
@@ -4,32 +4,38 @@ import {
|
|
|
4
4
|
buildResultIndexArtifact,
|
|
5
5
|
writeArtifactsFromResults,
|
|
6
6
|
writePerTestArtifacts
|
|
7
|
-
} from "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
10
|
-
import "./chunk-RCIEHFQ2.js";
|
|
7
|
+
} from "./chunk-CBBBRTQ3.js";
|
|
8
|
+
import "./chunk-GJWPQXHN.js";
|
|
9
|
+
import "./chunk-PEUTJS7B.js";
|
|
11
10
|
import {
|
|
12
11
|
RESULT_INDEX_FILENAME,
|
|
12
|
+
RUN_CONFIG_FILENAME,
|
|
13
13
|
RUN_SUMMARY_FILENAME,
|
|
14
14
|
aggregateRunDir,
|
|
15
15
|
buildAggregateGradingArtifact,
|
|
16
|
+
buildEvalTestTargetKey,
|
|
17
|
+
buildEvaluationResultTargetKey,
|
|
16
18
|
buildGradingArtifact,
|
|
17
19
|
buildRunSummaryArtifact,
|
|
18
20
|
buildTestTargetKey,
|
|
19
21
|
buildTimingArtifact,
|
|
20
22
|
deduplicateByTestIdTarget,
|
|
21
23
|
parseJsonlResults,
|
|
24
|
+
readRunConfigArtifact,
|
|
22
25
|
writeArtifacts,
|
|
23
26
|
writeInitialRunSummaryArtifact
|
|
24
|
-
} from "./chunk-
|
|
25
|
-
import "./chunk-
|
|
27
|
+
} from "./chunk-ZGPCX334.js";
|
|
28
|
+
import "./chunk-7BGERE6L.js";
|
|
26
29
|
import "./chunk-M7BUKBAF.js";
|
|
27
|
-
import "./chunk-
|
|
30
|
+
import "./chunk-QMRVH5ZP.js";
|
|
28
31
|
export {
|
|
29
32
|
RESULT_INDEX_FILENAME,
|
|
33
|
+
RUN_CONFIG_FILENAME,
|
|
30
34
|
RUN_SUMMARY_FILENAME,
|
|
31
35
|
aggregateRunDir,
|
|
32
36
|
buildAggregateGradingArtifact,
|
|
37
|
+
buildEvalTestTargetKey,
|
|
38
|
+
buildEvaluationResultTargetKey,
|
|
33
39
|
buildGradingArtifact,
|
|
34
40
|
buildIndexArtifactEntry,
|
|
35
41
|
buildResultIndexArtifact,
|
|
@@ -38,9 +44,10 @@ export {
|
|
|
38
44
|
buildTimingArtifact,
|
|
39
45
|
deduplicateByTestIdTarget,
|
|
40
46
|
parseJsonlResults,
|
|
47
|
+
readRunConfigArtifact,
|
|
41
48
|
writeArtifacts,
|
|
42
49
|
writeArtifactsFromResults,
|
|
43
50
|
writeInitialRunSummaryArtifact,
|
|
44
51
|
writePerTestArtifacts
|
|
45
52
|
};
|
|
46
|
-
//# sourceMappingURL=artifact-writer-
|
|
53
|
+
//# sourceMappingURL=artifact-writer-IORG7CW5.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
|
|
2
2
|
|
|
3
|
-
// ../../packages/core/dist/chunk-
|
|
3
|
+
// ../../packages/core/dist/chunk-F4TEFL4T.js
|
|
4
4
|
function shellEscapePath(value) {
|
|
5
5
|
if (process.platform === "win32") {
|
|
6
6
|
return `"${value.replaceAll('"', '""')}"`;
|
|
@@ -107,4 +107,4 @@ export {
|
|
|
107
107
|
execFileWithStdin,
|
|
108
108
|
execShellWithStdin
|
|
109
109
|
};
|
|
110
|
-
//# sourceMappingURL=chunk-
|
|
110
|
+
//# sourceMappingURL=chunk-7BGERE6L.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../packages/core/src/runtime/exec.ts"],"sourcesContent":["interface ExecOptions {\n readonly cwd?: string;\n readonly timeoutMs?: number;\n /** Additional environment variables to pass to the subprocess */\n readonly env?: Record<string, string>;\n}\n\nfunction shellEscapePath(value: string): string {\n if (process.platform === 'win32') {\n // Very small escape helper for file paths in cmd.exe context.\n // Wrap in double-quotes and escape existing double-quotes.\n return `\"${value.replaceAll('\"', '\"\"')}\"`;\n }\n // POSIX: single-quote escape (close/open around embedded single quotes).\n return `'${value.replaceAll(\"'\", `'\\\"'\\\"'`)}'`;\n}\n\n// IPC protocol: stdin/stdout JSON was chosen over JSON-RPC (overkill for one-shot eval),\n// HTTP (unnecessary network overhead for local scripts), and in-process (no isolation,\n// single-language only). Each evaluation is isolated by design for reproducibility and\n// safe parallelization. Process spawn overhead (~10-50ms) is acceptable for eval workloads.\nexport async function execFileWithStdin(\n argv: readonly string[],\n stdinPayload: string,\n options: ExecOptions = {},\n): Promise<{\n readonly stdout: string;\n readonly stderr: string;\n readonly exitCode: number;\n}> {\n if (argv.length === 0) {\n throw new Error('Executable argv must include at least one entry');\n }\n\n // Use Node's child_process path even under Bun. Bun 1.3.3 can hang when a\n // Bun.spawn child is a Node process reading stdin, which breaks script grader\n // and workspace hook execution on CI's pinned Bun version.\n return execFileWithStdinNode(argv, stdinPayload, options);\n}\n\n/**\n * Node.js implementation using child_process.spawn\n */\nasync function execFileWithStdinNode(\n argv: readonly string[],\n stdinPayload: string,\n options: ExecOptions,\n): Promise<{\n readonly stdout: string;\n readonly stderr: string;\n readonly exitCode: number;\n}> {\n const { spawn } = await import('node:child_process');\n\n return new Promise((resolve, reject) => {\n const [cmd, ...args] = argv;\n const child = spawn(cmd, args, {\n cwd: options.cwd,\n stdio: ['pipe', 'pipe', 'pipe'],\n // Merge additional env vars with process.env\n env: options.env ? { ...process.env, ...options.env } : process.env,\n });\n\n const stdoutChunks: Buffer[] = [];\n const stderrChunks: Buffer[] = [];\n\n child.stdout?.on('data', (chunk: Buffer) => stdoutChunks.push(chunk));\n child.stderr?.on('data', (chunk: Buffer) => stderrChunks.push(chunk));\n\n let timedOut = false;\n const timeout =\n options.timeoutMs !== undefined\n ? setTimeout(() => {\n timedOut = true;\n child.kill('SIGKILL');\n }, options.timeoutMs)\n : undefined;\n\n child.on('error', (error) => {\n if (timeout !== undefined) clearTimeout(timeout);\n reject(error);\n });\n\n child.on('close', (code) => {\n if (timeout !== undefined) clearTimeout(timeout);\n\n if (timedOut) {\n reject(new Error(`Process timed out after ${options.timeoutMs}ms`));\n return;\n }\n\n const stdout = Buffer.concat(stdoutChunks).toString('utf8').replace(/\\r\\n/g, '\\n');\n const stderr = Buffer.concat(stderrChunks).toString('utf8').replace(/\\r\\n/g, '\\n');\n\n resolve({\n stdout,\n stderr,\n exitCode: code ?? 0,\n });\n });\n\n // Write stdin and close\n if (child.stdin) {\n child.stdin.write(stdinPayload);\n child.stdin.end();\n }\n });\n}\n\n/**\n * Execute a shell command with the given stdin payload.\n *\n * Why this exists:\n * - Some providers/scripts (notably Node.js) must receive stdin reliably.\n * - In some Bun environments, `Bun.spawn` does not forward stdin to Node correctly.\n * - Capture stdout/stderr via temp files to avoid pipe incompatibilities.\n */\nexport async function execShellWithStdin(\n command: string,\n stdinPayload: string,\n options: ExecOptions = {},\n): Promise<{\n readonly stdout: string;\n readonly stderr: string;\n readonly exitCode: number;\n}> {\n const { mkdir, readFile, rm, writeFile } = await import('node:fs/promises');\n const { tmpdir } = await import('node:os');\n const path = await import('node:path');\n const { randomUUID } = await import('node:crypto');\n\n const dir = path.join(tmpdir(), `agentv-exec-${randomUUID()}`);\n await mkdir(dir, { recursive: true });\n\n const stdinPath = path.join(dir, 'stdin.txt');\n const stdoutPath = path.join(dir, 'stdout.txt');\n const stderrPath = path.join(dir, 'stderr.txt');\n\n await writeFile(stdinPath, stdinPayload, 'utf8');\n\n const wrappedCommand =\n process.platform === 'win32'\n ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`\n : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;\n\n const { spawn } = await import('node:child_process');\n try {\n const exitCode = await new Promise<number>((resolve, reject) => {\n const child = spawn(wrappedCommand, {\n shell: true,\n cwd: options.cwd,\n stdio: ['ignore', 'ignore', 'ignore'],\n // Merge additional env vars with process.env\n env: options.env ? { ...process.env, ...options.env } : process.env,\n });\n\n const timeout = options.timeoutMs\n ? setTimeout(() => {\n child.kill();\n reject(new Error(`Process timed out after ${options.timeoutMs}ms`));\n }, options.timeoutMs)\n : undefined;\n\n child.on('error', (error) => {\n if (timeout !== undefined) {\n clearTimeout(timeout);\n }\n reject(error);\n });\n\n child.on('exit', (code) => {\n if (timeout !== undefined) {\n clearTimeout(timeout);\n }\n resolve(code ?? 0);\n });\n });\n\n const stdout = (await readFile(stdoutPath, 'utf8')).replace(/\\r\\n/g, '\\n');\n const stderr = (await readFile(stderrPath, 'utf8')).replace(/\\r\\n/g, '\\n');\n return { stdout, stderr, exitCode };\n } finally {\n await rm(dir, { recursive: true, force: true });\n }\n}\n"],"mappings":";;;AAOA,SAAS,gBAAgB,OAAuB;AAC9C,MAAI,QAAQ,aAAa,SAAS;AAGhC,WAAO,IAAI,MAAM,WAAW,KAAK,IAAI,CAAC;EACxC;AAEA,SAAO,IAAI,MAAM,WAAW,KAAK,OAAS,CAAC;AAC7C;AAMA,eAAsB,kBACpB,MACA,cACA,UAAuB,CAAC,GAKvB;AACD,MAAI,KAAK,WAAW,GAAG;AACrB,UAAM,IAAI,MAAM,iDAAiD;EACnE;AAKA,SAAO,sBAAsB,MAAM,cAAc,OAAO;AAC1D;AAKA,eAAe,sBACb,MACA,cACA,SAKC;AACD,QAAM,EAAE,MAAM,IAAI,MAAM,OAAO,oBAAoB;AAEnD,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,CAAC,KAAK,GAAG,IAAI,IAAI;AACvB,UAAM,QAAQ,MAAM,KAAK,MAAM;MAC7B,KAAK,QAAQ;MACb,OAAO,CAAC,QAAQ,QAAQ,MAAM;;MAE9B,KAAK,QAAQ,MAAM,EAAE,GAAG,QAAQ,KAAK,GAAG,QAAQ,IAAI,IAAI,QAAQ;IAClE,CAAC;AAED,UAAM,eAAyB,CAAC;AAChC,UAAM,eAAyB,CAAC;AAEhC,UAAM,QAAQ,GAAG,QAAQ,CAAC,UAAkB,aAAa,KAAK,KAAK,CAAC;AACpE,UAAM,QAAQ,GAAG,QAAQ,CAAC,UAAkB,aAAa,KAAK,KAAK,CAAC;AAEpE,QAAI,WAAW;AACf,UAAM,UACJ,QAAQ,cAAc,SAClB,WAAW,MAAM;AACf,iBAAW;AACX,YAAM,KAAK,SAAS;IACtB,GAAG,QAAQ,SAAS,IACpB;AAEN,UAAM,GAAG,SAAS,CAAC,UAAU;AAC3B,UAAI,YAAY,OAAW,cAAa,OAAO;AAC/C,aAAO,KAAK;IACd,CAAC;AAED,UAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,UAAI,YAAY,OAAW,cAAa,OAAO;AAE/C,UAAI,UAAU;AACZ,eAAO,IAAI,MAAM,2BAA2B,QAAQ,SAAS,IAAI,CAAC;AAClE;MACF;AAEA,YAAM,SAAS,OAAO,OAAO,YAAY,EAAE,SAAS,MAAM,EAAE,QAAQ,SAAS,IAAI;AACjF,YAAM,SAAS,OAAO,OAAO,YAAY,EAAE,SAAS,MAAM,EAAE,QAAQ,SAAS,IAAI;AAEjF,cAAQ;QACN;QACA;QACA,UAAU,QAAQ;MACpB,CAAC;IACH,CAAC;AAGD,QAAI,MAAM,OAAO;AACf,YAAM,MAAM,MAAM,YAAY;AAC9B,YAAM,MAAM,IAAI;IAClB;EACF,CAAC;AACH;AAUA,eAAsB,mBACpB,SACA,cACA,UAAuB,CAAC,GAKvB;AACD,QAAM,EAAE,OAAO,UAAU,IAAI,UAAU,IAAI,MAAM,OAAO,kBAAkB;AAC1E,QAAM,EAAE,OAAO,IAAI,MAAM,OAAO,SAAS;AACzC,QAAM,OAAO,MAAM,OAAO,WAAW;AACrC,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AAEjD,QAAM,MAAM,KAAK,KAAK,OAAO,GAAG,eAAe,WAAW,CAAC,EAAE;AAC7D,QAAM,MAAM,KAAK,EAAE,WAAW,KAAK,CAAC;AAEpC,QAAM,YAAY,KAAK,KAAK,KAAK,WAAW;AAC5C,QAAM,aAAa,KAAK,KAAK,KAAK,YAAY;AAC9C,QAAM,aAAa,KAAK,KAAK,KAAK,YAAY;AAE9C,QAAM,UAAU,WAAW,cAAc,MAAM;AAE/C,QAAM,iBACJ,QAAQ,aAAa,UACjB,IAAI,OAAO,OAAO,gBAAgB,SAAS,CAAC,MAAM,gBAAgB,UAAU,CAAC,OAAO,gBAAgB,UAAU,CAAC,KAC/G,IAAI,OAAO,OAAO,gBAAgB,SAAS,CAAC,MAAM,gBAAgB,UAAU,CAAC,OAAO,gBAAgB,UAAU,CAAC;AAErH,QAAM,EAAE,MAAM,IAAI,MAAM,OAAO,oBAAoB;AACnD,MAAI;AACF,UAAM,WAAW,MAAM,IAAI,QAAgB,CAAC,SAAS,WAAW;AAC9D,YAAM,QAAQ,MAAM,gBAAgB;QAClC,OAAO;QACP,KAAK,QAAQ;QACb,OAAO,CAAC,UAAU,UAAU,QAAQ;;QAEpC,KAAK,QAAQ,MAAM,EAAE,GAAG,QAAQ,KAAK,GAAG,QAAQ,IAAI,IAAI,QAAQ;MAClE,CAAC;AAED,YAAM,UAAU,QAAQ,YACpB,WAAW,MAAM;AACf,cAAM,KAAK;AACX,eAAO,IAAI,MAAM,2BAA2B,QAAQ,SAAS,IAAI,CAAC;MACpE,GAAG,QAAQ,SAAS,IACpB;AAEJ,YAAM,GAAG,SAAS,CAAC,UAAU;AAC3B,YAAI,YAAY,QAAW;AACzB,uBAAa,OAAO;QACtB;AACA,eAAO,KAAK;MACd,CAAC;AAED,YAAM,GAAG,QAAQ,CAAC,SAAS;AACzB,YAAI,YAAY,QAAW;AACzB,uBAAa,OAAO;QACtB;AACA,gBAAQ,QAAQ,CAAC;MACnB,CAAC;IACH,CAAC;AAED,UAAM,UAAU,MAAM,SAAS,YAAY,MAAM,GAAG,QAAQ,SAAS,IAAI;AACzE,UAAM,UAAU,MAAM,SAAS,YAAY,MAAM,GAAG,QAAQ,SAAS,IAAI;AACzE,WAAO,EAAE,QAAQ,QAAQ,SAAS;EACpC,UAAA;AACE,UAAM,GAAG,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;EAChD;AACF;","names":[]}
|