agentv 2.8.0-next.1 → 2.9.0-next.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -2
- package/dist/{chunk-RCFB5QFS.js → chunk-3INJ7ISP.js} +46 -85
- package/dist/chunk-3INJ7ISP.js.map +1 -0
- package/dist/{chunk-2SXGPQVR.js → chunk-PC3FAOHT.js} +4 -4
- package/dist/chunk-PC3FAOHT.js.map +1 -0
- package/dist/{chunk-DJCMBVB3.js → chunk-RJWTL3VS.js} +166 -75
- package/dist/chunk-RJWTL3VS.js.map +1 -0
- package/dist/cli.js +3 -3
- package/dist/{dist-T7REAXNS.js → dist-BGRU67HI.js} +2 -2
- package/dist/index.js +3 -3
- package/dist/{interactive-TE5SJPJW.js → interactive-7KFUCBIP.js} +3 -3
- package/dist/templates/.agents/skills/agentv-eval-builder/SKILL.md +1 -1
- package/dist/templates/.agents/skills/agentv-eval-builder/references/custom-evaluators.md +8 -8
- package/dist/templates/.agents/skills/agentv-eval-builder/references/eval-schema.json +12683 -237
- package/dist/templates/.agentv/config.yaml +1 -1
- package/dist/templates/.agentv/targets.yaml +10 -13
- package/package.json +2 -2
- package/dist/chunk-2SXGPQVR.js.map +0 -1
- package/dist/chunk-DJCMBVB3.js.map +0 -1
- package/dist/chunk-RCFB5QFS.js.map +0 -1
- package/dist/templates/.claude/skills/agentv-eval-builder/SKILL.md +0 -202
- package/dist/templates/.claude/skills/agentv-eval-builder/references/batch-cli-evaluator.md +0 -316
- package/dist/templates/.claude/skills/agentv-eval-builder/references/compare-command.md +0 -137
- package/dist/templates/.claude/skills/agentv-eval-builder/references/composite-evaluator.md +0 -215
- package/dist/templates/.claude/skills/agentv-eval-builder/references/config-schema.json +0 -27
- package/dist/templates/.claude/skills/agentv-eval-builder/references/custom-evaluators.md +0 -118
- package/dist/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json +0 -278
- package/dist/templates/.claude/skills/agentv-eval-builder/references/example-evals.md +0 -333
- package/dist/templates/.claude/skills/agentv-eval-builder/references/rubric-evaluator.md +0 -77
- package/dist/templates/.claude/skills/agentv-eval-builder/references/structured-data-evaluators.md +0 -121
- package/dist/templates/.claude/skills/agentv-eval-builder/references/tool-trajectory-evaluator.md +0 -298
- package/dist/templates/.claude/skills/agentv-prompt-optimizer/SKILL.md +0 -78
- package/dist/templates/.github/prompts/agentv-eval-build.prompt.md +0 -5
- package/dist/templates/.github/prompts/agentv-optimize.prompt.md +0 -4
- /package/dist/{dist-T7REAXNS.js.map → dist-BGRU67HI.js.map} +0 -0
- /package/dist/{interactive-TE5SJPJW.js.map → interactive-7KFUCBIP.js.map} +0 -0
package/dist/cli.js
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
|
|
3
3
|
import {
|
|
4
4
|
runCli
|
|
5
|
-
} from "./chunk-
|
|
6
|
-
import "./chunk-
|
|
7
|
-
import "./chunk-
|
|
5
|
+
} from "./chunk-3INJ7ISP.js";
|
|
6
|
+
import "./chunk-PC3FAOHT.js";
|
|
7
|
+
import "./chunk-RJWTL3VS.js";
|
|
8
8
|
import "./chunk-JK6V4KVD.js";
|
|
9
9
|
import "./chunk-C5GOHBQM.js";
|
|
10
10
|
import "./chunk-3L2L5GIL.js";
|
|
@@ -120,7 +120,7 @@ import {
|
|
|
120
120
|
toSnakeCaseDeep,
|
|
121
121
|
tokensPerTool,
|
|
122
122
|
trimBaselineResult
|
|
123
|
-
} from "./chunk-
|
|
123
|
+
} from "./chunk-RJWTL3VS.js";
|
|
124
124
|
import {
|
|
125
125
|
SimpleTraceFileExporter
|
|
126
126
|
} from "./chunk-JK6V4KVD.js";
|
|
@@ -254,4 +254,4 @@ export {
|
|
|
254
254
|
tokensPerTool,
|
|
255
255
|
trimBaselineResult
|
|
256
256
|
};
|
|
257
|
-
//# sourceMappingURL=dist-
|
|
257
|
+
//# sourceMappingURL=dist-BGRU67HI.js.map
|
package/dist/index.js
CHANGED
|
@@ -3,9 +3,9 @@ import {
|
|
|
3
3
|
app,
|
|
4
4
|
preprocessArgv,
|
|
5
5
|
runCli
|
|
6
|
-
} from "./chunk-
|
|
7
|
-
import "./chunk-
|
|
8
|
-
import "./chunk-
|
|
6
|
+
} from "./chunk-3INJ7ISP.js";
|
|
7
|
+
import "./chunk-PC3FAOHT.js";
|
|
8
|
+
import "./chunk-RJWTL3VS.js";
|
|
9
9
|
import "./chunk-JK6V4KVD.js";
|
|
10
10
|
import "./chunk-C5GOHBQM.js";
|
|
11
11
|
import "./chunk-3L2L5GIL.js";
|
|
@@ -4,13 +4,13 @@ import {
|
|
|
4
4
|
fileExists,
|
|
5
5
|
findRepoRoot,
|
|
6
6
|
runEvalCommand
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-PC3FAOHT.js";
|
|
8
8
|
import {
|
|
9
9
|
DEFAULT_EVAL_PATTERNS,
|
|
10
10
|
listTargetNames,
|
|
11
11
|
loadConfig,
|
|
12
12
|
readTargetDefinitions
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-RJWTL3VS.js";
|
|
14
14
|
import "./chunk-JK6V4KVD.js";
|
|
15
15
|
import "./chunk-C5GOHBQM.js";
|
|
16
16
|
import "./chunk-3L2L5GIL.js";
|
|
@@ -330,4 +330,4 @@ async function executeConfig(config) {
|
|
|
330
330
|
export {
|
|
331
331
|
launchInteractiveWizard
|
|
332
332
|
};
|
|
333
|
-
//# sourceMappingURL=interactive-
|
|
333
|
+
//# sourceMappingURL=interactive-7KFUCBIP.js.map
|
|
@@ -193,7 +193,7 @@ Configure via `assert` array. Multiple evaluators produce a weighted average sco
|
|
|
193
193
|
target: {} # optional: enable LLM target proxy (max_calls: 50)
|
|
194
194
|
```
|
|
195
195
|
Contract: stdin JSON -> stdout JSON `{score, hits, misses, reasoning}`
|
|
196
|
-
Input includes: `question`, `criteria`, `answer`, `reference_answer`, `output`, `trace`, `file_changes`, `workspace_path`, `config`
|
|
196
|
+
Input includes: `question`, `criteria`, `answer`, `reference_answer`, `output`, `trace`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config`
|
|
197
197
|
When `workspace_template` is configured, `workspace_path` is the absolute path to the workspace dir (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the workspace).
|
|
198
198
|
See docs at https://agentv.dev/evaluators/code-judges/
|
|
199
199
|
|
|
@@ -20,13 +20,13 @@
|
|
|
20
20
|
"tool_names": ["fetch"],
|
|
21
21
|
"tool_calls_by_name": {"fetch": 1},
|
|
22
22
|
"error_count": 0,
|
|
23
|
-
"llm_call_count": 2
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
23
|
+
"llm_call_count": 2
|
|
24
|
+
},
|
|
25
|
+
"token_usage": {"input": 1000, "output": 500},
|
|
26
|
+
"cost_usd": 0.0015,
|
|
27
|
+
"duration_ms": 3500,
|
|
28
|
+
"start_time": "2026-02-13T10:00:00.000Z",
|
|
29
|
+
"end_time": "2026-02-13T10:00:03.500Z"
|
|
30
30
|
}
|
|
31
31
|
```
|
|
32
32
|
|
|
@@ -54,7 +54,7 @@ import { defineCodeJudge, createTargetClient, definePromptTemplate } from '@agen
|
|
|
54
54
|
- `.invoke({question, systemPrompt})` - Single LLM call
|
|
55
55
|
- `.invokeBatch(requests)` - Batch LLM calls
|
|
56
56
|
- `definePromptTemplate(fn)` - Wraps prompt generation function
|
|
57
|
-
- Context fields: `question`, `answer`, `referenceAnswer`, `criteria`, `expectedOutput`, `output`, `config`, `trace`
|
|
57
|
+
- Context fields: `question`, `answer`, `referenceAnswer`, `criteria`, `expectedOutput`, `output`, `config`, `trace`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`
|
|
58
58
|
|
|
59
59
|
## Python Example
|
|
60
60
|
|