npm - agentv - Versions diffs - 5.3.0-next.1 → 5.3.1-next.1 - Mend

agentv 5.3.0-next.1 → 5.3.1-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (43) hide show

package/dist/dashboard/index.html CHANGED Viewed

@@ -4,7 +4,7 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>AgentV</title>
-    <script type="module" crossorigin src="/assets/index-r_jSJmlw.js"></script>
+    <script type="module" crossorigin src="/assets/index-DTA6-l7q.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-D_bokML8.css">
   </head>
   <body class="bg-gray-950 text-gray-100">

package/dist/{dist-6Z7U473R.js → dist-NMXMI5SK.js} RENAMED Viewed

@@ -26,6 +26,7 @@ import {
   getResultsRepoSyncStatus,
   listGitRuns,
   listGitRunsCached,
+  loadEnvPathFiles,
   loadTsConfig,
   materializeGitRun,
   normalizeResultsConfig,
@@ -34,6 +35,8 @@ import {
   parseCodexSession,
   parseCopilotEvents,
   parseEnvOutput,
+  parseJsonEnv,
+  parseShellExportsEnv,
   prepareEvalWorkspace,
   prepareResultsRepoBranch,
   pushResultsRepoBranch,
@@ -44,6 +47,7 @@ import {
   resolveResultsRepoRunsDir,
   resolveResultsRepoUrl,
   runBeforeSessionHook,
+  runEnvFromEntries,
   setupWipWorktree,
   stageResultsArtifacts,
   syncProject,
@@ -54,7 +58,7 @@ import {
   transpileEvalYaml,
   transpileEvalYamlFile,
   trimBaselineResult
-} from "./chunk-V52ATPTT.js";
+} from "./chunk-LXBI3SPX.js";
 import {
   AGENTV_CONFIG_FILE_NAME,
   AGENTV_CONFIG_YML_FILE_NAME,
@@ -219,7 +223,7 @@ import {
   extractFailOnError,
   extractImageBlocks,
   extractJsonBlob,
-  extractLastAssistantContent,
+  extractLastAssistantContent2,
   extractTargetFromSuite,
   extractTargetRefsFromSuite,
   extractTargetsFromSuite,
@@ -266,6 +270,7 @@ import {
   isPlainConfigObject,
   isTestMessage,
   isTestMessageRole,
+  isTypeScriptEvalConfigFileName,
   killAllTrackedChildren,
   listTargetNames,
   loadConfig,
@@ -373,11 +378,12 @@ import {
   trackChild,
   trackedChildCount,
   transcriptReplayRecordToProviderResponse,
+  typeScriptEvalConfigGlob,
   writeArtifacts,
   writeArtifactsFromResults,
   writeInitialRunSummaryArtifact,
   writePerTestArtifacts
-} from "./chunk-6ZZCDZPD.js";
+} from "./chunk-ELCJ23K4.js";
 import "./chunk-7BGERE6L.js";
 import {
   DockerWorkspaceProvider
@@ -568,7 +574,7 @@ export {
   extractFailOnError,
   extractImageBlocks,
   extractJsonBlob,
-  extractLastAssistantContent,
+  extractLastAssistantContent2 as extractLastAssistantContent,
   extractTargetFromSuite,
   extractTargetRefsFromSuite,
   extractTargetsFromSuite,
@@ -620,11 +626,13 @@ export {
   isPlainConfigObject,
   isTestMessage,
   isTestMessageRole,
+  isTypeScriptEvalConfigFileName,
   killAllTrackedChildren,
   listGitRuns,
   listGitRunsCached,
   listTargetNames,
   loadConfig,
+  loadEnvPathFiles,
   loadEvalCaseById,
   loadEvalCases,
   loadEvalSuite,
@@ -654,9 +662,11 @@ export {
   parseCopilotEvents,
   parseEnvOutput,
   parseEvaluationResultBoundary,
+  parseJsonEnv,
   parseJsonFromText,
   parseJsonSafe,
   parseJsonlResults,
+  parseShellExportsEnv,
   parseTraceBoundary,
   parseTraceSummaryBoundary,
   parseYamlValue,
@@ -695,6 +705,7 @@ export {
   runContainsAnyAssertion,
   runContainsAssertion,
   runEndsWithAssertion,
+  runEnvFromEntries,
   runEqualsAssertion,
   runEvalCase,
   runEvaluation,
@@ -757,9 +768,10 @@ export {
   transpileEvalYaml,
   transpileEvalYamlFile,
   trimBaselineResult,
+  typeScriptEvalConfigGlob,
   writeArtifacts,
   writeArtifactsFromResults,
   writeInitialRunSummaryArtifact,
   writePerTestArtifacts
 };
-//# sourceMappingURL=dist-6Z7U473R.js.map
+//# sourceMappingURL=dist-NMXMI5SK.js.map

package/dist/index.js CHANGED Viewed

@@ -6,11 +6,11 @@ import {
   runCli,
   shouldRunBeforeSessionHook,
   usesDeprecatedStudioAlias
-} from "./chunk-T32NL3E6.js";
-import "./chunk-FVR4RQFK.js";
-import "./chunk-BHKQHG26.js";
-import "./chunk-V52ATPTT.js";
-import "./chunk-6ZZCDZPD.js";
+} from "./chunk-ASIGJIOJ.js";
+import "./chunk-LKGARI3W.js";
+import "./chunk-RKE7SSET.js";
+import "./chunk-LXBI3SPX.js";
+import "./chunk-ELCJ23K4.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";

package/dist/{interactive-RUY3OCBI.js → interactive-BN527UV3.js} RENAMED Viewed

@@ -8,14 +8,14 @@ import {
   getCategories,
   resolveExistingRunPrimaryPath,
   runEvalCommand
-} from "./chunk-FVR4RQFK.js";
-import "./chunk-BHKQHG26.js";
-import "./chunk-V52ATPTT.js";
+} from "./chunk-LKGARI3W.js";
+import "./chunk-RKE7SSET.js";
+import "./chunk-LXBI3SPX.js";
 import {
   getAgentvConfigDir,
   listTargetNames,
   readTargetDefinitions
-} from "./chunk-6ZZCDZPD.js";
+} from "./chunk-ELCJ23K4.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";
@@ -344,4 +344,4 @@ ${ANSI_DIM}Retrying execution errors...${ANSI_RESET}
 export {
   launchInteractiveWizard
 };
-//# sourceMappingURL=interactive-RUY3OCBI.js.map
+//# sourceMappingURL=interactive-BN527UV3.js.map

package/dist/skills/agentv-bench/SKILL.md CHANGED Viewed

@@ -49,7 +49,7 @@ Before running or optimizing, understand what you're working with.
 2. **Identify success criteria** — what does "good" look like for this agent? What are the edge cases? What would a failure look like? Talk to the user if this isn't clear from the artifacts alone.
-3. **Understand the target harness** — which provider runs the agent (Claude, GPT, Copilot CLI, Gemini, custom CLI)? This affects what grader types are available and how to run tests. Targets are configured in `.agentv/targets.yaml` (canonical location, searched from the eval file directory upward). Sensitive values like `api_key` must use `${{ ENV_VAR }}` syntax — literal secrets are rejected as a security guardrail.
+3. **Understand the target harness** — which provider runs the agent (Claude, GPT, Copilot CLI, Gemini, custom CLI)? This affects what grader types are available and how to run tests. Targets are configured in `.agentv/targets.yaml` (canonical location, searched from the eval file directory upward). Sensitive values like `api_key` must use `{{ env.ENV_VAR }}` references — literal secrets are rejected as a security guardrail.
 4. **Challenge assumptions** — if evals already exist, review their quality before running:
    - Are the test cases testing the right things?
@@ -65,7 +65,7 @@ Before running or optimizing, understand what you're working with.
 AgentV supports two evaluation formats:
-**EVAL.yaml** (native, full features) — supports environment recipes, script graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, and multi-provider targets. Use this for agent evaluation.
+**EVAL.yaml** (native, full features) — supports prompt matrices, environment recipes, script graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, and multi-provider targets. Use this for agent evaluation.
 ```yaml
 # example.eval.yaml
@@ -89,7 +89,7 @@ tests:
       - Review identifies the null pointer bug and suggests a concrete fix
 ```
-Multi-skill evaluation is handled naturally via input messages — describe the task in the test input, and the agent uses whatever skills it needs.
+Multi-skill evaluation is handled naturally via prompt messages — describe the task in `prompts`, put row-specific data in `tests[].vars`, and the agent uses whatever skills it needs.
 **evals.json** (skill-creator compatible) — auto-promoted to EVAL-equivalent format:
 - `prompt` → input messages
@@ -127,7 +127,7 @@ Prefer deterministic graders over LLM graders whenever possible. If an assertion
 This section is one continuous sequence — don't stop partway through.
-Each run produces a new `.agentv/results/default/<timestamp>/` directory automatically. Use timestamps to identify iterations when comparing runs.
+Each run produces a new canonical `.agentv/results/<run_id>/` directory automatically. Use the printed run id and `.internal/index.jsonl` when comparing runs.
 ### Choosing a run mode
@@ -228,11 +228,11 @@ Each grader subagent (operating under `agents/grader.md` instructions):
 2. Reads `<test-id>/response.md` for the candidate output
 3. Grades the response against the prompt criteria
 4. **Writes its result to disk**: `<run-dir>/<evalset>/<test-id>/llm_grader_results/<name>.json`
-5. Returns score (0.0–1.0) and per-assertion evidence to the orchestrator
+5. Returns score (0.0–1.0) and per-assertion notes to the orchestrator
-**Writing to disk is critical.** Assertion arrays are lost if accumulated only in the orchestrator's context across multiple batches (context summarization drops detail). Writing per-test results to `llm_grader_results/<name>.json` makes grading resumable and assertion evidence durable.
+**Writing to disk is critical.** Intermediate assertion arrays are lost if accumulated only in the orchestrator's context across multiple batches (context summarization drops detail). Writing per-test results to `llm_grader_results/<name>.json` makes grading resumable. Public `grading.json` artifacts normalize these details into recursive `component_results`.
-The result file format is:
+The intermediate subagent result file format is:
 ```json
 { "score": 0.85, "assertions": [{"text": "...", "passed": true, "evidence": "..."}] }
 ```
@@ -246,18 +246,19 @@ agentv pipeline bench <run-dir>
 agentv results validate <run-dir>
 ```
-`pipeline bench` reads LLM grader results from `llm_grader_results/<name>.json` per test automatically, merges with script-grader scores, computes weighted pass_rate, and writes `grading.json` + `index.jsonl` + `summary.json`.
+`pipeline bench` reads LLM grader results from `llm_grader_results/<name>.json` per test automatically, merges with script-grader scores, computes aggregate scores, and writes the canonical run bundle with `.internal/index.jsonl`, `summary.json`, and per-attempt `grading.json`.
 > **Diagnosing `pass_rate=0`:** If `pipeline bench` reports `pass_rate=0` across the board, do **not** assume the tests genuinely failed. First verify the grading pipeline ran correctly: check that `<test-id>/llm_grader_results/<name>.json` exists and is non-empty for each test. If these files are absent or empty, the grader subagents failed to produce output (most common cause: `agents/grader.md` was not embedded in the subagent prompts — see Phase 2). Treat `pass_rate=0` as a real signal only after confirming grader results exist.
 ### Artifacts
 All artifacts use established schemas — see `references/schemas.md` for the full definitions. Do not modify the structure. Key artifacts per run:
-- **grading.json**: per-test assertions with `{text, passed, evidence}`, plus summary
-- **timing.json**: `{total_tokens, duration_ms, total_duration_seconds}`
-- **summary.json**: per-target aggregate `{pass_rate, time_seconds, tokens}`
+- **.internal/index.jsonl**: canonical row index for compare, Dashboard, rerun, and artifact discovery
+- **grading.json**: aggregate `pass`, `score`, `reason`, recursive `component_results`, and optional `assertion`, `named_scores`, and `metadata`
+- **metrics.json**: token, cost, timing, tool, and transcript metrics for the attempt
+- **summary.json**: run-level aggregate metadata and pass/score rollups
-Write artifacts to `.agentv/artifacts/` or the iteration directory.
+Write artifacts to `.agentv/results/<run_id>/` through the CLI unless a task explicitly needs a fixed output path.
 ### Environment features (EVAL.yaml only)
@@ -350,7 +351,7 @@ If a prompt file is referenced in both task input and grader configs, optimize f
 After improving:
 1. Apply your changes to the agent's prompts/skills/config
-2. Re-run all test cases (agentv creates a new `.agentv/results/default/<timestamp>/` directory automatically)
+2. Re-run all test cases (agentv creates a new `.agentv/results/<run_id>/` directory automatically)
 3. Compare against the previous iteration (Step 4). If running in automated mode, use the **automated keep/discard** logic below instead of manual judgment — it will decide whether to keep or revert the change for you.
 4. Present results to the user (or log the decision if running automated keep/discard)
 5. Stop when ANY of:

package/dist/skills/agentv-bench/agents/analyzer.md CHANGED Viewed

@@ -22,7 +22,7 @@ You are an eval-quality analyst for AgentV. Your job is to read JSONL evaluation
 Read every line of the JSONL results file. Each line is a JSON object with:
 - `test_id`, `suite`, `score`, `assertions`, `reasoning`, `target`
-- `scores` (optional): Array of per-grader breakdowns with `name`, `type`, `score`, `weight`, `verdict`, `assertions`, `reasoning`
+- `scores` (optional): Array of per-grader breakdowns with `name`, `type`, `score`, `weight`, `pass`, `assertions`, `reasoning`
 If `eval-path` is provided, also read the EVAL.yaml to understand grader configurations.

package/dist/skills/agentv-bench/agents/executor.md CHANGED Viewed

@@ -14,7 +14,7 @@ You are the executor for an AgentV evaluation test case. Your job is to **perfor
 You are the target agent being evaluated. Do the task to the best of your ability — your output will be graded by a separate grader agent.
 **You will receive these parameters:**
-- `test-dir`: Path to the test case directory (e.g., `.agentv/results/default/<timestamp>/<test-id>/`)
+- `test-dir`: Path to the prepared test case directory inside the run bundle (for example `.agentv/results/<run_id>/<result-dir>/`)
 ## Process

package/dist/skills/agentv-bench/references/autoresearch.md CHANGED Viewed

@@ -56,14 +56,14 @@ Before proceeding to the next iteration, log the decision and rationale so the u
 Iteration 2: KEEP
   wins=3, losses=1, ties=6, meanDelta=+0.05
   Rationale: candidate wins outweigh losses (3 > 1)
-  Baseline promoted: .agentv/results/default/20250101-120000/index.jsonl
+  Baseline promoted: .agentv/results/2025-01-01T12-00-00-000Z/.internal/index.jsonl
 ```
 ```
 Iteration 3: DISCARD
   wins=1, losses=2, ties=7, meanDelta=-0.03
   Rationale: candidate losses outweigh wins (2 > 1)
-  Reverted to baseline: .agentv/results/default/20250101-110000/index.jsonl
+  Reverted to baseline: .agentv/results/2025-01-01T11-00-00-000Z/.internal/index.jsonl
   Next: try a different mutation
 ```
@@ -210,20 +210,20 @@ agentv eval <eval-path> --experiment autoresearch-<name>
 **b. Extract scores (bash only — do NOT read result files into your context)**
-Find the latest timestamped directory in the experiment folder. Use bash/jq to extract small structured values:
+Find the latest run directory. Use bash/jq to extract small structured values:
 ```bash
 # Find latest run dir
-RUN_DIR=$(ls -td <experiment-dir>/20*/ | head -1)
+RUN_DIR=$(ls -td .agentv/results/20*/ | head -1)
 # Overall score (mean of all scores in index.jsonl)
-SCORE=$(jq -sr '[.[].scores[].score] | add / length' "$RUN_DIR/index.jsonl")
+SCORE=$(jq -sr '[.[].score] | add / length' "$RUN_DIR/.internal/index.jsonl")
-# Per-assertion pass rates as JSON object
-PASS_RATES=$(jq -sr '[.[].scores[]] | group_by(.type) | map({key: .[0].type, value: (map(.score) | add / length)}) | from_entries' "$RUN_DIR/index.jsonl")
+# Per-target mean scores as JSON object
+TARGET_SCORES=$(jq -sr 'group_by(.target) | map({key: .[0].target, value: (map(.score) | add / length)}) | from_entries' "$RUN_DIR/.internal/index.jsonl")
-# Cost (if timing.json exists)
-COST=$(jq -r '.cost_usd // 0' "$RUN_DIR/timing.json" 2>/dev/null || echo 0)
+# Cost (if present in summary.json)
+COST=$(jq -r '.cost_usd // .total_cost_usd // 0' "$RUN_DIR/summary.json" 2>/dev/null || echo 0)
 ```
 Capture only these small outputs (`SCORE`, `PASS_RATES`, `COST`) — never read the full JSONL into context.

package/dist/skills/agentv-bench/references/environment-adaptation.md CHANGED Viewed

@@ -21,7 +21,7 @@ for each test case, show the prompt and output. Ask for feedback inline. Skip be
 - **Codex**: Supports skills via `.agents/` or `.codex/` folders. Emits `command_execution`
   and `file_change` tool calls.
 - **Custom CLI**: Needs `command` and output file pattern in target config
-- **Target config**: Uses `${{ ENV_VAR }}` syntax (not `${ENV_VAR}`) for API keys
+- **Target config**: Uses `{{ env.ENV_VAR }}` references for API keys
 **Note**: "Description Optimization" (see `references/description-optimization.md`) applies
 to any platform with skill-discovery mechanisms. All listed providers support skills.
@@ -39,15 +39,15 @@ tool calls; reserve `output` for final-answer text checks.
 ```yaml
 # Example: script-grader for Codex skill-use detection
 tests:
-  - id: should-trigger-codex
+  - id: codex-skill-use
     input: "Analyze this CSV file"
     assert:
       - type: script
-        command: [bun, run, ./judges/codex-skill-trigger.ts]
+        command: [bun, run, ./judges/codex-skill-use.ts]
 ```
 ```typescript
-// judges/codex-skill-trigger.ts
+// judges/codex-skill-use.ts
 import { defineScriptGrader } from '@agentv/sdk';
 export default defineScriptGrader(({ messages }) => {

package/dist/skills/agentv-bench/references/eval-yaml-spec.md CHANGED Viewed

@@ -20,7 +20,7 @@ The grader agent uses this to evaluate assertions without the CLI.
 - `id` (string, required) — unique test identifier
 - `vars` (object, required when prompts need row data) — prompt-template variables for this row
-- `expected_output` (string | Message[], optional) — passive reference answer. String shorthand expands to `[{role: assistant, content: "..."}]`. It is available to declared graders, but does not add an implicit grader when `assertions` is present.
+- `vars.expected_output` (string | Message[], optional) — passive reference answer data. It is available to declared graders, but does not add an implicit grader when `assert` is present.
 - `criteria` (string, optional) — human-readable success criteria
 - `assert` (array, optional) — grader assertions
 - `environment` (object | `file://...`, optional) — per-case testbed override
@@ -31,8 +31,8 @@ If `assert` already states the grading contract, omit `criteria` instead of
 duplicating the same rubric. Prefer plain assertion strings for semantic checks
 when the default LLM rubric grader can judge them; use multiple named
 `type: llm-rubric` blocks only for custom prompts, custom grader targets, or
-intentional grader panels. Write `expected_output` as a golden/reference answer,
-not as criteria or scoring instructions.
+intentional grader panels. Write `vars.expected_output` as a golden/reference
+answer, not as criteria or scoring instructions.
 For historical or repo-state evals, materialize the repository through
 `environment.setup.command` and pass the repo/ref as argv inputs. A SHA in
@@ -45,11 +45,12 @@ checkout. `setup.command` is a non-empty string array. Put the executable at
 ### Default grader contract
-When a test has no `assertions`, AgentV uses the default `llm-rubric` with the case context,
-including `criteria` and `expected_output` when present.
+When a test has no `assert`, AgentV can use configured suite/default graders
+with the case context, including `criteria` and `vars.expected_output` when
+present.
-When `assertions` is present, the list is explicit: run only the declared
-assertions/graders. `expected_output` remains reference data for graders that consume it,
+When `assert` is present, the list is explicit: run only the declared
+assertions/graders. `vars.expected_output` remains reference data for graders that consume it,
 such as `llm-rubric`, `script`, or `field-accuracy`; it does not trigger an additional
 default `llm-rubric`.
 When the declared assertion strings fully express the semantic contract, do not
@@ -211,18 +212,20 @@ Same as contains variants but explicitly case-insensitive.
 ### Script-based assertions
-#### `script-grader`
+#### `script`
-- **Fields:** `path` (string, required — path to script), `command` (string[], optional — custom command)
+- **Fields:** `command` (string[], required — command to execute), optional `cwd`, `config`, and `target`
 - **Script SDK:** Use `defineScriptGrader` from `@agentv/sdk`:
   ```typescript
   import { defineScriptGrader } from '@agentv/sdk';
   export default defineScriptGrader(({ output, trace }) => ({
+    pass: (output ?? '').includes('expected'),
     score: (output ?? '').includes('expected') ? 1 : 0,
-    assert: [{ text: 'Contains expected', passed: (output ?? '').includes('expected') }],
+    reason: 'Checks for the expected token',
+    checks: [{ text: 'Contains expected', pass: (output ?? '').includes('expected'), reason: 'Substring check' }],
   }));
   ```
-- **Recipe:** The CLI runs the script, passing canonical JSON on stdin (`{output, input, expected_output, ...}`). Script returns `{"score": N, "assertions": [...]}`
+- **Recipe:** The CLI runs the script, passing canonical JSON on stdin (`{output, input, expected_output, ...}`). Script returns `{pass, score, reason, checks?}`; public artifacts normalize `checks` into recursive `component_results`.
 - **PASS:** score >= 0.5 (or as configured).
 ### Assertion groups
@@ -252,42 +255,22 @@ All assertion types support:
 - `negate` (boolean, optional) — invert result
 - `threshold` (number, optional) — minimum score to pass (for LLM types)
-## 5. AgentV JSONL Output Format
+## 5. AgentV Run Bundle Output
-Each line in the results JSONL file is an `EvaluationResult` object. In JSONL, field names use snake_case (applied by `toSnakeCaseDeep()`).
+Each run writes `.agentv/results/<run_id>/`. Public persisted fields use
+snake_case. The canonical discovery file is `.internal/index.jsonl`; per-attempt
+sidecars live under each result directory.
-### Required fields
+Key artifacts:
-- `timestamp` (string, ISO-8601)
-- `test_id` (string)
-- `score` (number, 0.0-1.0, weighted average of all assertion scores)
-- `assertions` (array of `{text, passed, evidence?}`)
-- `output` (string) — final answer/scored result; transcript evidence is available through captured trace/messages when present
-- `execution_status` (string: `ok` | `quality_failure` | `execution_error`)
-### Optional fields
-- `scores` (array of EvaluatorResult) — per-grader breakdown
-- `input` (Message[]) — input messages
-- `token_usage` (object: `{prompt_tokens, completion_tokens, total_tokens}`)
-- `cost_usd` (number)
-- `duration_ms` (number)
-- `target` (string)
-- `eval_set` (string)
-- `error` (string)
-- `file_changes` (string — unified diff)
-- `mode` (string — `agent` for agent mode)
-### `scores[]` entries (EvaluatorResult)
-- `name` (string) — grader name
-- `type` (string) — grader kind (kebab-case)
-- `score` (number, 0.0-1.0)
-- `assertions` (array of `{text, passed, evidence?}`)
-- `weight` (number, optional)
-- `verdict` (string: `pass` | `fail` | `skip`)
-- `details` (object, optional — structured data from script graders)
-- `reasoning` (string, optional)
+- `.internal/index.jsonl` — one row per result aggregate with identity fields,
+  status, score, target metadata, and explicit paths to sidecars.
+- `summary.json` — run-level metadata and aggregate pass/score rollups.
+- `sample-N/grading.json` — aggregate `pass`, `score`, `reason`, recursive
+  `component_results`, and optional `assertion`, `named_scores`, and `metadata`.
+- `sample-N/metrics.json` — token, cost, timing, tool, and transcript metrics.
+- `sample-N/transcript.json` and `sample-N/transcript-raw.jsonl` — normalized
+  and raw transcript data when available.
 ## 6. Eval Set Support
@@ -366,9 +349,9 @@ LLM grader results are read from disk at `<test-id>/llm_grader_results/<name>.js
 ```
 **Output:**
-- `<test-id>/run-1/grading.json` — merged grading with `graders`, `assertions`, `summary.pass_rate`
-- `index.jsonl` — one JSON line per test: `{test_id, score, pass, graders: [...]}`
-- `summary.json` — aggregate stats: `{metadata: {targets}, run_summary: {<target>: {mean, stddev, n}}}`
+- `.internal/index.jsonl` — canonical row index
+- `summary.json` — aggregate run summary
+- per-attempt `grading.json` — aggregate `pass`, `score`, `reason`, and recursive `component_results`
 ### Agent-Mode Workflow

package/dist/skills/agentv-bench/references/schemas.md CHANGED Viewed

@@ -85,80 +85,64 @@ Tracks version progression in Improve mode. Located at workspace root.
 ## grading.json
-Output from the grader agent. Located at `<case-dir>/run-N/grading.json`.
+Public per-attempt grading output. Located at
+`.agentv/results/<run_id>/<result_dir>/sample-N/grading.json`.
-**Important:** The `assertions` array must use the fields `text`, `passed`, and `evidence` — downstream tooling depends on these exact field names.
+`checks` and other SDK/script convenience shapes are normalized into recursive
+`component_results`. Do not document `assertions`, `passed`, `graders`, or
+`evidence` as public grading artifact fields.
 ```json
 {
-  "assertions": [
+  "pass": false,
+  "score": 0.67,
+  "reason": "One required spreadsheet check failed.",
+  "component_results": [
     {
-      "text": "The output includes the name 'John Smith'",
-      "passed": true,
-      "evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'"
+      "name": "name-check",
+      "type": "llm-rubric",
+      "pass": true,
+      "score": 1,
+      "reason": "The answer includes the expected name.",
+      "assertion": {
+        "text": "The output includes the expected name."
+      }
     },
     {
-      "text": "The spreadsheet has a SUM formula in cell B10",
-      "passed": false,
-      "evidence": "No spreadsheet was created. The output was a text file."
-    }
-  ],
-  "summary": {
-    "passed": 2,
-    "failed": 1,
-    "total": 3,
-    "pass_rate": 0.67
-  },
-  "execution_metrics": {
-    "tool_calls": {
-      "Read": 5,
-      "Write": 2,
-      "Bash": 8
-    },
-    "total_tool_calls": 15,
-    "total_steps": 6,
-    "errors_encountered": 0,
-    "output_chars": 12450,
-    "transcript_chars": 3200
-  },
-  "timing": {
-    "executor_duration_seconds": 165.0,
-    "grader_duration_seconds": 26.0,
-    "total_duration_seconds": 191.0
-  },
-  "claims": [
-    {
-      "claim": "The form has 12 fillable fields",
-      "type": "factual",
-      "verified": true,
-      "evidence": "Counted 12 fields in field_info.json"
+      "name": "spreadsheet-check",
+      "type": "script",
+      "pass": false,
+      "score": 0,
+      "reason": "No spreadsheet was created.",
+      "component_results": [
+        {
+          "pass": false,
+          "score": 0,
+          "reason": "Cell B10 did not contain a SUM formula.",
+          "assertion": {
+            "text": "Spreadsheet has SUM formula in B10"
+          }
+        }
+      ]
     }
   ],
-  "user_notes_summary": {
-    "uncertainties": ["Used 2023 data, may be stale"],
-    "needs_review": [],
-    "workarounds": ["Fell back to text overlay for non-fillable fields"]
+  "named_scores": {
+    "spreadsheet": 0
   },
-  "eval_feedback": {
-    "suggestions": [
-      {
-        "assertion": "The output includes the name 'John Smith'",
-        "reason": "A hallucinated document that mentions the name would also pass"
-      }
-    ],
-    "overall": "Assertions check presence but not correctness."
+  "metadata": {
+    "grader_target": "local-openai-grader"
   }
 }
 ```
 **Fields:**
-- `assertions[]`: Graded assertion results with evidence
-- `summary`: Aggregate pass/fail counts
-- `execution_metrics`: Tool usage and output size (from executor's metrics.json)
-- `timing`: Wall clock timing (from timing.json)
-- `claims`: Extracted and verified claims from the output
-- `user_notes_summary`: Issues flagged by the executor
-- `eval_feedback`: (optional) Improvement suggestions for the evals, only present when the grader identifies issues worth raising
+- `pass`: Aggregate pass/fail for the grading component
+- `score`: Aggregate score from 0 to 1
+- `reason`: Human-readable explanation for the aggregate result
+- `component_results`: Recursive child grading components
+- `assertion`: Optional assertion metadata for a leaf component
+- `named_scores`: Optional metric-name to score map
+- `metadata`: Optional grader/runtime metadata
 ---
@@ -198,7 +182,7 @@ Output from the executor agent. Located at `<case-dir>/run-N/metrics.json`.
 ## timing.json
-Wall clock timing for a run attempt. Located at `<case-dir>/run-N/timing.json`.
+Wall clock timing for a run sample. Located at `<case-dir>/sample-N/timing.json`.
 **How to capture:** When a subagent task completes, the task notification includes `total_tokens` and `duration_ms`. Save these immediately — they are not persisted anywhere else and cannot be recovered after the fact.