agentv 5.2.0-next.1 → 5.3.1-next.1

package/README.md

@@ -40,18 +40,18 @@ targets:
     runtime: host
     config:
       api_format: chat
-      base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
-      api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
-      model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
+      base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
+      api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
+      model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}"
 
 graders:
   - id: local-openai-grader
     provider: openai
     config:
       api_format: chat
-      base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
-      api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
-      model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
+      base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
+      api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
+      model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}"
 
 defaults:
   target: local-openai
@@ -110,11 +110,11 @@ tests:
 ```
 
 Plain assertion strings are short-form rubric criteria: AgentV groups them into
-`llm-rubric` and writes each criterion to `grading.json.assertion_results` for the
-Dashboard. Use explicit `type: llm-rubric` when you need weights, required flags, or
-`score_ranges`, or when you need a custom grader prompt, grader target, or
-output transforms; use string `value` for free-form rubric checks. Executable
-graders use `type: script`.
+`llm-rubric` and writes grader detail to `grading.json.component_results` for
+the Dashboard. Use explicit `type: llm-rubric` when you need weights, required
+flags, `score_ranges`, a custom grader prompt, a grader target, or output
+transforms; use string `value` for free-form rubric checks. Executable graders
+use `type: script`.
 
 The target can be an eval-local object when this eval needs target settings of its own:
 
@@ -128,8 +128,8 @@ target:
   runtime: host
   config:
     api_format: chat
-    base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
-    api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
+    base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
+    api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
     model: gpt-5.4-mini
 evaluate_options:
   repeat:
@@ -193,14 +193,14 @@ Run bundle layout:
 ├── 2026-06-30T08-30-00-000Z/     # <run_id> — one committed run bundle
 │   ├── summary.json              # run rollup: metadata, pass rate, counts, cost
 │   ├── fizzbuzz--a1b2c3d4/       # <result_dir> for one test/target row
-│   │   ├── summary.json          # optional per-case rollup across attempts
+│   │   ├── summary.json          # optional per-case rollup across samples
 │   │   ├── test/                 # generated test bundle: frozen inputs for reproducibility
 │   │   │   ├── EVAL.yaml         #   resolved eval spec
 │   │   │   ├── targets.yaml      #   resolved target config
 │   │   │   └── graders/          #   grader files used
 │   │   └── sample-1/             # one materialized sample
-│   │       ├── result.json       # compact attempt manifest
-│   │       ├── grading.json      # assertion_results and grader evidence
+│   │       ├── result.json       # compact sample manifest
+│   │       ├── grading.json      # pass, score, reason, component_results
 │   │       ├── metrics.json      # tool calls, transcript stats, behavior metrics
 │   │       ├── transcript.json        # normalized agent transcript
 │   │       ├── transcript-raw.jsonl   # raw agent output (debugging)
@@ -240,12 +240,12 @@ const { results, summary } = await evaluate({
 console.log(`${summary.passed}/${summary.total} passed`);
 ```
 
-Use `defineEval()` when you want AgentV to run the TypeScript eval file:
+Use `*.eval.ts` when you want AgentV to run a TypeScript eval config:
 
 ```typescript
-import { defineEval } from '@agentv/sdk';
+import type { EvalConfig } from '@agentv/sdk';
 
-export default defineEval({
+const config: EvalConfig = {
   description: 'Code generation quality',
   tags: { experiment: 'with-skills' },
   target: {
@@ -285,7 +285,9 @@ export default defineEval({
       ],
     },
   ],
-});
+};
+
+export default config;
 ```
 
 ## Documentation

package/dist/{artifact-writer-SN6UNL5G.js → artifact-writer-7NBCOAYC.js}

@@ -4,8 +4,8 @@ import {
   buildResultIndexArtifact,
   writeArtifactsFromResults,
   writePerTestArtifacts
-} from "./chunk-46K2OET3.js";
-import "./chunk-ALVZQUZP.js";
+} from "./chunk-RKE7SSET.js";
+import "./chunk-LXBI3SPX.js";
 import {
   RESULT_INDEX_FILENAME,
   RUN_CONFIG_FILENAME,
@@ -23,7 +23,7 @@ import {
   readRunConfigArtifact,
   writeArtifacts,
   writeInitialRunSummaryArtifact
-} from "./chunk-3S6A2RKR.js";
+} from "./chunk-ELCJ23K4.js";
 import "./chunk-7BGERE6L.js";
 import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";
@@ -50,4 +50,4 @@ export {
   writeInitialRunSummaryArtifact,
   writePerTestArtifacts
 };
-//# sourceMappingURL=artifact-writer-SN6UNL5G.js.map
+//# sourceMappingURL=artifact-writer-7NBCOAYC.js.map

package/dist/{chunk-3X357HS4.js → chunk-ASIGJIOJ.js}

@@ -60,16 +60,18 @@ import {
   validateEvalFile,
   validateFileReferences,
   validateTargetsFile,
+  validateTypeScriptEvalConfigFile,
   validateWorkspacePaths
-} from "./chunk-T4AD3H3Y.js";
+} from "./chunk-LKGARI3W.js";
 import {
   materializeEvalBundle,
   writeArtifactsFromResults as writeArtifactsFromResults2
-} from "./chunk-46K2OET3.js";
+} from "./chunk-RKE7SSET.js";
 import {
   discoverClaudeSessions,
   discoverCodexSessions,
   getOutputFilenames,
+  loadEnvPathFiles,
   normalizeTraceArtifactToTraceSessionResponse,
   parseClaudeSession,
   parseCodexSession,
@@ -77,15 +79,18 @@ import {
   prepareEvalWorkspace,
   readGitResultArtifact,
   runBeforeSessionHook,
+  runEnvFromEntries,
   syncProjects,
   transpileEvalYamlFile,
   trimBaselineResult
-} from "./chunk-ALVZQUZP.js";
+} from "./chunk-LXBI3SPX.js";
 import {
   AGENTV_RESULTS_ARTIFACTS_REF,
   DEFAULT_CATEGORY,
   DEFAULT_THRESHOLD,
   EXECUTION_TRACE_SCHEMA_VERSION,
+  LEGACY_METRICS_SCHEMA_VERSION,
+  METRICS_SCHEMA_VERSION,
   RESULT_INDEX_FILENAME,
   addProject,
   aggregateRunDir,
@@ -104,12 +109,14 @@ import {
   getLocalConfigPath,
   getProject,
   gradePreparedEvalCase,
+  isTypeScriptEvalConfigFileName,
   listTargetNames,
   loadConfig,
   loadProjectRegistry,
   loadTestSuite,
   mergeConfigObjects,
   normalizeCategoryPath,
+  normalizeMetricsArtifactWire,
   normalizeResultRow,
   omitExternalTraceMetadataKeys,
   parseYamlValue,
@@ -139,7 +146,7 @@ import {
   traceEnvelopeToTranscriptMessages,
   traceFromTranscriptJsonLines,
   writeArtifactsFromResults
-} from "./chunk-3S6A2RKR.js";
+} from "./chunk-ELCJ23K4.js";
 import {
   __commonJS,
   __require,
@@ -12836,8 +12843,10 @@ function formatSummary(results, grading) {
   let passRate;
   if (grading) {
     const metadata = grading.metadata ?? {};
-    passed = typeof metadata.pass_count === "number" ? metadata.pass_count : grading.pass ? total : 0;
-    failed = typeof metadata.sample_count === "number" && typeof metadata.pass_count === "number" ? metadata.sample_count - metadata.pass_count : total - passed;
+    const passedSamples = typeof metadata.passed_samples === "number" ? metadata.passed_samples : typeof metadata.pass_count === "number" ? metadata.pass_count : void 0;
+    const totalSamples = typeof metadata.total_samples === "number" ? metadata.total_samples : typeof metadata.sample_count === "number" ? metadata.sample_count : void 0;
+    passed = passedSamples ?? (grading.pass ? total : 0);
+    failed = totalSamples !== void 0 && passedSamples !== void 0 ? totalSamples - passed : total - passed;
     passRate = typeof metadata.pass_rate === "number" ? metadata.pass_rate : total > 0 ? Math.round(passed / total * 1e3) / 1e3 : 0;
   } else {
     passed = results.filter((r) => r.score >= 1).length;
@@ -12947,6 +12956,44 @@ function validateNoLegacyGradingFields(value, pathLabel) {
   }
   return errors;
 }
+function validateMetricsArtifact(value) {
+  if (!isRecord2(value)) {
+    return [{ severity: "error", message: "metrics.json must be an object" }];
+  }
+  const diagnostics = [];
+  const schemaVersion = value.schema_version;
+  const isCurrentSchema = schemaVersion === METRICS_SCHEMA_VERSION;
+  const isLegacySchema = schemaVersion === LEGACY_METRICS_SCHEMA_VERSION;
+  if (isCurrentSchema && Object.hasOwn(value, "trace")) {
+    diagnostics.push({
+      severity: "error",
+      message: "metrics.json must not include trace in agentv.metrics.v2"
+    });
+  } else if (isLegacySchema && Object.hasOwn(value, "trace")) {
+    diagnostics.push({
+      severity: "warning",
+      message: "metrics.json uses legacy trace identity; readers ignore it"
+    });
+  }
+  if (isCurrentSchema && Object.hasOwn(value, "metrics")) {
+    diagnostics.push({
+      severity: "error",
+      message: "metrics.json must not include nested metrics in agentv.metrics.v2"
+    });
+  } else if (isLegacySchema && Object.hasOwn(value, "metrics")) {
+    diagnostics.push({
+      severity: "warning",
+      message: "metrics.json uses legacy nested metrics; readers flatten it"
+    });
+  }
+  if (!normalizeMetricsArtifactWire(value)) {
+    diagnostics.push({
+      severity: "error",
+      message: "metrics.json does not match a supported metrics artifact shape"
+    });
+  }
+  return diagnostics;
+}
 function checkDirectoryNaming(runDir) {
   const dirName = path24.basename(runDir);
   const pathSegments = path24.normalize(runDir).split(path24.sep).filter(Boolean);
@@ -13108,24 +13155,24 @@ function checkArtifactFiles(runDir, entries2) {
         });
       }
     }
-    for (const attempt of entry.attempts ?? entry.trials ?? []) {
-      const attemptPath = attempt.attempt_path ?? attempt.run_path;
-      if (!entry.result_dir || !attemptPath) {
+    for (const sample of entry.samples ?? entry.attempts ?? entry.trials ?? []) {
+      const samplePath = sample.sample_path ?? sample.attempt_path ?? sample.run_path;
+      if (!entry.result_dir || !samplePath) {
         continue;
       }
-      const runDirPath = path24.join(runDir, entry.result_dir, attemptPath);
+      const runDirPath = path24.join(runDir, entry.result_dir, samplePath);
       const resultPath = path24.join(runDirPath, "result.json");
       const gradingPath = path24.join(runDirPath, "grading.json");
       if (!existsSync14(resultPath)) {
         diagnostics.push({
           severity: "error",
-          message: `${testId}: result.json not found at '${path24.posix.join(entry.result_dir, attemptPath, "result.json")}'`
+          message: `${testId}: result.json not found at '${path24.posix.join(entry.result_dir, samplePath, "result.json")}'`
         });
       }
       if (!existsSync14(gradingPath)) {
         diagnostics.push({
           severity: "error",
-          message: `${testId}: grading.json not found at '${path24.posix.join(entry.result_dir, attemptPath, "grading.json")}'`
+          message: `${testId}: grading.json not found at '${path24.posix.join(entry.result_dir, samplePath, "grading.json")}'`
         });
       }
     }
@@ -13163,6 +13210,21 @@ function checkArtifactFiles(runDir, entries2) {
           severity: "warning",
           message: `${testId}: metrics.json not found at '${entry.metrics_path}'`
         });
+      } else {
+        try {
+          const metrics = JSON.parse(readFileSync12(metricsPath, "utf8"));
+          for (const diagnostic of validateMetricsArtifact(metrics)) {
+            diagnostics.push({
+              severity: diagnostic.severity,
+              message: `${testId}: ${diagnostic.message}`
+            });
+          }
+        } catch {
+          diagnostics.push({
+            severity: "error",
+            message: `${testId}: metrics.json is not valid JSON`
+          });
+        }
       }
     }
   }
@@ -14320,7 +14382,7 @@ function displayPathFromArtifactKey(key, runPath) {
 function addTrialRunCatalogEntries(entries2, seen, record) {
   const resultDir = record.result_dir ? normalizeArtifactRelativePath(record.result_dir) : void 0;
   if (!resultDir) return;
-  for (const trial of record.attempts ?? record.trials ?? []) {
+  for (const trial of record.samples ?? record.attempts ?? record.trials ?? []) {
     const rawPath = typeof trial.sample_path === "string" ? trial.sample_path : typeof trial.attempt_path === "string" ? trial.attempt_path : trial.run_path;
     const runPath = rawPath ? normalizeArtifactRelativePath(rawPath) : void 0;
     if (!runPath) continue;
@@ -14536,10 +14598,10 @@ function caseTrialArtifactPath(resultDir, runPath, filePath) {
   return path27.posix.join(resultDir, runPath, filePath);
 }
 function buildRepeatTrialReadModels(baseDir, record) {
-  const attempts = record.attempts ?? record.trials;
-  if (!attempts || attempts.length === 0) return void 0;
+  const samples = record.samples ?? record.attempts ?? record.trials;
+  if (!samples || samples.length === 0) return void 0;
   const resultDir = record.result_dir ? normalizeArtifactRelativePath(record.result_dir) : void 0;
-  return attempts.map((trial) => {
+  return samples.map((trial) => {
     const rawPath = typeof trial.sample_path === "string" ? trial.sample_path : typeof trial.attempt_path === "string" ? trial.attempt_path : trial.run_path;
     const runPath = rawPath ? normalizeArtifactRelativePath(rawPath) : void 0;
     const metricsPath = caseTrialArtifactPath(resultDir, runPath, "metrics.json");
@@ -14550,7 +14612,7 @@ function buildRepeatTrialReadModels(baseDir, record) {
     const answerPath = caseTrialArtifactPath(resultDir, runPath, "outputs/answer.md");
     const resultPath = caseTrialArtifactPath(resultDir, runPath, "result.json");
     const runResult = readArtifactJsonObject(baseDir, resultPath);
-    const metrics = readArtifactJsonObject(baseDir, metricsPath);
+    const metrics = normalizeMetricsArtifactWire(readArtifactJsonObject(baseDir, metricsPath));
     const timing = readArtifactJsonObject(baseDir, timingPath);
     const toolCalls = objectField(metrics, "tool_calls");
     const tokenUsage = objectField(metrics, "tokens") ?? objectField(timing, "token_usage");
@@ -14596,7 +14658,7 @@ function attachRunDetailReadModelFields(results, records, baseDir) {
   return results.map((result, index) => {
     const record = records[index];
     if (!record) return result;
-    const attempts = buildRepeatTrialReadModels(baseDir, record);
+    const samples = buildRepeatTrialReadModels(baseDir, record);
     return {
       ...result,
       ...record.aggregation && { aggregation: record.aggregation },
@@ -14611,7 +14673,7 @@ function attachRunDetailReadModelFields(results, records, baseDir) {
       ...record.transcript_raw_path && { transcript_raw_path: record.transcript_raw_path },
       ...record.output_path && { output_path: record.output_path },
       ...record.answer_path && { answer_path: record.answer_path },
-      ...attempts && { attempts }
+      ...samples && { samples }
     };
   });
 }
@@ -17544,6 +17606,9 @@ async function validateFiles(paths) {
 }
 async function validateSingleFile(filePath) {
   const absolutePath = path31.resolve(filePath);
+  if (isTypeScriptEvalConfigFileName(absolutePath)) {
+    return validateTypeScriptEvalConfigFile(absolutePath);
+  }
   const fileType = await detectFileType(absolutePath);
   let result;
   if (fileType === "eval") {
@@ -17592,12 +17657,14 @@ async function expandPaths(paths) {
       await access2(absolutePath, constants2.F_OK);
       const stats = await stat2(absolutePath);
       if (stats.isFile()) {
-        if (isYamlFile(absolutePath)) expanded.add(absolutePath);
+        if (isYamlFile(absolutePath) || isTypeScriptEvalConfigFileName(absolutePath)) {
+          expanded.add(absolutePath);
+        }
         continue;
       }
       if (stats.isDirectory()) {
-        const yamlFiles = await findYamlFiles(absolutePath);
-        for (const f of yamlFiles) expanded.add(f);
+        const evalFiles = await findEvalFiles(absolutePath);
+        for (const f of evalFiles) expanded.add(f);
         continue;
       }
     } catch {
@@ -17611,17 +17678,19 @@ async function expandPaths(paths) {
       dot: false,
       followSymbolicLinks: true
     });
-    const yamlMatches = matches.filter((f) => isYamlFile(f));
-    if (yamlMatches.length === 0) {
-      console.warn(`Warning: No YAML files matched pattern: ${inputPath}`);
+    const evalMatches = matches.filter((f) => isYamlFile(f) || isTypeScriptEvalConfigFileName(f));
+    if (evalMatches.length === 0) {
+      console.warn(
+        `Warning: No eval YAML or TypeScript config files matched pattern: ${inputPath}`
+      );
     }
-    for (const f of yamlMatches) expanded.add(path31.normalize(f));
+    for (const f of evalMatches) expanded.add(path31.normalize(f));
   }
   const sorted = Array.from(expanded);
   sorted.sort();
   return sorted;
 }
-async function findYamlFiles(dirPath) {
+async function findEvalFiles(dirPath) {
   const results = [];
   try {
     const entries2 = await readdir4(dirPath, { withFileTypes: true });
@@ -17631,9 +17700,9 @@ async function findYamlFiles(dirPath) {
         if (entry.name === "node_modules" || entry.name.startsWith(".")) {
           continue;
         }
-        const subFiles = await findYamlFiles(fullPath);
+        const subFiles = await findEvalFiles(fullPath);
         results.push(...subFiles);
-      } else if (entry.isFile() && isEvalYamlFile(entry.name)) {
+      } else if (entry.isFile() && (isEvalYamlFile(entry.name) || isTypeScriptEvalConfigFileName(entry.name))) {
         results.push(fullPath);
       }
     }
@@ -17678,7 +17747,7 @@ async function runValidateCommand(paths, maxWarnings) {
 }
 var validateCommand = command({
   name: "validate",
-  description: "Validate AgentV eval and targets YAML files",
+  description: "Validate AgentV eval, TypeScript config, and targets files",
   args: {
     paths: restPositionals({
       type: string,
@@ -17898,6 +17967,13 @@ ${updateNotice}
     const cwd = process.cwd();
     const repoRoot = await findRepoRoot(cwd);
     const sessionConfig = await loadConfig(path32.join(cwd, "_"), repoRoot);
+    const configDir = sessionConfig?.configDir ?? repoRoot;
+    if (sessionConfig?.env_path) {
+      await loadEnvPathFiles(sessionConfig.env_path, configDir);
+    }
+    if (sessionConfig?.env_from) {
+      await runEnvFromEntries(sessionConfig.env_from, { cwd: configDir });
+    }
     const beforeSessionCommand = sessionConfig?.hooks?.before_session;
     if (beforeSessionCommand) {
       runBeforeSessionHook(beforeSessionCommand);
@@ -17914,4 +17990,4 @@ export {
   preprocessArgv,
   runCli
 };
-//# sourceMappingURL=chunk-3X357HS4.js.map
+//# sourceMappingURL=chunk-ASIGJIOJ.js.map