agentv 5.1.0-next.1 → 5.3.0-next.1

package/dist/{chunk-AM2IPHKS.js → chunk-FVR4RQFK.js}

@@ -1,7 +1,7 @@
 import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
 import {
   writeArtifactsFromResults
-} from "./chunk-CBBBRTQ3.js";
+} from "./chunk-BHKQHG26.js";
 import {
   RunBudgetTracker,
   buildWipBranchName,
@@ -18,7 +18,7 @@ import {
   resolveResultsRepoRunsDir,
   setupWipWorktree,
   syncResultsRepoForProject
-} from "./chunk-GJWPQXHN.js";
+} from "./chunk-V52ATPTT.js";
 import {
   AGENTV_LOCAL_CONFIG_FILE_NAME,
   AGENTV_LOCAL_CONFIG_YML_FILE_NAME,
@@ -59,6 +59,7 @@ import {
   loadTestSuite,
   loadTestSuiteFromYamlObject,
   normalizeComposableConfigGraph,
+  normalizeMetricsArtifactWire,
   normalizeResultRow,
   normalizeTargetDefinition,
   parseJsonlResults,
@@ -66,6 +67,7 @@ import {
   readTargetDefinitions,
   readTestSuiteMetadata,
   resolveConfigFieldReferences,
+  resolveEnvironmentRecipe,
   resolveFileReference,
   resolveResultsConfigForProject,
   resolveTargetDefinition,
@@ -82,7 +84,7 @@ import {
   traceEnvelopeToTraceSummary,
   traceEnvelopeToTranscriptMessages,
   writeInitialRunSummaryArtifact
-} from "./chunk-ZGPCX334.js";
+} from "./chunk-6ZZCDZPD.js";
 
 // src/commands/eval/shared.ts
 import { constants } from "node:fs";
@@ -345,7 +347,7 @@ async function findRepoRoot(start) {
 // package.json
 var package_default = {
   name: "agentv",
-  version: "5.1.0-next.1",
+  version: "5.3.0-next.1",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -386,7 +388,7 @@ var package_default = {
     yaml: "^2.8.3"
   },
   peerDependencies: {
-    "@anthropic-ai/claude-agent-sdk": "^0.2.88",
+    "@anthropic-ai/claude-agent-sdk": "^0.2.89",
     "@github/copilot-sdk": "^1.0.3",
     "@openai/codex-sdk": "^0.136.0",
     "@earendil-works/pi-coding-agent": "^0.74.0"
@@ -405,6 +407,12 @@ var package_default = {
       optional: true
     }
   },
+  optionalDependencies: {
+    "@anthropic-ai/claude-agent-sdk": "^0.2.89",
+    "@github/copilot-sdk": "^1.0.3",
+    "@openai/codex-sdk": "^0.136.0",
+    "@earendil-works/pi-coding-agent": "^0.74.0"
+  },
   devDependencies: {
     "@agentv/core": "workspace:*",
     "@agentv/sdk": "workspace:*",
@@ -867,6 +875,9 @@ function readGradingAssertionResults(record) {
   );
 }
 function readNestedGradingScores(record) {
+  if (Array.isArray(record.component_results)) {
+    return record.component_results;
+  }
   if (Array.isArray(record.scores)) {
     return record.scores;
   }
@@ -878,14 +889,56 @@ function readNestedGradingScores(record) {
   }
   return void 0;
 }
+function componentLabel(component) {
+  const assertion = component.assertion;
+  if (assertion && typeof assertion === "object" && !Array.isArray(assertion)) {
+    const record = assertion;
+    for (const key of ["value", "name", "id", "type"]) {
+      const value = record[key];
+      if (typeof value === "string" && value.trim().length > 0) {
+        return value;
+      }
+    }
+  }
+  return typeof component.reason === "string" ? component.reason : "grading component";
+}
+function mapComponentAssertion(component) {
+  return {
+    text: componentLabel(component),
+    passed: component.pass === true,
+    evidence: typeof component.reason === "string" ? component.reason : void 0
+  };
+}
+function collectComponentAssertions(value) {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  return value.flatMap((component) => {
+    if (!component || typeof component !== "object" || Array.isArray(component)) {
+      return [];
+    }
+    const record = component;
+    const nested = collectComponentAssertions(record.component_results);
+    return nested.length > 0 ? nested : [mapComponentAssertion(record)];
+  });
+}
 function mapGradingEvaluator(evaluator) {
-  const verdict = evaluator.verdict === "pass" || evaluator.verdict === "fail" || evaluator.verdict === "skip" ? evaluator.verdict : void 0;
+  const pass = typeof evaluator.pass === "boolean" ? evaluator.pass : evaluator.verdict === "pass" || typeof evaluator.score === "number" && evaluator.score >= 0.8;
+  const verdict = pass ? "pass" : "fail";
   const details = evaluator.details && typeof evaluator.details === "object" && !Array.isArray(evaluator.details) ? evaluator.details : void 0;
+  const assertion = evaluator.assertion && typeof evaluator.assertion === "object" && !Array.isArray(evaluator.assertion) ? evaluator.assertion : void 0;
   return {
-    name: String(evaluator.name ?? ""),
-    type: String(evaluator.type ?? ""),
+    name: String(assertion?.name ?? assertion?.id ?? evaluator.name ?? componentLabel(evaluator)),
+    type: String(assertion?.type ?? evaluator.type ?? "llm-grader"),
     score: typeof evaluator.score === "number" ? evaluator.score : 0,
-    assertions: readGradingAssertionResults(evaluator) ?? [],
+    reason: typeof evaluator.reason === "string" ? evaluator.reason : void 0,
+    assertions: (() => {
+      const nestedAssertions = collectComponentAssertions(evaluator.component_results);
+      if (nestedAssertions.length > 0) {
+        return nestedAssertions;
+      }
+      return readGradingAssertionResults(evaluator) ?? [mapComponentAssertion(evaluator)];
+    })(),
     scores: mapGradingEvaluators(readNestedGradingScores(evaluator)),
     weight: typeof evaluator.weight === "number" ? evaluator.weight : void 0,
     verdict,
@@ -1001,12 +1054,15 @@ function hydrateTrace(baseDir, record, options) {
 }
 function hydrateManifestRecord(baseDir, record, options) {
   const grading = readOptionalJson(baseDir, record.grading_path);
-  const metrics = readOptionalJson(baseDir, record.metrics_path);
+  const metrics = normalizeMetricsArtifactWire(
+    readOptionalJson(baseDir, record.metrics_path)
+  );
   const timing = metrics ?? readOptionalJson(baseDir, record.timing_path);
   const testId = record.test_id ?? "unknown";
-  const gradingAssertions = grading ? readGradingAssertionResults(grading) : void 0;
+  const gradingAssertions = grading ? collectComponentAssertions(grading.component_results) : void 0;
+  const gradingRecord = grading;
   const gradingScores = mapGradingEvaluators(
-    grading?.graders ?? grading?.evaluators
+    gradingRecord?.component_results ?? gradingRecord?.graders ?? gradingRecord?.evaluators
   );
   return {
     timestamp: record.timestamp,
@@ -2800,18 +2856,63 @@ var UNSUPPORTED_PROMPTFOO_ASSERTION_TYPES = /* @__PURE__ */ new Set([
   "human",
   "max-score",
   "tool-call-f1",
-  "skill-used",
-  "trajectory:goal-success",
-  "trajectory:tool-args-match",
-  "trajectory:step-count",
-  "trajectory:tool-sequence",
-  "trajectory:tool-used",
   "trace-error-spans",
   "trace-span-count",
   "trace-span-duration",
   "search-rubric",
   "word-count"
 ]);
+function formatJsonValue(value) {
+  if (typeof value === "string") return value;
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return String(value);
+  }
+}
+function hasToolTrajectoryLatencyCheck(value) {
+  const expected = value.expected;
+  return Array.isArray(expected) && expected.some(
+    (item) => isObject(item) && (item.max_duration_ms !== void 0 || item.maxDurationMs !== void 0)
+  );
+}
+function staleSkillTriggerMessage(value) {
+  const skill = typeof value.skill === "string" && value.skill.trim() ? value.skill.trim() : "";
+  const shouldTrigger = value.should_trigger !== false;
+  if (!skill) {
+    return "Authored assertion type 'skill-trigger' has been removed. Use 'skill-used' with value: <skill> for expected skill use, or 'not-skill-used' with value: <skill> when the skill must not be used.";
+  }
+  const replacementType = shouldTrigger ? "skill-used" : "not-skill-used";
+  return `Authored assertion type 'skill-trigger' has been removed. Replace skill: ${skill} with type: ${replacementType}, value: ${skill}.`;
+}
+function staleToolTrajectoryMessage(value) {
+  const mode = typeof value.mode === "string" ? value.mode : void 0;
+  if (hasToolTrajectoryLatencyCheck(value)) {
+    return "Authored assertion type 'tool-trajectory' has been removed. Per-tool latency checks such as max_duration_ms have no Promptfoo trajectory:* equivalent in AgentV yet; use a custom script assertion or track this as unsupported future scope.";
+  }
+  if (mode === "any_order" && isObject(value.minimums)) {
+    const [toolName, min] = Object.entries(value.minimums)[0] ?? [];
+    const example = typeof toolName === "string" ? ` For example: type: trajectory:tool-used, value: { name: ${toolName}, min: ${formatJsonValue(min)} }.` : "";
+    return `Authored assertion type 'tool-trajectory' has been removed. Replace mode: any_order minimums with one Promptfoo trajectory:tool-used assertion per tool.${example}`;
+  }
+  if ((mode === "in_order" || mode === "exact") && Array.isArray(value.expected)) {
+    const steps = value.expected.filter(isObject).map((item) => item.tool).filter((tool) => typeof tool === "string");
+    const hasArgs = value.expected.some(
+      (item) => isObject(item) && item.args !== void 0 && item.args !== "any"
+    );
+    const sequenceHint = steps.length > 0 ? ` Use type: trajectory:tool-sequence, value: { mode: ${mode}, steps: ${formatJsonValue(steps)} }.` : " Use type: trajectory:tool-sequence with value: { mode, steps }.";
+    const argsHint = hasArgs ? " Move expected args checks to trajectory:tool-args-match with Promptfoo mode: partial or exact." : "";
+    return `Authored assertion type 'tool-trajectory' has been removed.${sequenceHint}${argsHint}`;
+  }
+  return "Authored assertion type 'tool-trajectory' has been removed. Use Promptfoo trajectory:* assertions: trajectory:tool-used for tool presence/counts, trajectory:tool-sequence for in_order/exact steps, and trajectory:tool-args-match for argument checks. AgentV-specific latency checks are unsupported future scope.";
+}
+function staleAuthoredAssertionMessage(value) {
+  if (typeof value.type !== "string") return void 0;
+  const type = value.type.replace(/_/g, "-");
+  if (type === "skill-trigger") return staleSkillTriggerMessage(value);
+  if (type === "tool-trajectory") return staleToolTrajectoryMessage(value);
+  return void 0;
+}
 var VALID_TEST_FILE_EXTENSIONS = /* @__PURE__ */ new Set([
   ".csv",
   ".cjs",
@@ -2837,10 +2938,10 @@ var KNOWN_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
   "input",
   "input_files",
   "prompts",
-  "imports",
   "tests",
   "graders",
   "defaults",
+  "environment",
   "target",
   "targets",
   "model",
@@ -2867,7 +2968,6 @@ var KNOWN_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
   "governance"
 ]);
 var KNOWN_INCLUDE_FIELDS = /* @__PURE__ */ new Set(["include", "type", "select", "run"]);
-var KNOWN_IMPORT_FIELDS = /* @__PURE__ */ new Set(["path", "select", "run"]);
 var KNOWN_RUN_OVERRIDE_FIELDS = /* @__PURE__ */ new Set(["threshold", "repeat", "timeout_seconds", "budget_usd"]);
 var KNOWN_DEFAULT_TEST_FIELDS = /* @__PURE__ */ new Set([
   "vars",
@@ -2907,6 +3007,10 @@ var KNOWN_TEST_EXECUTION_FIELDS = /* @__PURE__ */ new Set([
   "workspace"
 ]);
 var REMOVED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Map([
+  [
+    "imports",
+    "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds."
+  ],
   [
     "expected_output",
     "Top-level 'expected_output' has been removed from authored eval YAML. Put reference answers in default_test.vars.expected_output or tests[].vars.expected_output and consume them with an explicit assertion such as { type: 'llm-rubric', value: 'Matches the reference answer: {{ expected_output }}' }."
@@ -2966,6 +3070,7 @@ var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
   "threshold",
   "execution",
   "run",
+  "environment",
   "workspace",
   "metadata",
   "conversation_id",
@@ -2979,6 +3084,17 @@ var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
   "window_size"
 ]);
 var SUPPORTED_WORKSPACE_REPO_FIELDS = /* @__PURE__ */ new Set(["path", "repo", "commit", "ancestor", "sparse"]);
+var KNOWN_REMOVED_WORKSPACE_FIELDS = /* @__PURE__ */ new Set([
+  "template",
+  "repos",
+  "scope",
+  "docker",
+  "hooks",
+  "env",
+  "mode",
+  "isolation",
+  "path"
+]);
 var REMOVED_TEST_FIELDS = /* @__PURE__ */ new Map([]);
 REMOVED_TEST_FIELDS.set(
   "input",
@@ -3108,13 +3224,15 @@ async function validateEvalFile(filePath) {
       });
     }
   }
+  await validateEnvironmentConfig(parsed.environment, absolutePath, errors, "environment");
   await validateSuiteWorkspaceConfigs(parsed, absolutePath, errors);
+  validateTargetTestbedFields(parsed.target, "target", absolutePath, errors);
+  validateTargetsTestbedFields(parsed.targets, "targets", absolutePath, errors);
   validateAuthoredWorkers(parsed, absolutePath, errors);
   validateExecutionPolicy(parsed.execution, "execution", absolutePath, errors);
   validateEvaluateOptions(parsed.evaluate_options, "evaluate_options", absolutePath, errors);
   validateAssertArray(parsed.assert, "assert", absolutePath, errors, customAssertionTypes);
   validateDefaultTest(parsed.default_test, absolutePath, errors, customAssertionTypes);
-  await validateImportsField(parsed.imports, absolutePath, errors);
   const cases = parsed.tests;
   const hasImports = collectImportEntries(parsed).length > 0;
   const hasScenarios = Array.isArray(parsed.scenarios);
@@ -3144,7 +3262,7 @@ async function validateEvalFile(filePath) {
       severity: "error",
       filePath: absolutePath,
       location: "tests",
-      message: "Missing or invalid 'tests' field (must be an array or a file path string, unless imports are provided)"
+      message: "Missing or invalid 'tests' field (must be an array or a file path string, unless scenarios are provided)"
     });
     return {
       valid: errors.length === 0,
@@ -3264,6 +3382,12 @@ async function validateEvalFile(filePath) {
     );
     validateRunOverride(evalCase.run, `${location}.run`, absolutePath, errors);
     validateTestOptions(evalCase.options, `${location}.options`, absolutePath, errors);
+    await validateEnvironmentConfig(
+      evalCase.environment,
+      absolutePath,
+      errors,
+      `${location}.environment`
+    );
     validateConversationMode(evalCase, location, absolutePath, errors);
     await validateWorkspaceConfig(
       evalCase.workspace,
@@ -3292,6 +3416,43 @@ async function validateSuiteWorkspaceConfigs(parsed, absolutePath, errors) {
     );
   }
 }
+async function validateEnvironmentConfig(environment, evalFilePath, errors, location) {
+  if (environment === void 0) {
+    return;
+  }
+  try {
+    await resolveEnvironmentRecipe(environment, path22.dirname(evalFilePath), location);
+  } catch (error) {
+    errors.push({
+      severity: "error",
+      filePath: evalFilePath,
+      location,
+      message: error.message
+    });
+  }
+}
+function validateTargetTestbedFields(target, location, filePath, errors) {
+  if (!isObject(target)) {
+    return;
+  }
+  for (const field of ["environment", "container", "install"]) {
+    if (target[field] === void 0) {
+      continue;
+    }
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.${field}`,
+      message: field === "environment" ? "Target definitions cannot include environment; author environment at suite/test/case scope." : `Target definitions cannot include ${field}; use an environment recipe for testbed setup.`
+    });
+  }
+}
+function validateTargetsTestbedFields(targets, location, filePath, errors) {
+  const entries = Array.isArray(targets) ? targets : targets === void 0 ? [] : [targets];
+  entries.forEach((entry, index) => {
+    validateTargetTestbedFields(entry, `${location}[${index}]`, filePath, errors);
+  });
+}
 function validateTestExecutionFields(caseExecution, filePath, errors, location) {
   for (const key of Object.keys(caseExecution)) {
     if (key === "workers") {
@@ -3413,87 +3574,11 @@ function rejectRuntimeWorkspaceConfig(workspace, filePath, errors, location) {
     severity: "error",
     filePath,
     location,
-    message: `${location} has been removed from eval YAML. Put machine-local workspace_path in .agentv/config.local.yaml under execution, or pass --workspace-path. Keep portable task setup in top-level workspace.`
+    message: `${location} has been removed from eval YAML. Put machine-local workspace_path in .agentv/config.local.yaml under execution, or pass --workspace-path. Keep portable coding-agent testbed setup in environment.`
   });
 }
-function importEntryPath(value) {
-  if (typeof value === "string" && value.trim().length > 0) {
-    return value.trim();
-  }
-  if (!isObject(value)) {
-    return void 0;
-  }
-  return typeof value.path === "string" && value.path.trim().length > 0 ? value.path.trim() : void 0;
-}
-async function validateImportsField(imports, filePath, errors) {
-  if (imports === void 0) {
-    return;
-  }
-  if (!isObject(imports)) {
-    errors.push({
-      severity: "error",
-      filePath,
-      location: "imports",
-      message: "Invalid 'imports' field. Use imports.suites and/or imports.tests."
-    });
-    return;
-  }
-  for (const key of Object.keys(imports)) {
-    if (key !== "suites" && key !== "tests") {
-      errors.push({
-        severity: "warning",
-        filePath,
-        location: `imports.${key}`,
-        message: `Unknown imports field '${key}'. Use imports.suites or imports.tests.`
-      });
-    }
-  }
-  await validateImportGroup(imports.suites, "suite", "imports.suites", filePath, errors);
-  await validateImportGroup(imports.tests, "tests", "imports.tests", filePath, errors);
-}
-async function validateImportGroup(group, type, location, filePath, errors) {
-  if (group === void 0) {
-    return;
-  }
-  const entries = Array.isArray(group) ? group : [group];
-  for (let i = 0; i < entries.length; i++) {
-    const entry = entries[i];
-    const entryLocation = `${location}[${i}]`;
-    const entryPath = importEntryPath(entry);
-    if (!entryPath) {
-      errors.push({
-        severity: "error",
-        filePath,
-        location: entryLocation,
-        message: "Invalid import entry. Use a path string or an object with a non-empty 'path'."
-      });
-      continue;
-    }
-    if (type === "tests" && !/\.eval\.ya?ml$/i.test(entryPath)) {
-      await validateRawCaseImportPath(entryPath, filePath, `${entryLocation}.path`, errors);
-    }
-    if (isObject(entry)) {
-      for (const key of Object.keys(entry)) {
-        if (!KNOWN_IMPORT_FIELDS.has(key)) {
-          errors.push({
-            severity: "warning",
-            filePath,
-            location: `${entryLocation}.${key}`,
-            message: `Unknown field '${key}'. This field will be ignored.`
-          });
-        }
-      }
-      validateIncludeSelect(entry.select, `${entryLocation}.select`, filePath, errors);
-      validateRunOverride(entry.run, `${entryLocation}.run`, filePath, errors);
-    }
-  }
-}
 function collectImportEntries(parsed) {
   const entries = [];
-  if (isObject(parsed.imports)) {
-    entries.push(...collectImportGroup(parsed.imports.suites, "suite", "imports.suites"));
-    entries.push(...collectImportGroup(parsed.imports.tests, "tests", "imports.tests"));
-  }
   const tests = parsed.tests;
   if (Array.isArray(tests)) {
     for (let i = 0; i < tests.length; i++) {
@@ -3516,42 +3601,24 @@ function collectImportEntries(parsed) {
   }
   return entries;
 }
-function collectImportGroup(group, type, location) {
-  if (group === void 0) {
-    return [];
-  }
-  const entries = Array.isArray(group) ? group : [group];
-  return entries.flatMap((entry, index) => {
-    const pathValue = importEntryPath(entry);
-    return pathValue ? [
-      {
-        path: pathValue,
-        type,
-        location: `${location}[${index}].path`,
-        ...isObject(entry) && entry.select !== void 0 ? { select: entry.select } : {},
-        ...isObject(entry) && entry.run !== void 0 ? { run: entry.run } : {}
-      }
-    ] : [];
-  });
-}
 async function validateCompositionDiagnostics(filePath, parsed, errors) {
-  const imports = collectImportEntries(parsed);
-  if (imports.length === 0) {
+  const includeEntries = collectImportEntries(parsed);
+  if (includeEntries.length === 0) {
     return;
   }
   const parentHasRuntime = hasWrapperRuntimeControls(parsed);
-  const hasSuiteImport = imports.some((entry) => entry.type === "suite");
+  const hasSuiteImport = includeEntries.some((entry) => entry.type === "suite");
   if (hasSuiteImport) {
     for (const location of parentWorkspaceLocations(parsed)) {
       errors.push({
         severity: "error",
         filePath,
         location,
-        message: "Parent workspace is not allowed when an eval imports suites with type: suite. A wrapper eval owns target and run controls, while imported suites own task environment. Move workspace into the child suite, or import raw cases with type: tests when you intentionally want parent workspace context."
+        message: location === "environment" ? "Parent environment is not allowed with legacy tests[].include suite entries. Run eval files directly, or use tests: file://... for raw cases that should use the parent environment." : "Parent workspace is not allowed with legacy tests[].include suite entries. Run eval files directly, or use tests: file://... for raw cases that should use the parent workspace."
       });
     }
   }
-  for (const entry of imports) {
+  for (const entry of includeEntries) {
     const resolvedSuites = await resolveSuiteIncludePaths(entry.path, path22.dirname(filePath));
     if (entry.type === "suite") {
       for (const resolvedSuite of resolvedSuites) {
@@ -3567,7 +3634,7 @@ async function validateCompositionDiagnostics(filePath, parsed, errors) {
           severity: "warning",
           filePath,
           location: entry.location,
-          message: parentHasRuntime ? `Imported suite '${resolvedSuite.displayPath}' defines ${runtimeFields.join(", ")}, but child target and run controls are ignored for imports.suites. The parent eval owns wrapper target and run controls; move them to the parent eval or use import run overrides for per-case thresholds, timeouts, and budgets.` : `Imported suite '${resolvedSuite.displayPath}' defines ${runtimeFields.join(", ")}, but child target and run controls are ignored for imports.suites. The parent eval owns wrapper target and run controls, and this parent has none, so no child target or run controls are applied. Add parent target/run controls or use import run overrides for per-case thresholds, timeouts, and budgets.`
+          message: parentHasRuntime ? `Imported suite '${resolvedSuite.displayPath}' defines ${runtimeFields.join(", ")}, but child target and run controls are ignored by legacy tests[].include suite imports. Prefer running eval files directly with CLI multi-file selection and tags.` : `Imported suite '${resolvedSuite.displayPath}' defines ${runtimeFields.join(", ")}, but child target and run controls are ignored by legacy tests[].include suite imports. Prefer running eval files directly with CLI multi-file selection and tags.`
         });
       }
       continue;
@@ -3581,7 +3648,7 @@ async function validateCompositionDiagnostics(filePath, parsed, errors) {
           severity: "warning",
           filePath,
           location: entry.location,
-          message: `imports.tests imports raw cases from eval suite '${resolvedSuite.displayPath}' and drops suite context, including child workspace, input, assertions, metadata, target, and run controls. Parent suite context applies. Use imports.suites to preserve child test and workspace semantics.`
+          message: `Legacy tests[].include with type: tests imports raw cases from eval suite '${resolvedSuite.displayPath}' and drops suite context, including child environment, workspace, input, assertions, metadata, target, and run controls. Prefer tests: file://... for raw case files.`
         });
       }
     }
@@ -3620,6 +3687,9 @@ function parentWorkspaceLocations(parsed) {
   if (parsed.workspace !== void 0) {
     locations.push("workspace");
   }
+  if (parsed.environment !== void 0) {
+    locations.push("environment");
+  }
   if (isObject(parsed.execution) && parsed.execution.workspace !== void 0) {
     locations.push("execution.workspace");
   }
@@ -3634,12 +3704,11 @@ async function readImportedSuite(filePath) {
   }
 }
 function validateIncludeEntry(entry, location, filePath, errors) {
-  const mode = entry.type === "suite" ? "suites" : entry.type === "tests" ? "tests" : void 0;
   errors.push({
     severity: "warning",
     filePath,
     location,
-    message: mode ? `tests[].include is deprecated. Use imports.${mode} entries with path instead.` : "tests[].include is deprecated. Use imports.suites or imports.tests entries with path instead."
+    message: entry.type === "suite" ? "tests[].include with type: suite is deprecated. Run eval files directly with CLI multi-file selection and tags instead." : "tests[].include is deprecated. Use tests: file://... or string entries inside tests: for raw case files."
   });
   for (const key of Object.keys(entry)) {
     if (!KNOWN_INCLUDE_FIELDS.has(key)) {
@@ -4041,35 +4110,18 @@ async function validateWorkspaceConfig(workspace, evalFilePath, errors, location
   if (workspace === void 0) {
     return;
   }
-  if (isObject(workspace)) {
-    validateWorkspaceRepoConfig(workspace, evalFilePath, errors, location);
-    return;
-  }
-  if (typeof workspace !== "string") {
-    return;
-  }
-  const workspacePath = path22.resolve(path22.dirname(evalFilePath), workspace);
-  try {
-    const workspaceContent = await readFile22(workspacePath, "utf8");
-    const parsedWorkspace = interpolateEnv(parseYamlValue(workspaceContent), process.env);
-    if (!isObject(parsedWorkspace)) {
-      errors.push({
-        severity: "error",
-        filePath: evalFilePath,
-        location,
-        message: `External workspace file must contain a YAML object: ${workspace}`
-      });
-      return;
-    }
-    validateWorkspaceRepoConfig(parsedWorkspace, workspacePath, errors, location);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
+  if (typeof workspace === "string") {
     errors.push({
       severity: "error",
       filePath: evalFilePath,
       location,
-      message: `Failed to load external workspace file '${workspace}': ${message}`
+      message: `${location} has been removed from public eval YAML. Use environment: file://... for reusable coding-agent testbed recipes.`
     });
+    return;
+  }
+  if (isObject(workspace)) {
+    validateWorkspaceRepoConfig(workspace, evalFilePath, errors, location);
+    return;
   }
 }
 function validateWorkspaceRepoConfig(workspace, filePath, errors, location) {
@@ -4078,12 +4130,79 @@ function validateWorkspaceRepoConfig(workspace, filePath, errors, location) {
   const afterEachHook = isObject(hooks) ? hooks.after_each : void 0;
   const scope = workspace.scope;
   const docker = workspace.docker;
+  let hasRemovedTestbedField = false;
+  if (Object.keys(workspace).length === 0) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location,
+      message: "workspace has been removed from public eval YAML. Use environment for coding-agent testbed setup."
+    });
+    return;
+  }
+  if ("template" in workspace) {
+    hasRemovedTestbedField = true;
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.template`,
+      message: "workspace.template has been removed from public eval YAML. Use environment.workdir and environment.setup for authored testbed setup."
+    });
+  }
+  if ("repos" in workspace) {
+    hasRemovedTestbedField = true;
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.repos`,
+      message: "workspace.repos has been removed from public eval YAML. Use environment.setup.command argv to materialize repositories."
+    });
+  }
+  if ("scope" in workspace) {
+    hasRemovedTestbedField = true;
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.scope`,
+      message: "workspace.scope has been removed from public eval YAML. Use environment at suite/test scope and let runtime manage workspace lifetime."
+    });
+  }
+  if ("docker" in workspace) {
+    hasRemovedTestbedField = true;
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.docker`,
+      message: "workspace.docker has been removed from public eval YAML. Use environment.type: docker with image or context."
+    });
+  }
+  if ("hooks" in workspace) {
+    hasRemovedTestbedField = true;
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.hooks`,
+      message: "workspace.hooks has been removed from public eval YAML. Use environment.setup for testbed setup and extensions for lifecycle instrumentation."
+    });
+  }
+  if ("env" in workspace) {
+    hasRemovedTestbedField = true;
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.env`,
+      message: "workspace.env has been removed from public eval YAML. Use top-level env for provider/eval variables or environment.env for environment-scoped variables."
+    });
+  }
+  if (hasRemovedTestbedField) {
+    return;
+  }
   if ("mode" in workspace) {
     errors.push({
       severity: "error",
       filePath,
       location: `${location}.mode`,
-      message: "workspace.mode has been removed from eval YAML. Use workspace.scope: suite|attempt."
+      message: "workspace.mode has been removed from eval YAML. Use --workspace-path or execution.workspace_path for machine-local existing directories, and environment for portable testbed setup."
     });
   }
   if ("isolation" in workspace) {
@@ -4091,7 +4210,7 @@ function validateWorkspaceRepoConfig(workspace, filePath, errors, location) {
       severity: "error",
       filePath,
       location: `${location}.isolation`,
-      message: "workspace.isolation has been removed. Use workspace.scope: suite|attempt."
+      message: "workspace.isolation has been removed from eval YAML. Use environment at suite/test scope and let AgentV manage runtime isolation."
     });
   }
   if ("path" in workspace) {
@@ -4099,7 +4218,18 @@ function validateWorkspaceRepoConfig(workspace, filePath, errors, location) {
       severity: "error",
       filePath,
       location: `${location}.path`,
-      message: "workspace.path has been removed from eval YAML. Put existing workspace paths in .agentv/config.local.yaml execution.workspace_path or pass --workspace-path."
+      message: "workspace.path has been removed from eval YAML. Use environment.workdir for portable testbed cwd; put existing machine-local workspace paths in .agentv/config.local.yaml execution.workspace_path or pass --workspace-path."
+    });
+  }
+  for (const key of Object.keys(workspace)) {
+    if (KNOWN_REMOVED_WORKSPACE_FIELDS.has(key)) {
+      continue;
+    }
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.${key}`,
+      message: `${location}.${key} has been removed from public eval YAML. Use environment for portable coding-agent testbed setup.`
     });
   }
   if (scope !== void 0 && scope !== "suite" && scope !== "attempt") {
@@ -4412,11 +4542,11 @@ async function validateSuiteImportCycles(filePath, parsed, errors) {
   await validateSuiteImportCyclesFromParsed(filePath, parsed, [root], errors);
 }
 async function validateSuiteImportCyclesFromParsed(currentFilePath, parsed, stack, errors) {
-  const imports = collectImportEntries(parsed);
-  if (imports.length === 0) {
+  const includeEntries = collectImportEntries(parsed);
+  if (includeEntries.length === 0) {
     return;
   }
-  for (const entry of imports) {
+  for (const entry of includeEntries) {
     if (entry.type !== "suite") {
       continue;
     }
@@ -4529,6 +4659,16 @@ function validateAssertArray(assertField, location, filePath, errors, customAsse
       continue;
     }
     const typeValue = rawTypeValue.replace(/_/g, "-");
+    const staleMessage = staleAuthoredAssertionMessage(item);
+    if (staleMessage) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: `${itemLocation}.type`,
+        message: staleMessage
+      });
+      continue;
+    }
     const replacement = REMOVED_ASSERTION_TYPE_REPLACEMENTS.get(typeValue);
     if (replacement) {
       errors.push({
@@ -4549,7 +4689,7 @@ function validateAssertArray(assertField, location, filePath, errors, customAsse
       });
       continue;
     }
-    if (!isGraderKind(typeValue) && !PROMPTFOO_ASSERTION_TYPES.has(typeValue) && !customAssertionTypes.has(typeValue)) {
+    if (!isGraderKind(typeValue) && !isGraderKind(baseTypeValue) && !PROMPTFOO_ASSERTION_TYPES.has(typeValue) && !customAssertionTypes.has(typeValue)) {
       errors.push({
         severity: "warning",
         filePath,
@@ -5092,7 +5232,7 @@ function validateUnknownSettings(target, provider, absolutePath, location, error
         severity: "warning",
         filePath: absolutePath,
         location: `${location}.${key}`,
-        message: "workspace_template has been removed from targets. Use eval-level workspace.template instead."
+        message: "workspace_template has been removed from targets. Use eval-level environment.workdir and environment.setup instead."
       });
       continue;
     }
@@ -7445,7 +7585,7 @@ async function runEvalCommand(input) {
   const resolvedExperiment = resolveExperimentForRun(options.experiment);
   const evalPathInputs = input.testFiles.length > 0 ? [...input.testFiles] : [];
   if (evalPathInputs.length === 0 && process.stdin.isTTY) {
-    const { launchInteractiveWizard } = await import("./interactive-V4SEHLNY.js");
+    const { launchInteractiveWizard } = await import("./interactive-RUY3OCBI.js");
     await launchInteractiveWizard();
     return void 0;
   }
@@ -7785,7 +7925,7 @@ async function runEvalCommand(input) {
   });
   let transcriptProviderFactory;
   if (options.transcript) {
-    const { TranscriptProvider } = await import("./dist-2D2GBHZV.js");
+    const { TranscriptProvider } = await import("./dist-6Z7U473R.js");
     const transcriptProvider = await TranscriptProvider.fromFile(options.transcript);
     const totalTests = [...fileMetadata.values()].reduce(
       (sum, meta) => sum + meta.testCases.length,
@@ -8051,7 +8191,7 @@ ${formatRequiredVersionFailureNote(requiredVersionCheck)}`);
       const sourceTests = activeSourceTests;
       const taskBundleTargets = buildTaskBundleTargetSelections(activeTestFiles, fileMetadata);
       if (isResumeAppend) {
-        const { writePerTestArtifacts } = await import("./artifact-writer-IORG7CW5.js");
+        const { writePerTestArtifacts } = await import("./artifact-writer-JFNIPMKW.js");
         await writePerTestArtifacts(allResults, runDir, {
           experiment: normalizeExperimentName(options.experiment),
           runId: path12.basename(runDir),
@@ -8269,4 +8409,4 @@ export {
   filterByCategory,
   runEvalCommand
 };
-//# sourceMappingURL=chunk-AM2IPHKS.js.map
+//# sourceMappingURL=chunk-FVR4RQFK.js.map