agentv 5.3.0-next.1 → 5.3.1-next.1

package/dist/{chunk-FVR4RQFK.js → chunk-LKGARI3W.js}

@@ -1,7 +1,7 @@
 import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
 import {
   writeArtifactsFromResults
-} from "./chunk-BHKQHG26.js";
+} from "./chunk-RKE7SSET.js";
 import {
   RunBudgetTracker,
   buildWipBranchName,
@@ -18,7 +18,7 @@ import {
   resolveResultsRepoRunsDir,
   setupWipWorktree,
   syncResultsRepoForProject
-} from "./chunk-V52ATPTT.js";
+} from "./chunk-LXBI3SPX.js";
 import {
   AGENTV_LOCAL_CONFIG_FILE_NAME,
   AGENTV_LOCAL_CONFIG_YML_FILE_NAME,
@@ -51,6 +51,7 @@ import {
   interpolateEnv,
   isAgentVConfigFileName,
   isGraderKind,
+  isTypeScriptEvalConfigFileName,
   listTargetNames,
   loadCasesFromDirectory,
   loadCasesFromFile,
@@ -83,8 +84,9 @@ import {
   toSnakeCaseDeep,
   traceEnvelopeToTraceSummary,
   traceEnvelopeToTranscriptMessages,
+  typeScriptEvalConfigGlob,
   writeInitialRunSummaryArtifact
-} from "./chunk-6ZZCDZPD.js";
+} from "./chunk-ELCJ23K4.js";
 
 // src/commands/eval/shared.ts
 import { constants } from "node:fs";
@@ -215,7 +217,7 @@ function agentSkillsToAgentVYamlObject(suite) {
 
 // src/commands/eval/shared.ts
 function isNativeEvalFile(filePath) {
-  return /\.(ya?ml|jsonl|[cm]?ts)$/i.test(filePath);
+  return /\.(ya?ml|jsonl)$/i.test(filePath) || isTypeScriptEvalConfigFileName(filePath);
 }
 function shouldInspectJsonPath(filePath) {
   const base = path.basename(filePath).toLowerCase();
@@ -265,7 +267,7 @@ async function resolveEvalPaths(evalPaths, cwd, options = {}) {
         continue;
       }
       if (candidateStats.isDirectory()) {
-        const filePattern = options.allowReadAdapters ? "{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts,evals.json,*.evals.json}" : "{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts}";
+        const filePattern = options.allowReadAdapters ? `{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,${typeScriptEvalConfigGlob()},evals.json,*.evals.json}` : `{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,${typeScriptEvalConfigGlob()}}`;
         const dirGlob = path.posix.join(candidatePath.replace(/\\/g, "/"), `**/${filePattern}`);
         const dirMatches = await fg(dirGlob, {
           absolute: true,
@@ -347,7 +349,7 @@ async function findRepoRoot(start) {
 // package.json
 var package_default = {
   name: "agentv",
-  version: "5.3.0-next.1",
+  version: "5.3.1-next.1",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -3032,6 +3034,14 @@ var REMOVED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Map([
     "providers",
     "Top-level 'providers' is not a runtime alias in AgentV eval YAML. Use 'targets' for systems under test; provider names backend kind inside each target."
   ],
+  [
+    "eval_cases",
+    "Top-level 'eval_cases' has been removed from authored eval YAML. Use 'tests' instead."
+  ],
+  [
+    "evalcases",
+    "Top-level 'evalcases' has been removed from authored eval YAML. Use 'tests' instead."
+  ],
   ["repeat", "Top-level 'repeat' has been removed. Use evaluate_options.repeat instead."],
   ["runs", "Top-level 'runs' has been removed. Use evaluate_options.repeat.count instead."],
   [
@@ -3046,11 +3056,11 @@ var REMOVED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Map([
   [
     "preprocessors",
     "Top-level 'preprocessors' has been removed from authored eval YAML. Use default_test.options.transform or assertion-level transform instead."
-  ]
+  ],
+  ["providerPromptMap", "Top-level 'providerPromptMap' is not supported. Use 'targets'."],
+  ["provider_prompt_map", "Top-level 'provider_prompt_map' is not supported. Use 'targets'."]
 ]);
 var DEPRECATED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Map([
-  ["eval_cases", "'eval_cases' is deprecated. Use 'tests' instead."],
-  ["evalcases", "'evalcases' is deprecated. Use 'tests' instead."],
   ["evaluator", "'evaluator' is deprecated. Use 'assert' instead."]
 ]);
 var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
@@ -3083,6 +3093,7 @@ var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
   "on_turn_failure",
   "window_size"
 ]);
+var KNOWN_SCENARIO_FIELDS = /* @__PURE__ */ new Set(["description", "config", "tests"]);
 var SUPPORTED_WORKSPACE_REPO_FIELDS = /* @__PURE__ */ new Set(["path", "repo", "commit", "ancestor", "sparse"]);
 var KNOWN_REMOVED_WORKSPACE_FIELDS = /* @__PURE__ */ new Set([
   "template",
@@ -3100,6 +3111,22 @@ REMOVED_TEST_FIELDS.set(
   "input",
   "tests[].input has been removed from authored eval YAML. Put prompt text or chat/system/user messages in top-level 'prompts' and put row-specific data in tests[].vars."
 );
+REMOVED_TEST_FIELDS.set(
+  "preprocessors",
+  "tests[].preprocessors has been removed from authored eval YAML. Use tests[].options.transform or assertion-level transform instead."
+);
+REMOVED_TEST_FIELDS.set(
+  "postprocess",
+  "tests[].postprocess has been removed from authored eval YAML. Use tests[].options.transform instead."
+);
+REMOVED_TEST_FIELDS.set(
+  "eval_cases",
+  "tests[].eval_cases has been removed from authored eval YAML. Use 'tests' rows directly."
+);
+REMOVED_TEST_FIELDS.set(
+  "evalcases",
+  "tests[].evalcases has been removed from authored eval YAML. Use 'tests' rows directly."
+);
 var DEPRECATED_TEST_FIELDS = /* @__PURE__ */ new Map([
   ["evaluator", "'evaluator' is deprecated. Use 'assert' instead."],
   ["expected_outcome", "'expected_outcome' is deprecated. Use 'assert' instead."]
@@ -3233,6 +3260,7 @@ async function validateEvalFile(filePath) {
   validateEvaluateOptions(parsed.evaluate_options, "evaluate_options", absolutePath, errors);
   validateAssertArray(parsed.assert, "assert", absolutePath, errors, customAssertionTypes);
   validateDefaultTest(parsed.default_test, absolutePath, errors, customAssertionTypes);
+  await validateScenarios(parsed.scenarios, parsed, absolutePath, errors, customAssertionTypes);
   const cases = parsed.tests;
   const hasImports = collectImportEntries(parsed).length > 0;
   const hasScenarios = Array.isArray(parsed.scenarios);
@@ -3405,6 +3433,32 @@ async function validateEvalFile(filePath) {
     errors
   };
 }
+async function validateTypeScriptEvalConfigFile(filePath) {
+  const absolutePath = path22.resolve(filePath);
+  try {
+    const { loadTsEvalSuite } = await import("./ts-eval-loader-2RFVZHCT-7CZ3DCDD.js");
+    await loadTsEvalSuite(absolutePath, process.cwd());
+    return {
+      valid: true,
+      filePath: absolutePath,
+      fileType: "eval",
+      errors: []
+    };
+  } catch (error) {
+    return {
+      valid: false,
+      filePath: absolutePath,
+      fileType: "eval",
+      errors: [
+        {
+          severity: "error",
+          filePath: absolutePath,
+          message: error.message
+        }
+      ]
+    };
+  }
+}
 async function validateSuiteWorkspaceConfigs(parsed, absolutePath, errors) {
   await validateWorkspaceConfig(parsed.workspace, absolutePath, errors, "workspace");
   if (isObject(parsed.execution)) {
@@ -3891,6 +3945,182 @@ function validateDefaultTest(defaultTest, filePath, errors, customAssertionTypes
     });
   }
 }
+async function validateScenarios(scenarios, parsed, filePath, errors, customAssertionTypes) {
+  if (scenarios === void 0) {
+    return;
+  }
+  if (!Array.isArray(scenarios)) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: "scenarios",
+      message: "Invalid 'scenarios' field (must be an array)"
+    });
+    return;
+  }
+  for (let scenarioIndex = 0; scenarioIndex < scenarios.length; scenarioIndex++) {
+    const scenario = scenarios[scenarioIndex];
+    const scenarioLocation = `scenarios[${scenarioIndex}]`;
+    if (!isObject(scenario)) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: scenarioLocation,
+        message: "Scenario must be an object"
+      });
+      continue;
+    }
+    for (const key of Object.keys(scenario)) {
+      if (!KNOWN_SCENARIO_FIELDS.has(key)) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: `${scenarioLocation}.${key}`,
+          message: `Unknown scenario field '${key}'.`
+        });
+      }
+    }
+    if (!Array.isArray(scenario.config)) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: `${scenarioLocation}.config`,
+        message: "Invalid 'config' field (must be an array)"
+      });
+    } else {
+      for (let configIndex = 0; configIndex < scenario.config.length; configIndex++) {
+        await validateScenarioTestLikeRow(
+          scenario.config[configIndex],
+          `${scenarioLocation}.config[${configIndex}]`,
+          filePath,
+          errors,
+          customAssertionTypes,
+          { requireInput: false }
+        );
+      }
+    }
+    const scenarioConfigs = Array.isArray(scenario.config) ? scenario.config : [];
+    if (!Array.isArray(scenario.tests)) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: `${scenarioLocation}.tests`,
+        message: "Invalid 'tests' field (must be an array)"
+      });
+      continue;
+    }
+    for (let testIndex = 0; testIndex < scenario.tests.length; testIndex++) {
+      const test = scenario.tests[testIndex];
+      const everyConfigProvidesInput = scenarioConfigs.length > 0 && scenarioConfigs.every(
+        (config) => isObject(config) && (config.prompts !== void 0 || config.provider_output !== void 0)
+      );
+      const requireInput = isObject(test) && parsed.prompts === void 0 && test.prompts === void 0 && test.provider_output === void 0 && !everyConfigProvidesInput;
+      await validateScenarioTestLikeRow(
+        test,
+        `${scenarioLocation}.tests[${testIndex}]`,
+        filePath,
+        errors,
+        customAssertionTypes,
+        { requireInput }
+      );
+    }
+  }
+}
+async function validateScenarioTestLikeRow(row, location, filePath, errors, customAssertionTypes, options) {
+  if (!isObject(row)) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location,
+      message: "Scenario row must be an object"
+    });
+    return;
+  }
+  for (const key of Object.keys(row)) {
+    const removedMessage = REMOVED_TEST_FIELDS.get(key);
+    if (removedMessage) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: `${location}.${key}`,
+        message: removedMessage.replaceAll("tests[]", location)
+      });
+      continue;
+    }
+    const deprecationMessage = DEPRECATED_TEST_FIELDS.get(key);
+    if (deprecationMessage) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: `${location}.${key}`,
+        message: deprecationMessage
+      });
+    } else if (!KNOWN_TEST_FIELDS.has(key)) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: `${location}.${key}`,
+        message: `Unknown test field '${key}'.`
+      });
+    }
+  }
+  const id = row.id;
+  if (id !== void 0 && (typeof id !== "string" || id.trim().length === 0)) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.id`,
+      message: "Invalid 'id' field (must be a non-empty string when provided)"
+    });
+  }
+  const criteria = row.criteria;
+  if (criteria !== void 0 && (typeof criteria !== "string" || criteria.trim().length === 0)) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.criteria`,
+      message: "Invalid 'criteria' field (must be a non-empty string if provided)"
+    });
+  } else if (criteria !== void 0 && row.assert !== void 0) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.criteria`,
+      message: "Do not combine test-level 'criteria' with 'assert'. Put human-readable case descriptions in tests[].description, or express grading text as an explicit assertion such as { type: 'llm-rubric', value: ... }."
+    });
+  }
+  validateInputField(row.input, `${location}.input`, filePath, errors, {
+    required: options.requireInput
+  });
+  if (row.expected_output !== void 0) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.expected_output`,
+      message: "tests[].expected_output has been removed from authored eval YAML. Put the reference answer in tests[].vars.expected_output and consume it with an explicit assertion, for example { type: 'llm-rubric', value: 'Matches the reference answer: {{ expected_output }}' }."
+    });
+  }
+  validateAssertArray(row.assert, `${location}.assert`, filePath, errors, customAssertionTypes);
+  validateRunOverride(row.run, `${location}.run`, filePath, errors);
+  validateTestOptions(row.options, `${location}.options`, filePath, errors);
+  await validateEnvironmentConfig(row.environment, filePath, errors, `${location}.environment`);
+  validateConversationMode(row, location, filePath, errors);
+  await validateWorkspaceConfig(row.workspace, filePath, errors, `${location}.workspace`);
+  const caseExecution = isObject(row.execution) ? row.execution : void 0;
+  if (caseExecution) {
+    validateTestExecutionFields(caseExecution, filePath, errors, location);
+    rejectRuntimeWorkspaceConfig(
+      caseExecution.workspace,
+      filePath,
+      errors,
+      `${location}.execution.workspace`
+    );
+  }
+  const targets = row.providers !== void 0 ? row.providers : row.provider;
+  if (targets !== void 0) {
+    validateTargetTestbedFields(targets, `${location}.providers`, filePath, errors);
+  }
+}
 function validateEvaluateOptions(evaluateOptions, location, filePath, errors) {
   if (evaluateOptions === void 0) {
     return;
@@ -5212,6 +5442,7 @@ function validateUnknownSettings(target, provider, absolutePath, location, error
     "env",
     "grader_target",
     "judge_target",
+    "use_target",
     "workers",
     "$schema",
     "targets"
@@ -5494,15 +5725,22 @@ async function validateTargetsFile(filePath) {
     const provider = effectiveTarget.provider;
     const rawTarget = rawTargets[i];
     const rawUseTarget = isObject3(rawTarget) ? rawTarget.use_target : void 0;
-    const hasUseTarget = isNonEmptyString(effectiveTarget.use_target) || isNonEmptyString(rawUseTarget);
+    if (effectiveTarget.use_target !== void 0 || rawUseTarget !== void 0) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.use_target`,
+        message: "The 'use_target' field has been removed from authored targets.yaml target definitions. Define a concrete target with 'provider' instead."
+      });
+    }
     const providerValue = typeof provider === "string" ? provider.trim().toLowerCase() : void 0;
     const isTemplated = isEnvTemplated(provider);
-    if (!hasUseTarget && (typeof provider !== "string" || provider.trim().length === 0)) {
+    if (typeof provider !== "string" || provider.trim().length === 0) {
       errors.push({
         severity: "error",
         filePath: absolutePath,
         location: `${location}.provider`,
-        message: "Missing or invalid 'provider' field (must be a non-empty string, or use use_target for delegation)"
+        message: "Missing or invalid 'provider' field (must be a non-empty string)"
       });
     } else if (!isTemplated && (providerValue === "claude" || providerValue === "copilot")) {
       errors.push({
@@ -5637,6 +5875,9 @@ async function validateConfigFile(filePath, options = {}) {
     validateRefsConfig(errors, filePath, config.refs);
     validateComposableGraph(errors, filePath, config);
     validateDashboardConfig(errors, filePath, config.dashboard);
+    validateHooksConfig(errors, filePath, config.hooks);
+    validateEnvPathConfig(errors, filePath, config.env_path);
+    validateEnvFromConfig(errors, filePath, config.env_from);
     const projects = config.projects;
     if (projects !== void 0) {
       if (scope === "project") {
@@ -5671,7 +5912,10 @@ async function validateConfigFile(filePath, options = {}) {
       "defaults",
       "projects",
       "dashboard",
-      "studio"
+      "studio",
+      "hooks",
+      "env_path",
+      "env_from"
     ]);
     const unexpectedFields = Object.keys(config).filter((key) => !allowedFields.has(key));
     if (unexpectedFields.length > 0) {
@@ -5720,6 +5964,76 @@ function validateDashboardConfig(errors, filePath, rawDashboard) {
     );
   }
 }
+var ENV_FROM_FORMATS = /* @__PURE__ */ new Set(["shell_exports", "json"]);
+function validateHooksConfig(errors, filePath, rawHooks) {
+  if (rawHooks === void 0 || rawHooks === null) {
+    return;
+  }
+  if (!isPlainObject(rawHooks)) {
+    addError(errors, filePath, "hooks", "Field 'hooks' must be an object");
+    return;
+  }
+  if (rawHooks.before_session !== void 0) {
+    validateOptionalString(errors, filePath, rawHooks.before_session, "hooks.before_session");
+    errors.push({
+      severity: "warning",
+      filePath,
+      location: "hooks.before_session",
+      message: "'hooks.before_session' is deprecated; use 'env_path' and/or 'env_from' to load environment variables before validation and eval."
+    });
+  }
+}
+function validateEnvPathConfig(errors, filePath, rawEnvPath) {
+  if (rawEnvPath === void 0 || rawEnvPath === null) {
+    return;
+  }
+  const isList = Array.isArray(rawEnvPath);
+  const entries = isList ? rawEnvPath : [rawEnvPath];
+  entries.forEach((entry, index) => {
+    const location = isList ? `env_path[${index}]` : "env_path";
+    if (typeof entry !== "string" || entry.trim().length === 0) {
+      addError(errors, filePath, location, `Field '${location}' must be a non-empty string`);
+    }
+  });
+}
+function validateEnvFromConfig(errors, filePath, rawEnvFrom) {
+  if (rawEnvFrom === void 0 || rawEnvFrom === null) {
+    return;
+  }
+  const isList = Array.isArray(rawEnvFrom);
+  const entries = isList ? rawEnvFrom : [rawEnvFrom];
+  entries.forEach((entry, index) => {
+    const location = isList ? `env_from[${index}]` : "env_from";
+    if (!isPlainObject(entry)) {
+      addError(errors, filePath, location, `Field '${location}' must be an object`);
+      return;
+    }
+    const command = entry.command;
+    if (typeof command === "string") {
+      addError(
+        errors,
+        filePath,
+        `${location}.command`,
+        `Field '${location}.command' must be an argv array of strings, not a shell command string. Use e.g. ["bun", "scripts/load-secrets.ts"].`
+      );
+    } else if (!Array.isArray(command) || command.length === 0 || !command.every((part) => typeof part === "string" && part.length > 0)) {
+      addError(
+        errors,
+        filePath,
+        `${location}.command`,
+        `Field '${location}.command' must be a non-empty array of strings`
+      );
+    }
+    if (entry.format !== void 0 && !ENV_FROM_FORMATS.has(entry.format)) {
+      addError(
+        errors,
+        filePath,
+        `${location}.format`,
+        `Field '${location}.format' must be "shell_exports" or "json"`
+      );
+    }
+  });
+}
 function validateRefsConfig(errors, filePath, rawRefs) {
   if (rawRefs === void 0) {
     return;
@@ -5932,13 +6246,7 @@ async function validateFileReferences(evalFilePath) {
   if (!isObject4(parsed)) {
     return errors;
   }
-  let cases = parsed.tests;
-  if (cases === void 0 && "eval_cases" in parsed) {
-    cases = parsed.eval_cases;
-  }
-  if (cases === void 0 && "evalcases" in parsed) {
-    cases = parsed.evalcases;
-  }
+  const cases = parsed.tests;
   if (!Array.isArray(cases)) {
     return errors;
   }
@@ -7585,7 +7893,7 @@ async function runEvalCommand(input) {
   const resolvedExperiment = resolveExperimentForRun(options.experiment);
   const evalPathInputs = input.testFiles.length > 0 ? [...input.testFiles] : [];
   if (evalPathInputs.length === 0 && process.stdin.isTTY) {
-    const { launchInteractiveWizard } = await import("./interactive-RUY3OCBI.js");
+    const { launchInteractiveWizard } = await import("./interactive-BN527UV3.js");
     await launchInteractiveWizard();
     return void 0;
   }
@@ -7925,7 +8233,7 @@ async function runEvalCommand(input) {
   });
   let transcriptProviderFactory;
   if (options.transcript) {
-    const { TranscriptProvider } = await import("./dist-6Z7U473R.js");
+    const { TranscriptProvider } = await import("./dist-NMXMI5SK.js");
     const transcriptProvider = await TranscriptProvider.fromFile(options.transcript);
     const totalTests = [...fileMetadata.values()].reduce(
       (sum, meta) => sum + meta.testCases.length,
@@ -8191,7 +8499,7 @@ ${formatRequiredVersionFailureNote(requiredVersionCheck)}`);
       const sourceTests = activeSourceTests;
       const taskBundleTargets = buildTaskBundleTargetSelections(activeTestFiles, fileMetadata);
       if (isResumeAppend) {
-        const { writePerTestArtifacts } = await import("./artifact-writer-JFNIPMKW.js");
+        const { writePerTestArtifacts } = await import("./artifact-writer-7NBCOAYC.js");
         await writePerTestArtifacts(allResults, runDir, {
           experiment: normalizeExperimentName(options.experiment),
           runId: path12.basename(runDir),
@@ -8371,6 +8679,7 @@ export {
   loadEnvFromHierarchy,
   detectFileType,
   validateEvalFile,
+  validateTypeScriptEvalConfigFile,
   validateCasesFile,
   validateTargetsFile,
   validateConfigFile,
@@ -8409,4 +8718,4 @@ export {
   filterByCategory,
   runEvalCommand
 };
-//# sourceMappingURL=chunk-FVR4RQFK.js.map
+//# sourceMappingURL=chunk-LKGARI3W.js.map