@agentv/core 4.22.0 → 4.24.0-next.1

package/dist/index.cjs

@@ -277,6 +277,20 @@ var init_trace = __esm({
   }
 });
 
+// src/evaluation/yaml-loader.ts
+function parseYamlValue(content) {
+  return (0, import_yaml.parse)(content, PARSE_OPTIONS);
+}
+var import_yaml, PARSE_OPTIONS;
+var init_yaml_loader = __esm({
+  "src/evaluation/yaml-loader.ts"() {
+    "use strict";
+    init_cjs_shims();
+    import_yaml = require("yaml");
+    PARSE_OPTIONS = { merge: true };
+  }
+});
+
 // src/evaluation/input-message-utils.ts
 function flattenInputMessages(messages) {
   return messages.flatMap((message) => extractContentSegments(message.content));
@@ -472,7 +486,7 @@ function isGlobPattern(filePath) {
   return filePath.includes("*") || filePath.includes("?") || filePath.includes("{");
 }
 function parseYamlCases(content, filePath) {
-  const raw = (0, import_yaml.parse)(content);
+  const raw = parseYamlValue(content);
   const parsed = interpolateEnv(raw, process.env);
   if (!Array.isArray(parsed)) {
     throw new Error(
@@ -591,7 +605,7 @@ async function loadCasesFromDirectory(dirPath) {
       throw new Error(`Cannot read case file: ${caseFilePath}
   ${message}`);
     }
-    const raw = (0, import_yaml.parse)(content);
+    const raw = parseYamlValue(content);
     const parsed = interpolateEnv(raw, process.env);
     if (!isJsonObject(parsed)) {
       throw new Error(
@@ -628,7 +642,7 @@ async function expandFileReferences(tests, evalFileDir) {
   }
   return expanded;
 }
-var import_promises2, import_node_path2, import_fast_glob, import_yaml, ANSI_YELLOW, ANSI_RESET2, FILE_PROTOCOL;
+var import_promises2, import_node_path2, import_fast_glob, ANSI_YELLOW, ANSI_RESET2, FILE_PROTOCOL;
 var init_case_file_loader = __esm({
   "src/evaluation/loaders/case-file-loader.ts"() {
     "use strict";
@@ -636,9 +650,9 @@ var init_case_file_loader = __esm({
     import_promises2 = require("fs/promises");
     import_node_path2 = __toESM(require("path"), 1);
     import_fast_glob = __toESM(require("fast-glob"), 1);
-    import_yaml = require("yaml");
     init_interpolation();
     init_types();
+    init_yaml_loader();
     ANSI_YELLOW = "\x1B[33m";
     ANSI_RESET2 = "\x1B[0m";
     FILE_PROTOCOL = "file://";
@@ -763,7 +777,7 @@ async function loadConfig(evalFilePath, repoRoot) {
     }
     try {
       const rawConfig = await (0, import_promises4.readFile)(configPath, "utf8");
-      const parsed = interpolateEnv((0, import_yaml2.parse)(rawConfig), process.env);
+      const parsed = interpolateEnv(parseYamlValue(rawConfig), process.env);
       if (!isJsonObject(parsed)) {
         logWarning(`Invalid .agentv/config.yaml format at ${configPath}`);
         continue;
@@ -1163,16 +1177,16 @@ function parseHooksConfig(raw, configPath) {
 function logWarning(message) {
   console.warn(`${ANSI_YELLOW2}Warning: ${message}${ANSI_RESET3}`);
 }
-var import_promises4, import_node_path4, import_yaml2, ANSI_YELLOW2, ANSI_RESET3, DEFAULT_EVAL_PATTERNS, VALID_TRIAL_STRATEGIES;
+var import_promises4, import_node_path4, ANSI_YELLOW2, ANSI_RESET3, DEFAULT_EVAL_PATTERNS, VALID_TRIAL_STRATEGIES;
 var init_config_loader = __esm({
   "src/evaluation/loaders/config-loader.ts"() {
     "use strict";
     init_cjs_shims();
     import_promises4 = require("fs/promises");
     import_node_path4 = __toESM(require("path"), 1);
-    import_yaml2 = require("yaml");
     init_interpolation();
     init_types();
+    init_yaml_loader();
     init_file_resolver();
     ANSI_YELLOW2 = "\x1B[33m";
     ANSI_RESET3 = "\x1B[0m";
@@ -1689,7 +1703,7 @@ ${resolved.attempted.map((attempt) => `  Tried: ${attempt}`).join("\n")}` : "";
     throw new Error(`Assertion template cycle detected in '${evalId}': ${cycle}`);
   }
   const content = await (0, import_promises7.readFile)(resolved.resolvedPath, "utf8");
-  const parsed = interpolateEnv((0, import_yaml3.parse)(content), process.env);
+  const parsed = interpolateEnv(parseYamlValue(content), process.env);
   if (!isJsonObject2(parsed)) {
     throw new Error(
       `Invalid assertion template file in '${evalId}': ${resolved.resolvedPath} (expected a YAML object with an assertions array)`
@@ -3296,18 +3310,18 @@ function parseInlineRubrics(rawRubrics) {
     rubrics: rubricItems
   };
 }
-var import_promises7, import_node_path6, import_yaml3, ANSI_YELLOW4, ANSI_RESET5, MAX_ASSERTION_INCLUDE_DEPTH, PROMPT_FILE_PREFIX, VALID_FIELD_MATCH_TYPES, VALID_FIELD_AGGREGATION_TYPES;
+var import_promises7, import_node_path6, ANSI_YELLOW4, ANSI_RESET5, MAX_ASSERTION_INCLUDE_DEPTH, PROMPT_FILE_PREFIX, VALID_FIELD_MATCH_TYPES, VALID_FIELD_AGGREGATION_TYPES;
 var init_grader_parser = __esm({
   "src/evaluation/loaders/grader-parser.ts"() {
     "use strict";
     init_cjs_shims();
     import_promises7 = require("fs/promises");
     import_node_path6 = __toESM(require("path"), 1);
-    import_yaml3 = require("yaml");
     init_content_preprocessor();
     init_interpolation();
     init_types();
     init_prompt_validator();
+    init_yaml_loader();
     init_file_resolver();
     ANSI_YELLOW4 = "\x1B[33m";
     ANSI_RESET5 = "\x1B[0m";
@@ -3759,7 +3773,7 @@ async function loadSidecarMetadata(jsonlPath, verbose) {
   }
   try {
     const content = await (0, import_promises9.readFile)(sidecarPath, "utf8");
-    const parsed = interpolateEnv((0, import_yaml4.parse)(content), process.env);
+    const parsed = interpolateEnv(parseYamlValue(content), process.env);
     if (!isJsonObject(parsed)) {
       logWarning4(`Invalid sidecar metadata format in ${sidecarPath}`);
       return {};
@@ -3938,7 +3952,7 @@ ${detailBlock}${ANSI_RESET7}`);
     console.error(`${ANSI_RED2}Error: ${message}${ANSI_RESET7}`);
   }
 }
-var import_promises9, import_node_path8, import_micromatch, import_yaml4, ANSI_YELLOW6, ANSI_RED2, ANSI_RESET7;
+var import_promises9, import_node_path8, import_micromatch, ANSI_YELLOW6, ANSI_RED2, ANSI_RESET7;
 var init_jsonl_parser = __esm({
   "src/evaluation/loaders/jsonl-parser.ts"() {
     "use strict";
@@ -3946,10 +3960,10 @@ var init_jsonl_parser = __esm({
     import_promises9 = require("fs/promises");
     import_node_path8 = __toESM(require("path"), 1);
     import_micromatch = __toESM(require("micromatch"), 1);
-    import_yaml4 = require("yaml");
     init_input_message_utils();
     init_interpolation();
     init_types();
+    init_yaml_loader();
     init_file_resolver();
     init_grader_parser();
     init_message_processor();
@@ -3962,9 +3976,7 @@ var init_jsonl_parser = __esm({
 
 // src/evaluation/metadata.ts
 function parseMetadata(suite) {
-  const hasName = typeof suite.name === "string";
-  const hasDescription = typeof suite.description === "string";
-  if (!hasName) {
+  if (typeof suite.name !== "string") {
     return void 0;
   }
   return MetadataSchema.parse({
@@ -3984,7 +3996,7 @@ var init_metadata = __esm({
     init_cjs_shims();
     import_zod = require("zod");
     MetadataSchema = import_zod.z.object({
-      name: import_zod.z.string().min(1).max(64).regex(/^[a-z0-9-]+$/),
+      name: import_zod.z.string().min(1).max(64).regex(/^[a-z0-9-]+$/).optional(),
       description: import_zod.z.string().min(1).max(1024).optional(),
       version: import_zod.z.string().optional(),
       author: import_zod.z.string().optional(),
@@ -12363,7 +12375,7 @@ async function discoverCopilotSessions(opts) {
     const eventsPath = import_node_path21.default.join(sessionDir, "events.jsonl");
     try {
       const workspaceContent = await (0, import_promises19.readFile)(workspacePath, "utf8");
-      const workspace = (0, import_yaml5.parse)(workspaceContent) ?? {};
+      const workspace = parseYamlValue(workspaceContent) ?? {};
       const cwd = String(workspace.cwd ?? "");
       let updatedAt;
       try {
@@ -12407,7 +12419,7 @@ async function discoverCopilotSessions(opts) {
   filtered.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime());
   return filtered.slice(0, limit);
 }
-var import_promises19, import_node_os6, import_node_path21, import_yaml5, DEFAULT_SESSION_STATE_DIR;
+var import_promises19, import_node_os6, import_node_path21, DEFAULT_SESSION_STATE_DIR;
 var init_copilot_session_discovery = __esm({
   "src/evaluation/providers/copilot-session-discovery.ts"() {
     "use strict";
@@ -12415,7 +12427,7 @@ var init_copilot_session_discovery = __esm({
     import_promises19 = require("fs/promises");
     import_node_os6 = require("os");
     import_node_path21 = __toESM(require("path"), 1);
-    import_yaml5 = require("yaml");
+    init_yaml_loader();
     DEFAULT_SESSION_STATE_DIR = () => import_node_path21.default.join((0, import_node_os6.homedir)(), ".copilot", "session-state");
   }
 });
@@ -17644,7 +17656,7 @@ async function readTargetDefinitions(filePath) {
     throw new Error(`targets.yaml not found at ${absolutePath}`);
   }
   const raw = await (0, import_promises31.readFile)(absolutePath, "utf8");
-  const parsed = (0, import_yaml6.parse)(raw);
+  const parsed = parseYamlValue(raw);
   if (!isRecord(parsed)) {
     throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
   }
@@ -17657,7 +17669,7 @@ async function readTargetDefinitions(filePath) {
 function listTargetNames(definitions) {
   return definitions.map((definition) => definition.name);
 }
-var import_node_fs13, import_promises31, import_node_path38, import_yaml6;
+var import_node_fs13, import_promises31, import_node_path38;
 var init_targets_file = __esm({
   "src/evaluation/providers/targets-file.ts"() {
     "use strict";
@@ -17665,7 +17677,7 @@ var init_targets_file = __esm({
     import_node_fs13 = require("fs");
     import_promises31 = require("fs/promises");
     import_node_path38 = __toESM(require("path"), 1);
-    import_yaml6 = require("yaml");
+    init_yaml_loader();
   }
 });
 
@@ -20080,6 +20092,9 @@ async function runEvaluation(options) {
           result = { ...result, beforeAllOutput };
           beforeAllOutputAttached = true;
         }
+        if (evalCase.metadata && !result.metadata) {
+          result = { ...result, metadata: evalCase.metadata };
+        }
         if (onProgress) {
           await onProgress({
             workerId,
@@ -20090,7 +20105,9 @@ async function runEvaluation(options) {
             completedAt: Date.now(),
             error: result.error,
             score: result.score,
-            executionStatus: result.executionStatus
+            executionStatus: result.executionStatus,
+            durationMs: result.durationMs,
+            evalRunDurationMs: result.evalRun?.durationMs
           });
         }
         if (onResult) {
@@ -20404,7 +20421,8 @@ async function runBatchEvaluation(options) {
           completedAt: Date.now(),
           error: error instanceof Error ? error.message : String(error),
           score: errorResult.score,
-          executionStatus: errorResult.executionStatus
+          executionStatus: errorResult.executionStatus,
+          evalRunDurationMs: errorResult.evalRun?.durationMs
         });
       }
       continue;
@@ -20422,7 +20440,9 @@ async function runBatchEvaluation(options) {
         completedAt: Date.now(),
         error: result.error,
         score: result.score,
-        executionStatus: result.executionStatus
+        executionStatus: result.executionStatus,
+        durationMs: result.durationMs,
+        evalRunDurationMs: result.evalRun?.durationMs
       });
     }
   }
@@ -22487,7 +22507,7 @@ async function readTestSuiteMetadata(testFilePath) {
   try {
     const absolutePath = import_node_path50.default.resolve(testFilePath);
     const content = await (0, import_promises36.readFile)(absolutePath, "utf8");
-    const parsed = interpolateEnv((0, import_yaml7.parse)(content), process.env);
+    const parsed = interpolateEnv(parseYamlValue(content), process.env);
     if (!isJsonObject(parsed)) {
       return {};
     }
@@ -22559,7 +22579,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   const searchRoots = buildSearchRoots(absoluteTestPath, repoRootPath);
   const config = await loadConfig(absoluteTestPath, repoRootPath);
   const rawFile = await (0, import_promises36.readFile)(absoluteTestPath, "utf8");
-  const interpolated = interpolateEnv((0, import_yaml7.parse)(rawFile), process.env);
+  const interpolated = interpolateEnv(parseYamlValue(rawFile), process.env);
   if (!isJsonObject(interpolated)) {
     throw new Error(`Invalid test file format: ${evalFilePath}`);
   }
@@ -22596,6 +22616,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     throw new Error(`Invalid test file format: ${evalFilePath} - missing 'tests' field`);
   }
   const suiteWorkspace = await resolveWorkspaceConfig(suite.workspace, evalFileDir);
+  const suiteGovernance = extractSuiteGovernance(suite);
   const suiteInputMessages = expandInputShorthand(suite.input);
   const suiteInputFiles = suite.input_files;
   const rawGlobalExecution = isJsonObject(suite.execution) ? suite.execution : void 0;
@@ -22705,7 +22726,9 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     const userFilePaths = collectResolvedInputFilePaths(inputMessages);
     const caseWorkspace = await resolveWorkspaceConfig(testCaseConfig.workspace, evalFileDir);
     const mergedWorkspace = mergeWorkspaceConfigs(suiteWorkspace, caseWorkspace);
-    const metadata = isJsonObject(testCaseConfig.metadata) ? testCaseConfig.metadata : void 0;
+    const rawCaseMetadata = isJsonObject(testCaseConfig.metadata) ? testCaseConfig.metadata : void 0;
+    const suitePayload = suiteGovernance !== void 0 ? { governance: suiteGovernance } : void 0;
+    const metadata = mergeSuiteMetadataPayload(rawCaseMetadata, suitePayload);
     const caseTargets = extractTargetsFromTestCase(testCaseConfig);
     const dependsOn = Array.isArray(testCaseConfig.depends_on) ? testCaseConfig.depends_on.filter(
       (v) => typeof v === "string"
@@ -22845,7 +22868,7 @@ async function resolveWorkspaceConfig(raw, evalFileDir) {
     } catch {
       throw new Error(`Workspace file not found: ${raw} (resolved to ${workspaceFilePath})`);
     }
-    const parsed = interpolateEnv((0, import_yaml7.parse)(content), process.env);
+    const parsed = interpolateEnv(parseYamlValue(content), process.env);
     if (!isJsonObject(parsed)) {
       throw new Error(
         `Invalid workspace file format: ${workspaceFilePath} (expected a YAML object)`
@@ -22952,6 +22975,47 @@ function mergeWorkspaceConfigs(suiteLevel, caseLevel) {
 function asString5(value) {
   return typeof value === "string" ? value : void 0;
 }
+function extractSuiteGovernance(suite) {
+  const top = suite.governance;
+  if (isJsonObject(top)) {
+    return top;
+  }
+  const wrapper = suite.metadata;
+  if (isJsonObject(wrapper)) {
+    const nested = wrapper.governance;
+    if (isJsonObject(nested)) {
+      return nested;
+    }
+  }
+  return void 0;
+}
+function mergeSuiteMetadataPayload(caseMetadata, suitePayload) {
+  if (!suitePayload) return caseMetadata;
+  const result = { ...caseMetadata ?? {} };
+  for (const [key, suiteVal] of Object.entries(suitePayload)) {
+    const caseVal = result[key];
+    if (Array.isArray(suiteVal) && Array.isArray(caseVal)) {
+      const seen = /* @__PURE__ */ new Set();
+      const out = [];
+      for (const v of [...suiteVal, ...caseVal]) {
+        const k = typeof v === "string" ? v : JSON.stringify(v);
+        if (!seen.has(k)) {
+          seen.add(k);
+          out.push(v);
+        }
+      }
+      result[key] = out;
+    } else if (isJsonObject(suiteVal) && isJsonObject(caseVal)) {
+      result[key] = mergeSuiteMetadataPayload(
+        caseVal,
+        suiteVal
+      );
+    } else if (caseVal === void 0) {
+      result[key] = suiteVal;
+    }
+  }
+  return result;
+}
 function logWarning5(message, details) {
   if (details && details.length > 0) {
     const detailBlock = details.join("\n");
@@ -22970,7 +23034,7 @@ ${detailBlock}${ANSI_RESET9}`);
     console.error(`${ANSI_RED3}Error: ${message}${ANSI_RESET9}`);
   }
 }
-var import_promises36, import_node_path50, import_micromatch4, import_yaml7, ANSI_YELLOW8, ANSI_RED3, ANSI_RESET9, loadEvalSuite, loadEvalCases, loadEvalCaseById;
+var import_promises36, import_node_path50, import_micromatch4, ANSI_YELLOW8, ANSI_RED3, ANSI_RESET9, loadEvalSuite, loadEvalCases, loadEvalCaseById;
 var init_yaml_parser = __esm({
   "src/evaluation/yaml-parser.ts"() {
     "use strict";
@@ -22978,7 +23042,6 @@ var init_yaml_parser = __esm({
     import_promises36 = require("fs/promises");
     import_node_path50 = __toESM(require("path"), 1);
     import_micromatch4 = __toESM(require("micromatch"), 1);
-    import_yaml7 = require("yaml");
     init_input_message_utils();
     init_interpolation();
     init_agent_skills_parser();
@@ -22992,6 +23055,7 @@ var init_yaml_parser = __esm({
     init_metadata();
     init_types();
     init_repo_config_parser();
+    init_yaml_loader();
     init_prompt_builder();
     init_config_loader();
     init_jsonl_parser();
@@ -24464,6 +24528,7 @@ __export(index_exports, {
   parseEnvOutput: () => parseEnvOutput,
   parseJsonFromText: () => parseJsonFromText,
   parseJsonSafe: () => parseJsonSafe,
+  parseYamlValue: () => parseYamlValue,
   prepareResultsRepoBranch: () => prepareResultsRepoBranch,
   pushResultsRepoBranch: () => pushResultsRepoBranch,
   readJsonFile: () => readJsonFile,
@@ -24523,6 +24588,7 @@ init_cjs_shims();
 init_content();
 init_types();
 init_trace();
+init_yaml_loader();
 init_yaml_parser();
 init_agent_skills_parser();
 init_config_loader();
@@ -24532,7 +24598,7 @@ init_ts_eval_loader();
 init_cjs_shims();
 var import_node_fs18 = require("fs");
 var import_node_path51 = __toESM(require("path"), 1);
-var import_yaml8 = require("yaml");
+init_yaml_loader();
 function codeGraderInstruction(graderName, description) {
   const desc = description ? ` This grader: ${description}.` : "";
   return `Run \`agentv eval assert ${graderName} --agent-output <agent_output> --agent-input <original_prompt>\` and check the result.${desc} The command accepts --agent-output (the agent's full response text) and --agent-input (the original user prompt). It returns JSON on stdout: {"score": 0-1, "reasoning": "..."}. A score >= 0.5 means pass (exit 0); below 0.5 means fail (exit 1).`;
@@ -24771,7 +24837,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
 }
 function transpileEvalYamlFile(evalYamlPath) {
   const content = (0, import_node_fs18.readFileSync)(evalYamlPath, "utf8");
-  const parsed = (0, import_yaml8.parse)(content);
+  const parsed = parseYamlValue(content);
   return transpileEvalYaml(parsed, import_node_path51.default.basename(evalYamlPath));
 }
 function getOutputFilenames(result) {
@@ -24978,8 +25044,8 @@ init_pool_manager();
 init_cjs_shims();
 var import_promises37 = require("fs/promises");
 var import_node_path52 = __toESM(require("path"), 1);
-var import_yaml9 = require("yaml");
 init_interpolation();
+init_yaml_loader();
 init_repo_config_parser();
 function normalizeGitUrl(url) {
   let normalized = url.replace(/\.git$/, "");
@@ -25027,7 +25093,7 @@ async function scanRepoDeps(evalFilePaths) {
 }
 async function extractReposFromEvalFile(filePath) {
   const content = await (0, import_promises37.readFile)(filePath, "utf8");
-  const parsed = interpolateEnv((0, import_yaml9.parse)(content), process.env);
+  const parsed = interpolateEnv(parseYamlValue(content), process.env);
   if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
   const obj = parsed;
   const evalFileDir = import_node_path52.default.dirname(import_node_path52.default.resolve(filePath));
@@ -25048,7 +25114,7 @@ async function extractReposFromWorkspaceRaw(raw, evalFileDir) {
   if (typeof raw === "string") {
     const workspaceFilePath = import_node_path52.default.resolve(evalFileDir, raw);
     const content = await (0, import_promises37.readFile)(workspaceFilePath, "utf8");
-    const parsed = interpolateEnv((0, import_yaml9.parse)(content), process.env);
+    const parsed = interpolateEnv(parseYamlValue(content), process.env);
     if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
     return extractReposFromObject(parsed);
   }
@@ -25412,7 +25478,8 @@ init_paths();
 init_cjs_shims();
 var import_node_fs20 = require("fs");
 var import_node_path55 = __toESM(require("path"), 1);
-var import_yaml10 = require("yaml");
+var import_yaml2 = require("yaml");
+init_yaml_loader();
 init_paths();
 function getBenchmarksRegistryPath() {
   return import_node_path55.default.join(getAgentvConfigDir(), "benchmarks.yaml");
@@ -25447,7 +25514,7 @@ function loadBenchmarkRegistry() {
   }
   try {
     const raw = (0, import_node_fs20.readFileSync)(registryPath, "utf-8");
-    const parsed = (0, import_yaml10.parse)(raw);
+    const parsed = parseYamlValue(raw);
     if (!parsed || typeof parsed !== "object") {
       return { benchmarks: [] };
     }
@@ -25464,7 +25531,7 @@ function saveBenchmarkRegistry(registry) {
     (0, import_node_fs20.mkdirSync)(dir, { recursive: true });
   }
   const payload = { benchmarks: registry.benchmarks.map(toYaml) };
-  (0, import_node_fs20.writeFileSync)(registryPath, (0, import_yaml10.stringify)(payload), "utf-8");
+  (0, import_node_fs20.writeFileSync)(registryPath, (0, import_yaml2.stringify)(payload), "utf-8");
 }
 function deriveBenchmarkId(dirPath, existingIds) {
   const base = import_node_path55.default.basename(dirPath).toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
@@ -26991,6 +27058,7 @@ function createAgentKernel() {
   parseEnvOutput,
   parseJsonFromText,
   parseJsonSafe,
+  parseYamlValue,
   prepareResultsRepoBranch,
   pushResultsRepoBranch,
   readJsonFile,