agentv 2.17.0 → 2.17.2

package/dist/{chunk-UJMO2T4J.js → chunk-SO4O4O2B.js}

@@ -148,7 +148,7 @@ var require_dist = __commonJS({
   }
 });
 
-// ../../packages/core/dist/chunk-CPPYERD2.js
+// ../../packages/core/dist/chunk-PSYFRPNT.js
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
 import path from "node:path";
@@ -4195,7 +4195,7 @@ var coerce = {
 };
 var NEVER = INVALID;
 
-// ../../packages/core/dist/chunk-CPPYERD2.js
+// ../../packages/core/dist/chunk-PSYFRPNT.js
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
 var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
 var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
@@ -34107,6 +34107,23 @@ function mergeExecutionMetrics(computed, metrics) {
     endTime: metrics.endTime ?? computed.endTime
   };
 }
+var ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
+function interpolateEnv(value, env) {
+  if (typeof value === "string") {
+    return value.replace(ENV_VAR_PATTERN, (_, varName) => env[varName] ?? "");
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => interpolateEnv(item, env));
+  }
+  if (value !== null && typeof value === "object") {
+    const result = {};
+    for (const [key, val] of Object.entries(value)) {
+      result[key] = interpolateEnv(val, env);
+    }
+    return result;
+  }
+  return value;
+}
 var ANSI_YELLOW = "\x1B[33m";
 var ANSI_RESET = "\x1B[0m";
 var FILE_PROTOCOL = "file://";
@@ -34120,7 +34137,8 @@ function isGlobPattern(filePath) {
   return filePath.includes("*") || filePath.includes("?") || filePath.includes("{");
 }
 function parseYamlCases(content, filePath) {
-  const parsed = parseYaml(content);
+  const raw = parseYaml(content);
+  const parsed = interpolateEnv(raw, process.env);
   if (!Array.isArray(parsed)) {
     throw new Error(
       `External test file must contain a YAML array, got ${typeof parsed}: ${filePath}`
@@ -34142,7 +34160,8 @@ function parseJsonlCases(content, filePath) {
     const line = lines[i].trim();
     if (line === "") continue;
     try {
-      const parsed = JSON.parse(line);
+      const raw = JSON.parse(line);
+      const parsed = interpolateEnv(raw, process.env);
       if (!isJsonObject(parsed)) {
         throw new Error("Expected JSON object");
       }
@@ -36247,7 +36266,7 @@ async function loadSidecarMetadata(jsonlPath, verbose) {
   }
   try {
     const content = await readFile5(sidecarPath, "utf8");
-    const parsed = parseYaml2(content);
+    const parsed = interpolateEnv(parseYaml2(content), process.env);
     if (!isJsonObject(parsed)) {
       logWarning4(`Invalid sidecar metadata format in ${sidecarPath}`);
       return {};
@@ -36270,7 +36289,8 @@ function parseJsonlContent(content, filePath) {
     const line = lines[i].trim();
     if (line === "") continue;
     try {
-      const parsed = JSON.parse(line);
+      const raw = JSON.parse(line);
+      const parsed = interpolateEnv(raw, process.env);
       if (!isJsonObject(parsed)) {
         throw new Error("Expected JSON object");
       }
@@ -36327,9 +36347,10 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
     }
     const inputMessages = resolveInputMessages(evalcase);
     const expectedMessages = resolveExpectedMessages(evalcase) ?? [];
-    if (!id || !outcome || !inputMessages || inputMessages.length === 0) {
+    const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || evalcase.assert !== void 0;
+    if (!id || !hasEvaluationSpec || !inputMessages || inputMessages.length === 0) {
       logError(
-        `Skipping incomplete test at line ${lineNumber}: ${id ?? "unknown"}. Missing required fields: id, criteria, and/or input`
+        `Skipping incomplete test at line ${lineNumber}: ${id ?? "unknown"}. Missing required fields: id, input, and at least one of criteria/expected_output/assert`
       );
       continue;
     }
@@ -36407,7 +36428,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
       guideline_paths: guidelinePaths.map((guidelinePath) => path6.resolve(guidelinePath)),
       guideline_patterns: guidelinePatterns,
       file_paths: allFilePaths,
-      criteria: outcome,
+      criteria: outcome ?? "",
       evaluator: evalCaseEvaluatorKind,
       evaluators
     };
@@ -36711,7 +36732,7 @@ async function readTestSuiteMetadata(testFilePath) {
   try {
     const absolutePath = path8.resolve(testFilePath);
     const content = await readFile7(absolutePath, "utf8");
-    const parsed = parse22(content);
+    const parsed = interpolateEnv(parse22(content), process.env);
     if (!isJsonObject(parsed)) {
       return {};
     }
@@ -36761,11 +36782,11 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   const config2 = await loadConfig(absoluteTestPath, repoRootPath);
   const guidelinePatterns = config2?.guideline_patterns;
   const rawFile = await readFile7(absoluteTestPath, "utf8");
-  const parsed = parse22(rawFile);
-  if (!isJsonObject(parsed)) {
+  const interpolated = interpolateEnv(parse22(rawFile), process.env);
+  if (!isJsonObject(interpolated)) {
     throw new Error(`Invalid test file format: ${evalFilePath}`);
   }
-  const suite = parsed;
+  const suite = interpolated;
   const datasetNameFromSuite = asString6(suite.dataset)?.trim();
   const fallbackDataset = path8.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
   const datasetName = datasetNameFromSuite && datasetNameFromSuite.length > 0 ? datasetNameFromSuite : fallbackDataset;
@@ -36809,9 +36830,10 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     }
     const testInputMessages = resolveInputMessages(evalcase);
     const expectedMessages = resolveExpectedMessages(evalcase) ?? [];
-    if (!id || !outcome || !testInputMessages || testInputMessages.length === 0) {
+    const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || evalcase.assert !== void 0;
+    if (!id || !hasEvaluationSpec || !testInputMessages || testInputMessages.length === 0) {
       logError2(
-        `Skipping incomplete test: ${id ?? "unknown"}. Missing required fields: id, criteria, and/or input`
+        `Skipping incomplete test: ${id ?? "unknown"}. Missing required fields: id, input, and at least one of criteria/expected_output/assert`
       );
       continue;
     }
@@ -36907,7 +36929,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       guideline_paths: guidelinePaths.map((guidelinePath) => path8.resolve(guidelinePath)),
       guideline_patterns: guidelinePatterns,
       file_paths: allFilePaths,
-      criteria: outcome,
+      criteria: outcome ?? "",
       evaluator: evalCaseEvaluatorKind,
       evaluators,
       workspace: mergedWorkspace,
@@ -37047,7 +37069,7 @@ async function resolveWorkspaceConfig(raw, evalFileDir) {
     } catch {
       throw new Error(`Workspace file not found: ${raw} (resolved to ${workspaceFilePath})`);
     }
-    const parsed = parse22(content);
+    const parsed = interpolateEnv(parse22(content), process.env);
     if (!isJsonObject(parsed)) {
       throw new Error(
         `Invalid workspace file format: ${workspaceFilePath} (expected a YAML object)`
@@ -48473,9 +48495,11 @@ async function runEvaluatorList(options) {
     registry: typeRegistry
   };
   for (const evaluatorConfig of evaluators ?? []) {
+    const startedAt = /* @__PURE__ */ new Date();
     try {
       const evaluatorInstance = await typeRegistry.create(evaluatorConfig, dispatchContext);
       const score2 = await evaluatorInstance.evaluate(evalContext);
+      const endedAt = /* @__PURE__ */ new Date();
       const weight = evaluatorConfig.weight ?? 1;
       scored.push({
         score: score2,
@@ -48496,9 +48520,13 @@ async function runEvaluatorList(options) {
         evaluatorProviderRequest: score2.evaluatorRawRequest,
         details: score2.details,
         scores: mapChildResults(score2.scores),
-        tokenUsage: score2.tokenUsage
+        tokenUsage: score2.tokenUsage,
+        durationMs: endedAt.getTime() - startedAt.getTime(),
+        startedAt: startedAt.toISOString(),
+        endedAt: endedAt.toISOString()
       });
     } catch (error40) {
+      const endedAt = /* @__PURE__ */ new Date();
       const message = error40 instanceof Error ? error40.message : String(error40);
       const fallbackScore = {
         score: 0,
@@ -48524,7 +48552,10 @@ async function runEvaluatorList(options) {
         verdict: "fail",
         hits: [],
         misses: [`Evaluator '${evaluatorConfig.name ?? "unknown"}' failed: ${message}`],
-        reasoning: message
+        reasoning: message,
+        durationMs: endedAt.getTime() - startedAt.getTime(),
+        startedAt: startedAt.toISOString(),
+        endedAt: endedAt.toISOString()
       });
     }
     if (evaluatorConfig.negate === true && scored.length > 0) {
@@ -49681,4 +49712,4 @@ export {
   OtelStreamingObserver,
   createAgentKernel
 };
-//# sourceMappingURL=chunk-UJMO2T4J.js.map
+//# sourceMappingURL=chunk-SO4O4O2B.js.map