@agentv/core 2.17.0 → 2.17.1-next.1

package/dist/evaluation/validation/index.js

@@ -6,7 +6,7 @@ import {
   findGitRoot,
   isEvaluatorKind,
   resolveFileReference
-} from "../../chunk-CPPYERD2.js";
+} from "../../chunk-PSYFRPNT.js";
 
 // src/evaluation/validation/file-type.ts
 import { readFile } from "node:fs/promises";

package/dist/index.cjs

@@ -1777,6 +1777,25 @@ var import_node_path8 = __toESM(require("path"), 1);
 var import_micromatch3 = __toESM(require("micromatch"), 1);
 var import_yaml4 = require("yaml");
 
+// src/evaluation/interpolation.ts
+var ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
+function interpolateEnv(value, env) {
+  if (typeof value === "string") {
+    return value.replace(ENV_VAR_PATTERN, (_, varName) => env[varName] ?? "");
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => interpolateEnv(item, env));
+  }
+  if (value !== null && typeof value === "object") {
+    const result = {};
+    for (const [key, val] of Object.entries(value)) {
+      result[key] = interpolateEnv(val, env);
+    }
+    return result;
+  }
+  return value;
+}
+
 // src/evaluation/loaders/case-file-loader.ts
 var import_promises = require("fs/promises");
 var import_node_path = __toESM(require("path"), 1);
@@ -1795,7 +1814,8 @@ function isGlobPattern(filePath) {
   return filePath.includes("*") || filePath.includes("?") || filePath.includes("{");
 }
 function parseYamlCases(content, filePath) {
-  const parsed = (0, import_yaml.parse)(content);
+  const raw = (0, import_yaml.parse)(content);
+  const parsed = interpolateEnv(raw, process.env);
   if (!Array.isArray(parsed)) {
     throw new Error(
       `External test file must contain a YAML array, got ${typeof parsed}: ${filePath}`
@@ -1817,7 +1837,8 @@ function parseJsonlCases(content, filePath) {
     const line = lines[i].trim();
     if (line === "") continue;
     try {
-      const parsed = JSON.parse(line);
+      const raw = JSON.parse(line);
+      const parsed = interpolateEnv(raw, process.env);
       if (!isJsonObject(parsed)) {
         throw new Error("Expected JSON object");
       }
@@ -3966,7 +3987,7 @@ async function loadSidecarMetadata(jsonlPath, verbose) {
   }
   try {
     const content = await (0, import_promises6.readFile)(sidecarPath, "utf8");
-    const parsed = (0, import_yaml3.parse)(content);
+    const parsed = interpolateEnv((0, import_yaml3.parse)(content), process.env);
     if (!isJsonObject(parsed)) {
       logWarning4(`Invalid sidecar metadata format in ${sidecarPath}`);
       return {};
@@ -3989,7 +4010,8 @@ function parseJsonlContent(content, filePath) {
     const line = lines[i].trim();
     if (line === "") continue;
     try {
-      const parsed = JSON.parse(line);
+      const raw = JSON.parse(line);
+      const parsed = interpolateEnv(raw, process.env);
       if (!isJsonObject(parsed)) {
         throw new Error("Expected JSON object");
       }
@@ -4046,9 +4068,10 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
     }
     const inputMessages = resolveInputMessages(evalcase);
     const expectedMessages = resolveExpectedMessages(evalcase) ?? [];
-    if (!id || !outcome || !inputMessages || inputMessages.length === 0) {
+    const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || evalcase.assert !== void 0;
+    if (!id || !hasEvaluationSpec || !inputMessages || inputMessages.length === 0) {
       logError(
-        `Skipping incomplete test at line ${lineNumber}: ${id ?? "unknown"}. Missing required fields: id, criteria, and/or input`
+        `Skipping incomplete test at line ${lineNumber}: ${id ?? "unknown"}. Missing required fields: id, input, and at least one of criteria/expected_output/assert`
       );
       continue;
     }
@@ -4126,7 +4149,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
       guideline_paths: guidelinePaths.map((guidelinePath) => import_node_path6.default.resolve(guidelinePath)),
       guideline_patterns: guidelinePatterns,
       file_paths: allFilePaths,
-      criteria: outcome,
+      criteria: outcome ?? "",
       evaluator: evalCaseEvaluatorKind,
       evaluators
     };
@@ -4439,7 +4462,7 @@ async function readTestSuiteMetadata(testFilePath) {
   try {
     const absolutePath = import_node_path8.default.resolve(testFilePath);
     const content = await (0, import_promises8.readFile)(absolutePath, "utf8");
-    const parsed = (0, import_yaml4.parse)(content);
+    const parsed = interpolateEnv((0, import_yaml4.parse)(content), process.env);
     if (!isJsonObject(parsed)) {
       return {};
     }
@@ -4489,11 +4512,11 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   const config = await loadConfig(absoluteTestPath, repoRootPath);
   const guidelinePatterns = config?.guideline_patterns;
   const rawFile = await (0, import_promises8.readFile)(absoluteTestPath, "utf8");
-  const parsed = (0, import_yaml4.parse)(rawFile);
-  if (!isJsonObject(parsed)) {
+  const interpolated = interpolateEnv((0, import_yaml4.parse)(rawFile), process.env);
+  if (!isJsonObject(interpolated)) {
     throw new Error(`Invalid test file format: ${evalFilePath}`);
   }
-  const suite = parsed;
+  const suite = interpolated;
   const datasetNameFromSuite = asString6(suite.dataset)?.trim();
   const fallbackDataset = import_node_path8.default.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
   const datasetName = datasetNameFromSuite && datasetNameFromSuite.length > 0 ? datasetNameFromSuite : fallbackDataset;
@@ -4537,9 +4560,10 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     }
     const testInputMessages = resolveInputMessages(evalcase);
     const expectedMessages = resolveExpectedMessages(evalcase) ?? [];
-    if (!id || !outcome || !testInputMessages || testInputMessages.length === 0) {
+    const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || evalcase.assert !== void 0;
+    if (!id || !hasEvaluationSpec || !testInputMessages || testInputMessages.length === 0) {
       logError2(
-        `Skipping incomplete test: ${id ?? "unknown"}. Missing required fields: id, criteria, and/or input`
+        `Skipping incomplete test: ${id ?? "unknown"}. Missing required fields: id, input, and at least one of criteria/expected_output/assert`
       );
       continue;
     }
@@ -4635,7 +4659,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       guideline_paths: guidelinePaths.map((guidelinePath) => import_node_path8.default.resolve(guidelinePath)),
       guideline_patterns: guidelinePatterns,
       file_paths: allFilePaths,
-      criteria: outcome,
+      criteria: outcome ?? "",
       evaluator: evalCaseEvaluatorKind,
       evaluators,
       workspace: mergedWorkspace,
@@ -4775,7 +4799,7 @@ async function resolveWorkspaceConfig(raw, evalFileDir) {
     } catch {
       throw new Error(`Workspace file not found: ${raw} (resolved to ${workspaceFilePath})`);
     }
-    const parsed = (0, import_yaml4.parse)(content);
+    const parsed = interpolateEnv((0, import_yaml4.parse)(content), process.env);
     if (!isJsonObject(parsed)) {
       throw new Error(
         `Invalid workspace file format: ${workspaceFilePath} (expected a YAML object)`
@@ -17714,9 +17738,11 @@ async function runEvaluatorList(options) {
     registry: typeRegistry
   };
   for (const evaluatorConfig of evaluators ?? []) {
+    const startedAt = /* @__PURE__ */ new Date();
     try {
       const evaluatorInstance = await typeRegistry.create(evaluatorConfig, dispatchContext);
       const score2 = await evaluatorInstance.evaluate(evalContext);
+      const endedAt = /* @__PURE__ */ new Date();
       const weight = evaluatorConfig.weight ?? 1;
       scored.push({
         score: score2,
@@ -17737,9 +17763,13 @@ async function runEvaluatorList(options) {
         evaluatorProviderRequest: score2.evaluatorRawRequest,
         details: score2.details,
         scores: mapChildResults(score2.scores),
-        tokenUsage: score2.tokenUsage
+        tokenUsage: score2.tokenUsage,
+        durationMs: endedAt.getTime() - startedAt.getTime(),
+        startedAt: startedAt.toISOString(),
+        endedAt: endedAt.toISOString()
       });
     } catch (error) {
+      const endedAt = /* @__PURE__ */ new Date();
       const message = error instanceof Error ? error.message : String(error);
       const fallbackScore = {
         score: 0,
@@ -17765,7 +17795,10 @@ async function runEvaluatorList(options) {
         verdict: "fail",
         hits: [],
         misses: [`Evaluator '${evaluatorConfig.name ?? "unknown"}' failed: ${message}`],
-        reasoning: message
+        reasoning: message,
+        durationMs: endedAt.getTime() - startedAt.getTime(),
+        startedAt: startedAt.toISOString(),
+        endedAt: endedAt.toISOString()
       });
     }
     if (evaluatorConfig.negate === true && scored.length > 0) {