@agentv/core 2.16.0 → 2.17.1-next.1

package/dist/index.cjs

@@ -1777,6 +1777,25 @@ var import_node_path8 = __toESM(require("path"), 1);
 var import_micromatch3 = __toESM(require("micromatch"), 1);
 var import_yaml4 = require("yaml");
 
+// src/evaluation/interpolation.ts
+var ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
+function interpolateEnv(value, env) {
+  if (typeof value === "string") {
+    return value.replace(ENV_VAR_PATTERN, (_, varName) => env[varName] ?? "");
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => interpolateEnv(item, env));
+  }
+  if (value !== null && typeof value === "object") {
+    const result = {};
+    for (const [key, val] of Object.entries(value)) {
+      result[key] = interpolateEnv(val, env);
+    }
+    return result;
+  }
+  return value;
+}
+
 // src/evaluation/loaders/case-file-loader.ts
 var import_promises = require("fs/promises");
 var import_node_path = __toESM(require("path"), 1);
@@ -1795,7 +1814,8 @@ function isGlobPattern(filePath) {
   return filePath.includes("*") || filePath.includes("?") || filePath.includes("{");
 }
 function parseYamlCases(content, filePath) {
-  const parsed = (0, import_yaml.parse)(content);
+  const raw = (0, import_yaml.parse)(content);
+  const parsed = interpolateEnv(raw, process.env);
   if (!Array.isArray(parsed)) {
     throw new Error(
       `External test file must contain a YAML array, got ${typeof parsed}: ${filePath}`
@@ -1817,7 +1837,8 @@ function parseJsonlCases(content, filePath) {
     const line = lines[i].trim();
     if (line === "") continue;
     try {
-      const parsed = JSON.parse(line);
+      const raw = JSON.parse(line);
+      const parsed = interpolateEnv(raw, process.env);
       if (!isJsonObject(parsed)) {
         throw new Error("Expected JSON object");
       }
@@ -3966,7 +3987,7 @@ async function loadSidecarMetadata(jsonlPath, verbose) {
   }
   try {
     const content = await (0, import_promises6.readFile)(sidecarPath, "utf8");
-    const parsed = (0, import_yaml3.parse)(content);
+    const parsed = interpolateEnv((0, import_yaml3.parse)(content), process.env);
     if (!isJsonObject(parsed)) {
       logWarning4(`Invalid sidecar metadata format in ${sidecarPath}`);
       return {};
@@ -3989,7 +4010,8 @@ function parseJsonlContent(content, filePath) {
     const line = lines[i].trim();
     if (line === "") continue;
     try {
-      const parsed = JSON.parse(line);
+      const raw = JSON.parse(line);
+      const parsed = interpolateEnv(raw, process.env);
       if (!isJsonObject(parsed)) {
         throw new Error("Expected JSON object");
       }
@@ -4046,9 +4068,10 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
     }
     const inputMessages = resolveInputMessages(evalcase);
     const expectedMessages = resolveExpectedMessages(evalcase) ?? [];
-    if (!id || !outcome || !inputMessages || inputMessages.length === 0) {
+    const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || evalcase.assert !== void 0;
+    if (!id || !hasEvaluationSpec || !inputMessages || inputMessages.length === 0) {
       logError(
-        `Skipping incomplete test at line ${lineNumber}: ${id ?? "unknown"}. Missing required fields: id, criteria, and/or input`
+        `Skipping incomplete test at line ${lineNumber}: ${id ?? "unknown"}. Missing required fields: id, input, and at least one of criteria/expected_output/assert`
       );
       continue;
     }
@@ -4126,7 +4149,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
       guideline_paths: guidelinePaths.map((guidelinePath) => import_node_path6.default.resolve(guidelinePath)),
       guideline_patterns: guidelinePatterns,
       file_paths: allFilePaths,
-      criteria: outcome,
+      criteria: outcome ?? "",
       evaluator: evalCaseEvaluatorKind,
       evaluators
     };
@@ -4439,7 +4462,7 @@ async function readTestSuiteMetadata(testFilePath) {
   try {
     const absolutePath = import_node_path8.default.resolve(testFilePath);
     const content = await (0, import_promises8.readFile)(absolutePath, "utf8");
-    const parsed = (0, import_yaml4.parse)(content);
+    const parsed = interpolateEnv((0, import_yaml4.parse)(content), process.env);
     if (!isJsonObject(parsed)) {
       return {};
     }
@@ -4489,11 +4512,11 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   const config = await loadConfig(absoluteTestPath, repoRootPath);
   const guidelinePatterns = config?.guideline_patterns;
   const rawFile = await (0, import_promises8.readFile)(absoluteTestPath, "utf8");
-  const parsed = (0, import_yaml4.parse)(rawFile);
-  if (!isJsonObject(parsed)) {
+  const interpolated = interpolateEnv((0, import_yaml4.parse)(rawFile), process.env);
+  if (!isJsonObject(interpolated)) {
     throw new Error(`Invalid test file format: ${evalFilePath}`);
   }
-  const suite = parsed;
+  const suite = interpolated;
   const datasetNameFromSuite = asString6(suite.dataset)?.trim();
   const fallbackDataset = import_node_path8.default.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
   const datasetName = datasetNameFromSuite && datasetNameFromSuite.length > 0 ? datasetNameFromSuite : fallbackDataset;
@@ -4537,9 +4560,10 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     }
     const testInputMessages = resolveInputMessages(evalcase);
     const expectedMessages = resolveExpectedMessages(evalcase) ?? [];
-    if (!id || !outcome || !testInputMessages || testInputMessages.length === 0) {
+    const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || evalcase.assert !== void 0;
+    if (!id || !hasEvaluationSpec || !testInputMessages || testInputMessages.length === 0) {
       logError2(
-        `Skipping incomplete test: ${id ?? "unknown"}. Missing required fields: id, criteria, and/or input`
+        `Skipping incomplete test: ${id ?? "unknown"}. Missing required fields: id, input, and at least one of criteria/expected_output/assert`
       );
       continue;
     }
@@ -4635,7 +4659,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       guideline_paths: guidelinePaths.map((guidelinePath) => import_node_path8.default.resolve(guidelinePath)),
       guideline_patterns: guidelinePatterns,
       file_paths: allFilePaths,
-      criteria: outcome,
+      criteria: outcome ?? "",
       evaluator: evalCaseEvaluatorKind,
       evaluators,
       workspace: mergedWorkspace,
@@ -4745,30 +4769,24 @@ function parseWorkspaceHookConfig(raw, evalFileDir) {
   const script = parseWorkspaceScriptConfig(raw, evalFileDir);
   const obj = raw;
   const reset = obj.reset === "none" || obj.reset === "fast" || obj.reset === "strict" ? obj.reset : void 0;
-  const clean = obj.clean === "always" || obj.clean === "on_success" || obj.clean === "on_failure" || obj.clean === "never" ? obj.clean : void 0;
-  if (!script && !reset && !clean) return void 0;
+  if (!script && !reset) return void 0;
   return {
     ...script ?? {},
-    ...reset !== void 0 && { reset },
-    ...clean !== void 0 && { clean }
+    ...reset !== void 0 && { reset }
   };
 }
 function parseWorkspaceHooksConfig(raw, evalFileDir) {
   if (!isJsonObject(raw)) return void 0;
   const obj = raw;
-  const beforeAllTests = parseWorkspaceHookConfig(obj.before_all_tests, evalFileDir);
-  const beforeEachTest = parseWorkspaceHookConfig(obj.before_each_test, evalFileDir);
-  const afterEachTest = parseWorkspaceHookConfig(obj.after_each_test, evalFileDir);
-  const afterAllTests = parseWorkspaceHookConfig(obj.after_all_tests, evalFileDir);
-  const onReuse = parseWorkspaceHookConfig(obj.on_reuse, evalFileDir);
-  const onFinish = parseWorkspaceHookConfig(obj.on_finish, evalFileDir);
+  const beforeAll = parseWorkspaceHookConfig(obj.before_all, evalFileDir);
+  const beforeEach = parseWorkspaceHookConfig(obj.before_each, evalFileDir);
+  const afterEach = parseWorkspaceHookConfig(obj.after_each, evalFileDir);
+  const afterAll = parseWorkspaceHookConfig(obj.after_all, evalFileDir);
   const hooks = {
-    ...beforeAllTests !== void 0 && { before_all_tests: beforeAllTests },
-    ...beforeEachTest !== void 0 && { before_each_test: beforeEachTest },
-    ...afterEachTest !== void 0 && { after_each_test: afterEachTest },
-    ...afterAllTests !== void 0 && { after_all_tests: afterAllTests },
-    ...onReuse !== void 0 && { on_reuse: onReuse },
-    ...onFinish !== void 0 && { on_finish: onFinish }
+    ...beforeAll !== void 0 && { before_all: beforeAll },
+    ...beforeEach !== void 0 && { before_each: beforeEach },
+    ...afterEach !== void 0 && { after_each: afterEach },
+    ...afterAll !== void 0 && { after_all: afterAll }
   };
   return Object.keys(hooks).length > 0 ? hooks : void 0;
 }
@@ -4781,7 +4799,7 @@ async function resolveWorkspaceConfig(raw, evalFileDir) {
     } catch {
       throw new Error(`Workspace file not found: ${raw} (resolved to ${workspaceFilePath})`);
     }
-    const parsed = (0, import_yaml4.parse)(content);
+    const parsed = interpolateEnv((0, import_yaml4.parse)(content), process.env);
     if (!isJsonObject(parsed)) {
       throw new Error(
         `Invalid workspace file format: ${workspaceFilePath} (expected a YAML object)`
@@ -4829,18 +4847,10 @@ function mergeWorkspaceConfigs(suiteLevel, caseLevel) {
     };
   };
   const mergedHooks = {
-    before_all_tests: mergeHook(
-      suiteLevel.hooks?.before_all_tests,
-      caseLevel.hooks?.before_all_tests
-    ),
-    before_each_test: mergeHook(
-      suiteLevel.hooks?.before_each_test,
-      caseLevel.hooks?.before_each_test
-    ),
-    after_each_test: mergeHook(suiteLevel.hooks?.after_each_test, caseLevel.hooks?.after_each_test),
-    after_all_tests: mergeHook(suiteLevel.hooks?.after_all_tests, caseLevel.hooks?.after_all_tests),
-    on_reuse: mergeHook(suiteLevel.hooks?.on_reuse, caseLevel.hooks?.on_reuse),
-    on_finish: mergeHook(suiteLevel.hooks?.on_finish, caseLevel.hooks?.on_finish)
+    before_all: mergeHook(suiteLevel.hooks?.before_all, caseLevel.hooks?.before_all),
+    before_each: mergeHook(suiteLevel.hooks?.before_each, caseLevel.hooks?.before_each),
+    after_each: mergeHook(suiteLevel.hooks?.after_each, caseLevel.hooks?.after_each),
+    after_all: mergeHook(suiteLevel.hooks?.after_all, caseLevel.hooks?.after_all)
   };
   const hasHooks = Object.values(mergedHooks).some((hook) => hook !== void 0);
   return {
@@ -7086,6 +7096,7 @@ var CopilotCliProvider = class {
     const agentProcess = (0, import_node_child_process2.spawn)(executable, args, {
       stdio: ["pipe", "pipe", "inherit"]
     });
+    await waitForProcessSpawn(agentProcess, executable, this.targetName);
     const toolCallsInProgress = /* @__PURE__ */ new Map();
     const completedToolCalls = [];
     let finalContent = "";
@@ -7365,6 +7376,47 @@ var CopilotCliProvider = class {
     }
   }
 };
+async function waitForProcessSpawn(proc, executable, targetName) {
+  if (proc.pid) {
+    return;
+  }
+  await new Promise((resolve, reject) => {
+    const onSpawn = () => {
+      cleanup();
+      resolve();
+    };
+    const onError = (error) => {
+      cleanup();
+      reject(new Error(formatCopilotSpawnError(error, executable, targetName)));
+    };
+    const cleanup = () => {
+      proc.off("spawn", onSpawn);
+      proc.off("error", onError);
+    };
+    proc.once("spawn", onSpawn);
+    proc.once("error", onError);
+  });
+}
+function formatCopilotSpawnError(error, executable, targetName) {
+  const code = error.code;
+  const base = `Failed to start Copilot CLI executable '${executable}' for target '${targetName}'. ${error.message}`;
+  if (process.platform !== "win32") {
+    return base;
+  }
+  if (code !== "ENOENT" && code !== "EINVAL") {
+    return base;
+  }
+  return `${base}
+
+On Windows, shell commands like 'copilot -h' can work via .ps1/.bat shims, but AgentV launches a subprocess that needs a directly spawnable executable path.
+
+Fix options:
+1) Install native Copilot binary package:
+   npm install -g @github/copilot-win32-x64
+2) Set explicit executable for Copilot targets:
+   - In .env: COPILOT_EXE=C:\\Users\\<you>\\AppData\\Roaming\\npm\\node_modules\\@github\\copilot-win32-x64\\copilot.exe
+  - In .agentv/targets.yaml: executable: \${{ COPILOT_EXE }}`;
+}
 function summarizeAcpEvent(eventType, data) {
   if (!data || typeof data !== "object") {
     return eventType;
@@ -16424,9 +16476,8 @@ async function runEvaluation(options) {
   const hasSharedWorkspace = !!(useStaticWorkspace || workspaceTemplate || suiteWorkspace?.hooks || suiteWorkspace?.repos?.length && !isPerTestIsolation);
   const poolEnabled = configuredMode === "pooled" ? true : configuredMode === "ephemeral" || useStaticWorkspace ? false : suiteWorkspace?.pool ?? poolWorkspaces ?? true;
   const usePool = poolEnabled !== false && !!suiteWorkspace?.repos?.length && !isPerTestIsolation && !useStaticWorkspace;
-  const finishCleanPolicy = suiteWorkspace?.hooks?.on_finish?.clean;
-  const resolvedRetainOnSuccess = (finishCleanPolicy === "always" || finishCleanPolicy === "on_success" ? "cleanup" : finishCleanPolicy === "on_failure" || finishCleanPolicy === "never" ? "keep" : void 0) ?? retainOnSuccess ?? (keepWorkspaces ? "keep" : "cleanup");
-  const resolvedRetainOnFailure = (finishCleanPolicy === "always" || finishCleanPolicy === "on_failure" ? "cleanup" : finishCleanPolicy === "on_success" || finishCleanPolicy === "never" ? "keep" : void 0) ?? retainOnFailure ?? (cleanupWorkspaces ? "cleanup" : "keep");
+  const resolvedRetainOnSuccess = retainOnSuccess ?? (keepWorkspaces ? "keep" : "cleanup");
+  const resolvedRetainOnFailure = retainOnFailure ?? (cleanupWorkspaces ? "cleanup" : "keep");
   const requestedWorkers = options.maxConcurrency ?? target.workers ?? 1;
   const workers = hasSharedWorkspace && !usePool ? 1 : requestedWorkers;
   setupLog(
@@ -16461,7 +16512,7 @@ async function runEvaluation(options) {
         repos: suiteWorkspace.repos,
         maxSlots: poolMaxSlots,
         repoManager: poolRepoManager,
-        poolReset: (workspaceClean === "full" ? "strict" : workspaceClean === "standard" ? "fast" : null) ?? suiteWorkspace.hooks?.on_reuse?.reset ?? "fast"
+        poolReset: (workspaceClean === "full" ? "strict" : workspaceClean === "standard" ? "fast" : null) ?? "fast"
       });
       poolSlots.push(slot);
       setupLog(`pool slot ${i} acquired at: ${slot.path} (existing=${slot.isExisting})`);
@@ -16512,7 +16563,7 @@ async function runEvaluation(options) {
         throw new Error(`Failed to materialize repos: ${message}`);
       }
     }
-    const suiteBeforeAllHook = suiteWorkspace?.hooks?.before_all_tests;
+    const suiteBeforeAllHook = suiteWorkspace?.hooks?.before_all;
     if (sharedWorkspacePath && hasHookCommand(suiteBeforeAllHook)) {
       const beforeAllHook = suiteBeforeAllHook;
       const beforeAllCommand = (beforeAllHook.command ?? beforeAllHook.script ?? []).join(" ");
@@ -16527,7 +16578,7 @@ async function runEvaluation(options) {
       };
       try {
         beforeAllOutput = await executeWorkspaceScript(
-          toScriptConfig(beforeAllHook, "before_all_tests", "suite workspace"),
+          toScriptConfig(beforeAllHook, "before_all", "suite workspace"),
           scriptContext
         );
         setupLog("shared before_all completed");
@@ -16552,7 +16603,7 @@ async function runEvaluation(options) {
         };
         try {
           const output = await executeWorkspaceScript(
-            toScriptConfig(beforeAllHook, "before_all_tests", "suite workspace"),
+            toScriptConfig(beforeAllHook, "before_all", "suite workspace"),
             scriptContext
           );
           if (!beforeAllOutput) beforeAllOutput = output;
@@ -16781,7 +16832,7 @@ async function runEvaluation(options) {
       }
     }
     const afterAllWorkspaces = poolSlots.length > 1 ? poolSlots.map((s) => s.path) : sharedWorkspacePath ? [sharedWorkspacePath] : [];
-    const suiteAfterAllHook = suiteWorkspace?.hooks?.after_all_tests;
+    const suiteAfterAllHook = suiteWorkspace?.hooks?.after_all;
     if (afterAllWorkspaces.length > 0 && hasHookCommand(suiteAfterAllHook)) {
       const afterAllHook = suiteAfterAllHook;
       for (const wsPath of afterAllWorkspaces) {
@@ -16793,7 +16844,7 @@ async function runEvaluation(options) {
         };
         try {
           const afterAllOutput = await executeWorkspaceScript(
-            toScriptConfig(afterAllHook, "after_all_tests", "suite workspace"),
+            toScriptConfig(afterAllHook, "after_all", "suite workspace"),
             scriptContext,
             "warn"
           );
@@ -17090,7 +17141,7 @@ async function runEvalCase(options) {
         );
       }
     }
-    const caseBeforeAllHook = evalCase.workspace?.hooks?.before_all_tests;
+    const caseBeforeAllHook = evalCase.workspace?.hooks?.before_all;
     if (workspacePath && hasHookCommand(caseBeforeAllHook)) {
       const beforeAllHook = caseBeforeAllHook;
       const beforeAllCommand = (beforeAllHook.command ?? beforeAllHook.script ?? []).join(" ");
@@ -17109,7 +17160,7 @@ async function runEvalCase(options) {
       };
       try {
         beforeAllOutput = await executeWorkspaceScript(
-          toScriptConfig(beforeAllHook, "before_all_tests", `test '${evalCase.id}'`),
+          toScriptConfig(beforeAllHook, "before_all", `test '${evalCase.id}'`),
           scriptContext
         );
         if (setupDebug) {
@@ -17134,7 +17185,7 @@ async function runEvalCase(options) {
       }
     }
   }
-  const caseBeforeEachHook = evalCase.workspace?.hooks?.before_each_test;
+  const caseBeforeEachHook = evalCase.workspace?.hooks?.before_each;
   if (workspacePath && hasHookCommand(caseBeforeEachHook)) {
     const beforeEachHook = caseBeforeEachHook;
     const scriptContext = {
@@ -17147,7 +17198,7 @@ async function runEvalCase(options) {
     };
     try {
       beforeEachOutput = await executeWorkspaceScript(
-        toScriptConfig(beforeEachHook, "before_each_test", `test '${evalCase.id}'`),
+        toScriptConfig(beforeEachHook, "before_each", `test '${evalCase.id}'`),
         scriptContext
       );
     } catch (error) {
@@ -17264,17 +17315,17 @@ async function runEvalCase(options) {
     }
   }
   const providerError = extractProviderError(providerResponse);
-  if (repoManager && workspacePath && evalCase.workspace?.hooks?.after_each_test?.reset && evalCase.workspace.hooks.after_each_test.reset !== "none" && evalCase.workspace.repos) {
+  if (repoManager && workspacePath && evalCase.workspace?.hooks?.after_each?.reset && evalCase.workspace.hooks.after_each.reset !== "none" && evalCase.workspace.repos) {
     try {
       await repoManager.reset(
         evalCase.workspace.repos,
         workspacePath,
-        evalCase.workspace.hooks.after_each_test.reset
+        evalCase.workspace.hooks.after_each.reset
       );
     } catch {
     }
   }
-  const caseAfterEachHook = evalCase.workspace?.hooks?.after_each_test;
+  const caseAfterEachHook = evalCase.workspace?.hooks?.after_each;
   if (workspacePath && hasHookCommand(caseAfterEachHook)) {
     const afterEachHook = caseAfterEachHook;
     const scriptContext = {
@@ -17287,7 +17338,7 @@ async function runEvalCase(options) {
     };
     try {
       afterEachOutput = await executeWorkspaceScript(
-        toScriptConfig(afterEachHook, "after_each_test", `test '${evalCase.id}'`),
+        toScriptConfig(afterEachHook, "after_each", `test '${evalCase.id}'`),
         scriptContext,
         "warn"
       );
@@ -17687,9 +17738,11 @@ async function runEvaluatorList(options) {
     registry: typeRegistry
   };
   for (const evaluatorConfig of evaluators ?? []) {
+    const startedAt = /* @__PURE__ */ new Date();
     try {
       const evaluatorInstance = await typeRegistry.create(evaluatorConfig, dispatchContext);
       const score2 = await evaluatorInstance.evaluate(evalContext);
+      const endedAt = /* @__PURE__ */ new Date();
       const weight = evaluatorConfig.weight ?? 1;
       scored.push({
         score: score2,
@@ -17710,9 +17763,13 @@ async function runEvaluatorList(options) {
         evaluatorProviderRequest: score2.evaluatorRawRequest,
         details: score2.details,
         scores: mapChildResults(score2.scores),
-        tokenUsage: score2.tokenUsage
+        tokenUsage: score2.tokenUsage,
+        durationMs: endedAt.getTime() - startedAt.getTime(),
+        startedAt: startedAt.toISOString(),
+        endedAt: endedAt.toISOString()
       });
     } catch (error) {
+      const endedAt = /* @__PURE__ */ new Date();
       const message = error instanceof Error ? error.message : String(error);
       const fallbackScore = {
         score: 0,
@@ -17738,7 +17795,10 @@ async function runEvaluatorList(options) {
         verdict: "fail",
         hits: [],
         misses: [`Evaluator '${evaluatorConfig.name ?? "unknown"}' failed: ${message}`],
-        reasoning: message
+        reasoning: message,
+        durationMs: endedAt.getTime() - startedAt.getTime(),
+        startedAt: startedAt.toISOString(),
+        endedAt: endedAt.toISOString()
       });
     }
     if (evaluatorConfig.negate === true && scored.length > 0) {