@agentv/core 0.7.0 → 0.7.3

package/dist/index.cjs

@@ -382,6 +382,7 @@ async function processMessages(options) {
 }
 async function loadEvalCases(evalFilePath, repoRoot, options) {
   const verbose = options?.verbose ?? false;
+  const evalIdFilter = options?.evalId;
   const absoluteTestPath = import_node_path2.default.resolve(evalFilePath);
   if (!await fileExists2(absoluteTestPath)) {
     throw new Error(`Test file not found: ${evalFilePath}`);
@@ -413,62 +414,39 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
   const results = [];
   for (const rawEvalcase of rawTestcases) {
     if (!isJsonObject(rawEvalcase)) {
-      logWarning("Skipping invalid test case entry (expected object)");
+      logWarning("Skipping invalid eval case entry (expected object)");
       continue;
     }
     const evalcase = rawEvalcase;
     const id = asString(evalcase.id);
+    if (evalIdFilter && id !== evalIdFilter) {
+      continue;
+    }
     const conversationId = asString(evalcase.conversation_id);
     const outcome = asString(evalcase.outcome);
     const inputMessagesValue = evalcase.input_messages;
     const expectedMessagesValue = evalcase.expected_messages;
     if (!id || !outcome || !Array.isArray(inputMessagesValue)) {
-      logWarning(`Skipping incomplete test case: ${id ?? "unknown"}`);
+      logWarning(`Skipping incomplete eval case: ${id ?? "unknown"}`);
       continue;
     }
     if (!Array.isArray(expectedMessagesValue)) {
-      logWarning(`Test case '${id}' missing expected_messages array`);
+      logWarning(`Eval case '${id}' missing expected_messages array`);
       continue;
     }
     const inputMessages = inputMessagesValue.filter((msg) => isTestMessage(msg));
     const expectedMessages = expectedMessagesValue.filter((msg) => isTestMessage(msg));
-    const assistantMessages = expectedMessages.filter((message) => message.role === "assistant");
-    const userMessages = inputMessages.filter((message) => message.role === "user");
-    const systemMessages = inputMessages.filter((message) => message.role === "system");
-    if (assistantMessages.length === 0) {
-      logWarning(`No assistant message found for test case: ${id}`);
+    if (expectedMessages.length === 0) {
+      logWarning(`No expected message found for eval case: ${id}`);
       continue;
     }
-    if (assistantMessages.length > 1) {
-      logWarning(`Multiple assistant messages found for test case: ${id}, using first`);
-    }
-    if (systemMessages.length > 1) {
-      logWarning(`Multiple system messages found for test case: ${id}, using first`);
-    }
-    let systemMessageContent;
-    if (systemMessages.length > 0) {
-      const content = systemMessages[0]?.content;
-      if (typeof content === "string") {
-        systemMessageContent = content;
-      } else if (Array.isArray(content)) {
-        const textParts = [];
-        for (const segment of content) {
-          if (isJsonObject(segment)) {
-            const value = segment.value;
-            if (typeof value === "string") {
-              textParts.push(value);
-            }
-          }
-        }
-        if (textParts.length > 0) {
-          systemMessageContent = textParts.join("\n\n");
-        }
-      }
+    if (expectedMessages.length > 1) {
+      logWarning(`Multiple expected messages found for eval case: ${id}, using first`);
     }
     const guidelinePaths = [];
     const inputTextParts = [];
     const inputSegments = await processMessages({
-      messages: userMessages,
+      messages: inputMessages,
       searchRoots,
       repoRootPath,
       guidelinePatterns,
@@ -478,7 +456,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
       verbose
     });
     const outputSegments = await processMessages({
-      messages: assistantMessages,
+      messages: expectedMessages,
       searchRoots,
       repoRootPath,
       guidelinePatterns,
@@ -486,10 +464,10 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
       verbose
     });
     const codeSnippets = extractCodeBlocks(inputSegments);
-    const assistantContent = assistantMessages[0]?.content;
-    const referenceAnswer = await resolveAssistantContent(assistantContent, searchRoots, verbose);
+    const expectedContent = expectedMessages[0]?.content;
+    const referenceAnswer = await resolveAssistantContent(expectedContent, searchRoots, verbose);
     const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
-    const testCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
+    const evalCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
     const evaluators = await parseEvaluators(evalcase, searchRoots, id ?? "unknown");
     const userFilePaths = [];
     for (const segment of inputSegments) {
@@ -508,19 +486,18 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
       question,
       input_segments: inputSegments,
       output_segments: outputSegments,
-      system_message: systemMessageContent,
       reference_answer: referenceAnswer,
       guideline_paths: guidelinePaths.map((guidelinePath) => import_node_path2.default.resolve(guidelinePath)),
       guideline_patterns: guidelinePatterns,
       file_paths: allFilePaths,
       code_snippets: codeSnippets,
       expected_outcome: outcome,
-      evaluator: testCaseEvaluatorKind,
+      evaluator: evalCaseEvaluatorKind,
       evaluators
     };
     if (verbose) {
       console.log(`
-[Test Case: ${id}]`);
+[Eval Case: ${id}]`);
       if (testCase.guideline_paths.length > 0) {
         console.log(`  Guidelines used: ${testCase.guideline_paths.length}`);
         for (const guidelinePath of testCase.guideline_paths) {
@@ -579,7 +556,7 @@ ${body}`);
   }
   const question = questionParts.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
   const guidelines = guidelineContents.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
-  return { question, guidelines, systemMessage: testCase.system_message };
+  return { question, guidelines };
 }
 async function fileExists2(absolutePath) {
   try {
@@ -1338,7 +1315,6 @@ function pathToFileUri(filePath) {
 var execAsync2 = (0, import_node_util2.promisify)(import_node_child_process2.exec);
 var WORKSPACE_PREFIX = "agentv-codex-";
 var PROMPT_FILENAME = "prompt.md";
-var FILES_DIR = "files";
 var JSONL_TYPE_ITEM_COMPLETED = "item.completed";
 var CodexProvider = class {
   id;
@@ -1361,21 +1337,10 @@ var CodexProvider = class {
     }
     await this.ensureEnvironmentReady();
     const inputFiles = normalizeInputFiles2(request.inputFiles);
-    const originalGuidelines = new Set(
-      collectGuidelineFiles(inputFiles, request.guideline_patterns).map((file) => import_node_path5.default.resolve(file))
-    );
     const workspaceRoot = await this.createWorkspace();
     const logger = await this.createStreamLogger(request).catch(() => void 0);
     try {
-      const { mirroredInputFiles, guidelineMirrors } = await this.mirrorInputFiles(
-        inputFiles,
-        workspaceRoot,
-        originalGuidelines
-      );
-      const promptContent = buildPromptDocument(request, mirroredInputFiles, {
-        guidelinePatterns: request.guideline_patterns,
-        guidelineOverrides: guidelineMirrors
-      });
+      const promptContent = buildPromptDocument(request, inputFiles);
       const promptFile = import_node_path5.default.join(workspaceRoot, PROMPT_FILENAME);
       await (0, import_promises3.writeFile)(promptFile, promptContent, "utf8");
       const args = this.buildCodexArgs();
@@ -1404,7 +1369,7 @@ var CodexProvider = class {
           executable: this.resolvedExecutable ?? this.config.executable,
           promptFile,
           workspace: workspaceRoot,
-          inputFiles: mirroredInputFiles,
+          inputFiles,
           logFile: logger?.filePath
         }
       };
@@ -1459,37 +1424,6 @@ var CodexProvider = class {
       throw error;
     }
   }
-  async mirrorInputFiles(inputFiles, workspaceRoot, guidelineOriginals) {
-    if (!inputFiles || inputFiles.length === 0) {
-      return {
-        mirroredInputFiles: void 0,
-        guidelineMirrors: /* @__PURE__ */ new Set()
-      };
-    }
-    const filesRoot = import_node_path5.default.join(workspaceRoot, FILES_DIR);
-    await (0, import_promises3.mkdir)(filesRoot, { recursive: true });
-    const mirrored = [];
-    const guidelineMirrors = /* @__PURE__ */ new Set();
-    const nameCounts = /* @__PURE__ */ new Map();
-    for (const inputFile of inputFiles) {
-      const absoluteSource = import_node_path5.default.resolve(inputFile);
-      const baseName = import_node_path5.default.basename(absoluteSource);
-      const count = nameCounts.get(baseName) ?? 0;
-      nameCounts.set(baseName, count + 1);
-      const finalName = count === 0 ? baseName : `${baseName}.${count}`;
-      const destination = import_node_path5.default.join(filesRoot, finalName);
-      await (0, import_promises3.copyFile)(absoluteSource, destination);
-      const resolvedDestination = import_node_path5.default.resolve(destination);
-      mirrored.push(resolvedDestination);
-      if (guidelineOriginals.has(absoluteSource)) {
-        guidelineMirrors.add(resolvedDestination);
-      }
-    }
-    return {
-      mirroredInputFiles: mirrored,
-      guidelineMirrors
-    };
-  }
   async createWorkspace() {
     return await (0, import_promises3.mkdtemp)(import_node_path5.default.join((0, import_node_os.tmpdir)(), WORKSPACE_PREFIX));
   }
@@ -2460,23 +2394,25 @@ function resolveOptionalString(source, env, description, options) {
   if (trimmed.length === 0) {
     return void 0;
   }
-  const envValue = env[trimmed];
-  if (envValue !== void 0) {
-    if (envValue.trim().length === 0) {
-      throw new Error(`Environment variable '${trimmed}' for ${description} is empty`);
+  const envVarMatch = trimmed.match(/^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i);
+  if (envVarMatch) {
+    const varName = envVarMatch[1];
+    const envValue = env[varName];
+    if (envValue !== void 0) {
+      if (envValue.trim().length === 0) {
+        throw new Error(`Environment variable '${varName}' for ${description} is empty`);
+      }
+      return envValue;
     }
-    return envValue;
-  }
-  const allowLiteral = options?.allowLiteral ?? false;
-  const optionalEnv = options?.optionalEnv ?? false;
-  const looksLikeEnv = isLikelyEnvReference(trimmed);
-  if (looksLikeEnv) {
+    const optionalEnv = options?.optionalEnv ?? false;
     if (optionalEnv) {
       return void 0;
     }
-    if (!allowLiteral) {
-      throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
-    }
+    throw new Error(`Environment variable '${varName}' required for ${description} is not set`);
+  }
+  const allowLiteral = options?.allowLiteral ?? false;
+  if (!allowLiteral) {
+    throw new Error(`${description} must use \${{ VARIABLE_NAME }} syntax for environment variables or be marked as allowing literals`);
   }
   return trimmed;
 }
@@ -2523,9 +2459,6 @@ function resolveOptionalBoolean(source) {
   }
   throw new Error("expected boolean value");
 }
-function isLikelyEnvReference(value) {
-  return /^[A-Z0-9_]+$/.test(value);
-}
 function resolveOptionalStringArray(source, env, description) {
   if (source === void 0 || source === null) {
     return void 0;
@@ -2546,21 +2479,25 @@ function resolveOptionalStringArray(source, env, description) {
     if (trimmed.length === 0) {
       throw new Error(`${description}[${i}] cannot be empty`);
     }
-    const envValue = env[trimmed];
-    if (envValue !== void 0) {
-      if (envValue.trim().length === 0) {
-        throw new Error(`Environment variable '${trimmed}' for ${description}[${i}] is empty`);
+    const envVarMatch = trimmed.match(/^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i);
+    if (envVarMatch) {
+      const varName = envVarMatch[1];
+      const envValue = env[varName];
+      if (envValue !== void 0) {
+        if (envValue.trim().length === 0) {
+          throw new Error(`Environment variable '${varName}' for ${description}[${i}] is empty`);
+        }
+        resolved.push(envValue);
+        continue;
       }
-      resolved.push(envValue);
-    } else {
-      resolved.push(trimmed);
+      throw new Error(`Environment variable '${varName}' for ${description}[${i}] is not set`);
     }
+    resolved.push(trimmed);
   }
   return resolved.length > 0 ? resolved : void 0;
 }
 
 // src/evaluation/providers/vscode.ts
-var import_promises4 = require("fs/promises");
 var import_node_path6 = __toESM(require("path"), 1);
 var import_subagent = require("subagent");
 var VSCodeProvider = class {
@@ -2604,7 +2541,7 @@ var VSCodeProvider = class {
         }
       };
     }
-    const responseText = await (0, import_promises4.readFile)(session.responseFile, "utf8");
+    const responseText = await readTextFile(session.responseFile);
     return {
       text: responseText,
       raw: {
@@ -2658,7 +2595,7 @@ var VSCodeProvider = class {
     }
     const responses = [];
     for (const [index, responseFile] of session.responseFiles.entries()) {
-      const responseText = await (0, import_promises4.readFile)(responseFile, "utf8");
+      const responseText = await readTextFile(responseFile);
       responses.push({
         text: responseText,
         raw: {
@@ -2808,12 +2745,20 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
 
 // src/evaluation/providers/targets-file.ts
 var import_node_fs4 = require("fs");
-var import_promises5 = require("fs/promises");
+var import_promises4 = require("fs/promises");
 var import_node_path7 = __toESM(require("path"), 1);
 var import_yaml2 = require("yaml");
 
 // src/evaluation/providers/types.ts
-var TARGETS_SCHEMA_V2 = "agentv-targets-v2";
+var AGENT_PROVIDER_KINDS = [
+  "codex",
+  "vscode",
+  "vscode-insiders"
+];
+var TARGETS_SCHEMA_V2 = "agentv-targets-v2.1";
+function isAgentProvider(provider) {
+  return provider ? AGENT_PROVIDER_KINDS.includes(provider.kind) : false;
+}
 
 // src/evaluation/providers/targets-file.ts
 function isRecord(value) {
@@ -2870,7 +2815,7 @@ function assertTargetDefinition(value, index, filePath) {
 }
 async function fileExists3(filePath) {
   try {
-    await (0, import_promises5.access)(filePath, import_node_fs4.constants.F_OK);
+    await (0, import_promises4.access)(filePath, import_node_fs4.constants.F_OK);
     return true;
   } catch {
     return false;
@@ -2881,7 +2826,7 @@ async function readTargetDefinitions(filePath) {
   if (!await fileExists3(absolutePath)) {
     throw new Error(`targets.yaml not found at ${absolutePath}`);
   }
-  const raw = await (0, import_promises5.readFile)(absolutePath, "utf8");
+  const raw = await (0, import_promises4.readFile)(absolutePath, "utf8");
   const parsed = (0, import_yaml2.parse)(raw);
   if (!isRecord(parsed)) {
     throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with '$schema' and 'targets' fields`);
@@ -3127,7 +3072,6 @@ var CodeEvaluator = class {
         expected_outcome: context.evalCase.expected_outcome,
         reference_answer: context.evalCase.reference_answer,
         candidate_answer: context.candidate,
-        system_message: context.promptInputs.systemMessage ?? "",
         guideline_paths: context.evalCase.guideline_paths,
         input_files: context.evalCase.file_paths,
         input_segments: context.evalCase.input_segments
@@ -3227,7 +3171,7 @@ function substituteVariables(template, variables) {
 
 // src/evaluation/orchestrator.ts
 var import_node_crypto3 = require("crypto");
-var import_promises6 = require("fs/promises");
+var import_promises5 = require("fs/promises");
 var import_node_path8 = __toESM(require("path"), 1);
 
 // ../../node_modules/.pnpm/yocto-queue@1.2.1/node_modules/yocto-queue/index.js
@@ -3369,7 +3313,7 @@ function validateConcurrency(concurrency) {
 // src/evaluation/orchestrator.ts
 async function runEvaluation(options) {
   const {
-    testFilePath,
+    testFilePath: evalFilePath,
     repoRoot,
     target,
     targets,
@@ -3388,11 +3332,11 @@ async function runEvaluation(options) {
     onProgress
   } = options;
   const load = loadEvalCases;
-  const evalCases = await load(testFilePath, repoRoot, { verbose });
+  const evalCases = await load(evalFilePath, repoRoot, { verbose, evalId });
   const filteredEvalCases = filterEvalCases(evalCases, evalId);
   if (filteredEvalCases.length === 0) {
     if (evalId) {
-      throw new Error(`Test case with id '${evalId}' not found in ${testFilePath}`);
+      throw new Error(`Eval case with id '${evalId}' not found in ${evalFilePath}`);
     }
     return [];
   }
@@ -3546,7 +3490,8 @@ async function runEvaluation(options) {
         target.name,
         (now ?? (() => /* @__PURE__ */ new Date()))(),
         outcome.reason,
-        promptInputs
+        promptInputs,
+        primaryProvider
       );
       results.push(errorResult);
       if (onResult) {
@@ -3630,7 +3575,7 @@ async function runBatchEvaluation(options) {
         agentTimeoutMs
       });
     } catch (error) {
-      const errorResult = buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
+      const errorResult = buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs, provider);
       results.push(errorResult);
       if (onResult) {
         await onResult(errorResult);
@@ -3707,7 +3652,7 @@ async function runEvalCase(options) {
         attempt += 1;
         continue;
       }
-      return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
+      return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs, provider);
     }
   }
   if (!providerResponse) {
@@ -3716,7 +3661,8 @@ async function runEvalCase(options) {
       target.name,
       nowFn(),
       lastError ?? new Error("Provider did not return a response"),
-      promptInputs
+      promptInputs,
+      provider
     );
   }
   if (cacheKey && cache && !cachedResponse) {
@@ -3736,7 +3682,7 @@ async function runEvalCase(options) {
       agentTimeoutMs
     });
   } catch (error) {
-    return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
+    return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs, provider);
   }
 }
 async function evaluateCandidate(options) {
@@ -3768,9 +3714,8 @@ async function evaluateCandidate(options) {
   const completedAt = nowFn();
   const rawRequest = {
     question: promptInputs.question,
-    guidelines: promptInputs.guidelines,
-    guideline_paths: evalCase.guideline_paths,
-    system_message: promptInputs.systemMessage ?? ""
+    ...isAgentProvider(provider) ? {} : { guidelines: promptInputs.guidelines },
+    guideline_paths: evalCase.guideline_paths
   };
   return {
     eval_id: evalCase.id,
@@ -3986,14 +3931,14 @@ async function dumpPrompt(directory, evalCase, promptInputs) {
   const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
   const filename = `${timestamp}_${sanitizeFilename(evalCase.id)}.json`;
   const filePath = import_node_path8.default.resolve(directory, filename);
-  await (0, import_promises6.mkdir)(import_node_path8.default.dirname(filePath), { recursive: true });
+  await (0, import_promises5.mkdir)(import_node_path8.default.dirname(filePath), { recursive: true });
   const payload = {
     eval_id: evalCase.id,
     question: promptInputs.question,
     guidelines: promptInputs.guidelines,
     guideline_paths: evalCase.guideline_paths
   };
-  await (0, import_promises6.writeFile)(filePath, JSON.stringify(payload, null, 2), "utf8");
+  await (0, import_promises5.writeFile)(filePath, JSON.stringify(payload, null, 2), "utf8");
 }
 function sanitizeFilename(value) {
   if (!value) {
@@ -4028,13 +3973,12 @@ async function invokeProvider(provider, options) {
     }
   }
 }
-function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs) {
+function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs, provider) {
   const message = error instanceof Error ? error.message : String(error);
   const rawRequest = {
     question: promptInputs.question,
-    guidelines: promptInputs.guidelines,
+    ...isAgentProvider(provider) ? {} : { guidelines: promptInputs.guidelines },
     guideline_paths: evalCase.guideline_paths,
-    system_message: promptInputs.systemMessage ?? "",
     error: message
   };
   return {