@agentv/core 2.6.0 → 2.7.1-next.1

package/dist/{chunk-SSPAANFZ.js → chunk-6W5E3VR6.js}

@@ -1,3 +1,76 @@
+// src/evaluation/types.ts
+var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
+var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
+var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
+function isTestMessageRole(value) {
+  return typeof value === "string" && TEST_MESSAGE_ROLE_SET.has(value);
+}
+function isJsonObject(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return false;
+  }
+  return Object.values(value).every(isJsonValue);
+}
+function isJsonValue(value) {
+  if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+    return true;
+  }
+  if (Array.isArray(value)) {
+    return value.every(isJsonValue);
+  }
+  if (typeof value === "object") {
+    return isJsonObject(value);
+  }
+  return false;
+}
+function isTestMessage(value) {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+  const candidate = value;
+  if (!isTestMessageRole(candidate.role)) {
+    return false;
+  }
+  if (typeof candidate.content === "string") {
+    return true;
+  }
+  if (Array.isArray(candidate.content) && candidate.content.every(isJsonObject)) {
+    return true;
+  }
+  if (Array.isArray(candidate.tool_calls) && candidate.tool_calls.length > 0) {
+    return true;
+  }
+  if (isJsonObject(candidate.content)) {
+    return true;
+  }
+  return false;
+}
+var EVALUATOR_KIND_VALUES = [
+  "code_judge",
+  "llm_judge",
+  "rubric",
+  "composite",
+  "tool_trajectory",
+  "field_accuracy",
+  "latency",
+  "cost",
+  "token_usage",
+  "execution_metrics",
+  "agent_judge",
+  "contains",
+  "regex",
+  "is_json",
+  "equals",
+  "rubrics"
+];
+var EVALUATOR_KIND_SET = new Set(EVALUATOR_KIND_VALUES);
+function isEvaluatorKind(value) {
+  return typeof value === "string" && EVALUATOR_KIND_SET.has(value);
+}
+function getHitCount(result) {
+  return result.hits.length;
+}
+
 // src/evaluation/file-utils.ts
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
@@ -147,6 +220,9 @@ var CliTargetInputSchema = z.object({
   attachmentsFormat: z.string().optional(),
   // Working directory - optional
   cwd: z.string().optional(),
+  // Workspace template directory - optional (mutually exclusive with cwd)
+  workspace_template: z.string().optional(),
+  workspaceTemplate: z.string().optional(),
   // Timeout in seconds - optional
   timeout_seconds: z.number().positive().optional(),
   timeoutSeconds: z.number().positive().optional(),
@@ -188,6 +264,7 @@ var CliTargetConfigSchema = z.object({
   commandTemplate: z.string().min(1),
   filesFormat: z.string().optional(),
   cwd: z.string().optional(),
+  workspaceTemplate: z.string().optional(),
   timeoutMs: z.number().positive().optional(),
   healthcheck: CliHealthcheckSchema.optional(),
   verbose: z.boolean().optional(),
@@ -247,6 +324,19 @@ function normalizeCliTargetInput(input, env, evalFilePath) {
   );
   const filesFormatSource = input.files_format ?? input.filesFormat ?? input.attachments_format ?? input.attachmentsFormat;
   const filesFormat = resolveOptionalLiteralString(filesFormatSource);
+  const workspaceTemplateSource = input.workspace_template ?? input.workspaceTemplate;
+  let workspaceTemplate = resolveOptionalString(
+    workspaceTemplateSource,
+    env,
+    `${targetName} workspace template`,
+    {
+      allowLiteral: true,
+      optionalEnv: true
+    }
+  );
+  if (workspaceTemplate && evalFilePath && !path2.isAbsolute(workspaceTemplate)) {
+    workspaceTemplate = path2.resolve(path2.dirname(path2.resolve(evalFilePath)), workspaceTemplate);
+  }
   let cwd = resolveOptionalString(input.cwd, env, `${targetName} working directory`, {
     allowLiteral: true,
     optionalEnv: true
@@ -254,7 +344,12 @@ function normalizeCliTargetInput(input, env, evalFilePath) {
   if (cwd && evalFilePath && !path2.isAbsolute(cwd)) {
     cwd = path2.resolve(path2.dirname(path2.resolve(evalFilePath)), cwd);
   }
-  if (!cwd && evalFilePath) {
+  if (cwd && workspaceTemplate) {
+    throw new Error(
+      `${targetName}: 'cwd' and 'workspace_template' are mutually exclusive. Use 'cwd' to run in an existing directory, or 'workspace_template' to copy a template to a temp location.`
+    );
+  }
+  if (!cwd && !workspaceTemplate && evalFilePath) {
     cwd = path2.dirname(path2.resolve(evalFilePath));
   }
   const timeoutSeconds = input.timeout_seconds ?? input.timeoutSeconds;
@@ -268,6 +363,7 @@ function normalizeCliTargetInput(input, env, evalFilePath) {
     commandTemplate,
     filesFormat,
     cwd,
+    workspaceTemplate,
     timeoutMs,
     healthcheck,
     verbose,
@@ -286,7 +382,9 @@ var BASE_TARGET_SCHEMA = z.object({
   name: z.string().min(1, "target name is required"),
   provider: z.string().min(1, "provider is required"),
   judge_target: z.string().optional(),
-  workers: z.number().int().min(1).optional()
+  workers: z.number().int().min(1).optional(),
+  workspace_template: z.string().optional(),
+  workspaceTemplate: z.string().optional()
 }).passthrough();
 var DEFAULT_AZURE_API_VERSION = "2024-12-01-preview";
 function normalizeAzureApiVersion(value) {
@@ -377,7 +475,18 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
         providerBatching,
-        config: resolveCodexConfig(parsed, env)
+        config: resolveCodexConfig(parsed, env, evalFilePath)
+      };
+    case "copilot":
+    case "copilot-sdk":
+    case "copilot_sdk":
+      return {
+        kind: "copilot",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveCopilotSdkConfig(parsed, env, evalFilePath)
       };
     case "copilot-cli":
       return {
@@ -386,7 +495,7 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
         providerBatching,
-        config: resolveCopilotConfig(parsed, env)
+        config: resolveCopilotCliConfig(parsed, env, evalFilePath)
       };
     case "pi":
     case "pi-coding-agent":
@@ -396,7 +505,7 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
         providerBatching,
-        config: resolvePiCodingAgentConfig(parsed, env)
+        config: resolvePiCodingAgentConfig(parsed, env, evalFilePath)
       };
     case "pi-agent-sdk":
       return {
@@ -407,14 +516,16 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
         providerBatching,
         config: resolvePiAgentSdkConfig(parsed, env)
       };
+    case "claude":
     case "claude-code":
+    case "claude-sdk":
       return {
-        kind: "claude-code",
+        kind: "claude",
         name: parsed.name,
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
         providerBatching,
-        config: resolveClaudeCodeConfig(parsed, env)
+        config: resolveClaudeConfig(parsed, env, evalFilePath)
       };
     case "mock":
       return {
@@ -433,7 +544,7 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
         providerBatching,
-        config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
+        config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders", evalFilePath)
       };
     case "cli":
       return {
@@ -445,7 +556,14 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
         config: resolveCliConfig(parsed, env, evalFilePath)
       };
     default:
-      throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
+      return {
+        kind: "cli",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveDiscoveredProviderConfig(parsed, provider, env, evalFilePath)
+      };
   }
 }
 function resolveAzureConfig(target, env) {
@@ -517,14 +635,20 @@ function resolveGeminiConfig(target, env) {
     retry
   };
 }
-function resolveCodexConfig(target, env) {
+function resolveCodexConfig(target, env, evalFilePath) {
+  const modelSource = target.model;
   const executableSource = target.executable ?? target.command ?? target.binary;
   const argsSource = target.args ?? target.arguments;
   const cwdSource = target.cwd;
+  const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate;
   const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
   const logDirSource = target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory;
   const logFormatSource = target.log_format ?? target.logFormat ?? target.log_output_format ?? target.logOutputFormat ?? env.AGENTV_CODEX_LOG_FORMAT;
   const systemPromptSource = target.system_prompt ?? target.systemPrompt;
+  const model = resolveOptionalString(modelSource, env, `${target.name} codex model`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
   const executable = resolveOptionalString(executableSource, env, `${target.name} codex executable`, {
     allowLiteral: true,
     optionalEnv: true
@@ -534,6 +658,23 @@ function resolveCodexConfig(target, env) {
     allowLiteral: true,
     optionalEnv: true
   });
+  let workspaceTemplate = resolveOptionalString(
+    workspaceTemplateSource,
+    env,
+    `${target.name} codex workspace template`,
+    {
+      allowLiteral: true,
+      optionalEnv: true
+    }
+  );
+  if (workspaceTemplate && evalFilePath && !path2.isAbsolute(workspaceTemplate)) {
+    workspaceTemplate = path2.resolve(path2.dirname(path2.resolve(evalFilePath)), workspaceTemplate);
+  }
+  if (cwd && workspaceTemplate) {
+    throw new Error(
+      `${target.name}: 'cwd' and 'workspace_template' are mutually exclusive. Use 'cwd' to run in an existing directory, or 'workspace_template' to copy a template to a temp location.`
+    );
+  }
   const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} codex timeout`);
   const logDir = resolveOptionalString(logDirSource, env, `${target.name} codex log directory`, {
     allowLiteral: true,
@@ -542,9 +683,11 @@ function resolveCodexConfig(target, env) {
   const logFormat = normalizeCodexLogFormat(logFormatSource);
   const systemPrompt = typeof systemPromptSource === "string" && systemPromptSource.trim().length > 0 ? systemPromptSource.trim() : void 0;
   return {
+    model,
     executable,
     args,
     cwd,
+    workspaceTemplate,
     timeoutMs,
     logDir,
     logFormat,
@@ -564,36 +707,147 @@ function normalizeCodexLogFormat(value) {
   }
   throw new Error("codex log format must be 'summary' or 'json'");
 }
-function resolveCopilotConfig(target, env) {
-  const executableSource = target.executable ?? target.command ?? target.binary;
+function resolveCopilotSdkConfig(target, env, evalFilePath) {
+  const cliUrlSource = target.cli_url ?? target.cliUrl;
+  const cliPathSource = target.cli_path ?? target.cliPath;
+  const githubTokenSource = target.github_token ?? target.githubToken;
   const modelSource = target.model;
-  const argsSource = target.args ?? target.arguments;
   const cwdSource = target.cwd;
+  const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate;
   const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
   const logDirSource = target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory;
-  const logFormatSource = target.log_format ?? target.logFormat ?? target.log_output_format ?? target.logOutputFormat;
+  const logFormatSource = target.log_format ?? target.logFormat;
   const systemPromptSource = target.system_prompt ?? target.systemPrompt;
-  const executable = resolveOptionalString(executableSource, env, `${target.name} copilot executable`, {
+  const cliUrl = resolveOptionalString(cliUrlSource, env, `${target.name} copilot-sdk cli URL`, {
     allowLiteral: true,
     optionalEnv: true
-  }) ?? "copilot";
-  const model = resolveOptionalString(modelSource, env, `${target.name} copilot model`, {
+  });
+  const cliPath = resolveOptionalString(cliPathSource, env, `${target.name} copilot-sdk cli path`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const githubToken = resolveOptionalString(
+    githubTokenSource,
+    env,
+    `${target.name} copilot-sdk github token`,
+    {
+      allowLiteral: false,
+      optionalEnv: true
+    }
+  );
+  const model = resolveOptionalString(modelSource, env, `${target.name} copilot-sdk model`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const cwd = resolveOptionalString(cwdSource, env, `${target.name} copilot-sdk cwd`, {
     allowLiteral: true,
     optionalEnv: true
   });
-  const args = resolveOptionalStringArray(argsSource, env, `${target.name} copilot args`);
-  const cwd = resolveOptionalString(cwdSource, env, `${target.name} copilot cwd`, {
+  let workspaceTemplate = resolveOptionalString(
+    workspaceTemplateSource,
+    env,
+    `${target.name} copilot-sdk workspace template`,
+    {
+      allowLiteral: true,
+      optionalEnv: true
+    }
+  );
+  if (workspaceTemplate && evalFilePath && !path2.isAbsolute(workspaceTemplate)) {
+    workspaceTemplate = path2.resolve(path2.dirname(path2.resolve(evalFilePath)), workspaceTemplate);
+  }
+  if (cwd && workspaceTemplate) {
+    throw new Error(
+      `${target.name}: 'cwd' and 'workspace_template' are mutually exclusive. Use 'cwd' to run in an existing directory, or 'workspace_template' to copy a template to a temp location.`
+    );
+  }
+  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} copilot-sdk timeout`);
+  const logDir = resolveOptionalString(
+    logDirSource,
+    env,
+    `${target.name} copilot-sdk log directory`,
+    {
+      allowLiteral: true,
+      optionalEnv: true
+    }
+  );
+  const logFormat = normalizeCopilotLogFormat(logFormatSource);
+  const systemPrompt = typeof systemPromptSource === "string" && systemPromptSource.trim().length > 0 ? systemPromptSource.trim() : void 0;
+  return {
+    cliUrl,
+    cliPath,
+    githubToken,
+    model,
+    cwd,
+    workspaceTemplate,
+    timeoutMs,
+    logDir,
+    logFormat,
+    systemPrompt
+  };
+}
+function resolveCopilotCliConfig(target, env, evalFilePath) {
+  const executableSource = target.executable ?? target.command ?? target.binary;
+  const modelSource = target.model;
+  const argsSource = target.args ?? target.arguments;
+  const cwdSource = target.cwd;
+  const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate;
+  const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
+  const logDirSource = target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory;
+  const logFormatSource = target.log_format ?? target.logFormat;
+  const systemPromptSource = target.system_prompt ?? target.systemPrompt;
+  const executable = resolveOptionalString(executableSource, env, `${target.name} copilot-cli executable`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? "copilot";
+  const model = resolveOptionalString(modelSource, env, `${target.name} copilot-cli model`, {
     allowLiteral: true,
     optionalEnv: true
   });
-  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} copilot timeout`);
-  const logDir = resolveOptionalString(logDirSource, env, `${target.name} copilot log directory`, {
+  const args = resolveOptionalStringArray(argsSource, env, `${target.name} copilot-cli args`);
+  const cwd = resolveOptionalString(cwdSource, env, `${target.name} copilot-cli cwd`, {
     allowLiteral: true,
     optionalEnv: true
   });
+  let workspaceTemplate = resolveOptionalString(
+    workspaceTemplateSource,
+    env,
+    `${target.name} copilot-cli workspace template`,
+    {
+      allowLiteral: true,
+      optionalEnv: true
+    }
+  );
+  if (workspaceTemplate && evalFilePath && !path2.isAbsolute(workspaceTemplate)) {
+    workspaceTemplate = path2.resolve(path2.dirname(path2.resolve(evalFilePath)), workspaceTemplate);
+  }
+  if (cwd && workspaceTemplate) {
+    throw new Error(
+      `${target.name}: 'cwd' and 'workspace_template' are mutually exclusive. Use 'cwd' to run in an existing directory, or 'workspace_template' to copy a template to a temp location.`
+    );
+  }
+  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} copilot-cli timeout`);
+  const logDir = resolveOptionalString(
+    logDirSource,
+    env,
+    `${target.name} copilot-cli log directory`,
+    {
+      allowLiteral: true,
+      optionalEnv: true
+    }
+  );
   const logFormat = normalizeCopilotLogFormat(logFormatSource);
   const systemPrompt = typeof systemPromptSource === "string" && systemPromptSource.trim().length > 0 ? systemPromptSource.trim() : void 0;
-  return { executable, model, args, cwd, timeoutMs, logDir, logFormat, systemPrompt };
+  return {
+    executable,
+    model,
+    args,
+    cwd,
+    workspaceTemplate,
+    timeoutMs,
+    logDir,
+    logFormat,
+    systemPrompt
+  };
 }
 function normalizeCopilotLogFormat(value) {
   if (value === void 0 || value === null) return void 0;
@@ -602,7 +856,7 @@ function normalizeCopilotLogFormat(value) {
   if (normalized === "json" || normalized === "summary") return normalized;
   throw new Error("copilot log format must be 'summary' or 'json'");
 }
-function resolvePiCodingAgentConfig(target, env) {
+function resolvePiCodingAgentConfig(target, env, evalFilePath) {
   const executableSource = target.executable ?? target.command ?? target.binary;
   const providerSource = target.pi_provider ?? target.piProvider ?? target.llm_provider;
   const modelSource = target.model ?? target.pi_model ?? target.piModel;
@@ -611,6 +865,7 @@ function resolvePiCodingAgentConfig(target, env) {
   const thinkingSource = target.thinking ?? target.pi_thinking ?? target.piThinking;
   const argsSource = target.args ?? target.arguments;
   const cwdSource = target.cwd;
+  const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate;
   const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
   const logDirSource = target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory;
   const logFormatSource = target.log_format ?? target.logFormat;
@@ -644,6 +899,23 @@ function resolvePiCodingAgentConfig(target, env) {
     allowLiteral: true,
     optionalEnv: true
   });
+  let workspaceTemplate = resolveOptionalString(
+    workspaceTemplateSource,
+    env,
+    `${target.name} pi workspace template`,
+    {
+      allowLiteral: true,
+      optionalEnv: true
+    }
+  );
+  if (workspaceTemplate && evalFilePath && !path2.isAbsolute(workspaceTemplate)) {
+    workspaceTemplate = path2.resolve(path2.dirname(path2.resolve(evalFilePath)), workspaceTemplate);
+  }
+  if (cwd && workspaceTemplate) {
+    throw new Error(
+      `${target.name}: 'cwd' and 'workspace_template' are mutually exclusive. Use 'cwd' to run in an existing directory, or 'workspace_template' to copy a template to a temp location.`
+    );
+  }
   const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} pi timeout`);
   const logDir = resolveOptionalString(logDirSource, env, `${target.name} pi log directory`, {
     allowLiteral: true,
@@ -660,6 +932,7 @@ function resolvePiCodingAgentConfig(target, env) {
     thinking,
     args,
     cwd,
+    workspaceTemplate,
     timeoutMs,
     logDir,
     logFormat,
@@ -699,96 +972,114 @@ function resolvePiAgentSdkConfig(target, env) {
     systemPrompt
   };
 }
-function resolveClaudeCodeConfig(target, env) {
-  const executableSource = target.executable ?? target.command ?? target.binary;
+function resolveClaudeConfig(target, env, evalFilePath) {
   const modelSource = target.model;
-  const argsSource = target.args ?? target.arguments;
   const cwdSource = target.cwd;
+  const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate;
   const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
   const logDirSource = target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory;
-  const logFormatSource = target.log_format ?? target.logFormat ?? target.log_output_format ?? target.logOutputFormat ?? env.AGENTV_CLAUDE_CODE_LOG_FORMAT;
+  const logFormatSource = target.log_format ?? target.logFormat ?? target.log_output_format ?? target.logOutputFormat ?? env.AGENTV_CLAUDE_LOG_FORMAT;
   const systemPromptSource = target.system_prompt ?? target.systemPrompt;
-  const executable = resolveOptionalString(executableSource, env, `${target.name} claude-code executable`, {
-    allowLiteral: true,
-    optionalEnv: true
-  }) ?? "claude";
-  const model = resolveOptionalString(modelSource, env, `${target.name} claude-code model`, {
+  const model = resolveOptionalString(modelSource, env, `${target.name} claude model`, {
     allowLiteral: true,
     optionalEnv: true
   });
-  const args = resolveOptionalStringArray(argsSource, env, `${target.name} claude-code args`);
-  const cwd = resolveOptionalString(cwdSource, env, `${target.name} claude-code cwd`, {
+  const cwd = resolveOptionalString(cwdSource, env, `${target.name} claude cwd`, {
     allowLiteral: true,
     optionalEnv: true
   });
-  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} claude-code timeout`);
-  const logDir = resolveOptionalString(
-    logDirSource,
+  let workspaceTemplate = resolveOptionalString(
+    workspaceTemplateSource,
     env,
-    `${target.name} claude-code log directory`,
+    `${target.name} claude workspace template`,
     {
       allowLiteral: true,
       optionalEnv: true
     }
   );
-  const logFormat = normalizeClaudeCodeLogFormat(logFormatSource);
+  if (workspaceTemplate && evalFilePath && !path2.isAbsolute(workspaceTemplate)) {
+    workspaceTemplate = path2.resolve(path2.dirname(path2.resolve(evalFilePath)), workspaceTemplate);
+  }
+  if (cwd && workspaceTemplate) {
+    throw new Error(
+      `${target.name}: 'cwd' and 'workspace_template' are mutually exclusive. Use 'cwd' to run in an existing directory, or 'workspace_template' to copy a template to a temp location.`
+    );
+  }
+  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} claude timeout`);
+  const logDir = resolveOptionalString(logDirSource, env, `${target.name} claude log directory`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const logFormat = normalizeClaudeLogFormat(logFormatSource);
   const systemPrompt = typeof systemPromptSource === "string" && systemPromptSource.trim().length > 0 ? systemPromptSource.trim() : void 0;
+  const maxTurns = typeof target.max_turns === "number" ? target.max_turns : typeof target.maxTurns === "number" ? target.maxTurns : void 0;
+  const maxBudgetUsd = typeof target.max_budget_usd === "number" ? target.max_budget_usd : typeof target.maxBudgetUsd === "number" ? target.maxBudgetUsd : void 0;
   return {
-    executable,
     model,
     systemPrompt,
-    args,
     cwd,
+    workspaceTemplate,
     timeoutMs,
+    maxTurns,
+    maxBudgetUsd,
     logDir,
     logFormat
   };
 }
-function normalizeClaudeCodeLogFormat(value) {
+function normalizeClaudeLogFormat(value) {
   if (value === void 0 || value === null) {
     return void 0;
   }
   if (typeof value !== "string") {
-    throw new Error("claude-code log format must be 'summary' or 'json'");
+    throw new Error("claude log format must be 'summary' or 'json'");
   }
   const normalized = value.trim().toLowerCase();
   if (normalized === "json" || normalized === "summary") {
     return normalized;
   }
-  throw new Error("claude-code log format must be 'summary' or 'json'");
+  throw new Error("claude log format must be 'summary' or 'json'");
 }
 function resolveMockConfig(target) {
   const response = typeof target.response === "string" ? target.response : void 0;
   return { response };
 }
-function resolveVSCodeConfig(target, env, insiders) {
+function resolveVSCodeConfig(target, env, insiders, evalFilePath) {
   const workspaceTemplateEnvVar = resolveOptionalLiteralString(
     target.workspace_template ?? target.workspaceTemplate
   );
-  const workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString(
+  let workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString(
     workspaceTemplateEnvVar,
     env,
     `${target.name} workspace template path`,
     {
-      allowLiteral: false,
+      allowLiteral: true,
       optionalEnv: true
     }
   ) : void 0;
-  const commandSource = target.vscode_cmd ?? target.command;
+  if (workspaceTemplate && evalFilePath && !path2.isAbsolute(workspaceTemplate)) {
+    workspaceTemplate = path2.resolve(path2.dirname(path2.resolve(evalFilePath)), workspaceTemplate);
+  }
+  const executableSource = target.executable;
   const waitSource = target.wait;
   const dryRunSource = target.dry_run ?? target.dryRun;
   const subagentRootSource = target.subagent_root ?? target.subagentRoot;
+  const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
   const defaultCommand = insiders ? "code-insiders" : "code";
-  const command = resolveOptionalLiteralString(commandSource) ?? defaultCommand;
+  const executable = resolveOptionalString(executableSource, env, `${target.name} vscode executable`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? defaultCommand;
+  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} vscode timeout`);
   return {
-    command,
+    executable,
     waitForResponse: resolveOptionalBoolean(waitSource) ?? true,
     dryRun: resolveOptionalBoolean(dryRunSource) ?? false,
     subagentRoot: resolveOptionalString(subagentRootSource, env, `${target.name} subagent root`, {
       allowLiteral: true,
       optionalEnv: true
     }),
-    workspaceTemplate
+    workspaceTemplate,
+    timeoutMs
   };
 }
 var cliErrorMap = (issue, ctx) => {
@@ -821,6 +1112,27 @@ function resolveCliConfig(target, env, evalFilePath) {
   }
   return normalized;
 }
+function resolveDiscoveredProviderConfig(target, providerKind, env, evalFilePath) {
+  const commandTemplateSource = target.command_template ?? target.commandTemplate;
+  const commandTemplate = commandTemplateSource ? resolveString(commandTemplateSource, env, `${target.name} command template`, true) : `bun run .agentv/providers/${providerKind}.ts {PROMPT}`;
+  const timeoutSeconds = target.timeout_seconds ?? target.timeoutSeconds;
+  const timeoutMs = resolveTimeoutMs(timeoutSeconds, `${target.name} timeout`);
+  let cwd = resolveOptionalString(target.cwd, env, `${target.name} working directory`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  if (cwd && evalFilePath && !path2.isAbsolute(cwd)) {
+    cwd = path2.resolve(path2.dirname(path2.resolve(evalFilePath)), cwd);
+  }
+  if (!cwd && evalFilePath) {
+    cwd = path2.dirname(path2.resolve(evalFilePath));
+  }
+  return {
+    commandTemplate,
+    cwd,
+    timeoutMs
+  };
+}
 function resolveTimeoutMs(source, description) {
   const seconds = resolveOptionalNumber(source, `${description} (seconds)`);
   if (seconds === void 0) {
@@ -998,9 +1310,10 @@ function resolveOptionalNumberArray(source, description) {
 // src/evaluation/providers/types.ts
 var AGENT_PROVIDER_KINDS = [
   "codex",
+  "copilot",
   "copilot-cli",
   "pi-coding-agent",
-  "claude-code",
+  "claude",
   "vscode",
   "vscode-insiders"
 ];
@@ -1009,10 +1322,11 @@ var KNOWN_PROVIDERS = [
   "anthropic",
   "gemini",
   "codex",
+  "copilot",
   "copilot-cli",
   "pi-coding-agent",
   "pi-agent-sdk",
-  "claude-code",
+  "claude",
   "cli",
   "mock",
   "vscode",
@@ -1027,8 +1341,16 @@ var PROVIDER_ALIASES = [
   // alias for "gemini"
   "codex-cli",
   // alias for "codex"
+  "copilot-sdk",
+  // alias for "copilot"
+  "copilot_sdk",
+  // alias for "copilot" (underscore variant)
   "pi",
   // alias for "pi-coding-agent"
+  "claude-code",
+  // alias for "claude" (legacy)
+  "claude-sdk",
+  // alias for "claude"
   "openai",
   // legacy/future support
   "bedrock",
@@ -1056,6 +1378,13 @@ function isAgentProvider(provider) {
 }
 
 export {
+  TEST_MESSAGE_ROLES,
+  isTestMessageRole,
+  isJsonObject,
+  isJsonValue,
+  isTestMessage,
+  isEvaluatorKind,
+  getHitCount,
   fileExists,
   normalizeLineEndings,
   readTextFile,
@@ -1071,4 +1400,4 @@ export {
   extractLastAssistantContent,
   isAgentProvider
 };
-//# sourceMappingURL=chunk-SSPAANFZ.js.map
+//# sourceMappingURL=chunk-6W5E3VR6.js.map