@agentv/core 0.7.4 → 0.9.0

package/dist/index.cjs

@@ -52,6 +52,7 @@ __export(index_exports, {
   isTestMessageRole: () => isTestMessageRole,
   listTargetNames: () => listTargetNames,
   loadEvalCases: () => loadEvalCases,
+  normalizeLineEndings: () => normalizeLineEndings,
   readTargetDefinitions: () => readTargetDefinitions,
   readTextFile: () => readTextFile,
   resolveAndCreateProvider: () => resolveAndCreateProvider,
@@ -133,9 +134,12 @@ async function fileExists(filePath) {
     return false;
   }
 }
+function normalizeLineEndings(content) {
+  return content.replace(/\r\n/g, "\n");
+}
 async function readTextFile(filePath) {
   const content = await (0, import_promises.readFile)(filePath, "utf8");
-  return content.replace(/\r\n/g, "\n");
+  return normalizeLineEndings(content);
 }
 async function findGitRoot(startPath) {
   let currentDir = import_node_path.default.dirname(import_node_path.default.resolve(startPath));
@@ -824,6 +828,67 @@ function ensureChatResponse(result) {
   }
   return result;
 }
+function isRetryableError(error, retryableStatusCodes) {
+  if (!error || typeof error !== "object") {
+    return false;
+  }
+  if ("status" in error && typeof error.status === "number") {
+    return retryableStatusCodes.includes(error.status);
+  }
+  if ("message" in error && typeof error.message === "string") {
+    const match = error.message.match(/HTTP (\d{3})/);
+    if (match) {
+      const status = Number.parseInt(match[1], 10);
+      return retryableStatusCodes.includes(status);
+    }
+  }
+  if ("name" in error && error.name === "AxAIServiceNetworkError") {
+    return true;
+  }
+  return false;
+}
+function calculateRetryDelay(attempt, config) {
+  const delay = Math.min(
+    config.maxDelayMs,
+    config.initialDelayMs * config.backoffFactor ** attempt
+  );
+  return delay * (0.75 + Math.random() * 0.5);
+}
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function withRetry(fn, retryConfig, signal) {
+  const config = {
+    maxRetries: retryConfig?.maxRetries ?? 3,
+    initialDelayMs: retryConfig?.initialDelayMs ?? 1e3,
+    maxDelayMs: retryConfig?.maxDelayMs ?? 6e4,
+    backoffFactor: retryConfig?.backoffFactor ?? 2,
+    retryableStatusCodes: retryConfig?.retryableStatusCodes ?? [500, 408, 429, 502, 503, 504]
+  };
+  let lastError;
+  for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
+    if (signal?.aborted) {
+      throw new Error(`Request aborted: ${signal.reason ?? "Unknown reason"}`);
+    }
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      if (attempt >= config.maxRetries) {
+        break;
+      }
+      if (!isRetryableError(error, config.retryableStatusCodes)) {
+        throw error;
+      }
+      const delay = calculateRetryDelay(attempt, config);
+      await sleep(delay);
+      if (signal?.aborted) {
+        throw new Error(`Request aborted: ${signal.reason ?? "Unknown reason"}`);
+      }
+    }
+  }
+  throw lastError;
+}
 var AzureProvider = class {
   constructor(targetName, config) {
     this.config = config;
@@ -833,6 +898,7 @@ var AzureProvider = class {
       temperature: config.temperature,
       maxOutputTokens: config.maxOutputTokens
     };
+    this.retryConfig = config.retry;
     this.ai = import_ax.AxAI.create({
       name: "azure-openai",
       apiKey: config.apiKey,
@@ -849,16 +915,21 @@ var AzureProvider = class {
   targetName;
   ai;
   defaults;
+  retryConfig;
   async invoke(request) {
     const chatPrompt = buildChatPrompt(request);
     const modelConfig = extractModelConfig(request, this.defaults);
-    const response = await this.ai.chat(
-      {
-        chatPrompt,
-        model: this.config.deploymentName,
-        ...modelConfig ? { modelConfig } : {}
-      },
-      request.signal ? { abortSignal: request.signal } : void 0
+    const response = await withRetry(
+      async () => await this.ai.chat(
+        {
+          chatPrompt,
+          model: this.config.deploymentName,
+          ...modelConfig ? { modelConfig } : {}
+        },
+        request.signal ? { abortSignal: request.signal } : void 0
+      ),
+      this.retryConfig,
+      request.signal
     );
     return mapResponse(ensureChatResponse(response));
   }
@@ -876,6 +947,7 @@ var AnthropicProvider = class {
       maxOutputTokens: config.maxOutputTokens,
       thinkingBudget: config.thinkingBudget
     };
+    this.retryConfig = config.retry;
     this.ai = import_ax.AxAI.create({
       name: "anthropic",
       apiKey: config.apiKey
@@ -886,16 +958,21 @@ var AnthropicProvider = class {
   targetName;
   ai;
   defaults;
+  retryConfig;
   async invoke(request) {
     const chatPrompt = buildChatPrompt(request);
     const modelConfig = extractModelConfig(request, this.defaults);
-    const response = await this.ai.chat(
-      {
-        chatPrompt,
-        model: this.config.model,
-        ...modelConfig ? { modelConfig } : {}
-      },
-      request.signal ? { abortSignal: request.signal } : void 0
+    const response = await withRetry(
+      async () => await this.ai.chat(
+        {
+          chatPrompt,
+          model: this.config.model,
+          ...modelConfig ? { modelConfig } : {}
+        },
+        request.signal ? { abortSignal: request.signal } : void 0
+      ),
+      this.retryConfig,
+      request.signal
     );
     return mapResponse(ensureChatResponse(response));
   }
@@ -912,6 +989,7 @@ var GeminiProvider = class {
       temperature: config.temperature,
       maxOutputTokens: config.maxOutputTokens
     };
+    this.retryConfig = config.retry;
     this.ai = import_ax.AxAI.create({
       name: "google-gemini",
       apiKey: config.apiKey
@@ -922,16 +1000,21 @@ var GeminiProvider = class {
   targetName;
   ai;
   defaults;
+  retryConfig;
   async invoke(request) {
     const chatPrompt = buildChatPrompt(request);
     const modelConfig = extractModelConfig(request, this.defaults);
-    const response = await this.ai.chat(
-      {
-        chatPrompt,
-        model: this.config.model,
-        ...modelConfig ? { modelConfig } : {}
-      },
-      request.signal ? { abortSignal: request.signal } : void 0
+    const response = await withRetry(
+      async () => await this.ai.chat(
+        {
+          chatPrompt,
+          model: this.config.model,
+          ...modelConfig ? { modelConfig } : {}
+        },
+        request.signal ? { abortSignal: request.signal } : void 0
+      ),
+      this.retryConfig,
+      request.signal
     );
     return mapResponse(ensureChatResponse(response));
   }
@@ -959,7 +1042,6 @@ async function defaultCommandRunner(command, options) {
   };
   try {
     const { stdout, stderr } = await execAsync(command, execOptions);
-    console.error(`[CLI DEBUG] SUCCESS - stdout: ${stdout.length} bytes, stderr: ${stderr.length} bytes`);
     return {
       stdout,
       stderr,
@@ -970,8 +1052,6 @@ async function defaultCommandRunner(command, options) {
     };
   } catch (error) {
     const execError = error;
-    console.error(`[CLI DEBUG] ERROR - code: ${execError.code}, message: ${execError.message}`);
-    console.error(`[CLI DEBUG] stdout: ${execError.stdout?.length ?? 0} bytes, stderr: ${execError.stderr?.length ?? 0} bytes`);
     return {
       stdout: execError.stdout ?? "",
       stderr: execError.stderr ?? "",
@@ -1004,10 +1084,9 @@ var CliProvider = class {
     const outputFilePath = generateOutputFilePath(request.evalCaseId);
     const templateValues = buildTemplateValues(request, this.config, outputFilePath);
     const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
-    const env = this.config.env ? { ...process.env, ...this.config.env } : process.env;
     const result = await this.runCommand(renderedCommand, {
       cwd: this.config.cwd,
-      env,
+      env: process.env,
       timeoutMs: this.config.timeoutMs,
       signal: request.signal
     });
@@ -1039,7 +1118,7 @@ var CliProvider = class {
   }
   async readAndCleanupOutputFile(filePath) {
     try {
-      const content = await import_promises3.default.readFile(filePath, "utf-8");
+      const content = await readTextFile(filePath);
       return content;
     } catch (error) {
       const errorMsg = error instanceof Error ? error.message : String(error);
@@ -1096,10 +1175,9 @@ var CliProvider = class {
         generateOutputFilePath("healthcheck")
       )
     );
-    const env = this.config.env ? { ...process.env, ...this.config.env } : process.env;
     const result = await this.runCommand(renderedCommand, {
       cwd: healthcheck.cwd ?? this.config.cwd,
-      env,
+      env: process.env,
       timeoutMs,
       signal
     });
@@ -2050,10 +2128,9 @@ var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID
 var BASE_TARGET_SCHEMA = import_zod.z.object({
   name: import_zod.z.string().min(1, "target name is required"),
   provider: import_zod.z.string().min(1, "provider is required"),
-  settings: import_zod.z.record(import_zod.z.unknown()).optional(),
   judge_target: import_zod.z.string().optional(),
   workers: import_zod.z.number().int().min(1).optional()
-});
+}).passthrough();
 var DEFAULT_AZURE_API_VERSION = "2024-10-01-preview";
 function normalizeAzureApiVersion(value) {
   if (!value) {
@@ -2066,11 +2143,43 @@ function normalizeAzureApiVersion(value) {
   const withoutPrefix = trimmed.replace(/^api[-_]?version\s*=\s*/i, "").trim();
   return withoutPrefix.length > 0 ? withoutPrefix : DEFAULT_AZURE_API_VERSION;
 }
+function resolveRetryConfig(target) {
+  const maxRetries = resolveOptionalNumber(
+    target.max_retries ?? target.maxRetries,
+    `${target.name} max retries`
+  );
+  const initialDelayMs = resolveOptionalNumber(
+    target.retry_initial_delay_ms ?? target.retryInitialDelayMs,
+    `${target.name} retry initial delay`
+  );
+  const maxDelayMs = resolveOptionalNumber(
+    target.retry_max_delay_ms ?? target.retryMaxDelayMs,
+    `${target.name} retry max delay`
+  );
+  const backoffFactor = resolveOptionalNumber(
+    target.retry_backoff_factor ?? target.retryBackoffFactor,
+    `${target.name} retry backoff factor`
+  );
+  const retryableStatusCodes = resolveOptionalNumberArray(
+    target.retry_status_codes ?? target.retryStatusCodes,
+    `${target.name} retry status codes`
+  );
+  if (maxRetries === void 0 && initialDelayMs === void 0 && maxDelayMs === void 0 && backoffFactor === void 0 && retryableStatusCodes === void 0) {
+    return void 0;
+  }
+  return {
+    maxRetries,
+    initialDelayMs,
+    maxDelayMs,
+    backoffFactor,
+    retryableStatusCodes
+  };
+}
 function resolveTargetDefinition(definition, env = process.env) {
   const parsed = BASE_TARGET_SCHEMA.parse(definition);
   const provider = parsed.provider.toLowerCase();
   const providerBatching = resolveOptionalBoolean(
-    parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
+    parsed.provider_batching ?? parsed.providerBatching
   );
   switch (provider) {
     case "azure":
@@ -2146,13 +2255,12 @@ function resolveTargetDefinition(definition, env = process.env) {
   }
 }
 function resolveAzureConfig(target, env) {
-  const settings = target.settings ?? {};
-  const endpointSource = settings.endpoint ?? settings.resource ?? settings.resourceName;
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const deploymentSource = settings.deployment ?? settings.deploymentName ?? settings.model;
-  const versionSource = settings.version ?? settings.api_version;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const endpointSource = target.endpoint ?? target.resource ?? target.resourceName;
+  const apiKeySource = target.api_key ?? target.apiKey;
+  const deploymentSource = target.deployment ?? target.deploymentName ?? target.model;
+  const versionSource = target.version ?? target.api_version;
+  const temperatureSource = target.temperature;
+  const maxTokensSource = target.max_output_tokens ?? target.maxTokens;
   const resourceName = resolveString(endpointSource, env, `${target.name} endpoint`);
   const apiKey = resolveString(apiKeySource, env, `${target.name} api key`);
   const deploymentName = resolveString(deploymentSource, env, `${target.name} deployment`);
@@ -2164,58 +2272,61 @@ function resolveAzureConfig(target, env) {
     maxTokensSource,
     `${target.name} max output tokens`
   );
+  const retry = resolveRetryConfig(target);
   return {
     resourceName,
     deploymentName,
     apiKey,
     version,
     temperature,
-    maxOutputTokens
+    maxOutputTokens,
+    retry
   };
 }
 function resolveAnthropicConfig(target, env) {
-  const settings = target.settings ?? {};
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
-  const thinkingBudgetSource = settings.thinking_budget ?? settings.thinkingBudget;
+  const apiKeySource = target.api_key ?? target.apiKey;
+  const modelSource = target.model ?? target.deployment ?? target.variant;
+  const temperatureSource = target.temperature;
+  const maxTokensSource = target.max_output_tokens ?? target.maxTokens;
+  const thinkingBudgetSource = target.thinking_budget ?? target.thinkingBudget;
   const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`);
   const model = resolveString(modelSource, env, `${target.name} Anthropic model`);
+  const retry = resolveRetryConfig(target);
   return {
     apiKey,
     model,
     temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
     maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
-    thinkingBudget: resolveOptionalNumber(thinkingBudgetSource, `${target.name} thinking budget`)
+    thinkingBudget: resolveOptionalNumber(thinkingBudgetSource, `${target.name} thinking budget`),
+    retry
   };
 }
 function resolveGeminiConfig(target, env) {
-  const settings = target.settings ?? {};
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const apiKeySource = target.api_key ?? target.apiKey;
+  const modelSource = target.model ?? target.deployment ?? target.variant;
+  const temperatureSource = target.temperature;
+  const maxTokensSource = target.max_output_tokens ?? target.maxTokens;
   const apiKey = resolveString(apiKeySource, env, `${target.name} Google API key`);
   const model = resolveOptionalString(modelSource, env, `${target.name} Gemini model`, {
     allowLiteral: true,
     optionalEnv: true
   }) ?? "gemini-2.5-flash";
+  const retry = resolveRetryConfig(target);
   return {
     apiKey,
     model,
     temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
-    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
+    retry
   };
 }
 function resolveCodexConfig(target, env) {
-  const settings = target.settings ?? {};
-  const executableSource = settings.executable ?? settings.command ?? settings.binary;
-  const argsSource = settings.args ?? settings.arguments;
-  const cwdSource = settings.cwd;
-  const timeoutSource = settings.timeout_seconds ?? settings.timeoutSeconds;
-  const logDirSource = settings.log_dir ?? settings.logDir ?? settings.log_directory ?? settings.logDirectory;
-  const logFormatSource = settings.log_format ?? settings.logFormat ?? settings.log_output_format ?? settings.logOutputFormat ?? env.AGENTV_CODEX_LOG_FORMAT;
+  const executableSource = target.executable ?? target.command ?? target.binary;
+  const argsSource = target.args ?? target.arguments;
+  const cwdSource = target.cwd;
+  const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
+  const logDirSource = target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory;
+  const logFormatSource = target.log_format ?? target.logFormat ?? target.log_output_format ?? target.logOutputFormat ?? env.AGENTV_CODEX_LOG_FORMAT;
   const executable = resolveOptionalString(executableSource, env, `${target.name} codex executable`, {
     allowLiteral: true,
     optionalEnv: true
@@ -2254,21 +2365,19 @@ function normalizeCodexLogFormat(value) {
   throw new Error("codex log format must be 'summary' or 'json'");
 }
 function resolveMockConfig(target) {
-  const settings = target.settings ?? {};
-  const response = typeof settings.response === "string" ? settings.response : void 0;
+  const response = typeof target.response === "string" ? target.response : void 0;
   return { response };
 }
 function resolveVSCodeConfig(target, env, insiders) {
-  const settings = target.settings ?? {};
-  const workspaceTemplateEnvVar = resolveOptionalLiteralString(settings.workspace_template ?? settings.workspaceTemplate);
+  const workspaceTemplateEnvVar = resolveOptionalLiteralString(target.workspace_template ?? target.workspaceTemplate);
   const workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString(workspaceTemplateEnvVar, env, `${target.name} workspace template path`, {
     allowLiteral: false,
     optionalEnv: true
   }) : void 0;
-  const commandSource = settings.vscode_cmd ?? settings.command;
-  const waitSource = settings.wait;
-  const dryRunSource = settings.dry_run ?? settings.dryRun;
-  const subagentRootSource = settings.subagent_root ?? settings.subagentRoot;
+  const commandSource = target.vscode_cmd ?? target.command;
+  const waitSource = target.wait;
+  const dryRunSource = target.dry_run ?? target.dryRun;
+  const subagentRootSource = target.subagent_root ?? target.subagentRoot;
   const defaultCommand = insiders ? "code-insiders" : "code";
   const command = resolveOptionalLiteralString(commandSource) ?? defaultCommand;
   return {
@@ -2283,18 +2392,16 @@ function resolveVSCodeConfig(target, env, insiders) {
   };
 }
 function resolveCliConfig(target, env) {
-  const settings = target.settings ?? {};
-  const commandTemplateSource = settings.command_template ?? settings.commandTemplate;
+  const commandTemplateSource = target.command_template ?? target.commandTemplate;
   const filesFormat = resolveOptionalLiteralString(
-    settings.files_format ?? settings.filesFormat ?? settings.attachments_format ?? settings.attachmentsFormat
+    target.files_format ?? target.filesFormat ?? target.attachments_format ?? target.attachmentsFormat
   );
-  const cwd = resolveOptionalString(settings.cwd, env, `${target.name} working directory`, {
+  const cwd = resolveOptionalString(target.cwd, env, `${target.name} working directory`, {
     allowLiteral: true,
     optionalEnv: true
   });
-  const envOverrides = resolveEnvOverrides(settings.env, env, target.name);
-  const timeoutMs = resolveTimeoutMs(settings.timeout_seconds ?? settings.timeoutSeconds, `${target.name} timeout`);
-  const healthcheck = resolveCliHealthcheck(settings.healthcheck, env, target.name);
+  const timeoutMs = resolveTimeoutMs(target.timeout_seconds ?? target.timeoutSeconds, `${target.name} timeout`);
+  const healthcheck = resolveCliHealthcheck(target.healthcheck, env, target.name);
   const commandTemplate = resolveString(
     commandTemplateSource,
     env,
@@ -2306,29 +2413,10 @@ function resolveCliConfig(target, env) {
     commandTemplate,
     filesFormat,
     cwd,
-    env: envOverrides,
     timeoutMs,
     healthcheck
   };
 }
-function resolveEnvOverrides(source, env, targetName) {
-  if (source === void 0 || source === null) {
-    return void 0;
-  }
-  if (typeof source !== "object" || Array.isArray(source)) {
-    throw new Error(`${targetName} env overrides must be an object map of strings`);
-  }
-  const entries = Object.entries(source);
-  const resolved = {};
-  for (const [key, value] of entries) {
-    if (typeof value !== "string") {
-      throw new Error(`${targetName} env override '${key}' must be a string`);
-    }
-    const resolvedValue = resolveString(value, env, `${targetName} env override '${key}'`);
-    resolved[key] = resolvedValue;
-  }
-  return Object.keys(resolved).length > 0 ? resolved : void 0;
-}
 function resolveTimeoutMs(source, description) {
   const seconds = resolveOptionalNumber(source, `${description} (seconds)`);
   if (seconds === void 0) {
@@ -2524,6 +2612,26 @@ function resolveOptionalStringArray(source, env, description) {
   }
   return resolved.length > 0 ? resolved : void 0;
 }
+function resolveOptionalNumberArray(source, description) {
+  if (source === void 0 || source === null) {
+    return void 0;
+  }
+  if (!Array.isArray(source)) {
+    throw new Error(`${description} must be an array of numbers`);
+  }
+  if (source.length === 0) {
+    return void 0;
+  }
+  const resolved = [];
+  for (let i = 0; i < source.length; i++) {
+    const item = source[i];
+    if (typeof item !== "number" || !Number.isFinite(item)) {
+      throw new Error(`${description}[${i}] must be a number`);
+    }
+    resolved.push(item);
+  }
+  return resolved.length > 0 ? resolved : void 0;
+}
 
 // src/evaluation/providers/vscode.ts
 var import_node_path6 = __toESM(require("path"), 1);
@@ -2783,7 +2891,7 @@ var AGENT_PROVIDER_KINDS = [
   "vscode",
   "vscode-insiders"
 ];
-var TARGETS_SCHEMA_V2 = "agentv-targets-v2.1";
+var TARGETS_SCHEMA_V2 = "agentv-targets-v2.2";
 function isAgentProvider(provider) {
   return provider ? AGENT_PROVIDER_KINDS.includes(provider.kind) : false;
 }
@@ -2826,20 +2934,13 @@ function assertTargetDefinition(value, index, filePath) {
   }
   const name = value.name;
   const provider = value.provider;
-  const settings = value.settings;
-  const judgeTarget = value.judge_target;
   if (typeof name !== "string" || name.trim().length === 0) {
     throw new Error(`targets.yaml entry at index ${index} in ${filePath} is missing a valid 'name'`);
   }
   if (typeof provider !== "string" || provider.trim().length === 0) {
     throw new Error(`targets.yaml entry '${name}' in ${filePath} is missing a valid 'provider'`);
   }
-  return {
-    name,
-    provider,
-    settings: isRecord(settings) ? settings : void 0,
-    judge_target: typeof judgeTarget === "string" ? judgeTarget : void 0
-  };
+  return value;
 }
 async function fileExists3(filePath) {
   try {
@@ -3480,10 +3581,11 @@ async function runEvaluation(options) {
           await onProgress({
             workerId,
             evalId: evalCase.id,
-            status: "completed",
+            status: result.error ? "failed" : "completed",
             startedAt: 0,
             // Not used for completed status
-            completedAt: Date.now()
+            completedAt: Date.now(),
+            error: result.error
           });
         }
         if (onResult) {
@@ -4021,7 +4123,8 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
     target: targetName,
     timestamp: timestamp.toISOString(),
     raw_aspects: [],
-    raw_request: rawRequest
+    raw_request: rawRequest,
+    error: message
   };
 }
 function createCacheKey(provider, target, evalCase, promptInputs) {
@@ -4078,6 +4181,7 @@ function createAgentKernel() {
   isTestMessageRole,
   listTargetNames,
   loadEvalCases,
+  normalizeLineEndings,
   readTargetDefinitions,
   readTextFile,
   resolveAndCreateProvider,