@agentv/core 4.5.1 → 4.6.0

package/dist/index.cjs

@@ -2585,6 +2585,7 @@ function validateTemplateVariables(content, source) {
 // src/evaluation/loaders/evaluator-parser.ts
 var ANSI_YELLOW4 = "\x1B[33m";
 var ANSI_RESET5 = "\x1B[0m";
+var PROMPT_FILE_PREFIX = "file://";
 function normalizeEvaluatorType(type) {
   return type.replace(/_/g, "-");
 }
@@ -2883,12 +2884,23 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
           threshold: thresholdValue
         };
       } else {
-        const aggregatorPrompt = asString(rawAggregator.prompt);
+        const rawAggPrompt = asString(rawAggregator.prompt);
+        let aggregatorPrompt;
         let promptPath2;
-        if (aggregatorPrompt) {
-          const resolved = await resolveFileReference2(aggregatorPrompt, searchRoots);
-          if (resolved.resolvedPath) {
-            promptPath2 = import_node_path5.default.resolve(resolved.resolvedPath);
+        if (rawAggPrompt) {
+          if (rawAggPrompt.startsWith(PROMPT_FILE_PREFIX)) {
+            const fileRef = rawAggPrompt.slice(PROMPT_FILE_PREFIX.length);
+            aggregatorPrompt = fileRef;
+            const resolved = await resolveFileReference2(fileRef, searchRoots);
+            if (resolved.resolvedPath) {
+              promptPath2 = import_node_path5.default.resolve(resolved.resolvedPath);
+            } else {
+              throw new Error(
+                `Composite aggregator in '${evalId}': prompt file not found: ${resolved.displayPath}`
+              );
+            }
+          } else {
+            aggregatorPrompt = rawAggPrompt;
           }
         }
         aggregator = {
@@ -3468,21 +3480,25 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         promptScriptConfig = rawPrompt.config;
       }
     } else if (typeof rawPrompt === "string") {
-      prompt = rawPrompt;
-      const resolved = await resolveFileReference2(prompt, searchRoots);
-      if (resolved.resolvedPath) {
-        promptPath = import_node_path5.default.resolve(resolved.resolvedPath);
-        try {
-          await validateCustomPromptContent(promptPath);
-        } catch (error) {
-          const message = error instanceof Error ? error.message : String(error);
-          throw new Error(`Evaluator '${name}' template (${promptPath}): ${message}`);
+      if (rawPrompt.startsWith(PROMPT_FILE_PREFIX)) {
+        const fileRef = rawPrompt.slice(PROMPT_FILE_PREFIX.length);
+        prompt = fileRef;
+        const resolved = await resolveFileReference2(fileRef, searchRoots);
+        if (resolved.resolvedPath) {
+          promptPath = import_node_path5.default.resolve(resolved.resolvedPath);
+          try {
+            await validateCustomPromptContent(promptPath);
+          } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`Evaluator '${name}' template (${promptPath}): ${message}`);
+          }
+        } else {
+          throw new Error(
+            `Evaluator '${name}' in '${evalId}': prompt file not found: ${resolved.displayPath}`
+          );
         }
       } else {
-        logWarning2(
-          `Inline prompt used for evaluator '${name}' in '${evalId}' (file not found: ${resolved.displayPath})`,
-          resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => `  Tried: ${attempt}`) : void 0
-        );
+        prompt = rawPrompt;
       }
     }
     const _model = asString(rawEvaluator.model);
@@ -5519,7 +5535,7 @@ var OpenAIProvider = class {
       apiKey: config.apiKey,
       baseURL: config.baseURL
     });
-    this.model = openai(config.model);
+    this.model = config.apiFormat === "responses" ? openai(config.model) : openai.chat(config.model);
   }
   id;
   kind = "openai";
@@ -10752,21 +10768,27 @@ var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set([
   "OUTPUT_FILE"
 ]);
 var COMMON_TARGET_SETTINGS = [
+  "use_target",
   "provider_batching",
   "providerBatching",
   "subagent_mode_allowed",
-  "subagentModeAllowed"
+  "subagentModeAllowed",
+  "fallback_targets",
+  "fallbackTargets"
 ];
 var BASE_TARGET_SCHEMA = import_zod3.z.object({
   name: import_zod3.z.string().min(1, "target name is required"),
-  provider: import_zod3.z.string().min(1, "provider is required"),
+  provider: import_zod3.z.string().optional(),
+  use_target: import_zod3.z.string().optional(),
   grader_target: import_zod3.z.string().optional(),
   judge_target: import_zod3.z.string().optional(),
   // backward compat
   workers: import_zod3.z.number().int().min(1).optional(),
   workspace_template: import_zod3.z.string().optional(),
   workspaceTemplate: import_zod3.z.string().optional(),
-  subagent_mode_allowed: import_zod3.z.boolean().optional()
+  subagent_mode_allowed: import_zod3.z.boolean().optional(),
+  fallback_targets: import_zod3.z.array(import_zod3.z.string().min(1)).optional(),
+  fallbackTargets: import_zod3.z.array(import_zod3.z.string().min(1)).optional()
 }).passthrough();
 var DEFAULT_AZURE_API_VERSION = "2024-12-01-preview";
 var DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
@@ -10820,6 +10842,11 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
       `${parsed.name}: target-level workspace_template has been removed. Use eval-level workspace.template.`
     );
   }
+  if (!parsed.provider) {
+    throw new Error(
+      `${parsed.name}: 'provider' is required (targets with use_target must be resolved before calling resolveTargetDefinition)`
+    );
+  }
   const provider = resolveString(
     parsed.provider,
     env,
@@ -10832,12 +10859,14 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
   const subagentModeAllowed = resolveOptionalBoolean(
     parsed.subagent_mode_allowed ?? parsed.subagentModeAllowed
   );
+  const fallbackTargets = parsed.fallback_targets ?? parsed.fallbackTargets;
   const base = {
     name: parsed.name,
     graderTarget: parsed.grader_target ?? parsed.judge_target,
     workers: parsed.workers,
     providerBatching,
-    subagentModeAllowed
+    subagentModeAllowed,
+    ...fallbackTargets ? { fallbackTargets } : {}
   };
   switch (provider) {
     case "openai":
@@ -11011,6 +11040,14 @@ function resolveAzureConfig(target, env) {
     retry
   };
 }
+function resolveApiFormat(target, targetName) {
+  const raw = target.api_format ?? target.apiFormat;
+  if (raw === void 0) return void 0;
+  if (raw === "chat" || raw === "responses") return raw;
+  throw new Error(
+    `Invalid api_format '${raw}' for target '${targetName}'. Must be 'chat' or 'responses'.`
+  );
+}
 function resolveOpenAIConfig(target, env) {
   const endpointSource = target.endpoint ?? target.base_url ?? target.baseUrl;
   const apiKeySource = target.api_key ?? target.apiKey;
@@ -11030,6 +11067,7 @@ function resolveOpenAIConfig(target, env) {
     baseURL,
     apiKey,
     model,
+    apiFormat: resolveApiFormat(target, target.name),
     temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
     maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
     retry
@@ -13364,8 +13402,11 @@ function assertTargetDefinition(value, index, filePath) {
       `targets.yaml entry at index ${index} in ${filePath} is missing a valid 'name'`
     );
   }
-  if (typeof provider !== "string" || provider.trim().length === 0) {
-    throw new Error(`targets.yaml entry '${name}' in ${filePath} is missing a valid 'provider'`);
+  const hasUseTarget = typeof value.use_target === "string" && value.use_target.trim().length > 0;
+  if (!hasUseTarget && (typeof provider !== "string" || provider.trim().length === 0)) {
+    throw new Error(
+      `targets.yaml entry '${name}' in ${filePath} is missing a valid 'provider' (or use use_target for delegation)`
+    );
   }
   return value;
 }
@@ -18818,10 +18859,20 @@ async function runEvaluation(options) {
     if (resolvedTargetsByName.has(name)) {
       return resolvedTargetsByName.get(name);
     }
-    const definition = targetDefinitions.get(name);
+    let definition = targetDefinitions.get(name);
     if (!definition) {
       return void 0;
     }
+    for (let depth = 0; depth < 5; depth++) {
+      const useTarget = definition.use_target;
+      if (typeof useTarget !== "string" || useTarget.trim().length === 0) break;
+      const envMatch = useTarget.trim().match(/^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i);
+      const resolvedName = envMatch ? envLookup[envMatch[1]] ?? "" : useTarget.trim();
+      if (resolvedName.length === 0) break;
+      const next = targetDefinitions.get(resolvedName);
+      if (!next) break;
+      definition = next;
+    }
     const resolved = resolveTargetDefinition(definition, envLookup, evalFilePath);
     resolvedTargetsByName.set(name, resolved);
     return resolved;
@@ -19826,6 +19877,7 @@ async function runEvalCase(options) {
   let attempt = 0;
   let providerResponse = cachedResponse;
   let lastError;
+  let targetUsed;
   while (!providerResponse && attempt < attemptBudget) {
     try {
       providerResponse = await invokeProvider(provider, {
@@ -19848,25 +19900,33 @@ async function runEvalCase(options) {
         attempt += 1;
         continue;
       }
-      const errorResult = buildErrorResult(
-        evalCase,
-        target.name,
-        nowFn(),
-        error,
-        promptInputs,
-        provider,
-        "agent",
-        "provider_error",
-        verbose
-      );
-      if (workspacePath) {
-        if (forceCleanup) {
-          await cleanupWorkspace(workspacePath).catch(() => {
-          });
-        }
-        return { ...errorResult, workspacePath };
+      break;
+    }
+  }
+  if (!providerResponse && target.fallbackTargets?.length && targetResolver) {
+    for (const fallbackName of target.fallbackTargets) {
+      const fallbackProvider = targetResolver(fallbackName);
+      if (!fallbackProvider) {
+        continue;
+      }
+      try {
+        providerResponse = await invokeProvider(fallbackProvider, {
+          evalCase,
+          target,
+          promptInputs,
+          attempt: 0,
+          agentTimeoutMs,
+          signal,
+          cwd: workspacePath,
+          workspaceFile: caseWorkspaceFile ?? suiteWorkspaceFile,
+          captureFileChanges: !!baselineCommit,
+          streamCallbacks: options.streamCallbacks
+        });
+        targetUsed = fallbackName;
+        break;
+      } catch (error) {
+        lastError = error;
       }
-      return errorResult;
     }
   }
   if (!providerResponse) {
@@ -19992,8 +20052,10 @@ async function runEvalCase(options) {
     };
     const skippedEvaluatorError = buildSkippedEvaluatorError(result.scores);
     const executionStatus = providerError || skippedEvaluatorError ? "execution_error" : classifyQualityStatus(result.score, caseThreshold);
+    const targetUsedField = targetUsed ? { targetUsed } : {};
     const finalResult = providerError ? {
       ...result,
+      ...targetUsedField,
       evalRun,
       error: providerError,
       executionStatus,
@@ -20005,6 +20067,7 @@ async function runEvalCase(options) {
       afterEachOutput
     } : skippedEvaluatorError ? {
       ...result,
+      ...targetUsedField,
       score: 0,
       evalRun,
       error: skippedEvaluatorError,
@@ -20017,6 +20080,7 @@ async function runEvalCase(options) {
       afterEachOutput
     } : {
       ...result,
+      ...targetUsedField,
       evalRun,
       executionStatus,
       beforeAllOutput,