@agentv/core 0.13.0 → 0.15.0

package/dist/index.cjs

@@ -814,13 +814,12 @@ ${guidelineContent.trim()}`);
     const segments = segmentsByMessage[i];
     const contentParts = [];
     let role = message.role;
-    let name;
     if (role === "system") {
       role = "assistant";
       contentParts.push("@[System]:");
     } else if (role === "tool") {
-      role = "function";
-      name = "tool";
+      role = "assistant";
+      contentParts.push("@[Tool]:");
     }
     for (const segment of segments) {
       if (segment.type === "guideline_ref") {
@@ -838,10 +837,10 @@ ${guidelineContent.trim()}`);
     if (contentParts.length === 0) {
       continue;
     }
+    const content = contentParts.join("\n");
     chatPrompt.push({
       role,
-      content: contentParts.join("\n"),
-      ...name ? { name } : {}
+      content
     });
   }
   return chatPrompt.length > 0 ? chatPrompt : void 0;
@@ -1122,68 +1121,225 @@ async function resolveFileReference2(rawValue, searchRoots) {
   return { displayPath, attempted };
 }
 
-// src/evaluation/providers/ax.ts
-var import_ax = require("@ax-llm/ax");
+// src/evaluation/providers/ai-sdk.ts
+var import_anthropic = require("@ai-sdk/anthropic");
+var import_azure = require("@ai-sdk/azure");
+var import_google = require("@ai-sdk/google");
+var import_ai = require("ai");
 var DEFAULT_SYSTEM_PROMPT = "You are a careful assistant. Follow all provided instructions and do not fabricate results.";
+var AzureProvider = class {
+  constructor(targetName, config) {
+    this.config = config;
+    this.id = `azure:${targetName}`;
+    this.targetName = targetName;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens
+    };
+    this.retryConfig = config.retry;
+    const azure = (0, import_azure.createAzure)(buildAzureOptions(config));
+    this.model = azure(config.deploymentName);
+  }
+  id;
+  kind = "azure";
+  targetName;
+  model;
+  defaults;
+  retryConfig;
+  async invoke(request) {
+    return invokeModel({
+      model: this.model,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig
+    });
+  }
+};
+var AnthropicProvider = class {
+  constructor(targetName, config) {
+    this.config = config;
+    this.id = `anthropic:${targetName}`;
+    this.targetName = targetName;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens,
+      thinkingBudget: config.thinkingBudget
+    };
+    this.retryConfig = config.retry;
+    const anthropic = (0, import_anthropic.createAnthropic)({
+      apiKey: config.apiKey
+    });
+    this.model = anthropic(config.model);
+  }
+  id;
+  kind = "anthropic";
+  targetName;
+  model;
+  defaults;
+  retryConfig;
+  async invoke(request) {
+    const providerOptions = buildAnthropicProviderOptions(this.defaults);
+    return invokeModel({
+      model: this.model,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig,
+      providerOptions
+    });
+  }
+};
+var GeminiProvider = class {
+  constructor(targetName, config) {
+    this.config = config;
+    this.id = `gemini:${targetName}`;
+    this.targetName = targetName;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens
+    };
+    this.retryConfig = config.retry;
+    const google = (0, import_google.createGoogleGenerativeAI)({
+      apiKey: config.apiKey
+    });
+    this.model = google(config.model);
+  }
+  id;
+  kind = "gemini";
+  targetName;
+  model;
+  defaults;
+  retryConfig;
+  async invoke(request) {
+    return invokeModel({
+      model: this.model,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig
+    });
+  }
+};
+function buildAzureOptions(config) {
+  const options = {
+    apiKey: config.apiKey,
+    apiVersion: config.version,
+    useDeploymentBasedUrls: true
+  };
+  const baseURL = normalizeAzureBaseUrl(config.resourceName);
+  if (baseURL) {
+    options.baseURL = baseURL;
+  } else {
+    options.resourceName = config.resourceName;
+  }
+  return options;
+}
+function normalizeAzureBaseUrl(resourceName) {
+  const trimmed = resourceName.trim();
+  if (!/^https?:\/\//i.test(trimmed)) {
+    return void 0;
+  }
+  const withoutSlash = trimmed.replace(/\/+$/, "");
+  const normalized = withoutSlash.endsWith("/openai") ? withoutSlash : `${withoutSlash}/openai`;
+  return normalized;
+}
+function buildAnthropicProviderOptions(defaults) {
+  if (defaults.thinkingBudget === void 0) {
+    return void 0;
+  }
+  return {
+    anthropic: {
+      thinking: {
+        type: "enabled",
+        budgetTokens: defaults.thinkingBudget
+      }
+    }
+  };
+}
 function buildChatPrompt(request) {
-  if (request.chatPrompt) {
-    const hasSystemMessage = request.chatPrompt.some((message) => message.role === "system");
+  const provided = request.chatPrompt?.length ? request.chatPrompt : void 0;
+  if (provided) {
+    const hasSystemMessage = provided.some((message) => message.role === "system");
     if (hasSystemMessage) {
-      return request.chatPrompt;
+      return provided;
     }
-    const systemContent2 = resolveSystemContent(request);
-    return [{ role: "system", content: systemContent2 }, ...request.chatPrompt];
+    const systemContent2 = resolveSystemContent(request, false);
+    return [{ role: "system", content: systemContent2 }, ...provided];
   }
-  const systemContent = resolveSystemContent(request);
+  const systemContent = resolveSystemContent(request, true);
   const userContent = request.question.trim();
   const prompt = [
-    {
-      role: "system",
-      content: systemContent
-    },
-    {
-      role: "user",
-      content: userContent
-    }
+    { role: "system", content: systemContent },
+    { role: "user", content: userContent }
   ];
   return prompt;
 }
-function resolveSystemContent(request) {
+function resolveSystemContent(request, includeGuidelines) {
   const systemSegments = [];
   if (request.systemPrompt && request.systemPrompt.trim().length > 0) {
     systemSegments.push(request.systemPrompt.trim());
   } else {
     systemSegments.push(DEFAULT_SYSTEM_PROMPT);
   }
-  if (request.guidelines && request.guidelines.trim().length > 0) {
+  if (includeGuidelines && request.guidelines && request.guidelines.trim().length > 0) {
     systemSegments.push(`[[ ## Guidelines ## ]]
 
 ${request.guidelines.trim()}`);
   }
   return systemSegments.join("\n\n");
 }
-function extractModelConfig(request, defaults) {
+function toModelMessages(chatPrompt) {
+  return chatPrompt.map((message) => {
+    if (message.role === "tool" || message.role === "function") {
+      const prefix = message.name ? `@[${message.name}]: ` : "@[Tool]: ";
+      return {
+        role: "assistant",
+        content: `${prefix}${message.content}`
+      };
+    }
+    if (message.role === "assistant" || message.role === "system" || message.role === "user") {
+      return {
+        role: message.role,
+        content: message.content
+      };
+    }
+    return {
+      role: "user",
+      content: message.content
+    };
+  });
+}
+function resolveModelSettings(request, defaults) {
   const temperature = request.temperature ?? defaults.temperature;
-  const maxTokens = request.maxOutputTokens ?? defaults.maxOutputTokens;
-  const config = {};
-  if (temperature !== void 0) {
-    config.temperature = temperature;
-  }
-  if (maxTokens !== void 0) {
-    config.maxTokens = maxTokens;
-  }
-  return Object.keys(config).length > 0 ? config : void 0;
+  const maxOutputTokens = request.maxOutputTokens ?? defaults.maxOutputTokens;
+  return {
+    temperature,
+    maxOutputTokens
+  };
+}
+async function invokeModel(options) {
+  const { model, request, defaults, retryConfig, providerOptions } = options;
+  const chatPrompt = buildChatPrompt(request);
+  const { temperature, maxOutputTokens } = resolveModelSettings(request, defaults);
+  const result = await withRetry(
+    () => (0, import_ai.generateText)({
+      model,
+      messages: toModelMessages(chatPrompt),
+      temperature,
+      maxOutputTokens,
+      maxRetries: 0,
+      abortSignal: request.signal,
+      ...providerOptions ? { providerOptions } : {}
+    }),
+    retryConfig,
+    request.signal
+  );
+  return mapResponse(result);
 }
-function mapResponse(response) {
-  const primary = response.results[0];
-  const text = typeof primary?.content === "string" ? primary.content : "";
-  const reasoning = primary?.thought ?? primary?.thoughtBlock?.data;
-  const usage = toJsonObject(response.modelUsage);
+function mapResponse(result) {
   return {
-    text,
-    reasoning,
-    raw: response,
-    usage
+    text: result.text ?? "",
+    reasoning: result.reasoningText ?? void 0,
+    raw: result,
+    usage: toJsonObject(result.totalUsage ?? result.usage)
   };
 }
 function toJsonObject(value) {
@@ -1196,34 +1352,59 @@ function toJsonObject(value) {
     return void 0;
   }
 }
-function ensureChatResponse(result) {
-  if (typeof ReadableStream !== "undefined" && result instanceof ReadableStream) {
-    throw new Error("Streaming responses are not supported for this provider");
+function extractStatus(error) {
+  if (!error || typeof error !== "object") {
+    return void 0;
+  }
+  const candidate = error;
+  const directStatus = candidate.status ?? candidate.statusCode;
+  if (typeof directStatus === "number" && Number.isFinite(directStatus)) {
+    return directStatus;
   }
-  if (!result || typeof result !== "object" || !("results" in result)) {
-    throw new Error("Unexpected response type from AxAI provider");
+  const responseStatus = typeof candidate.response === "object" && candidate.response ? candidate.response.status : void 0;
+  if (typeof responseStatus === "number" && Number.isFinite(responseStatus)) {
+    return responseStatus;
   }
-  return result;
+  const message = typeof candidate.message === "string" ? candidate.message : void 0;
+  if (message) {
+    const match = message.match(/HTTP\s+(\d{3})/i);
+    if (match) {
+      const parsed = Number.parseInt(match[1], 10);
+      if (Number.isFinite(parsed)) {
+        return parsed;
+      }
+    }
+  }
+  return void 0;
 }
-function isRetryableError(error, retryableStatusCodes) {
+function isNetworkError(error) {
   if (!error || typeof error !== "object") {
     return false;
   }
-  if ("status" in error && typeof error.status === "number") {
-    return retryableStatusCodes.includes(error.status);
+  const candidate = error;
+  if (candidate.name === "AbortError") {
+    return false;
   }
-  if ("message" in error && typeof error.message === "string") {
-    const match = error.message.match(/HTTP (\d{3})/);
-    if (match) {
-      const status = Number.parseInt(match[1], 10);
-      return retryableStatusCodes.includes(status);
-    }
+  const code = candidate.code;
+  if (typeof code === "string" && /^E(AI|CONN|HOST|NET|PIPE|TIME|REFUSED|RESET)/i.test(code)) {
+    return true;
   }
-  if ("name" in error && error.name === "AxAIServiceNetworkError") {
+  const message = typeof candidate.message === "string" ? candidate.message : void 0;
+  if (message && /(network|fetch failed|ECONNRESET|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNREFUSED)/i.test(message)) {
     return true;
   }
   return false;
 }
+function isRetryableError(error, retryableStatusCodes) {
+  const status = extractStatus(error);
+  if (status === 401 || status === 403) {
+    return false;
+  }
+  if (typeof status === "number") {
+    return retryableStatusCodes.includes(status);
+  }
+  return isNetworkError(error);
+}
 function calculateRetryDelay(attempt, config) {
   const delay = Math.min(
     config.maxDelayMs,
@@ -1259,146 +1440,10 @@ async function withRetry(fn, retryConfig, signal) {
       }
       const delay = calculateRetryDelay(attempt, config);
       await sleep(delay);
-      if (signal?.aborted) {
-        throw new Error(`Request aborted: ${signal.reason ?? "Unknown reason"}`);
-      }
     }
   }
   throw lastError;
 }
-var AzureProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `azure:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens
-    };
-    this.retryConfig = config.retry;
-    this.ai = import_ax.AxAI.create({
-      name: "azure-openai",
-      apiKey: config.apiKey,
-      resourceName: config.resourceName,
-      deploymentName: config.deploymentName,
-      version: config.version,
-      config: {
-        stream: false
-      }
-    });
-  }
-  id;
-  kind = "azure";
-  targetName;
-  ai;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    const chatPrompt = buildChatPrompt(request);
-    const modelConfig = extractModelConfig(request, this.defaults);
-    const response = await withRetry(
-      async () => await this.ai.chat(
-        {
-          chatPrompt,
-          model: this.config.deploymentName,
-          ...modelConfig ? { modelConfig } : {}
-        },
-        request.signal ? { abortSignal: request.signal } : void 0
-      ),
-      this.retryConfig,
-      request.signal
-    );
-    return mapResponse(ensureChatResponse(response));
-  }
-  getAxAI() {
-    return this.ai;
-  }
-};
-var AnthropicProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `anthropic:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens,
-      thinkingBudget: config.thinkingBudget
-    };
-    this.retryConfig = config.retry;
-    this.ai = import_ax.AxAI.create({
-      name: "anthropic",
-      apiKey: config.apiKey
-    });
-  }
-  id;
-  kind = "anthropic";
-  targetName;
-  ai;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    const chatPrompt = buildChatPrompt(request);
-    const modelConfig = extractModelConfig(request, this.defaults);
-    const response = await withRetry(
-      async () => await this.ai.chat(
-        {
-          chatPrompt,
-          model: this.config.model,
-          ...modelConfig ? { modelConfig } : {}
-        },
-        request.signal ? { abortSignal: request.signal } : void 0
-      ),
-      this.retryConfig,
-      request.signal
-    );
-    return mapResponse(ensureChatResponse(response));
-  }
-  getAxAI() {
-    return this.ai;
-  }
-};
-var GeminiProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `gemini:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens
-    };
-    this.retryConfig = config.retry;
-    this.ai = import_ax.AxAI.create({
-      name: "google-gemini",
-      apiKey: config.apiKey
-    });
-  }
-  id;
-  kind = "gemini";
-  targetName;
-  ai;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    const chatPrompt = buildChatPrompt(request);
-    const modelConfig = extractModelConfig(request, this.defaults);
-    const response = await withRetry(
-      async () => await this.ai.chat(
-        {
-          chatPrompt,
-          model: this.config.model,
-          ...modelConfig ? { modelConfig } : {}
-        },
-        request.signal ? { abortSignal: request.signal } : void 0
-      ),
-      this.retryConfig,
-      request.signal
-    );
-    return mapResponse(ensureChatResponse(response));
-  }
-  getAxAI() {
-    return this.ai;
-  }
-};
 
 // src/evaluation/providers/cli.ts
 var import_node_child_process = require("child_process");
@@ -1446,12 +1491,14 @@ var CliProvider = class {
   supportsBatch = false;
   config;
   runCommand;
+  verbose;
   healthcheckPromise;
   constructor(targetName, config, runner = defaultCommandRunner) {
     this.targetName = targetName;
     this.id = `cli:${targetName}`;
     this.config = config;
     this.runCommand = runner;
+    this.verbose = config.verbose ?? false;
   }
   async invoke(request) {
     if (request.signal?.aborted) {
@@ -1552,6 +1599,11 @@ var CliProvider = class {
         generateOutputFilePath("healthcheck")
       )
     );
+    if (this.verbose) {
+      console.log(
+        `[cli-provider:${this.targetName}] (healthcheck) CLI_EVALS_DIR=${process.env.CLI_EVALS_DIR ?? ""} cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
+      );
+    }
     const result = await this.runCommand(renderedCommand, {
       cwd: healthcheck.cwd ?? this.config.cwd,
       env: process.env,