@agentv/core 0.13.0 → 0.15.0

package/dist/index.d.cts

@@ -1,5 +1,3 @@
-import { AxChatRequest, AxAI } from '@ax-llm/ax';
-
 /**
  * JSON primitive values appearing in AgentV payloads.
  */
@@ -149,7 +147,13 @@ interface EvaluatorResult {
  */
 declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
 
-type ChatPrompt = AxChatRequest["chatPrompt"];
+type ChatMessageRole = "system" | "user" | "assistant" | "tool" | "function";
+interface ChatMessage {
+    readonly role: ChatMessageRole;
+    readonly content: string;
+    readonly name?: string;
+}
+type ChatPrompt = readonly ChatMessage[];
 type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
     readonly question: string;
@@ -185,11 +189,6 @@ interface Provider {
      * the orchestrator may send multiple requests in a single provider session.
      */
     invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
-    /**
-     * Optional access to the underlying AxAI instance.
-     * This enables using advanced Ax features like structured output signatures.
-     */
-    getAxAI?(): AxAI;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -341,6 +340,9 @@ interface RetryConfig {
     readonly backoffFactor?: number;
     readonly retryableStatusCodes?: readonly number[];
 }
+/**
+ * Azure OpenAI settings used by the Vercel AI SDK.
+ */
 interface AzureResolvedConfig {
     readonly resourceName: string;
     readonly deploymentName: string;
@@ -350,6 +352,9 @@ interface AzureResolvedConfig {
     readonly maxOutputTokens?: number;
     readonly retry?: RetryConfig;
 }
+/**
+ * Anthropic Claude settings used by the Vercel AI SDK.
+ */
 interface AnthropicResolvedConfig {
     readonly apiKey: string;
     readonly model: string;
@@ -358,6 +363,9 @@ interface AnthropicResolvedConfig {
     readonly thinkingBudget?: number;
     readonly retry?: RetryConfig;
 }
+/**
+ * Google Gemini settings used by the Vercel AI SDK.
+ */
 interface GeminiResolvedConfig {
     readonly apiKey: string;
     readonly model: string;
@@ -402,6 +410,7 @@ interface CliResolvedConfig {
     readonly cwd?: string;
     readonly timeoutMs?: number;
     readonly healthcheck?: CliHealthcheck;
+    readonly verbose?: boolean;
 }
 type ResolvedTarget = {
     readonly kind: "azure";

package/dist/index.d.ts

@@ -1,5 +1,3 @@
-import { AxChatRequest, AxAI } from '@ax-llm/ax';
-
 /**
  * JSON primitive values appearing in AgentV payloads.
  */
@@ -149,7 +147,13 @@ interface EvaluatorResult {
  */
 declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
 
-type ChatPrompt = AxChatRequest["chatPrompt"];
+type ChatMessageRole = "system" | "user" | "assistant" | "tool" | "function";
+interface ChatMessage {
+    readonly role: ChatMessageRole;
+    readonly content: string;
+    readonly name?: string;
+}
+type ChatPrompt = readonly ChatMessage[];
 type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
     readonly question: string;
@@ -185,11 +189,6 @@ interface Provider {
      * the orchestrator may send multiple requests in a single provider session.
      */
     invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
-    /**
-     * Optional access to the underlying AxAI instance.
-     * This enables using advanced Ax features like structured output signatures.
-     */
-    getAxAI?(): AxAI;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -341,6 +340,9 @@ interface RetryConfig {
     readonly backoffFactor?: number;
     readonly retryableStatusCodes?: readonly number[];
 }
+/**
+ * Azure OpenAI settings used by the Vercel AI SDK.
+ */
 interface AzureResolvedConfig {
     readonly resourceName: string;
     readonly deploymentName: string;
@@ -350,6 +352,9 @@ interface AzureResolvedConfig {
     readonly maxOutputTokens?: number;
     readonly retry?: RetryConfig;
 }
+/**
+ * Anthropic Claude settings used by the Vercel AI SDK.
+ */
 interface AnthropicResolvedConfig {
     readonly apiKey: string;
     readonly model: string;
@@ -358,6 +363,9 @@ interface AnthropicResolvedConfig {
     readonly thinkingBudget?: number;
     readonly retry?: RetryConfig;
 }
+/**
+ * Google Gemini settings used by the Vercel AI SDK.
+ */
 interface GeminiResolvedConfig {
     readonly apiKey: string;
     readonly model: string;
@@ -402,6 +410,7 @@ interface CliResolvedConfig {
     readonly cwd?: string;
     readonly timeoutMs?: number;
     readonly healthcheck?: CliHealthcheck;
+    readonly verbose?: boolean;
 }
 type ResolvedTarget = {
     readonly kind: "azure";

package/dist/index.js

@@ -9,7 +9,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-U3GEJ3K7.js";
+} from "./chunk-IOCVST3R.js";
 
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
@@ -760,13 +760,12 @@ ${guidelineContent.trim()}`);
     const segments = segmentsByMessage[i];
     const contentParts = [];
     let role = message.role;
-    let name;
     if (role === "system") {
       role = "assistant";
       contentParts.push("@[System]:");
     } else if (role === "tool") {
-      role = "function";
-      name = "tool";
+      role = "assistant";
+      contentParts.push("@[Tool]:");
     }
     for (const segment of segments) {
       if (segment.type === "guideline_ref") {
@@ -784,10 +783,10 @@ ${guidelineContent.trim()}`);
     if (contentParts.length === 0) {
       continue;
     }
+    const content = contentParts.join("\n");
     chatPrompt.push({
       role,
-      content: contentParts.join("\n"),
-      ...name ? { name } : {}
+      content
     });
   }
   return chatPrompt.length > 0 ? chatPrompt : void 0;
@@ -957,68 +956,225 @@ ${detailBlock}${ANSI_RESET5}`);
   }
 }
 
-// src/evaluation/providers/ax.ts
-import { AxAI } from "@ax-llm/ax";
+// src/evaluation/providers/ai-sdk.ts
+import { createAnthropic } from "@ai-sdk/anthropic";
+import { createAzure } from "@ai-sdk/azure";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { generateText } from "ai";
 var DEFAULT_SYSTEM_PROMPT = "You are a careful assistant. Follow all provided instructions and do not fabricate results.";
+var AzureProvider = class {
+  constructor(targetName, config) {
+    this.config = config;
+    this.id = `azure:${targetName}`;
+    this.targetName = targetName;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens
+    };
+    this.retryConfig = config.retry;
+    const azure = createAzure(buildAzureOptions(config));
+    this.model = azure(config.deploymentName);
+  }
+  id;
+  kind = "azure";
+  targetName;
+  model;
+  defaults;
+  retryConfig;
+  async invoke(request) {
+    return invokeModel({
+      model: this.model,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig
+    });
+  }
+};
+var AnthropicProvider = class {
+  constructor(targetName, config) {
+    this.config = config;
+    this.id = `anthropic:${targetName}`;
+    this.targetName = targetName;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens,
+      thinkingBudget: config.thinkingBudget
+    };
+    this.retryConfig = config.retry;
+    const anthropic = createAnthropic({
+      apiKey: config.apiKey
+    });
+    this.model = anthropic(config.model);
+  }
+  id;
+  kind = "anthropic";
+  targetName;
+  model;
+  defaults;
+  retryConfig;
+  async invoke(request) {
+    const providerOptions = buildAnthropicProviderOptions(this.defaults);
+    return invokeModel({
+      model: this.model,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig,
+      providerOptions
+    });
+  }
+};
+var GeminiProvider = class {
+  constructor(targetName, config) {
+    this.config = config;
+    this.id = `gemini:${targetName}`;
+    this.targetName = targetName;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens
+    };
+    this.retryConfig = config.retry;
+    const google = createGoogleGenerativeAI({
+      apiKey: config.apiKey
+    });
+    this.model = google(config.model);
+  }
+  id;
+  kind = "gemini";
+  targetName;
+  model;
+  defaults;
+  retryConfig;
+  async invoke(request) {
+    return invokeModel({
+      model: this.model,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig
+    });
+  }
+};
+function buildAzureOptions(config) {
+  const options = {
+    apiKey: config.apiKey,
+    apiVersion: config.version,
+    useDeploymentBasedUrls: true
+  };
+  const baseURL = normalizeAzureBaseUrl(config.resourceName);
+  if (baseURL) {
+    options.baseURL = baseURL;
+  } else {
+    options.resourceName = config.resourceName;
+  }
+  return options;
+}
+function normalizeAzureBaseUrl(resourceName) {
+  const trimmed = resourceName.trim();
+  if (!/^https?:\/\//i.test(trimmed)) {
+    return void 0;
+  }
+  const withoutSlash = trimmed.replace(/\/+$/, "");
+  const normalized = withoutSlash.endsWith("/openai") ? withoutSlash : `${withoutSlash}/openai`;
+  return normalized;
+}
+function buildAnthropicProviderOptions(defaults) {
+  if (defaults.thinkingBudget === void 0) {
+    return void 0;
+  }
+  return {
+    anthropic: {
+      thinking: {
+        type: "enabled",
+        budgetTokens: defaults.thinkingBudget
+      }
+    }
+  };
+}
 function buildChatPrompt(request) {
-  if (request.chatPrompt) {
-    const hasSystemMessage = request.chatPrompt.some((message) => message.role === "system");
+  const provided = request.chatPrompt?.length ? request.chatPrompt : void 0;
+  if (provided) {
+    const hasSystemMessage = provided.some((message) => message.role === "system");
     if (hasSystemMessage) {
-      return request.chatPrompt;
+      return provided;
     }
-    const systemContent2 = resolveSystemContent(request);
-    return [{ role: "system", content: systemContent2 }, ...request.chatPrompt];
+    const systemContent2 = resolveSystemContent(request, false);
+    return [{ role: "system", content: systemContent2 }, ...provided];
   }
-  const systemContent = resolveSystemContent(request);
+  const systemContent = resolveSystemContent(request, true);
   const userContent = request.question.trim();
   const prompt = [
-    {
-      role: "system",
-      content: systemContent
-    },
-    {
-      role: "user",
-      content: userContent
-    }
+    { role: "system", content: systemContent },
+    { role: "user", content: userContent }
   ];
   return prompt;
 }
-function resolveSystemContent(request) {
+function resolveSystemContent(request, includeGuidelines) {
   const systemSegments = [];
   if (request.systemPrompt && request.systemPrompt.trim().length > 0) {
     systemSegments.push(request.systemPrompt.trim());
   } else {
     systemSegments.push(DEFAULT_SYSTEM_PROMPT);
   }
-  if (request.guidelines && request.guidelines.trim().length > 0) {
+  if (includeGuidelines && request.guidelines && request.guidelines.trim().length > 0) {
     systemSegments.push(`[[ ## Guidelines ## ]]
 
 ${request.guidelines.trim()}`);
   }
   return systemSegments.join("\n\n");
 }
-function extractModelConfig(request, defaults) {
+function toModelMessages(chatPrompt) {
+  return chatPrompt.map((message) => {
+    if (message.role === "tool" || message.role === "function") {
+      const prefix = message.name ? `@[${message.name}]: ` : "@[Tool]: ";
+      return {
+        role: "assistant",
+        content: `${prefix}${message.content}`
+      };
+    }
+    if (message.role === "assistant" || message.role === "system" || message.role === "user") {
+      return {
+        role: message.role,
+        content: message.content
+      };
+    }
+    return {
+      role: "user",
+      content: message.content
+    };
+  });
+}
+function resolveModelSettings(request, defaults) {
   const temperature = request.temperature ?? defaults.temperature;
-  const maxTokens = request.maxOutputTokens ?? defaults.maxOutputTokens;
-  const config = {};
-  if (temperature !== void 0) {
-    config.temperature = temperature;
-  }
-  if (maxTokens !== void 0) {
-    config.maxTokens = maxTokens;
-  }
-  return Object.keys(config).length > 0 ? config : void 0;
+  const maxOutputTokens = request.maxOutputTokens ?? defaults.maxOutputTokens;
+  return {
+    temperature,
+    maxOutputTokens
+  };
+}
+async function invokeModel(options) {
+  const { model, request, defaults, retryConfig, providerOptions } = options;
+  const chatPrompt = buildChatPrompt(request);
+  const { temperature, maxOutputTokens } = resolveModelSettings(request, defaults);
+  const result = await withRetry(
+    () => generateText({
+      model,
+      messages: toModelMessages(chatPrompt),
+      temperature,
+      maxOutputTokens,
+      maxRetries: 0,
+      abortSignal: request.signal,
+      ...providerOptions ? { providerOptions } : {}
+    }),
+    retryConfig,
+    request.signal
+  );
+  return mapResponse(result);
 }
-function mapResponse(response) {
-  const primary = response.results[0];
-  const text = typeof primary?.content === "string" ? primary.content : "";
-  const reasoning = primary?.thought ?? primary?.thoughtBlock?.data;
-  const usage = toJsonObject(response.modelUsage);
+function mapResponse(result) {
   return {
-    text,
-    reasoning,
-    raw: response,
-    usage
+    text: result.text ?? "",
+    reasoning: result.reasoningText ?? void 0,
+    raw: result,
+    usage: toJsonObject(result.totalUsage ?? result.usage)
   };
 }
 function toJsonObject(value) {
@@ -1031,34 +1187,59 @@ function toJsonObject(value) {
     return void 0;
   }
 }
-function ensureChatResponse(result) {
-  if (typeof ReadableStream !== "undefined" && result instanceof ReadableStream) {
-    throw new Error("Streaming responses are not supported for this provider");
+function extractStatus(error) {
+  if (!error || typeof error !== "object") {
+    return void 0;
   }
-  if (!result || typeof result !== "object" || !("results" in result)) {
-    throw new Error("Unexpected response type from AxAI provider");
+  const candidate = error;
+  const directStatus = candidate.status ?? candidate.statusCode;
+  if (typeof directStatus === "number" && Number.isFinite(directStatus)) {
+    return directStatus;
   }
-  return result;
+  const responseStatus = typeof candidate.response === "object" && candidate.response ? candidate.response.status : void 0;
+  if (typeof responseStatus === "number" && Number.isFinite(responseStatus)) {
+    return responseStatus;
+  }
+  const message = typeof candidate.message === "string" ? candidate.message : void 0;
+  if (message) {
+    const match = message.match(/HTTP\s+(\d{3})/i);
+    if (match) {
+      const parsed = Number.parseInt(match[1], 10);
+      if (Number.isFinite(parsed)) {
+        return parsed;
+      }
+    }
+  }
+  return void 0;
 }
-function isRetryableError(error, retryableStatusCodes) {
+function isNetworkError(error) {
   if (!error || typeof error !== "object") {
     return false;
   }
-  if ("status" in error && typeof error.status === "number") {
-    return retryableStatusCodes.includes(error.status);
+  const candidate = error;
+  if (candidate.name === "AbortError") {
+    return false;
   }
-  if ("message" in error && typeof error.message === "string") {
-    const match = error.message.match(/HTTP (\d{3})/);
-    if (match) {
-      const status = Number.parseInt(match[1], 10);
-      return retryableStatusCodes.includes(status);
-    }
+  const code = candidate.code;
+  if (typeof code === "string" && /^E(AI|CONN|HOST|NET|PIPE|TIME|REFUSED|RESET)/i.test(code)) {
+    return true;
   }
-  if ("name" in error && error.name === "AxAIServiceNetworkError") {
+  const message = typeof candidate.message === "string" ? candidate.message : void 0;
+  if (message && /(network|fetch failed|ECONNRESET|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNREFUSED)/i.test(message)) {
     return true;
   }
   return false;
 }
+function isRetryableError(error, retryableStatusCodes) {
+  const status = extractStatus(error);
+  if (status === 401 || status === 403) {
+    return false;
+  }
+  if (typeof status === "number") {
+    return retryableStatusCodes.includes(status);
+  }
+  return isNetworkError(error);
+}
 function calculateRetryDelay(attempt, config) {
   const delay = Math.min(
     config.maxDelayMs,
@@ -1094,146 +1275,10 @@ async function withRetry(fn, retryConfig, signal) {
       }
       const delay = calculateRetryDelay(attempt, config);
       await sleep(delay);
-      if (signal?.aborted) {
-        throw new Error(`Request aborted: ${signal.reason ?? "Unknown reason"}`);
-      }
     }
   }
   throw lastError;
 }
-var AzureProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `azure:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens
-    };
-    this.retryConfig = config.retry;
-    this.ai = AxAI.create({
-      name: "azure-openai",
-      apiKey: config.apiKey,
-      resourceName: config.resourceName,
-      deploymentName: config.deploymentName,
-      version: config.version,
-      config: {
-        stream: false
-      }
-    });
-  }
-  id;
-  kind = "azure";
-  targetName;
-  ai;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    const chatPrompt = buildChatPrompt(request);
-    const modelConfig = extractModelConfig(request, this.defaults);
-    const response = await withRetry(
-      async () => await this.ai.chat(
-        {
-          chatPrompt,
-          model: this.config.deploymentName,
-          ...modelConfig ? { modelConfig } : {}
-        },
-        request.signal ? { abortSignal: request.signal } : void 0
-      ),
-      this.retryConfig,
-      request.signal
-    );
-    return mapResponse(ensureChatResponse(response));
-  }
-  getAxAI() {
-    return this.ai;
-  }
-};
-var AnthropicProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `anthropic:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens,
-      thinkingBudget: config.thinkingBudget
-    };
-    this.retryConfig = config.retry;
-    this.ai = AxAI.create({
-      name: "anthropic",
-      apiKey: config.apiKey
-    });
-  }
-  id;
-  kind = "anthropic";
-  targetName;
-  ai;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    const chatPrompt = buildChatPrompt(request);
-    const modelConfig = extractModelConfig(request, this.defaults);
-    const response = await withRetry(
-      async () => await this.ai.chat(
-        {
-          chatPrompt,
-          model: this.config.model,
-          ...modelConfig ? { modelConfig } : {}
-        },
-        request.signal ? { abortSignal: request.signal } : void 0
-      ),
-      this.retryConfig,
-      request.signal
-    );
-    return mapResponse(ensureChatResponse(response));
-  }
-  getAxAI() {
-    return this.ai;
-  }
-};
-var GeminiProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `gemini:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens
-    };
-    this.retryConfig = config.retry;
-    this.ai = AxAI.create({
-      name: "google-gemini",
-      apiKey: config.apiKey
-    });
-  }
-  id;
-  kind = "gemini";
-  targetName;
-  ai;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    const chatPrompt = buildChatPrompt(request);
-    const modelConfig = extractModelConfig(request, this.defaults);
-    const response = await withRetry(
-      async () => await this.ai.chat(
-        {
-          chatPrompt,
-          model: this.config.model,
-          ...modelConfig ? { modelConfig } : {}
-        },
-        request.signal ? { abortSignal: request.signal } : void 0
-      ),
-      this.retryConfig,
-      request.signal
-    );
-    return mapResponse(ensureChatResponse(response));
-  }
-  getAxAI() {
-    return this.ai;
-  }
-};
 
 // src/evaluation/providers/cli.ts
 import { exec as execWithCallback } from "node:child_process";
@@ -1281,12 +1326,14 @@ var CliProvider = class {
   supportsBatch = false;
   config;
   runCommand;
+  verbose;
   healthcheckPromise;
   constructor(targetName, config, runner = defaultCommandRunner) {
     this.targetName = targetName;
     this.id = `cli:${targetName}`;
     this.config = config;
     this.runCommand = runner;
+    this.verbose = config.verbose ?? false;
   }
   async invoke(request) {
     if (request.signal?.aborted) {
@@ -1387,6 +1434,11 @@ var CliProvider = class {
         generateOutputFilePath("healthcheck")
       )
     );
+    if (this.verbose) {
+      console.log(
+        `[cli-provider:${this.targetName}] (healthcheck) CLI_EVALS_DIR=${process.env.CLI_EVALS_DIR ?? ""} cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
+      );
+    }
     const result = await this.runCommand(renderedCommand, {
       cwd: healthcheck.cwd ?? this.config.cwd,
       env: process.env,