@agentv/core 4.25.1 → 4.25.2-next.1

package/dist/agentv-provider-MUIGGIP3.js

@@ -0,0 +1,7 @@
+import {
+  AgentvProvider
+} from "./chunk-5XV3FAAD.js";
+export {
+  AgentvProvider
+};
+//# sourceMappingURL=agentv-provider-MUIGGIP3.js.map

package/dist/chunk-5XV3FAAD.js

@@ -0,0 +1,616 @@
+// src/evaluation/providers/llm-providers.ts
+import {
+  complete as piComplete,
+  getModel as piGetModel,
+  registerBuiltInApiProviders
+} from "@mariozechner/pi-ai";
+registerBuiltInApiProviders();
+var DEFAULT_SYSTEM_PROMPT = "You are a careful assistant. Follow all provided instructions and do not fabricate results.";
+var OpenAIProvider = class {
+  id;
+  kind = "openai";
+  targetName;
+  piModel;
+  defaults;
+  retryConfig;
+  apiKey;
+  constructor(targetName, config) {
+    this.id = `openai:${targetName}`;
+    this.targetName = targetName;
+    this.apiKey = config.apiKey;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens
+    };
+    this.retryConfig = config.retry;
+    this.piModel = resolvePiModel({
+      providerName: "openai",
+      apiId: config.apiFormat === "responses" ? "openai-responses" : "openai-completions",
+      modelId: config.model,
+      baseUrl: config.baseURL
+    });
+  }
+  async invoke(request) {
+    return invokePiAi({
+      model: this.piModel,
+      apiKey: this.apiKey,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig
+    });
+  }
+};
+var OpenRouterProvider = class {
+  id;
+  kind = "openrouter";
+  targetName;
+  piModel;
+  defaults;
+  retryConfig;
+  apiKey;
+  constructor(targetName, config) {
+    this.id = `openrouter:${targetName}`;
+    this.targetName = targetName;
+    this.apiKey = config.apiKey;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens
+    };
+    this.retryConfig = config.retry;
+    this.piModel = resolvePiModel({
+      providerName: "openrouter",
+      apiId: "openai-completions",
+      modelId: config.model,
+      baseUrl: "https://openrouter.ai/api/v1"
+    });
+  }
+  async invoke(request) {
+    return invokePiAi({
+      model: this.piModel,
+      apiKey: this.apiKey,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig
+    });
+  }
+};
+var AnthropicProvider = class {
+  id;
+  kind = "anthropic";
+  targetName;
+  piModel;
+  defaults;
+  retryConfig;
+  apiKey;
+  thinkingBudget;
+  constructor(targetName, config) {
+    this.id = `anthropic:${targetName}`;
+    this.targetName = targetName;
+    this.apiKey = config.apiKey;
+    this.thinkingBudget = config.thinkingBudget;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens,
+      thinkingBudget: config.thinkingBudget
+    };
+    this.retryConfig = config.retry;
+    this.piModel = resolvePiModel({
+      providerName: "anthropic",
+      apiId: "anthropic-messages",
+      modelId: config.model
+    });
+  }
+  async invoke(request) {
+    const providerOptions = this.thinkingBudget !== void 0 ? { thinkingEnabled: true, thinkingBudgetTokens: this.thinkingBudget } : void 0;
+    return invokePiAi({
+      model: this.piModel,
+      apiKey: this.apiKey,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig,
+      ...providerOptions ? { providerOptions } : {}
+    });
+  }
+};
+var GeminiProvider = class {
+  id;
+  kind = "gemini";
+  targetName;
+  piModel;
+  defaults;
+  retryConfig;
+  apiKey;
+  constructor(targetName, config) {
+    this.id = `gemini:${targetName}`;
+    this.targetName = targetName;
+    this.apiKey = config.apiKey;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens
+    };
+    this.retryConfig = config.retry;
+    this.piModel = resolvePiModel({
+      providerName: "google",
+      apiId: "google-generative-ai",
+      modelId: config.model
+    });
+  }
+  async invoke(request) {
+    return invokePiAi({
+      model: this.piModel,
+      apiKey: this.apiKey,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig
+    });
+  }
+};
+var AzureProvider = class {
+  id;
+  kind = "azure";
+  targetName;
+  piModel;
+  defaults;
+  retryConfig;
+  apiKey;
+  providerOptions;
+  constructor(targetName, config) {
+    this.id = `azure:${targetName}`;
+    this.targetName = targetName;
+    this.apiKey = config.apiKey;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens
+    };
+    this.retryConfig = config.retry;
+    const trimmed = config.resourceName.trim();
+    const isFullUrl = /^https?:\/\//i.test(trimmed);
+    const baseUrl = isFullUrl ? buildAzureBaseUrl(trimmed) : void 0;
+    this.providerOptions = {
+      ...baseUrl ? { azureBaseUrl: baseUrl } : { azureResourceName: trimmed },
+      ...config.version ? { azureApiVersion: config.version } : {}
+    };
+    this.piModel = resolvePiModel({
+      providerName: "azure-openai-responses",
+      apiId: "azure-openai-responses",
+      // The "model id" for Azure is the deployment name.
+      modelId: config.deploymentName,
+      ...baseUrl ? { baseUrl } : {}
+    });
+  }
+  async invoke(request) {
+    return invokePiAi({
+      model: this.piModel,
+      apiKey: this.apiKey,
+      request,
+      defaults: this.defaults,
+      retryConfig: this.retryConfig,
+      providerOptions: this.providerOptions
+    });
+  }
+};
+function buildAzureBaseUrl(input) {
+  const trimmed = input.replace(/\/+$/, "");
+  if (trimmed.endsWith("/openai/v1")) return trimmed;
+  if (trimmed.endsWith("/openai")) return `${trimmed}/v1`;
+  return `${trimmed}/openai/v1`;
+}
+async function invokePiAi(options) {
+  const { model, apiKey, request, defaults, retryConfig, providerOptions } = options;
+  const tools = request.tools && request.tools.length > 0 ? request.tools : void 0;
+  const maxSteps = tools ? Math.max(1, request.maxSteps ?? 1) : 1;
+  const { systemPrompt, messages } = chatPromptToPiContext(buildChatPrompt(request));
+  if (request.images && request.images.length > 0) {
+    attachImagesToLastUserMessage(messages, request.images);
+  }
+  const piTools = tools ? tools.map((t) => ({
+    name: t.name,
+    description: t.description,
+    parameters: t.parameters
+  })) : void 0;
+  const ctx = { systemPrompt, messages, ...piTools ? { tools: piTools } : {} };
+  const { temperature, maxOutputTokens } = resolveModelSettings(request, defaults);
+  const callOptions = {
+    ...apiKey !== void 0 ? { apiKey } : {},
+    temperature,
+    ...maxOutputTokens !== void 0 ? { maxTokens: maxOutputTokens } : {},
+    signal: request.signal,
+    ...providerOptions ?? {}
+  };
+  const startTime = (/* @__PURE__ */ new Date()).toISOString();
+  const startMs = Date.now();
+  const aggregateUsage = { input: 0, output: 0, cacheRead: 0, cost: 0 };
+  let stepCount = 0;
+  let toolCallCount = 0;
+  let result = await withRetry(
+    () => piComplete(model, ctx, callOptions),
+    retryConfig,
+    request.signal
+  );
+  ctx.messages.push(result);
+  stepCount = 1;
+  accumulateUsage(aggregateUsage, result.usage);
+  while (tools) {
+    const calls = result.content.filter(
+      (b) => b.type === "toolCall"
+    );
+    if (calls.length === 0) break;
+    if (stepCount >= maxSteps) break;
+    toolCallCount += calls.length;
+    for (const call of calls) {
+      const tool = tools.find((t) => t.name === call.name);
+      let output;
+      let isError = false;
+      try {
+        if (!tool) {
+          throw new Error(`pi-ai adapter: model called unknown tool '${call.name}'`);
+        }
+        output = await tool.execute(call.arguments);
+      } catch (err) {
+        output = err instanceof Error ? err.message : String(err);
+        isError = true;
+      }
+      ctx.messages.push({
+        role: "toolResult",
+        toolCallId: call.id,
+        toolName: call.name,
+        content: [
+          { type: "text", text: typeof output === "string" ? output : JSON.stringify(output) }
+        ],
+        isError,
+        timestamp: Date.now()
+      });
+    }
+    result = await withRetry(
+      () => piComplete(model, ctx, callOptions),
+      retryConfig,
+      request.signal
+    );
+    ctx.messages.push(result);
+    stepCount += 1;
+    accumulateUsage(aggregateUsage, result.usage);
+  }
+  const endTime = (/* @__PURE__ */ new Date()).toISOString();
+  const durationMs = Date.now() - startMs;
+  return mapPiResponse(result, {
+    durationMs,
+    startTime,
+    endTime,
+    aggregateUsage,
+    steps: tools ? { count: stepCount, toolCallCount } : void 0
+  });
+}
+function accumulateUsage(agg, u) {
+  agg.input += u.input;
+  agg.output += u.output;
+  agg.cacheRead += u.cacheRead;
+  agg.cost += u.cost.total;
+}
+function resolvePiModel(args) {
+  const { providerName, apiId, modelId, baseUrl } = args;
+  let model;
+  try {
+    model = piGetModel(providerName, modelId);
+  } catch {
+    model = void 0;
+  }
+  if (!model) {
+    const fallbackBaseUrl = baseUrl ?? defaultBaseUrlFor(providerName);
+    if (!fallbackBaseUrl) {
+      throw new Error(
+        `pi-ai adapter cannot resolve a baseUrl for provider '${providerName}' / model '${modelId}'. Either set the target's baseUrl/endpoint or use a model id pi-ai recognizes.`
+      );
+    }
+    model = {
+      id: modelId,
+      name: modelId,
+      api: apiId,
+      provider: providerName,
+      baseUrl: fallbackBaseUrl,
+      reasoning: false,
+      input: ["text"],
+      cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+      contextWindow: 128e3,
+      maxTokens: 16384
+    };
+  }
+  if (model.api !== apiId) {
+    model = { ...model, api: apiId };
+  }
+  if (baseUrl) {
+    model = { ...model, baseUrl };
+  }
+  return model;
+}
+function defaultBaseUrlFor(providerName) {
+  if (providerName === "openai") return "https://api.openai.com/v1";
+  if (providerName === "openrouter") return "https://openrouter.ai/api/v1";
+  return void 0;
+}
+function chatPromptToPiContext(chatPrompt) {
+  const systemSegments = [];
+  const messages = [];
+  const now = Date.now();
+  for (const message of chatPrompt) {
+    if (message.role === "system") {
+      systemSegments.push(message.content);
+      continue;
+    }
+    if (message.role === "user") {
+      messages.push({ role: "user", content: message.content, timestamp: now });
+      continue;
+    }
+    if (message.role === "assistant") {
+      messages.push({
+        role: "assistant",
+        content: [{ type: "text", text: message.content }],
+        api: "",
+        provider: "",
+        model: "",
+        usage: {
+          input: 0,
+          output: 0,
+          cacheRead: 0,
+          cacheWrite: 0,
+          totalTokens: 0,
+          cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }
+        },
+        stopReason: "stop",
+        timestamp: now
+      });
+      continue;
+    }
+    if (message.role === "tool" || message.role === "function") {
+      const prefix = message.name ? `@[${message.name}]: ` : "@[Tool]: ";
+      messages.push({
+        role: "assistant",
+        content: [{ type: "text", text: `${prefix}${message.content}` }],
+        api: "",
+        provider: "",
+        model: "",
+        usage: {
+          input: 0,
+          output: 0,
+          cacheRead: 0,
+          cacheWrite: 0,
+          totalTokens: 0,
+          cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }
+        },
+        stopReason: "stop",
+        timestamp: now
+      });
+      continue;
+    }
+    throw new Error(`pi-ai adapter received unsupported message role '${message.role}'.`);
+  }
+  return {
+    systemPrompt: systemSegments.length > 0 ? systemSegments.join("\n\n") : void 0,
+    messages
+  };
+}
+function attachImagesToLastUserMessage(messages, images) {
+  if (!images || images.length === 0) return;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const m = messages[i];
+    if (m.role !== "user") continue;
+    const text = typeof m.content === "string" ? m.content : "";
+    messages[i] = {
+      ...m,
+      content: [
+        ...text ? [{ type: "text", text }] : [],
+        ...images.map((img) => ({
+          type: "image",
+          data: img.source,
+          mimeType: img.media_type
+        }))
+      ]
+    };
+    return;
+  }
+  messages.push({
+    role: "user",
+    content: images.map((img) => ({
+      type: "image",
+      data: img.source,
+      mimeType: img.media_type
+    })),
+    timestamp: Date.now()
+  });
+}
+function mapPiResponse(result, timing) {
+  const text = result.content.filter((b) => b.type === "text").map((b) => b.text).join("");
+  const cached = timing.aggregateUsage.cacheRead > 0 ? timing.aggregateUsage.cacheRead : void 0;
+  const tokenUsage = {
+    input: timing.aggregateUsage.input,
+    output: timing.aggregateUsage.output,
+    ...cached !== void 0 ? { cached } : {}
+  };
+  const costUsd = timing.aggregateUsage.cost > 0 ? timing.aggregateUsage.cost : void 0;
+  return {
+    raw: result,
+    usage: toJsonObject(result.usage),
+    output: [{ role: "assistant", content: text }],
+    tokenUsage,
+    ...costUsd !== void 0 ? { costUsd } : {},
+    durationMs: timing.durationMs,
+    startTime: timing.startTime,
+    endTime: timing.endTime,
+    ...timing.steps ? { steps: timing.steps } : {}
+  };
+}
+function buildChatPrompt(request) {
+  const provided = request.chatPrompt?.length ? request.chatPrompt : void 0;
+  if (provided) {
+    const hasSystemMessage = provided.some((message) => message.role === "system");
+    if (hasSystemMessage) {
+      return provided;
+    }
+    const systemContent2 = resolveSystemContent(request);
+    return [{ role: "system", content: systemContent2 }, ...provided];
+  }
+  const systemContent = resolveSystemContent(request);
+  const userContent = request.question.trim();
+  return [
+    { role: "system", content: systemContent },
+    { role: "user", content: userContent }
+  ];
+}
+function resolveSystemContent(request) {
+  if (request.systemPrompt && request.systemPrompt.trim().length > 0) {
+    return request.systemPrompt.trim();
+  }
+  return DEFAULT_SYSTEM_PROMPT;
+}
+function resolveModelSettings(request, defaults) {
+  return {
+    temperature: request.temperature ?? defaults.temperature,
+    maxOutputTokens: request.maxOutputTokens ?? defaults.maxOutputTokens
+  };
+}
+function toJsonObject(value) {
+  if (!value || typeof value !== "object") {
+    return void 0;
+  }
+  try {
+    return JSON.parse(JSON.stringify(value));
+  } catch {
+    return void 0;
+  }
+}
+function extractStatus(error) {
+  if (!error || typeof error !== "object") return void 0;
+  const candidate = error;
+  const directStatus = candidate.status ?? candidate.statusCode;
+  if (typeof directStatus === "number" && Number.isFinite(directStatus)) {
+    return directStatus;
+  }
+  const responseStatus = typeof candidate.response === "object" && candidate.response ? candidate.response.status : void 0;
+  if (typeof responseStatus === "number" && Number.isFinite(responseStatus)) {
+    return responseStatus;
+  }
+  const message = typeof candidate.message === "string" ? candidate.message : void 0;
+  if (message) {
+    const match = message.match(/HTTP\s+(\d{3})/i);
+    if (match) {
+      const parsed = Number.parseInt(match[1], 10);
+      if (Number.isFinite(parsed)) return parsed;
+    }
+  }
+  return void 0;
+}
+function isNetworkError(error) {
+  if (!error || typeof error !== "object") return false;
+  const candidate = error;
+  if (candidate.name === "AbortError") return false;
+  const code = candidate.code;
+  if (typeof code === "string" && /^E(AI|CONN|HOST|NET|PIPE|TIME|REFUSED|RESET)/i.test(code)) {
+    return true;
+  }
+  const message = typeof candidate.message === "string" ? candidate.message : void 0;
+  if (message && /(network|fetch failed|ECONNRESET|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNREFUSED)/i.test(message)) {
+    return true;
+  }
+  return false;
+}
+function isRetryableError(error, retryableStatusCodes) {
+  const status = extractStatus(error);
+  if (status === 401 || status === 403) return false;
+  if (typeof status === "number") return retryableStatusCodes.includes(status);
+  return isNetworkError(error);
+}
+function calculateRetryDelay(attempt, config) {
+  const delay = Math.min(
+    config.maxDelayMs,
+    config.initialDelayMs * config.backoffFactor ** attempt
+  );
+  return delay * (0.75 + Math.random() * 0.5);
+}
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function withRetry(fn, retryConfig, signal) {
+  const config = {
+    maxRetries: retryConfig?.maxRetries ?? 3,
+    initialDelayMs: retryConfig?.initialDelayMs ?? 1e3,
+    maxDelayMs: retryConfig?.maxDelayMs ?? 6e4,
+    backoffFactor: retryConfig?.backoffFactor ?? 2,
+    retryableStatusCodes: retryConfig?.retryableStatusCodes ?? [500, 408, 429, 502, 503, 504]
+  };
+  let lastError;
+  for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
+    if (signal?.aborted) {
+      throw new Error(`Request aborted: ${signal.reason ?? "Unknown reason"}`);
+    }
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      if (attempt >= config.maxRetries) break;
+      if (!isRetryableError(error, config.retryableStatusCodes)) throw error;
+      const delay = calculateRetryDelay(attempt, config);
+      await sleep(delay);
+    }
+  }
+  throw lastError;
+}
+
+// src/evaluation/providers/agentv-provider.ts
+var AgentvProvider = class {
+  id;
+  kind = "agentv";
+  targetName;
+  piModel;
+  defaults;
+  constructor(targetName, config) {
+    this.id = `agentv:${targetName}`;
+    this.targetName = targetName;
+    const { providerName, apiId, modelId } = parseAgentvModel(config.model);
+    this.piModel = resolvePiModel({ providerName, apiId, modelId });
+    this.defaults = { temperature: config.temperature };
+  }
+  async invoke(request) {
+    return invokePiAi({
+      model: this.piModel,
+      request,
+      defaults: this.defaults
+    });
+  }
+};
+function parseAgentvModel(model) {
+  const colonIndex = model.indexOf(":");
+  if (colonIndex === -1) {
+    throw new Error(
+      `Invalid agentv model "${model}". Expected "provider:model" (e.g., "openai:gpt-5-mini").`
+    );
+  }
+  const provider = model.slice(0, colonIndex);
+  const modelId = model.slice(colonIndex + 1);
+  switch (provider) {
+    case "openai":
+      return { providerName: "openai", apiId: "openai-completions", modelId };
+    case "anthropic":
+      return { providerName: "anthropic", apiId: "anthropic-messages", modelId };
+    case "azure":
+      return {
+        providerName: "azure-openai-responses",
+        apiId: "azure-openai-responses",
+        modelId
+      };
+    case "google":
+      return { providerName: "google", apiId: "google-generative-ai", modelId };
+    default:
+      throw new Error(
+        `Unsupported agentv provider "${provider}" in "${model}". Supported: openai, anthropic, azure, google.`
+      );
+  }
+}
+
+export {
+  OpenAIProvider,
+  OpenRouterProvider,
+  AnthropicProvider,
+  GeminiProvider,
+  AzureProvider,
+  AgentvProvider
+};
+//# sourceMappingURL=chunk-5XV3FAAD.js.map