@agentv/core 0.7.3 → 0.7.5

package/dist/chunk-7XM7HYRS.js

@@ -0,0 +1,645 @@
+// src/evaluation/file-utils.ts
+import { constants } from "node:fs";
+import { access, readFile } from "node:fs/promises";
+import path from "node:path";
+async function fileExists(filePath) {
+  try {
+    await access(filePath, constants.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function normalizeLineEndings(content) {
+  return content.replace(/\r\n/g, "\n");
+}
+async function readTextFile(filePath) {
+  const content = await readFile(filePath, "utf8");
+  return normalizeLineEndings(content);
+}
+async function findGitRoot(startPath) {
+  let currentDir = path.dirname(path.resolve(startPath));
+  const root = path.parse(currentDir).root;
+  while (currentDir !== root) {
+    const gitPath = path.join(currentDir, ".git");
+    if (await fileExists(gitPath)) {
+      return currentDir;
+    }
+    const parentDir = path.dirname(currentDir);
+    if (parentDir === currentDir) {
+      break;
+    }
+    currentDir = parentDir;
+  }
+  return null;
+}
+function buildDirectoryChain(filePath, repoRoot) {
+  const directories = [];
+  const seen = /* @__PURE__ */ new Set();
+  const boundary = path.resolve(repoRoot);
+  let current = path.resolve(path.dirname(filePath));
+  while (current !== void 0) {
+    if (!seen.has(current)) {
+      directories.push(current);
+      seen.add(current);
+    }
+    if (current === boundary) {
+      break;
+    }
+    const parent = path.dirname(current);
+    if (parent === current) {
+      break;
+    }
+    current = parent;
+  }
+  if (!seen.has(boundary)) {
+    directories.push(boundary);
+  }
+  return directories;
+}
+function buildSearchRoots(evalPath, repoRoot) {
+  const uniqueRoots = [];
+  const addRoot = (root) => {
+    const normalized = path.resolve(root);
+    if (!uniqueRoots.includes(normalized)) {
+      uniqueRoots.push(normalized);
+    }
+  };
+  let currentDir = path.dirname(evalPath);
+  let reachedBoundary = false;
+  while (!reachedBoundary) {
+    addRoot(currentDir);
+    const parentDir = path.dirname(currentDir);
+    if (currentDir === repoRoot || parentDir === currentDir) {
+      reachedBoundary = true;
+    } else {
+      currentDir = parentDir;
+    }
+  }
+  addRoot(repoRoot);
+  addRoot(process.cwd());
+  return uniqueRoots;
+}
+function trimLeadingSeparators(value) {
+  const trimmed = value.replace(/^[/\\]+/, "");
+  return trimmed.length > 0 ? trimmed : value;
+}
+async function resolveFileReference(rawValue, searchRoots) {
+  const displayPath = trimLeadingSeparators(rawValue);
+  const potentialPaths = [];
+  if (path.isAbsolute(rawValue)) {
+    potentialPaths.push(path.normalize(rawValue));
+  }
+  for (const base of searchRoots) {
+    potentialPaths.push(path.resolve(base, displayPath));
+  }
+  const attempted = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const candidate of potentialPaths) {
+    const absoluteCandidate = path.resolve(candidate);
+    if (seen.has(absoluteCandidate)) {
+      continue;
+    }
+    seen.add(absoluteCandidate);
+    attempted.push(absoluteCandidate);
+    if (await fileExists(absoluteCandidate)) {
+      return { displayPath, resolvedPath: absoluteCandidate, attempted };
+    }
+  }
+  return { displayPath, attempted };
+}
+
+// src/evaluation/providers/targets.ts
+import { z } from "zod";
+var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES", "OUTPUT_FILE"]);
+var BASE_TARGET_SCHEMA = z.object({
+  name: z.string().min(1, "target name is required"),
+  provider: z.string().min(1, "provider is required"),
+  settings: z.record(z.unknown()).optional(),
+  judge_target: z.string().optional(),
+  workers: z.number().int().min(1).optional()
+});
+var DEFAULT_AZURE_API_VERSION = "2024-10-01-preview";
+function normalizeAzureApiVersion(value) {
+  if (!value) {
+    return DEFAULT_AZURE_API_VERSION;
+  }
+  const trimmed = value.trim();
+  if (trimmed.length === 0) {
+    return DEFAULT_AZURE_API_VERSION;
+  }
+  const withoutPrefix = trimmed.replace(/^api[-_]?version\s*=\s*/i, "").trim();
+  return withoutPrefix.length > 0 ? withoutPrefix : DEFAULT_AZURE_API_VERSION;
+}
+function resolveTargetDefinition(definition, env = process.env) {
+  const parsed = BASE_TARGET_SCHEMA.parse(definition);
+  const provider = parsed.provider.toLowerCase();
+  const providerBatching = resolveOptionalBoolean(
+    parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
+  );
+  switch (provider) {
+    case "azure":
+    case "azure-openai":
+      return {
+        kind: "azure",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveAzureConfig(parsed, env)
+      };
+    case "anthropic":
+      return {
+        kind: "anthropic",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveAnthropicConfig(parsed, env)
+      };
+    case "gemini":
+    case "google":
+    case "google-gemini":
+      return {
+        kind: "gemini",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveGeminiConfig(parsed, env)
+      };
+    case "codex":
+    case "codex-cli":
+      return {
+        kind: "codex",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveCodexConfig(parsed, env)
+      };
+    case "mock":
+      return {
+        kind: "mock",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveMockConfig(parsed)
+      };
+    case "vscode":
+    case "vscode-insiders":
+      return {
+        kind: provider,
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
+      };
+    case "cli":
+      return {
+        kind: "cli",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveCliConfig(parsed, env)
+      };
+    default:
+      throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
+  }
+}
+function resolveAzureConfig(target, env) {
+  const settings = target.settings ?? {};
+  const endpointSource = settings.endpoint ?? settings.resource ?? settings.resourceName;
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const deploymentSource = settings.deployment ?? settings.deploymentName ?? settings.model;
+  const versionSource = settings.version ?? settings.api_version;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const resourceName = resolveString(endpointSource, env, `${target.name} endpoint`);
+  const apiKey = resolveString(apiKeySource, env, `${target.name} api key`);
+  const deploymentName = resolveString(deploymentSource, env, `${target.name} deployment`);
+  const version = normalizeAzureApiVersion(
+    resolveOptionalString(versionSource, env, `${target.name} api version`)
+  );
+  const temperature = resolveOptionalNumber(temperatureSource, `${target.name} temperature`);
+  const maxOutputTokens = resolveOptionalNumber(
+    maxTokensSource,
+    `${target.name} max output tokens`
+  );
+  return {
+    resourceName,
+    deploymentName,
+    apiKey,
+    version,
+    temperature,
+    maxOutputTokens
+  };
+}
+function resolveAnthropicConfig(target, env) {
+  const settings = target.settings ?? {};
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const thinkingBudgetSource = settings.thinking_budget ?? settings.thinkingBudget;
+  const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`);
+  const model = resolveString(modelSource, env, `${target.name} Anthropic model`);
+  return {
+    apiKey,
+    model,
+    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
+    thinkingBudget: resolveOptionalNumber(thinkingBudgetSource, `${target.name} thinking budget`)
+  };
+}
+function resolveGeminiConfig(target, env) {
+  const settings = target.settings ?? {};
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const apiKey = resolveString(apiKeySource, env, `${target.name} Google API key`);
+  const model = resolveOptionalString(modelSource, env, `${target.name} Gemini model`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? "gemini-2.5-flash";
+  return {
+    apiKey,
+    model,
+    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
+  };
+}
+function resolveCodexConfig(target, env) {
+  const settings = target.settings ?? {};
+  const executableSource = settings.executable ?? settings.command ?? settings.binary;
+  const argsSource = settings.args ?? settings.arguments;
+  const cwdSource = settings.cwd;
+  const timeoutSource = settings.timeout_seconds ?? settings.timeoutSeconds;
+  const logDirSource = settings.log_dir ?? settings.logDir ?? settings.log_directory ?? settings.logDirectory;
+  const logFormatSource = settings.log_format ?? settings.logFormat ?? settings.log_output_format ?? settings.logOutputFormat ?? env.AGENTV_CODEX_LOG_FORMAT;
+  const executable = resolveOptionalString(executableSource, env, `${target.name} codex executable`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? "codex";
+  const args = resolveOptionalStringArray(argsSource, env, `${target.name} codex args`);
+  const cwd = resolveOptionalString(cwdSource, env, `${target.name} codex cwd`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} codex timeout`);
+  const logDir = resolveOptionalString(logDirSource, env, `${target.name} codex log directory`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const logFormat = normalizeCodexLogFormat(logFormatSource);
+  return {
+    executable,
+    args,
+    cwd,
+    timeoutMs,
+    logDir,
+    logFormat
+  };
+}
+function normalizeCodexLogFormat(value) {
+  if (value === void 0 || value === null) {
+    return void 0;
+  }
+  if (typeof value !== "string") {
+    throw new Error("codex log format must be 'summary' or 'json'");
+  }
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "json" || normalized === "summary") {
+    return normalized;
+  }
+  throw new Error("codex log format must be 'summary' or 'json'");
+}
+function resolveMockConfig(target) {
+  const settings = target.settings ?? {};
+  const response = typeof settings.response === "string" ? settings.response : void 0;
+  return { response };
+}
+function resolveVSCodeConfig(target, env, insiders) {
+  const settings = target.settings ?? {};
+  const workspaceTemplateEnvVar = resolveOptionalLiteralString(settings.workspace_template ?? settings.workspaceTemplate);
+  const workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString(workspaceTemplateEnvVar, env, `${target.name} workspace template path`, {
+    allowLiteral: false,
+    optionalEnv: true
+  }) : void 0;
+  const commandSource = settings.vscode_cmd ?? settings.command;
+  const waitSource = settings.wait;
+  const dryRunSource = settings.dry_run ?? settings.dryRun;
+  const subagentRootSource = settings.subagent_root ?? settings.subagentRoot;
+  const defaultCommand = insiders ? "code-insiders" : "code";
+  const command = resolveOptionalLiteralString(commandSource) ?? defaultCommand;
+  return {
+    command,
+    waitForResponse: resolveOptionalBoolean(waitSource) ?? true,
+    dryRun: resolveOptionalBoolean(dryRunSource) ?? false,
+    subagentRoot: resolveOptionalString(subagentRootSource, env, `${target.name} subagent root`, {
+      allowLiteral: true,
+      optionalEnv: true
+    }),
+    workspaceTemplate
+  };
+}
+function resolveCliConfig(target, env) {
+  const settings = target.settings ?? {};
+  const commandTemplateSource = settings.command_template ?? settings.commandTemplate;
+  const filesFormat = resolveOptionalLiteralString(
+    settings.files_format ?? settings.filesFormat ?? settings.attachments_format ?? settings.attachmentsFormat
+  );
+  const cwd = resolveOptionalString(settings.cwd, env, `${target.name} working directory`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const envOverrides = resolveEnvOverrides(settings.env, env, target.name);
+  const timeoutMs = resolveTimeoutMs(settings.timeout_seconds ?? settings.timeoutSeconds, `${target.name} timeout`);
+  const healthcheck = resolveCliHealthcheck(settings.healthcheck, env, target.name);
+  const commandTemplate = resolveString(
+    commandTemplateSource,
+    env,
+    `${target.name} CLI command template`,
+    true
+  );
+  assertSupportedCliPlaceholders(commandTemplate, `${target.name} CLI command template`);
+  return {
+    commandTemplate,
+    filesFormat,
+    cwd,
+    env: envOverrides,
+    timeoutMs,
+    healthcheck
+  };
+}
+function resolveEnvOverrides(source, env, targetName) {
+  if (source === void 0 || source === null) {
+    return void 0;
+  }
+  if (typeof source !== "object" || Array.isArray(source)) {
+    throw new Error(`${targetName} env overrides must be an object map of strings`);
+  }
+  const entries = Object.entries(source);
+  const resolved = {};
+  for (const [key, value] of entries) {
+    if (typeof value !== "string") {
+      throw new Error(`${targetName} env override '${key}' must be a string`);
+    }
+    const resolvedValue = resolveString(value, env, `${targetName} env override '${key}'`);
+    resolved[key] = resolvedValue;
+  }
+  return Object.keys(resolved).length > 0 ? resolved : void 0;
+}
+function resolveTimeoutMs(source, description) {
+  const seconds = resolveOptionalNumber(source, `${description} (seconds)`);
+  if (seconds === void 0) {
+    return void 0;
+  }
+  if (seconds <= 0) {
+    throw new Error(`${description} must be greater than zero seconds`);
+  }
+  return Math.floor(seconds * 1e3);
+}
+function resolveCliHealthcheck(source, env, targetName) {
+  if (source === void 0 || source === null) {
+    return void 0;
+  }
+  if (typeof source !== "object" || Array.isArray(source)) {
+    throw new Error(`${targetName} healthcheck must be an object`);
+  }
+  const candidate = source;
+  const type = candidate.type;
+  const timeoutMs = resolveTimeoutMs(
+    candidate.timeout_seconds ?? candidate.timeoutSeconds,
+    `${targetName} healthcheck timeout`
+  );
+  if (type === "http") {
+    const url = resolveString(candidate.url, env, `${targetName} healthcheck URL`);
+    return {
+      type: "http",
+      url,
+      timeoutMs
+    };
+  }
+  if (type === "command") {
+    const commandTemplate = resolveString(
+      candidate.command_template ?? candidate.commandTemplate,
+      env,
+      `${targetName} healthcheck command template`,
+      true
+    );
+    assertSupportedCliPlaceholders(commandTemplate, `${targetName} healthcheck command template`);
+    const cwd = resolveOptionalString(candidate.cwd, env, `${targetName} healthcheck cwd`, {
+      allowLiteral: true,
+      optionalEnv: true
+    });
+    return {
+      type: "command",
+      commandTemplate,
+      timeoutMs,
+      cwd
+    };
+  }
+  throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
+}
+function assertSupportedCliPlaceholders(template, description) {
+  const placeholders = extractCliPlaceholders(template);
+  for (const placeholder of placeholders) {
+    if (!CLI_PLACEHOLDERS.has(placeholder)) {
+      throw new Error(
+        `${description} includes unsupported placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(", ")}`
+      );
+    }
+  }
+}
+function extractCliPlaceholders(template) {
+  const matches = template.matchAll(/\{([A-Z_]+)\}/g);
+  const results = [];
+  for (const match of matches) {
+    if (match[1]) {
+      results.push(match[1]);
+    }
+  }
+  return results;
+}
+function resolveString(source, env, description, allowLiteral = false) {
+  const value = resolveOptionalString(source, env, description, {
+    allowLiteral,
+    optionalEnv: false
+  });
+  if (value === void 0) {
+    throw new Error(`${description} is required`);
+  }
+  return value;
+}
+function resolveOptionalString(source, env, description, options) {
+  if (source === void 0 || source === null) {
+    return void 0;
+  }
+  if (typeof source !== "string") {
+    throw new Error(`${description} must be a string`);
+  }
+  const trimmed = source.trim();
+  if (trimmed.length === 0) {
+    return void 0;
+  }
+  const envVarMatch = trimmed.match(/^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i);
+  if (envVarMatch) {
+    const varName = envVarMatch[1];
+    const envValue = env[varName];
+    if (envValue !== void 0) {
+      if (envValue.trim().length === 0) {
+        throw new Error(`Environment variable '${varName}' for ${description} is empty`);
+      }
+      return envValue;
+    }
+    const optionalEnv = options?.optionalEnv ?? false;
+    if (optionalEnv) {
+      return void 0;
+    }
+    throw new Error(`Environment variable '${varName}' required for ${description} is not set`);
+  }
+  const allowLiteral = options?.allowLiteral ?? false;
+  if (!allowLiteral) {
+    throw new Error(`${description} must use \${{ VARIABLE_NAME }} syntax for environment variables or be marked as allowing literals`);
+  }
+  return trimmed;
+}
+function resolveOptionalLiteralString(source) {
+  if (source === void 0 || source === null) {
+    return void 0;
+  }
+  if (typeof source !== "string") {
+    throw new Error("expected string value");
+  }
+  const trimmed = source.trim();
+  return trimmed.length > 0 ? trimmed : void 0;
+}
+function resolveOptionalNumber(source, description) {
+  if (source === void 0 || source === null || source === "") {
+    return void 0;
+  }
+  if (typeof source === "number") {
+    return Number.isFinite(source) ? source : void 0;
+  }
+  if (typeof source === "string") {
+    const numeric = Number(source);
+    if (Number.isFinite(numeric)) {
+      return numeric;
+    }
+  }
+  throw new Error(`${description} must be a number`);
+}
+function resolveOptionalBoolean(source) {
+  if (source === void 0 || source === null || source === "") {
+    return void 0;
+  }
+  if (typeof source === "boolean") {
+    return source;
+  }
+  if (typeof source === "string") {
+    const lowered = source.trim().toLowerCase();
+    if (lowered === "true" || lowered === "1") {
+      return true;
+    }
+    if (lowered === "false" || lowered === "0") {
+      return false;
+    }
+  }
+  throw new Error("expected boolean value");
+}
+function resolveOptionalStringArray(source, env, description) {
+  if (source === void 0 || source === null) {
+    return void 0;
+  }
+  if (!Array.isArray(source)) {
+    throw new Error(`${description} must be an array of strings`);
+  }
+  if (source.length === 0) {
+    return void 0;
+  }
+  const resolved = [];
+  for (let i = 0; i < source.length; i++) {
+    const item = source[i];
+    if (typeof item !== "string") {
+      throw new Error(`${description}[${i}] must be a string`);
+    }
+    const trimmed = item.trim();
+    if (trimmed.length === 0) {
+      throw new Error(`${description}[${i}] cannot be empty`);
+    }
+    const envVarMatch = trimmed.match(/^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i);
+    if (envVarMatch) {
+      const varName = envVarMatch[1];
+      const envValue = env[varName];
+      if (envValue !== void 0) {
+        if (envValue.trim().length === 0) {
+          throw new Error(`Environment variable '${varName}' for ${description}[${i}] is empty`);
+        }
+        resolved.push(envValue);
+        continue;
+      }
+      throw new Error(`Environment variable '${varName}' for ${description}[${i}] is not set`);
+    }
+    resolved.push(trimmed);
+  }
+  return resolved.length > 0 ? resolved : void 0;
+}
+
+// src/evaluation/providers/types.ts
+var AGENT_PROVIDER_KINDS = [
+  "codex",
+  "vscode",
+  "vscode-insiders"
+];
+var KNOWN_PROVIDERS = [
+  "azure",
+  "anthropic",
+  "gemini",
+  "codex",
+  "cli",
+  "mock",
+  "vscode",
+  "vscode-insiders"
+];
+var PROVIDER_ALIASES = [
+  "azure-openai",
+  // alias for "azure"
+  "google",
+  // alias for "gemini"
+  "google-gemini",
+  // alias for "gemini"
+  "codex-cli",
+  // alias for "codex"
+  "openai",
+  // legacy/future support
+  "bedrock",
+  // legacy/future support
+  "vertex"
+  // legacy/future support
+];
+var TARGETS_SCHEMA_V2 = "agentv-targets-v2.1";
+function isAgentProvider(provider) {
+  return provider ? AGENT_PROVIDER_KINDS.includes(provider.kind) : false;
+}
+
+export {
+  fileExists,
+  normalizeLineEndings,
+  readTextFile,
+  findGitRoot,
+  buildDirectoryChain,
+  buildSearchRoots,
+  resolveFileReference,
+  CLI_PLACEHOLDERS,
+  resolveTargetDefinition,
+  KNOWN_PROVIDERS,
+  PROVIDER_ALIASES,
+  TARGETS_SCHEMA_V2,
+  isAgentProvider
+};
+//# sourceMappingURL=chunk-7XM7HYRS.js.map