agentv 5.3.0-next.1 → 5.3.2-next.1

package/dist/chunk-TEEXVJWM.js

@@ -0,0 +1,2 @@
+import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
+//# sourceMappingURL=chunk-TEEXVJWM.js.map

package/dist/{chunk-BHKQHG26.js → chunk-WIAJ7JAV.js}

@@ -6,7 +6,7 @@ import {
   toSnakeCaseDeep,
   writeArtifactsFromResults,
   writePerTestArtifacts
-} from "./chunk-6ZZCDZPD.js";
+} from "./chunk-6262OKXM.js";
 
 // src/commands/eval/artifact-writer.ts
 import path2 from "node:path";
@@ -18,13 +18,15 @@ import path from "node:path";
 import { stringify as stringifyYaml } from "yaml";
 var TEST_BUNDLE_DIRNAME = "test";
 var TASK_EVAL_FILENAME = "EVAL.yaml";
-var TASK_TARGETS_FILENAME = "targets.yaml";
+var TASK_PROVIDERS_FILENAME = "providers.yaml";
 var TASK_FILES_DIRNAME = "files";
 var TASK_GRADERS_DIRNAME = "graders";
 var INPUT_PROMPT = "{{ input }}";
 var BUNDLE_EVALS_DIRNAME = "evals";
 var BUNDLE_MANIFEST_FILENAME = "agentv_bundle.json";
-var BUNDLE_TARGETS_FILENAME = "targets.yaml";
+var BUNDLE_PROVIDERS_FILENAME = "providers.yaml";
+var BUNDLE_CONFIG_DIRNAME = ".agentv";
+var BUNDLE_CONFIG_FILENAME = "config.yaml";
 var BUNDLE_WORKSPACES_DIRNAME = "workspaces";
 var BUNDLE_SCRIPTS_DIRNAME = "scripts";
 var REDACTED_SOURCE_VALUE = "[redacted]";
@@ -44,17 +46,20 @@ var SKIPPED_DIR_NAMES = /* @__PURE__ */ new Set([
 var AUTHORING_TOP_LEVEL_TARGET_FIELDS = /* @__PURE__ */ new Set([
   "label",
   "provider",
+  "provider_spec",
+  "environment",
   "prompts",
   "transform",
   "delay",
   "env",
   "model",
+  "runtime",
+  "hooks",
   "use_target",
   "fallback_targets",
   "grader_target",
   "max_budget_usd",
   "workers",
-  "batch_requests",
   "subagent_mode_allowed",
   "max_retries",
   "retry_initial_delay_ms",
@@ -420,7 +425,7 @@ function targetReferenceNames(target) {
   }
   return references;
 }
-function selectTargetDefinitions(targetName, definitions) {
+function selectTargetDefinitions(providerLabel, definitions) {
   const byName = new Map(definitions.map((definition) => [definition.name, definition]));
   const selected = [];
   const seen = /* @__PURE__ */ new Set();
@@ -438,7 +443,7 @@ function selectTargetDefinitions(targetName, definitions) {
       visit(referenceName);
     }
   }
-  visit(targetName);
+  visit(providerLabel);
   return selected;
 }
 async function writeYamlFile(filePath, value) {
@@ -457,11 +462,62 @@ function bundledEvalFileName(evalFilePath) {
   const safeName = withoutKnownExtension.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
   return `${safeName || "bundle"}.eval.yaml`;
 }
-function uniqueTargetDefinitions(selections) {
+function uniqueTargetDefinitions(selections, tests = []) {
   const selected = [];
   const seen = /* @__PURE__ */ new Set();
+  function addDefinitions(names, definitions) {
+    for (const name of names) {
+      for (const definition of selectTargetDefinitions(name, definitions)) {
+        if (seen.has(definition.name)) {
+          continue;
+        }
+        seen.add(definition.name);
+        selected.push(definition);
+      }
+    }
+  }
+  const graderTargetNames = collectGraderTargetNames(tests);
   for (const selection of selections) {
-    for (const definition of selectTargetDefinitions(selection.targetName, selection.definitions)) {
+    addDefinitions([selection.providerLabel, ...graderTargetNames], selection.definitions);
+  }
+  return selected;
+}
+function collectGraderTargetNames(tests) {
+  const names = [];
+  const seen = /* @__PURE__ */ new Set();
+  function collect(value, key) {
+    if (key === "target" && typeof value === "string") {
+      const target = value.trim();
+      if (target.length > 0 && !target.includes("${{") && !seen.has(target)) {
+        seen.add(target);
+        names.push(target);
+      }
+      return;
+    }
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        collect(item);
+      }
+      return;
+    }
+    if (isRecord(value)) {
+      for (const [childKey, childValue] of Object.entries(value)) {
+        collect(childValue, childKey);
+      }
+    }
+  }
+  for (const test of tests) {
+    for (const grader of test.source?.graderDefinitions ?? []) {
+      collect(grader.definition);
+    }
+  }
+  return names;
+}
+function selectTaskBundleTargetDefinitions(providerLabel, definitions, tests) {
+  const selected = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const name of [providerLabel, ...collectGraderTargetNames(tests)]) {
+    for (const definition of selectTargetDefinitions(name, definitions)) {
       if (seen.has(definition.name)) {
         continue;
       }
@@ -471,18 +527,24 @@ function uniqueTargetDefinitions(selections) {
   }
   return selected;
 }
-function serializeTargetDefinition(definition) {
-  const target = { id: definition.name };
-  if (definition.id !== void 0) {
-    target.id = definition.id;
+function serializeTargetDefinition(definition, rewrites) {
+  const providerSpec = typeof definition.provider_spec === "string" && definition.provider_spec.trim().length > 0 ? definition.provider_spec.trim() : typeof definition.provider === "string" && definition.provider.trim().length > 0 ? definition.provider.trim() : typeof definition.id === "string" && definition.id.trim().length > 0 ? definition.id.trim() : definition.name;
+  const provider = { id: providerSpec };
+  if (definition.name !== providerSpec) {
+    provider.label = definition.name;
   }
   const config = {};
   for (const [key, value] of Object.entries(definition)) {
-    if (value === void 0 || key === "name" || key === "id" || key === "label" || key === "config") {
+    if (value === void 0 || key === "name" || key === "id" || key === "provider" || key === "provider_spec" || key === "label" || key === "config") {
       continue;
     }
-    if (AUTHORING_TOP_LEVEL_TARGET_FIELDS.has(key)) {
-      target[key] = value;
+    if (key === "environment" && definition.environment) {
+      provider.environment = serializeEnvironment(
+        definition.environment,
+        rewrites
+      );
+    } else if (AUTHORING_TOP_LEVEL_TARGET_FIELDS.has(key)) {
+      provider[key] = value;
     } else {
       config[key] = value;
     }
@@ -495,22 +557,22 @@ function serializeTargetDefinition(definition) {
     }
   }
   if (Object.keys(config).length > 0) {
-    target.config = config;
+    provider.config = config;
   }
-  return target;
+  return provider;
 }
-function serializeTargetDefinitions(definitions) {
-  return definitions.map((definition) => serializeTargetDefinition(definition));
+function serializeTargetDefinitions(definitions, rewrites = /* @__PURE__ */ new Map()) {
+  return definitions.map((definition) => serializeTargetDefinition(definition, rewrites));
 }
 function uniqueTargetNames(selections) {
   const names = [];
   const seen = /* @__PURE__ */ new Set();
   for (const selection of selections) {
-    if (seen.has(selection.targetName)) {
+    if (seen.has(selection.providerLabel)) {
       continue;
     }
-    seen.add(selection.targetName);
-    names.push(selection.targetName);
+    seen.add(selection.providerLabel);
+    names.push(selection.providerLabel);
   }
   return names;
 }
@@ -710,6 +772,45 @@ async function collectEnvironmentReferences(tests, evalFileDir) {
   }
   return references;
 }
+async function collectProviderEnvironmentReferences(definitions, evalFileDir) {
+  const references = [];
+  for (const definition of definitions) {
+    const environment = definition.environment;
+    if (!environment) {
+      continue;
+    }
+    const label = `provider "${definition.name}"`;
+    if (environment.type === "host") {
+      references.push({
+        kind: "environment_workdir",
+        displayPath: environment.workdir,
+        resolvedPath: environment.workdir,
+        location: `${label}.environment.workdir`
+      });
+    }
+    const command = environment.setup?.command;
+    if (!command) {
+      continue;
+    }
+    const baseDir = environmentBaseDir(environment, evalFileDir);
+    for (const arg of command) {
+      const reference = await maybeWorkspaceHookCommandReference({
+        arg,
+        baseDir,
+        testId: definition.name,
+        hookName: "setup"
+      });
+      if (reference) {
+        references.push({
+          ...reference,
+          kind: "environment_setup_command",
+          location: `${label}.environment.setup.command`
+        });
+      }
+    }
+  }
+  return references;
+}
 async function collectWorkspaceReferences(tests, evalFileDir) {
   const references = [];
   const errors = [];
@@ -793,7 +894,7 @@ function bundleManifest(options) {
     created_at: options.createdAt,
     source_eval: options.evalFilePath,
     eval_path: relative(options.evalPath),
-    targets_path: relative(options.targetsPath),
+    providers_path: relative(options.providersPath),
     test_count: options.tests.length,
     targets: options.targetNames,
     ...hasCopiedBucket(options.copiedReferences, "files") ? { files_path: "evals/files" } : {},
@@ -806,27 +907,41 @@ async function materializeTaskBundle(options) {
   if (!options.test.source) {
     return void 0;
   }
-  const targetDefinitions = selectTargetDefinitions(options.targetName, options.targetDefinitions);
-  if (targetDefinitions.length === 0) {
+  const providerDefinitions = selectTaskBundleTargetDefinitions(
+    options.providerLabel,
+    options.providerDefinitions,
+    [options.test]
+  );
+  if (providerDefinitions.length === 0) {
     return void 0;
   }
   const testDir = path.join(options.outputDir, TEST_BUNDLE_DIRNAME);
   await mkdir(testDir, { recursive: true });
-  const copiedReferences = await copyReferences(options.test.source.references, testDir, options);
+  const providerEnvironmentReferences = await collectProviderEnvironmentReferences(
+    providerDefinitions,
+    path.dirname(options.test.source.evalFileAbsolutePath ?? options.test.source.evalFilePath)
+  );
+  const copiedReferences = await copyReferences(
+    [...options.test.source.references, ...providerEnvironmentReferences],
+    testDir,
+    options
+  );
   const rewrites = buildPathRewrites(copiedReferences);
   const evalCase = buildEvalCase(options.test, rewrites);
   const evalPath = path.join(testDir, TASK_EVAL_FILENAME);
-  const targetsPath = path.join(testDir, TASK_TARGETS_FILENAME);
+  const providersPath = path.join(testDir, TASK_PROVIDERS_FILENAME);
   await writeYamlFile(evalPath, {
-    target: options.targetName,
+    providers: [options.providerLabel],
     prompts: [INPUT_PROMPT],
     tests: [evalCase]
   });
-  await writeYamlFile(targetsPath, { targets: serializeTargetDefinitions(targetDefinitions) });
+  await writeYamlFile(providersPath, {
+    providers: serializeTargetDefinitions(providerDefinitions, rewrites)
+  });
   return {
     testDir,
     evalPath,
-    targetsPath,
+    providersPath,
     ...hasCopiedBucket(copiedReferences, "files") ? { filesPath: path.join(testDir, TASK_FILES_DIRNAME) } : {},
     ...hasCopiedBucket(copiedReferences, "graders") ? { gradersPath: path.join(testDir, TASK_GRADERS_DIRNAME) } : {}
   };
@@ -835,7 +950,7 @@ async function materializeEvalBundle(options) {
   if (options.tests.length === 0) {
     throw new Error("Cannot bundle eval with no runnable tests.");
   }
-  if (options.targetSelections.length === 0) {
+  if (options.providerSelections.length === 0) {
     throw new Error("Cannot bundle eval without a selected target.");
   }
   const missingSource = options.tests.find((test) => !test.source);
@@ -848,12 +963,18 @@ async function materializeEvalBundle(options) {
   const evalsDir = path.join(outputDir, BUNDLE_EVALS_DIRNAME);
   await mkdir(evalsDir, { recursive: true });
   const evalFileDir = path.dirname(path.resolve(options.evalFilePath));
+  const providerDefinitions = uniqueTargetDefinitions(options.providerSelections, options.tests);
   const workspaceReferences = await collectWorkspaceReferences(options.tests, evalFileDir);
   const environmentReferences = await collectEnvironmentReferences(options.tests, evalFileDir);
+  const providerEnvironmentReferences = await collectProviderEnvironmentReferences(
+    providerDefinitions,
+    evalFileDir
+  );
   const references = [
     ...options.tests.flatMap((test) => test.source?.references ?? []),
     ...collectExpectedOutputReferences(options.tests),
     ...environmentReferences,
+    ...providerEnvironmentReferences,
     ...workspaceReferences.references
   ];
   const { copied, failures } = await copyReferencesStrict(references, evalsDir, options);
@@ -863,24 +984,25 @@ async function materializeEvalBundle(options) {
 ${errors.map((error) => `- ${error}`).join("\n")}`);
   }
   const rewrites = buildPathRewrites(copied);
-  const targetNames = uniqueTargetNames(options.targetSelections);
+  const targetNames = uniqueTargetNames(options.providerSelections);
   const evalPath = path.join(evalsDir, bundledEvalFileName(options.evalFilePath));
-  const targetsPath = path.join(outputDir, BUNDLE_TARGETS_FILENAME);
+  const providersPath = path.join(outputDir, BUNDLE_PROVIDERS_FILENAME);
+  const configPath = path.join(outputDir, BUNDLE_CONFIG_DIRNAME, BUNDLE_CONFIG_FILENAME);
   const manifestPath = path.join(outputDir, BUNDLE_MANIFEST_FILENAME);
-  const runtime = options.runtime ?? (targetNames.length === 1 ? { target: targetNames[0] } : void 0);
+  const runtime = options.runtime ?? (targetNames.length > 0 ? { providers: targetNames } : void 0);
   await writeYamlFile(evalPath, {
     ...runtime ?? {},
     prompts: [INPUT_PROMPT],
     tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites))
   });
-  await writeYamlFile(targetsPath, {
-    targets: serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections))
-  });
+  await writeYamlFile(providersPath, serializeTargetDefinitions(providerDefinitions, rewrites));
+  await mkdir(path.dirname(configPath), { recursive: true });
+  await writeYamlFile(configPath, { providers: `file://../${BUNDLE_PROVIDERS_FILENAME}` });
   const manifest = bundleManifest({
     outputDir,
     evalFilePath: path.resolve(options.evalFilePath),
     evalPath,
-    targetsPath,
+    providersPath,
     copiedReferences: copied,
     tests: options.tests,
     targetNames,
@@ -892,7 +1014,7 @@ ${errors.map((error) => `- ${error}`).join("\n")}`);
     bundleDir: outputDir,
     evalsDir,
     evalPath,
-    targetsPath,
+    providersPath,
     manifestPath,
     ...hasCopiedBucket(copied, "files") ? { filesPath: path.join(evalsDir, TASK_FILES_DIRNAME) } : {},
     ...hasCopiedBucket(copied, "graders") ? { gradersPath: path.join(evalsDir, TASK_GRADERS_DIRNAME) } : {},
@@ -912,7 +1034,7 @@ function buildTaskBundleIndexFields(outputDir, taskBundle) {
   return {
     test_dir: toRelativeArtifactPath(outputDir, taskBundle.testDir),
     eval_path: toRelativeArtifactPath(outputDir, taskBundle.evalPath),
-    targets_path: toRelativeArtifactPath(outputDir, taskBundle.targetsPath),
+    providers_path: toRelativeArtifactPath(outputDir, taskBundle.providersPath),
     ...taskBundle.filesPath ? { files_path: toRelativeArtifactPath(outputDir, taskBundle.filesPath) } : {},
     ...taskBundle.gradersPath ? { graders_path: toRelativeArtifactPath(outputDir, taskBundle.gradersPath) } : {}
   };
@@ -928,56 +1050,56 @@ function buildResultIndexArtifact2(result, taskBundle) {
   const extraIndexFields = taskBundle ? {
     test_dir: path2.posix.join(artifactSubdir, "test"),
     eval_path: path2.posix.join(artifactSubdir, "test", "EVAL.yaml"),
-    targets_path: path2.posix.join(artifactSubdir, "test", "targets.yaml"),
+    providers_path: path2.posix.join(artifactSubdir, "test", "providers.yaml"),
     ...taskBundle.filesPath ? { files_path: path2.posix.join(artifactSubdir, "test", "files") } : {},
     ...taskBundle.gradersPath ? { graders_path: path2.posix.join(artifactSubdir, "test", "graders") } : {}
   } : void 0;
   return buildResultIndexArtifact(result, extraIndexFields);
 }
-function targetSelectionKey(evalFileAbsolutePath, targetName) {
-  return `${evalFileAbsolutePath ? path2.resolve(evalFileAbsolutePath) : ""}::${targetName}`;
+function providerSelectionKey(evalFileAbsolutePath, providerLabel) {
+  return `${evalFileAbsolutePath ? path2.resolve(evalFileAbsolutePath) : ""}::${providerLabel}`;
 }
-function buildTargetSelectionMap(selections) {
-  const targets = /* @__PURE__ */ new Map();
+function buildProviderSelectionMap(selections) {
+  const providers = /* @__PURE__ */ new Map();
   for (const selection of selections ?? []) {
-    targets.set(
-      targetSelectionKey(selection.evalFileAbsolutePath, selection.targetName),
+    providers.set(
+      providerSelectionKey(selection.evalFileAbsolutePath, selection.providerLabel),
       selection
     );
-    if (selection.resolvedTargetName) {
-      targets.set(
-        targetSelectionKey(selection.evalFileAbsolutePath, selection.resolvedTargetName),
+    if (selection.resolvedProviderName) {
+      providers.set(
+        providerSelectionKey(selection.evalFileAbsolutePath, selection.resolvedProviderName),
         selection
       );
     }
     if (!selection.evalFileAbsolutePath) {
-      targets.set(targetSelectionKey(void 0, selection.targetName), selection);
-      if (selection.resolvedTargetName) {
-        targets.set(targetSelectionKey(void 0, selection.resolvedTargetName), selection);
+      providers.set(providerSelectionKey(void 0, selection.providerLabel), selection);
+      if (selection.resolvedProviderName) {
+        providers.set(providerSelectionKey(void 0, selection.resolvedProviderName), selection);
       }
     }
   }
-  return targets;
+  return providers;
 }
-function findTargetSelection(result, test, targets) {
-  const targetName = result.target ?? "unknown";
+function findProviderSelection(result, test, providers) {
+  const providerLabel = result.target ?? "unknown";
   const evalFileAbsolutePath = test?.source?.evalFileAbsolutePath;
-  return targets.get(targetSelectionKey(evalFileAbsolutePath, targetName)) ?? targets.get(targetSelectionKey(void 0, targetName));
+  return providers.get(providerSelectionKey(evalFileAbsolutePath, providerLabel)) ?? providers.get(providerSelectionKey(void 0, providerLabel));
 }
 function createTaskBundleArtifactsWriter(options) {
-  const targetSelections = buildTargetSelectionMap(options?.taskBundleTargets);
-  if (targetSelections.size === 0) {
+  const providerSelections = buildProviderSelectionMap(options?.taskBundleTargets);
+  if (providerSelections.size === 0) {
     return void 0;
   }
   return async ({ outputDir, result, sourceTest, testDir }) => {
-    const targetSelection = findTargetSelection(result, sourceTest, targetSelections);
-    if (!sourceTest || !targetSelection) {
+    const providerSelection = findProviderSelection(result, sourceTest, providerSelections);
+    if (!sourceTest || !providerSelection) {
       return void 0;
     }
     const taskBundle = await materializeTaskBundle({
       test: sourceTest,
-      targetName: targetSelection.targetName,
-      targetDefinitions: targetSelection.definitions,
+      providerLabel: providerSelection.providerLabel,
+      providerDefinitions: providerSelection.definitions,
       outputDir: testDir,
       cwd: options?.cwd,
       repoRoot: options?.repoRoot
@@ -1020,4 +1142,4 @@ export {
   writePerTestArtifacts2 as writePerTestArtifacts,
   writeArtifactsFromResults2 as writeArtifactsFromResults
 };
-//# sourceMappingURL=chunk-BHKQHG26.js.map
+//# sourceMappingURL=chunk-WIAJ7JAV.js.map