agentv 4.41.1 → 4.41.4-next.1

package/dist/{chunk-A4J456KS.js → chunk-VBEWRK44.js}

@@ -5,7 +5,7 @@ import {
   parseYamlValue,
   writeArtifactsFromResults,
   writePerTestArtifacts
-} from "./chunk-ENHX2CCS.js";
+} from "./chunk-QCF3LUI7.js";
 
 // src/commands/eval/artifact-writer.ts
 import path2 from "node:path";
@@ -47,6 +47,11 @@ var TASK_EVAL_FILENAME = "EVAL.yaml";
 var TASK_TARGETS_FILENAME = "targets.yaml";
 var TASK_FILES_DIRNAME = "files";
 var TASK_GRADERS_DIRNAME = "graders";
+var BUNDLE_EVALS_DIRNAME = "evals";
+var BUNDLE_MANIFEST_FILENAME = "agentv_bundle.json";
+var BUNDLE_TARGETS_FILENAME = "targets.yaml";
+var BUNDLE_WORKSPACES_DIRNAME = "workspaces";
+var BUNDLE_SCRIPTS_DIRNAME = "scripts";
 var REDACTED_SOURCE_VALUE = "[redacted]";
 var SECRET_KEY_PATTERN = /(?:api[_-]?key|authorization|bearer|credential|password|private[_-]?key|secret|token)/i;
 var SOURCE_SECRET_LINE_PATTERN = /^(\s*[\w.-]*(?:api[_-]?key|authorization|bearer|credential|password|private[_-]?key|secret|token)[\w.-]*\s*:\s*).+$/gim;
@@ -96,7 +101,16 @@ function relativeReferencePath(reference, options) {
   return safeRelativePath(reference.displayPath);
 }
 function referenceBucket(reference) {
-  return reference.kind === "input_file" ? TASK_FILES_DIRNAME : TASK_GRADERS_DIRNAME;
+  if (reference.kind === "input_file" || reference.kind === "expected_output_file") {
+    return TASK_FILES_DIRNAME;
+  }
+  if (reference.kind === "workspace_template") {
+    return BUNDLE_WORKSPACES_DIRNAME;
+  }
+  if (reference.kind === "workspace_hook_command") {
+    return BUNDLE_SCRIPTS_DIRNAME;
+  }
+  return TASK_GRADERS_DIRNAME;
 }
 function isLikelyBinary(buffer) {
   return buffer.subarray(0, Math.min(buffer.length, 8e3)).includes(0);
@@ -195,13 +209,23 @@ function shouldCopyDirectory(reference) {
   }
   return !path.isAbsolute(reference.displayPath);
 }
-async function copyReference(reference, taskDir, options) {
+async function copyReferenceWithFailure(reference, taskDir, options) {
   if (!reference.resolvedPath) {
-    return void 0;
+    return {
+      failure: {
+        reference,
+        reason: `${reference.location ?? reference.kind} has no resolved path: ${reference.displayPath}`
+      }
+    };
   }
   const relPath = relativeReferencePath(reference, options);
   if (!relPath) {
-    return void 0;
+    return {
+      failure: {
+        reference,
+        reason: `${reference.location ?? reference.kind} could not be assigned a portable path: ${reference.displayPath}`
+      }
+    };
   }
   const bucket = referenceBucket(reference);
   const localPath = `${bucket}/${relPath}`;
@@ -209,23 +233,51 @@ async function copyReference(reference, taskDir, options) {
   const sourcePath = path.resolve(reference.resolvedPath);
   const sourceStat = await stat(sourcePath).catch(() => void 0);
   if (!sourceStat) {
-    return void 0;
+    return {
+      failure: {
+        reference,
+        reason: `${reference.location ?? reference.kind} not found: ${reference.displayPath} (resolved to ${sourcePath})`
+      }
+    };
   }
   if (sourceStat.isDirectory()) {
     if (!shouldCopyDirectory(reference)) {
-      return void 0;
+      return {
+        failure: {
+          reference,
+          reason: `${reference.location ?? reference.kind} uses an absolute directory that is not safe to copy automatically: ${reference.displayPath}`
+        }
+      };
     }
     if (!await copyDirectory(sourcePath, destinationPath)) {
-      return void 0;
+      return {
+        failure: {
+          reference,
+          reason: `${reference.location ?? reference.kind} directory contained no bundleable files: ${reference.displayPath}`
+        }
+      };
     }
   } else if (sourceStat.isFile()) {
     if (!await copyFileRedactingText(sourcePath, destinationPath)) {
-      return void 0;
+      return {
+        failure: {
+          reference,
+          reason: `${reference.location ?? reference.kind} could not be copied: ${reference.displayPath}`
+        }
+      };
     }
   } else {
-    return void 0;
+    return {
+      failure: {
+        reference,
+        reason: `${reference.location ?? reference.kind} is not a regular file or directory: ${reference.displayPath}`
+      }
+    };
   }
-  return { reference, localPath: localPath.split(path.sep).join("/"), destinationPath };
+  return { copied: { reference, localPath: localPath.split(path.sep).join("/"), destinationPath } };
+}
+async function copyReference(reference, taskDir, options) {
+  return (await copyReferenceWithFailure(reference, taskDir, options)).copied;
 }
 async function copyReferences(references, taskDir, options) {
   const copied = [];
@@ -240,6 +292,31 @@ async function copyReferences(references, taskDir, options) {
   }
   return copied;
 }
+async function copyReferencesStrict(references, taskDir, options) {
+  const copied = [];
+  const failures = [];
+  const seenDestinations = /* @__PURE__ */ new Set();
+  const seenFailures = /* @__PURE__ */ new Set();
+  for (const reference of references) {
+    const result = await copyReferenceWithFailure(reference, taskDir, options);
+    if (result.copied) {
+      if (seenDestinations.has(result.copied.destinationPath)) {
+        continue;
+      }
+      seenDestinations.add(result.copied.destinationPath);
+      copied.push(result.copied);
+      continue;
+    }
+    if (result.failure) {
+      const key = `${result.failure.reference.kind}:${result.failure.reference.displayPath}:${result.failure.reason}`;
+      if (!seenFailures.has(key)) {
+        seenFailures.add(key);
+        failures.push(result.failure);
+      }
+    }
+  }
+  return { copied, failures };
+}
 function addRewrite(rewrites, from, to) {
   if (!from || from.trim().length === 0) {
     return;
@@ -354,6 +431,279 @@ async function writeYamlFile(filePath, value) {
 function hasCopiedBucket(copied, bucket) {
   return copied.some((entry) => entry.localPath.startsWith(`${bucket}/`));
 }
+function bundledEvalFileName(evalFilePath) {
+  const basename = path.basename(evalFilePath);
+  const withoutKnownExtension = basename.replace(/\.eval\.[cm]?ts$/i, "").replace(/\.eval\.ya?ml$/i, "").replace(/\.[cm]?ts$/i, "").replace(/\.ya?ml$/i, "").replace(/\.jsonl?$/i, "");
+  const safeName = withoutKnownExtension.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+  return `${safeName || "bundle"}.eval.yaml`;
+}
+function uniqueTargetDefinitions(selections) {
+  const selected = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const selection of selections) {
+    for (const definition of selectTargetDefinitions(selection.targetName, selection.definitions)) {
+      if (seen.has(definition.name)) {
+        continue;
+      }
+      seen.add(definition.name);
+      selected.push(definition);
+    }
+  }
+  return selected;
+}
+function uniqueTargetNames(selections) {
+  const names = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const selection of selections) {
+    if (seen.has(selection.targetName)) {
+      continue;
+    }
+    seen.add(selection.targetName);
+    names.push(selection.targetName);
+  }
+  return names;
+}
+function rewritePathString(value, rewrites) {
+  const direct = rewrites.get(value);
+  if (direct) {
+    return direct.startsWith("file://") ? direct.slice("file://".length) : direct;
+  }
+  const fileUrl = rewrites.get(`file://${value}`);
+  if (fileUrl) {
+    return fileUrl.startsWith("file://") ? fileUrl.slice("file://".length) : fileUrl;
+  }
+  return value;
+}
+function serializeContentValue(value, rewrites) {
+  if (typeof value === "string") {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => serializeContentItem(item, rewrites));
+  }
+  if (isRecord(value)) {
+    return rewritePathsDeep(value, rewrites);
+  }
+  return value;
+}
+function serializeContentItem(value, rewrites) {
+  if (typeof value === "string") {
+    return value;
+  }
+  if (!isRecord(value)) {
+    return value;
+  }
+  const type = value.type;
+  if (type === "text") {
+    const text = typeof value.value === "string" ? value.value : value.text;
+    return { type: "text", value: typeof text === "string" ? text : "" };
+  }
+  if (type === "file") {
+    const source = typeof value.value === "string" ? value.value : typeof value.path === "string" ? value.path : typeof value.resolvedPath === "string" ? value.resolvedPath : void 0;
+    return source ? { type: "file", value: rewritePathString(source, rewrites) } : { type: "file", value: "" };
+  }
+  if (type === "image") {
+    const source = typeof value.value === "string" ? value.value : typeof value.path === "string" ? value.path : typeof value.resolvedPath === "string" ? value.resolvedPath : void 0;
+    if (source) {
+      return { type: "image", value: rewritePathString(source, rewrites) };
+    }
+  }
+  return rewritePathsDeep(value, rewrites);
+}
+function serializeMessage(message, rewrites) {
+  return {
+    role: message.role,
+    content: serializeContentValue(message.content, rewrites)
+  };
+}
+function serializeExpectedMessage(message, rewrites) {
+  const serialized = {};
+  for (const [key, value] of Object.entries(message)) {
+    serialized[key] = key === "content" ? serializeContentValue(value, rewrites) : value;
+  }
+  return rewritePathsDeep(serialized, rewrites);
+}
+function serializeWorkspace(workspace, rewrites) {
+  const {
+    workspaceFileDir: _workspaceFileDir,
+    path: _path,
+    mode,
+    ...portableWorkspace
+  } = workspace;
+  const withoutStaticMode = mode === "static" ? portableWorkspace : { ...portableWorkspace, ...mode ? { mode } : {} };
+  return rewritePathsDeep(withoutStaticMode, rewrites);
+}
+function buildPortableEvalCase(test, rewrites) {
+  const testCase = buildEvalCase(test, rewrites);
+  testCase.id = test.id;
+  testCase.input = test.input.map((message) => serializeMessage(message, rewrites));
+  if (test.criteria.trim().length > 0) {
+    testCase.criteria = test.criteria;
+  }
+  if (test.expected_output.length > 0) {
+    testCase.expected_output = test.expected_output.map(
+      (message) => serializeExpectedMessage(message, rewrites)
+    );
+  }
+  if (test.workspace) {
+    testCase.workspace = serializeWorkspace(test.workspace, rewrites);
+  }
+  if (test.metadata && Object.keys(test.metadata).length > 0) {
+    testCase.metadata = rewritePathsDeep(test.metadata, rewrites);
+  }
+  if (test.conversation_id) {
+    testCase.conversation_id = test.conversation_id;
+  }
+  if (test.targets && test.targets.length > 0) {
+    const existingExecution = isRecord(testCase.execution) ? testCase.execution : {};
+    testCase.execution = { ...existingExecution, targets: test.targets };
+  }
+  if (test.threshold !== void 0) {
+    const existingExecution = isRecord(testCase.execution) ? testCase.execution : {};
+    testCase.execution = { ...existingExecution, threshold: test.threshold };
+  }
+  if (test.mode) {
+    testCase.mode = test.mode;
+  }
+  if (test.turns && test.turns.length > 0) {
+    testCase.turns = rewritePathsDeep(test.turns, rewrites);
+  }
+  if (test.aggregation) {
+    testCase.aggregation = test.aggregation;
+  }
+  if (test.on_turn_failure) {
+    testCase.on_turn_failure = test.on_turn_failure;
+  }
+  if (test.window_size !== void 0) {
+    testCase.window_size = test.window_size;
+  }
+  if (test.depends_on && test.depends_on.length > 0) {
+    testCase.depends_on = test.depends_on;
+  }
+  if (test.on_dependency_failure) {
+    testCase.on_dependency_failure = test.on_dependency_failure;
+  }
+  return testCase;
+}
+function isLikelyCommandPath(value) {
+  if (value.trim().length === 0 || value.startsWith("-") || value.includes("${{")) {
+    return false;
+  }
+  return value.startsWith(".") || value.includes("/") || value.includes("\\") || path.extname(value).length > 0;
+}
+async function maybeWorkspaceHookCommandReference(options) {
+  if (!isLikelyCommandPath(options.arg)) {
+    return void 0;
+  }
+  const resolvedPath = path.isAbsolute(options.arg) ? path.resolve(options.arg) : path.resolve(options.baseDir, options.arg);
+  const sourceStat = await stat(resolvedPath).catch(() => void 0);
+  if (!sourceStat?.isFile()) {
+    return void 0;
+  }
+  return {
+    kind: "workspace_hook_command",
+    displayPath: options.arg,
+    resolvedPath,
+    location: `workspace.hooks.${options.hookName}.command for test "${options.testId}"`
+  };
+}
+async function collectWorkspaceReferences(tests, evalFileDir) {
+  const references = [];
+  const errors = [];
+  for (const test of tests) {
+    const workspace = test.workspace;
+    if (!workspace) {
+      continue;
+    }
+    if (workspace.path || workspace.mode === "static") {
+      errors.push(
+        `workspace.path for test "${test.id}" cannot be bundled because it points at an existing static workspace. Use workspace.template, workspace.repos, or workspace.hooks for portable bundles.`
+      );
+    }
+    if (workspace.template) {
+      references.push({
+        kind: "workspace_template",
+        displayPath: workspace.template,
+        resolvedPath: workspace.template,
+        location: `workspace.template for test "${test.id}"`
+      });
+    }
+    const hooks = workspace.hooks;
+    if (!hooks) {
+      continue;
+    }
+    for (const hookName of ["before_all", "before_each", "after_each", "after_all"]) {
+      const hook = hooks[hookName];
+      const command = hook?.command ?? hook?.script;
+      if (!command || command.length === 0) {
+        continue;
+      }
+      if (hook?.cwd) {
+        errors.push(
+          `workspace.hooks.${hookName}.cwd for test "${test.id}" cannot be bundled safely yet: ${hook.cwd}`
+        );
+        continue;
+      }
+      const baseDir = workspace.workspaceFileDir ?? evalFileDir;
+      for (const arg of command) {
+        const reference = await maybeWorkspaceHookCommandReference({
+          arg,
+          baseDir,
+          testId: test.id,
+          hookName
+        });
+        if (reference) {
+          references.push(reference);
+        }
+      }
+    }
+  }
+  return { references, errors };
+}
+function collectExpectedOutputReferences(tests) {
+  const references = [];
+  for (const test of tests) {
+    for (const message of test.expected_output) {
+      const content = message.content;
+      if (!Array.isArray(content)) {
+        continue;
+      }
+      for (const segment of content) {
+        if (!isRecord(segment) || segment.type !== "file") {
+          continue;
+        }
+        const resolvedPath = typeof segment.resolvedPath === "string" ? path.resolve(segment.resolvedPath) : void 0;
+        const displayPath = typeof segment.path === "string" ? segment.path : typeof segment.value === "string" ? segment.value : resolvedPath;
+        if (!displayPath) {
+          continue;
+        }
+        references.push({
+          kind: "expected_output_file",
+          displayPath,
+          ...resolvedPath ? { resolvedPath } : {},
+          location: `expected_output file for test "${test.id}"`
+        });
+      }
+    }
+  }
+  return references;
+}
+function bundleManifest(options) {
+  const relative = (filePath) => path.relative(options.outputDir, filePath).split(path.sep).join("/");
+  return {
+    schema_version: 1,
+    created_at: options.createdAt,
+    source_eval: options.evalFilePath,
+    eval_path: relative(options.evalPath),
+    targets_path: relative(options.targetsPath),
+    test_count: options.tests.length,
+    targets: options.targetNames,
+    ...hasCopiedBucket(options.copiedReferences, "files") ? { files_path: "evals/files" } : {},
+    ...hasCopiedBucket(options.copiedReferences, "graders") ? { graders_path: "evals/graders" } : {},
+    ...hasCopiedBucket(options.copiedReferences, "workspaces") ? { workspaces_path: "evals/workspaces" } : {},
+    ...hasCopiedBucket(options.copiedReferences, "scripts") ? { scripts_path: "evals/scripts" } : {}
+  };
+}
 async function materializeTaskBundle(options) {
   if (!options.test.source) {
     return void 0;
@@ -382,6 +732,72 @@ async function materializeTaskBundle(options) {
     ...hasCopiedBucket(copiedReferences, "graders") ? { gradersPath: path.join(taskDir, TASK_GRADERS_DIRNAME) } : {}
   };
 }
+async function materializeEvalBundle(options) {
+  if (options.tests.length === 0) {
+    throw new Error("Cannot bundle eval with no runnable tests.");
+  }
+  if (options.targetSelections.length === 0) {
+    throw new Error("Cannot bundle eval without a selected target.");
+  }
+  const missingSource = options.tests.find((test) => !test.source);
+  if (missingSource) {
+    throw new Error(
+      `Cannot bundle test "${missingSource.id}" because it has no source metadata. YAML evals are supported; programmatic evals with inline functions are not portable yet.`
+    );
+  }
+  const outputDir = path.resolve(options.outputDir);
+  const evalsDir = path.join(outputDir, BUNDLE_EVALS_DIRNAME);
+  await mkdir(evalsDir, { recursive: true });
+  const evalFileDir = path.dirname(path.resolve(options.evalFilePath));
+  const workspaceReferences = await collectWorkspaceReferences(options.tests, evalFileDir);
+  const references = [
+    ...options.tests.flatMap((test) => test.source?.references ?? []),
+    ...collectExpectedOutputReferences(options.tests),
+    ...workspaceReferences.references
+  ];
+  const { copied, failures } = await copyReferencesStrict(references, evalsDir, options);
+  const errors = [...workspaceReferences.errors, ...failures.map((failure) => failure.reason)];
+  if (errors.length > 0) {
+    throw new Error(`Cannot bundle eval:
+${errors.map((error) => `- ${error}`).join("\n")}`);
+  }
+  const rewrites = buildPathRewrites(copied);
+  const targetNames = uniqueTargetNames(options.targetSelections);
+  const evalPath = path.join(evalsDir, bundledEvalFileName(options.evalFilePath));
+  const targetsPath = path.join(outputDir, BUNDLE_TARGETS_FILENAME);
+  const manifestPath = path.join(outputDir, BUNDLE_MANIFEST_FILENAME);
+  const execution = options.execution ?? (targetNames.length === 1 ? { target: targetNames[0] } : { targets: targetNames });
+  await writeYamlFile(evalPath, {
+    execution,
+    tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites))
+  });
+  await writeYamlFile(targetsPath, {
+    targets: uniqueTargetDefinitions(options.targetSelections)
+  });
+  const manifest = bundleManifest({
+    outputDir,
+    evalFilePath: path.resolve(options.evalFilePath),
+    evalPath,
+    targetsPath,
+    copiedReferences: copied,
+    tests: options.tests,
+    targetNames,
+    createdAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
+  });
+  await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}
+`, "utf8");
+  return {
+    bundleDir: outputDir,
+    evalsDir,
+    evalPath,
+    targetsPath,
+    manifestPath,
+    ...hasCopiedBucket(copied, "files") ? { filesPath: path.join(evalsDir, TASK_FILES_DIRNAME) } : {},
+    ...hasCopiedBucket(copied, "graders") ? { gradersPath: path.join(evalsDir, TASK_GRADERS_DIRNAME) } : {},
+    ...hasCopiedBucket(copied, "workspaces") ? { workspacesPath: path.join(evalsDir, BUNDLE_WORKSPACES_DIRNAME) } : {},
+    ...hasCopiedBucket(copied, "scripts") ? { scriptsPath: path.join(evalsDir, BUNDLE_SCRIPTS_DIRNAME) } : {}
+  };
+}
 
 // src/commands/eval/artifact-writer.ts
 function toRelativeArtifactPath(outputDir, filePath) {
@@ -486,9 +902,10 @@ async function writeArtifactsFromResults2(results, outputDir, options) {
 
 export {
   toSnakeCaseDeep,
+  materializeEvalBundle,
   buildIndexArtifactEntry2 as buildIndexArtifactEntry,
   buildResultIndexArtifact2 as buildResultIndexArtifact,
   writePerTestArtifacts2 as writePerTestArtifacts,
   writeArtifactsFromResults2 as writeArtifactsFromResults
 };
-//# sourceMappingURL=chunk-A4J456KS.js.map
+//# sourceMappingURL=chunk-VBEWRK44.js.map