agentv 4.29.1-next.1 → 4.30.0-next.1

package/dist/{artifact-writer-RQXU4LZV.js → artifact-writer-JJLJUUUY.js}

@@ -14,11 +14,11 @@ import {
   writeArtifactsFromResults,
   writeInitialBenchmarkArtifact,
   writePerTestArtifacts
-} from "./chunk-OS67VZUO.js";
-import "./chunk-VZMGBDJD.js";
+} from "./chunk-YFVD6FC6.js";
+import "./chunk-LUZRCQNH.js";
 import "./chunk-QOBQ5XYF.js";
 import "./chunk-BPGJ4HBU.js";
-import "./chunk-VTOY4SMC.js";
+import "./chunk-L3FMZXCD.js";
 import "./chunk-NPVGBFF6.js";
 import "./chunk-M7BUKBAF.js";
 import "./chunk-5H446C7X.js";
@@ -38,4 +38,4 @@ export {
   writeInitialBenchmarkArtifact,
   writePerTestArtifacts
 };
-//# sourceMappingURL=artifact-writer-RQXU4LZV.js.map
+//# sourceMappingURL=artifact-writer-JJLJUUUY.js.map

package/dist/{chunk-VTOY4SMC.js → chunk-L3FMZXCD.js}

@@ -4056,7 +4056,7 @@ var coerce = {
 };
 var NEVER = INVALID;
 
-// ../../packages/core/dist/chunk-YFXMMBUG.js
+// ../../packages/core/dist/chunk-5RQMJZDJ.js
 import { parse } from "yaml";
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
@@ -5783,6 +5783,8 @@ function resolveOptionalNumberArray(source, description) {
   return resolved.length > 0 ? resolved : void 0;
 }
 var ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
+var TEMPLATE_VAR_PATTERN = /\{\{\s*([A-Za-z_][A-Za-z0-9_.]*)\s*\}\}/g;
+var WHOLE_TEMPLATE_VAR_PATTERN = /^\{\{\s*([A-Za-z_][A-Za-z0-9_.]*)\s*\}\}$/;
 var WHOLE_VAR_PATTERN = /^\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}$/;
 var PLAIN_NUMBER_PATTERN = /^-?(?:0|[1-9]\d*)(?:\.\d+)?$/;
 function coercePrimitive(value) {
@@ -5791,6 +5793,35 @@ function coercePrimitive(value) {
   if (PLAIN_NUMBER_PATTERN.test(value)) return Number(value);
   return value;
 }
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function cloneTemplateValue(value) {
+  if (Array.isArray(value)) {
+    return value.map((item) => cloneTemplateValue(item));
+  }
+  if (isPlainObject(value)) {
+    const result = {};
+    for (const [key, nested] of Object.entries(value)) {
+      result[key] = cloneTemplateValue(nested);
+    }
+    return result;
+  }
+  return value;
+}
+function stringifyTemplateValue(value) {
+  if (typeof value === "string") return value;
+  return JSON.stringify(value);
+}
+function lookupTemplateVar(vars, expression) {
+  if (!expression) return void 0;
+  return expression.split(".").reduce((current, segment) => {
+    if (!isPlainObject(current)) {
+      return void 0;
+    }
+    return current[segment];
+  }, vars);
+}
 function interpolateEnv(value, env) {
   if (typeof value === "string") {
     const wholeMatch = WHOLE_VAR_PATTERN.exec(value);
@@ -5812,6 +5843,30 @@ function interpolateEnv(value, env) {
   }
   return value;
 }
+function interpolateTemplateVars(value, vars) {
+  if (typeof value === "string") {
+    const wholeMatch = WHOLE_TEMPLATE_VAR_PATTERN.exec(value);
+    if (wholeMatch) {
+      const resolved = lookupTemplateVar(vars, wholeMatch[1]);
+      return resolved === void 0 ? value : cloneTemplateValue(resolved);
+    }
+    return value.replace(TEMPLATE_VAR_PATTERN, (match, expression) => {
+      const resolved = lookupTemplateVar(vars, expression);
+      return resolved === void 0 ? match : stringifyTemplateValue(resolved);
+    });
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => interpolateTemplateVars(item, vars));
+  }
+  if (isPlainObject(value)) {
+    const result = {};
+    for (const [key, nested] of Object.entries(value)) {
+      result[key] = interpolateTemplateVars(nested, vars);
+    }
+    return result;
+  }
+  return value;
+}
 var ANSI_YELLOW = "\x1B[33m";
 var ANSI_RESET = "\x1B[0m";
 var FILE_PROTOCOL = "file://";
@@ -5982,7 +6037,7 @@ async function expandFileReferences(tests, evalFileDir) {
   return expanded;
 }
 
-// ../../packages/core/dist/chunk-SCC35F3L.js
+// ../../packages/core/dist/chunk-Z2BBOGE4.js
 import path46 from "node:path";
 import { pathToFileURL as pathToFileURL2 } from "node:url";
 import { existsSync as existsSync6 } from "node:fs";
@@ -6609,7 +6664,7 @@ __export(util_exports, {
   getParsedType: () => getParsedType2,
   getSizableOrigin: () => getSizableOrigin,
   isObject: () => isObject,
-  isPlainObject: () => isPlainObject,
+  isPlainObject: () => isPlainObject2,
   issue: () => issue,
   joinValues: () => joinValues,
   jsonStringifyReplacer: () => jsonStringifyReplacer,
@@ -6757,7 +6812,7 @@ var allowsEval = cached(() => {
     return false;
   }
 });
-function isPlainObject(o) {
+function isPlainObject2(o) {
   if (isObject(o) === false)
     return false;
   const ctor = o.constructor;
@@ -6942,7 +6997,7 @@ function omit(schema, mask) {
   });
 }
 function extend(schema, shape) {
-  if (!isPlainObject(shape)) {
+  if (!isPlainObject2(shape)) {
     throw new Error("Invalid input to extend: expected a plain object");
   }
   const def = {
@@ -8939,7 +8994,7 @@ function mergeValues2(a, b) {
   if (a instanceof Date && b instanceof Date && +a === +b) {
     return { valid: true, data: a };
   }
-  if (isPlainObject(a) && isPlainObject(b)) {
+  if (isPlainObject2(a) && isPlainObject2(b)) {
     const bKeys = Object.keys(b);
     const sharedKeys = Object.keys(a).filter((key) => bKeys.indexOf(key) !== -1);
     const newObj = { ...a, ...b };
@@ -9069,7 +9124,7 @@ var $ZodRecord = /* @__PURE__ */ $constructor("$ZodRecord", (inst, def) => {
   $ZodType.init(inst, def);
   inst._zod.parse = (payload, ctx) => {
     const input = payload.value;
-    if (!isPlainObject(input)) {
+    if (!isPlainObject2(input)) {
       payload.issues.push({
         expected: "record",
         code: "invalid_type",
@@ -18649,7 +18704,7 @@ var RequestError = class _RequestError extends Error {
   }
 };
 
-// ../../packages/core/dist/chunk-SCC35F3L.js
+// ../../packages/core/dist/chunk-Z2BBOGE4.js
 import { exec as execCallback } from "node:child_process";
 import { readdirSync, statSync } from "node:fs";
 import { readFile as readFile22, readdir as readdir2, stat as stat2 } from "node:fs/promises";
@@ -34032,6 +34087,41 @@ function resolveTests(suite) {
   }
   return void 0;
 }
+function interpolateCaseField(value, vars) {
+  if (!vars || value === void 0) {
+    return value;
+  }
+  return interpolateTemplateVars(value, vars);
+}
+function interpolateCaseTurns(turns, vars) {
+  if (!vars || !Array.isArray(turns)) {
+    return turns;
+  }
+  return turns.map((rawTurn) => {
+    if (!isJsonObject(rawTurn)) {
+      return rawTurn;
+    }
+    return {
+      ...rawTurn,
+      input: interpolateCaseField(rawTurn.input, vars),
+      expected_output: interpolateCaseField(rawTurn.expected_output, vars)
+    };
+  });
+}
+function interpolateRawEvalCase(raw, vars) {
+  if (!vars) {
+    return raw;
+  }
+  return {
+    ...raw,
+    ...raw.criteria !== void 0 ? { criteria: interpolateCaseField(raw.criteria, vars) } : {},
+    ...raw.expected_outcome !== void 0 ? { expected_outcome: interpolateCaseField(raw.expected_outcome, vars) } : {},
+    ...raw.input !== void 0 ? { input: interpolateCaseField(raw.input, vars) } : {},
+    ...raw.input_files !== void 0 ? { input_files: interpolateCaseField(raw.input_files, vars) } : {},
+    ...raw.expected_output !== void 0 ? { expected_output: interpolateCaseField(raw.expected_output, vars) } : {},
+    ...raw.turns !== void 0 ? { turns: interpolateCaseTurns(raw.turns, vars) } : {}
+  };
+}
 async function readTestSuiteMetadata(testFilePath) {
   try {
     const absolutePath = path43.resolve(testFilePath);
@@ -34059,7 +34149,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
     return { tests: await loadTestsFromAgentSkills(evalFilePath) };
   }
   if (format === "typescript") {
-    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-EMSGL2BQ-P5RV35TE.js");
+    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-JL5DGTJL-U5LTKGOE.js");
     return loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
   }
   const { tests, parsed, suiteWorkspacePath } = await loadTestsFromYaml(
@@ -34094,7 +34184,7 @@ async function loadTests(evalFilePath, repoRoot, options) {
     return loadTestsFromAgentSkills(evalFilePath);
   }
   if (format === "typescript") {
-    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-EMSGL2BQ-P5RV35TE.js");
+    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-JL5DGTJL-U5LTKGOE.js");
     const suite = await loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
     return suite.tests;
   }
@@ -34148,8 +34238,8 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   }
   const suiteWorkspace = await resolveWorkspaceConfig(suite.workspace, evalFileDir);
   const suiteGovernance = extractSuiteGovernance(suite);
-  const suiteInputMessages = expandInputShorthand(suite.input);
-  const suiteInputFiles = suite.input_files;
+  const rawSuiteInput = suite.input;
+  const rawSuiteInputFiles = suite.input_files;
   const rawGlobalExecution = isJsonObject(suite.execution) ? suite.execution : void 0;
   const _globalTarget = asString5(rawGlobalExecution?.target) ?? asString5(suite.target);
   const suiteAssertions = suite.assertions ?? suite.assert;
@@ -34168,30 +34258,33 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     if (filterPattern && (!id || !matchesFilter2(id, filterPattern))) {
       continue;
     }
-    const conversationId = asString5(testCaseConfig.conversation_id);
-    let outcome = asString5(testCaseConfig.criteria);
-    if (!outcome && testCaseConfig.expected_outcome !== void 0) {
-      outcome = asString5(testCaseConfig.expected_outcome);
+    const caseVars = isJsonObject(testCaseConfig.vars) ? testCaseConfig.vars : void 0;
+    const renderedCase = interpolateRawEvalCase(testCaseConfig, caseVars);
+    const conversationId = asString5(renderedCase.conversation_id);
+    let outcome = asString5(renderedCase.criteria);
+    if (!outcome && renderedCase.expected_outcome !== void 0) {
+      outcome = asString5(renderedCase.expected_outcome);
       if (outcome) {
         logWarning5(
-          `Test '${asString5(testCaseConfig.id) ?? "unknown"}': 'expected_outcome' is deprecated. Use 'criteria' instead.`
+          `Test '${asString5(renderedCase.id) ?? "unknown"}': 'expected_outcome' is deprecated. Use 'criteria' instead.`
         );
       }
     }
-    const caseExecution = isJsonObject(testCaseConfig.execution) ? testCaseConfig.execution : void 0;
+    const caseExecution = isJsonObject(renderedCase.execution) ? renderedCase.execution : void 0;
     const skipDefaults = caseExecution?.skip_defaults === true;
     const caseThreshold = typeof caseExecution?.threshold === "number" && caseExecution.threshold >= 0 && caseExecution.threshold <= 1 ? caseExecution.threshold : void 0;
-    const effectiveSuiteInputFiles = suiteInputFiles && !skipDefaults ? suiteInputFiles : void 0;
-    const testInputMessages = resolveInputMessages(testCaseConfig, effectiveSuiteInputFiles);
-    const expectedMessages = resolveExpectedMessages(testCaseConfig) ?? [];
-    const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || testCaseConfig.assertions !== void 0 || testCaseConfig.assert !== void 0 || Array.isArray(testCaseConfig.turns) && testCaseConfig.turns.length > 0;
+    const effectiveSuiteInputFiles = rawSuiteInputFiles && !skipDefaults ? interpolateCaseField(rawSuiteInputFiles, caseVars) : void 0;
+    const testInputMessages = resolveInputMessages(renderedCase, effectiveSuiteInputFiles);
+    const expectedMessages = resolveExpectedMessages(renderedCase) ?? [];
+    const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || renderedCase.assertions !== void 0 || renderedCase.assert !== void 0 || Array.isArray(renderedCase.turns) && renderedCase.turns.length > 0;
     if (!id || !hasEvaluationSpec || !testInputMessages || testInputMessages.length === 0) {
       logError3(
         `Skipping incomplete test: ${id ?? "unknown"}. Missing required fields: id, input, and at least one of criteria/expected_output/assertions/turns`
       );
       continue;
     }
-    const effectiveSuiteInputMessages = suiteInputMessages && !skipDefaults ? suiteInputMessages : void 0;
+    const effectiveSuiteInputValue = rawSuiteInput && !skipDefaults ? interpolateCaseField(rawSuiteInput, caseVars) : void 0;
+    const effectiveSuiteInputMessages = expandInputShorthand(effectiveSuiteInputValue);
     const hasExpectedMessages = expectedMessages.length > 0;
     const inputTextParts = [];
     const suiteResolvedInputMessages = effectiveSuiteInputMessages ? await processMessages({
@@ -34231,11 +34324,11 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       }
     }
     const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
-    const testCaseEvaluatorKind = coerceEvaluator(testCaseConfig.evaluator, id) ?? globalEvaluator;
+    const testCaseEvaluatorKind = coerceEvaluator(renderedCase.evaluator, id) ?? globalEvaluator;
     let evaluators;
     try {
       evaluators = await parseGraders(
-        testCaseConfig,
+        renderedCase,
         globalExecution,
         searchRoots,
         id ?? "unknown",
@@ -34246,7 +34339,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       logError3(`Skipping test '${id}': ${message}`);
       continue;
     }
-    const inlineRubrics = testCaseConfig.rubrics;
+    const inlineRubrics = renderedCase.rubrics;
     if (inlineRubrics !== void 0 && Array.isArray(inlineRubrics)) {
       const rubricEvaluator = parseInlineRubrics(inlineRubrics);
       if (rubricEvaluator) {
@@ -34255,25 +34348,25 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     }
     warnUnconsumedCriteria(outcome, evaluators, id ?? "unknown");
     const userFilePaths = collectResolvedInputFilePaths(inputMessages);
-    const caseWorkspace = await resolveWorkspaceConfig(testCaseConfig.workspace, evalFileDir);
+    const caseWorkspace = await resolveWorkspaceConfig(renderedCase.workspace, evalFileDir);
     const mergedWorkspace = mergeWorkspaceConfigs(suiteWorkspace, caseWorkspace);
-    const rawCaseMetadata = isJsonObject(testCaseConfig.metadata) ? testCaseConfig.metadata : void 0;
+    const rawCaseMetadata = isJsonObject(renderedCase.metadata) ? renderedCase.metadata : void 0;
     const suitePayload = suiteGovernance !== void 0 ? { governance: suiteGovernance } : void 0;
     const metadata = mergeSuiteMetadataPayload(rawCaseMetadata, suitePayload);
-    const caseTargets = extractTargetsFromTestCase(testCaseConfig);
-    const dependsOn = Array.isArray(testCaseConfig.depends_on) ? testCaseConfig.depends_on.filter(
+    const caseTargets = extractTargetsFromTestCase(renderedCase);
+    const dependsOn = Array.isArray(renderedCase.depends_on) ? renderedCase.depends_on.filter(
       (v) => typeof v === "string"
     ) : void 0;
-    const onDependencyFailureRaw = asString5(testCaseConfig.on_dependency_failure);
+    const onDependencyFailureRaw = asString5(renderedCase.on_dependency_failure);
     const onDependencyFailure = onDependencyFailureRaw === "skip" || onDependencyFailureRaw === "fail" || onDependencyFailureRaw === "run" ? onDependencyFailureRaw : void 0;
-    const modeRaw = asString5(testCaseConfig.mode);
+    const modeRaw = asString5(renderedCase.mode);
     const mode = modeRaw === "conversation" ? "conversation" : void 0;
-    const turns = Array.isArray(testCaseConfig.turns) ? parseTurns(testCaseConfig.turns) : void 0;
-    const aggregationRaw = asString5(testCaseConfig.aggregation);
+    const turns = Array.isArray(renderedCase.turns) ? parseTurns(renderedCase.turns) : void 0;
+    const aggregationRaw = asString5(renderedCase.aggregation);
     const aggregation = aggregationRaw === "mean" || aggregationRaw === "min" || aggregationRaw === "max" ? aggregationRaw : void 0;
-    const onTurnFailureRaw = asString5(testCaseConfig.on_turn_failure);
+    const onTurnFailureRaw = asString5(renderedCase.on_turn_failure);
     const onTurnFailure = onTurnFailureRaw === "continue" || onTurnFailureRaw === "stop" ? onTurnFailureRaw : void 0;
-    const windowSize = typeof testCaseConfig.window_size === "number" && testCaseConfig.window_size >= 1 ? testCaseConfig.window_size : void 0;
+    const windowSize = typeof renderedCase.window_size === "number" && renderedCase.window_size >= 1 ? renderedCase.window_size : void 0;
     const testCase = {
       id,
       suite: suiteName,
@@ -37991,4 +38084,4 @@ export {
   loadTsEvalFile,
   loadTsEvalSuite
 };
-//# sourceMappingURL=chunk-VTOY4SMC.js.map
+//# sourceMappingURL=chunk-L3FMZXCD.js.map