@agentv/core 4.29.2-next.1 → 4.30.0-next.1

package/dist/index.cjs

@@ -369,6 +369,35 @@ function coercePrimitive(value) {
   if (PLAIN_NUMBER_PATTERN.test(value)) return Number(value);
   return value;
 }
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function cloneTemplateValue(value) {
+  if (Array.isArray(value)) {
+    return value.map((item) => cloneTemplateValue(item));
+  }
+  if (isPlainObject(value)) {
+    const result = {};
+    for (const [key, nested] of Object.entries(value)) {
+      result[key] = cloneTemplateValue(nested);
+    }
+    return result;
+  }
+  return value;
+}
+function stringifyTemplateValue(value) {
+  if (typeof value === "string") return value;
+  return JSON.stringify(value);
+}
+function lookupTemplateVar(vars, expression) {
+  if (!expression) return void 0;
+  return expression.split(".").reduce((current, segment) => {
+    if (!isPlainObject(current)) {
+      return void 0;
+    }
+    return current[segment];
+  }, vars);
+}
 function interpolateEnv(value, env) {
   if (typeof value === "string") {
     const wholeMatch = WHOLE_VAR_PATTERN.exec(value);
@@ -390,12 +419,38 @@ function interpolateEnv(value, env) {
   }
   return value;
 }
-var ENV_VAR_PATTERN, WHOLE_VAR_PATTERN, PLAIN_NUMBER_PATTERN;
+function interpolateTemplateVars(value, vars) {
+  if (typeof value === "string") {
+    const wholeMatch = WHOLE_TEMPLATE_VAR_PATTERN.exec(value);
+    if (wholeMatch) {
+      const resolved = lookupTemplateVar(vars, wholeMatch[1]);
+      return resolved === void 0 ? value : cloneTemplateValue(resolved);
+    }
+    return value.replace(TEMPLATE_VAR_PATTERN, (match, expression) => {
+      const resolved = lookupTemplateVar(vars, expression);
+      return resolved === void 0 ? match : stringifyTemplateValue(resolved);
+    });
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => interpolateTemplateVars(item, vars));
+  }
+  if (isPlainObject(value)) {
+    const result = {};
+    for (const [key, nested] of Object.entries(value)) {
+      result[key] = interpolateTemplateVars(nested, vars);
+    }
+    return result;
+  }
+  return value;
+}
+var ENV_VAR_PATTERN, TEMPLATE_VAR_PATTERN, WHOLE_TEMPLATE_VAR_PATTERN, WHOLE_VAR_PATTERN, PLAIN_NUMBER_PATTERN;
 var init_interpolation = __esm({
   "src/evaluation/interpolation.ts"() {
     "use strict";
     init_cjs_shims();
     ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
+    TEMPLATE_VAR_PATTERN = /\{\{\s*([A-Za-z_][A-Za-z0-9_.]*)\s*\}\}/g;
+    WHOLE_TEMPLATE_VAR_PATTERN = /^\{\{\s*([A-Za-z_][A-Za-z0-9_.]*)\s*\}\}$/;
     WHOLE_VAR_PATTERN = /^\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}$/;
     PLAIN_NUMBER_PATTERN = /^-?(?:0|[1-9]\d*)(?:\.\d+)?$/;
   }
@@ -22740,6 +22795,41 @@ function resolveTests(suite) {
   }
   return void 0;
 }
+function interpolateCaseField(value, vars) {
+  if (!vars || value === void 0) {
+    return value;
+  }
+  return interpolateTemplateVars(value, vars);
+}
+function interpolateCaseTurns(turns, vars) {
+  if (!vars || !Array.isArray(turns)) {
+    return turns;
+  }
+  return turns.map((rawTurn) => {
+    if (!isJsonObject(rawTurn)) {
+      return rawTurn;
+    }
+    return {
+      ...rawTurn,
+      input: interpolateCaseField(rawTurn.input, vars),
+      expected_output: interpolateCaseField(rawTurn.expected_output, vars)
+    };
+  });
+}
+function interpolateRawEvalCase(raw, vars) {
+  if (!vars) {
+    return raw;
+  }
+  return {
+    ...raw,
+    ...raw.criteria !== void 0 ? { criteria: interpolateCaseField(raw.criteria, vars) } : {},
+    ...raw.expected_outcome !== void 0 ? { expected_outcome: interpolateCaseField(raw.expected_outcome, vars) } : {},
+    ...raw.input !== void 0 ? { input: interpolateCaseField(raw.input, vars) } : {},
+    ...raw.input_files !== void 0 ? { input_files: interpolateCaseField(raw.input_files, vars) } : {},
+    ...raw.expected_output !== void 0 ? { expected_output: interpolateCaseField(raw.expected_output, vars) } : {},
+    ...raw.turns !== void 0 ? { turns: interpolateCaseTurns(raw.turns, vars) } : {}
+  };
+}
 async function readTestSuiteMetadata(testFilePath) {
   try {
     const absolutePath = import_node_path50.default.resolve(testFilePath);
@@ -22854,8 +22944,8 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   }
   const suiteWorkspace = await resolveWorkspaceConfig(suite.workspace, evalFileDir);
   const suiteGovernance = extractSuiteGovernance(suite);
-  const suiteInputMessages = expandInputShorthand(suite.input);
-  const suiteInputFiles = suite.input_files;
+  const rawSuiteInput = suite.input;
+  const rawSuiteInputFiles = suite.input_files;
   const rawGlobalExecution = isJsonObject(suite.execution) ? suite.execution : void 0;
   const _globalTarget = asString5(rawGlobalExecution?.target) ?? asString5(suite.target);
   const suiteAssertions = suite.assertions ?? suite.assert;
@@ -22874,30 +22964,33 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     if (filterPattern && (!id || !matchesFilter4(id, filterPattern))) {
       continue;
     }
-    const conversationId = asString5(testCaseConfig.conversation_id);
-    let outcome = asString5(testCaseConfig.criteria);
-    if (!outcome && testCaseConfig.expected_outcome !== void 0) {
-      outcome = asString5(testCaseConfig.expected_outcome);
+    const caseVars = isJsonObject(testCaseConfig.vars) ? testCaseConfig.vars : void 0;
+    const renderedCase = interpolateRawEvalCase(testCaseConfig, caseVars);
+    const conversationId = asString5(renderedCase.conversation_id);
+    let outcome = asString5(renderedCase.criteria);
+    if (!outcome && renderedCase.expected_outcome !== void 0) {
+      outcome = asString5(renderedCase.expected_outcome);
       if (outcome) {
         logWarning5(
-          `Test '${asString5(testCaseConfig.id) ?? "unknown"}': 'expected_outcome' is deprecated. Use 'criteria' instead.`
+          `Test '${asString5(renderedCase.id) ?? "unknown"}': 'expected_outcome' is deprecated. Use 'criteria' instead.`
         );
       }
     }
-    const caseExecution = isJsonObject(testCaseConfig.execution) ? testCaseConfig.execution : void 0;
+    const caseExecution = isJsonObject(renderedCase.execution) ? renderedCase.execution : void 0;
     const skipDefaults = caseExecution?.skip_defaults === true;
     const caseThreshold = typeof caseExecution?.threshold === "number" && caseExecution.threshold >= 0 && caseExecution.threshold <= 1 ? caseExecution.threshold : void 0;
-    const effectiveSuiteInputFiles = suiteInputFiles && !skipDefaults ? suiteInputFiles : void 0;
-    const testInputMessages = resolveInputMessages(testCaseConfig, effectiveSuiteInputFiles);
-    const expectedMessages = resolveExpectedMessages(testCaseConfig) ?? [];
-    const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || testCaseConfig.assertions !== void 0 || testCaseConfig.assert !== void 0 || Array.isArray(testCaseConfig.turns) && testCaseConfig.turns.length > 0;
+    const effectiveSuiteInputFiles = rawSuiteInputFiles && !skipDefaults ? interpolateCaseField(rawSuiteInputFiles, caseVars) : void 0;
+    const testInputMessages = resolveInputMessages(renderedCase, effectiveSuiteInputFiles);
+    const expectedMessages = resolveExpectedMessages(renderedCase) ?? [];
+    const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || renderedCase.assertions !== void 0 || renderedCase.assert !== void 0 || Array.isArray(renderedCase.turns) && renderedCase.turns.length > 0;
     if (!id || !hasEvaluationSpec || !testInputMessages || testInputMessages.length === 0) {
       logError3(
         `Skipping incomplete test: ${id ?? "unknown"}. Missing required fields: id, input, and at least one of criteria/expected_output/assertions/turns`
       );
       continue;
     }
-    const effectiveSuiteInputMessages = suiteInputMessages && !skipDefaults ? suiteInputMessages : void 0;
+    const effectiveSuiteInputValue = rawSuiteInput && !skipDefaults ? interpolateCaseField(rawSuiteInput, caseVars) : void 0;
+    const effectiveSuiteInputMessages = expandInputShorthand(effectiveSuiteInputValue);
     const hasExpectedMessages = expectedMessages.length > 0;
     const inputTextParts = [];
     const suiteResolvedInputMessages = effectiveSuiteInputMessages ? await processMessages({
@@ -22937,11 +23030,11 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       }
     }
     const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
-    const testCaseEvaluatorKind = coerceEvaluator(testCaseConfig.evaluator, id) ?? globalEvaluator;
+    const testCaseEvaluatorKind = coerceEvaluator(renderedCase.evaluator, id) ?? globalEvaluator;
     let evaluators;
     try {
       evaluators = await parseGraders(
-        testCaseConfig,
+        renderedCase,
         globalExecution,
         searchRoots,
         id ?? "unknown",
@@ -22952,7 +23045,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       logError3(`Skipping test '${id}': ${message}`);
       continue;
     }
-    const inlineRubrics = testCaseConfig.rubrics;
+    const inlineRubrics = renderedCase.rubrics;
     if (inlineRubrics !== void 0 && Array.isArray(inlineRubrics)) {
       const rubricEvaluator = parseInlineRubrics(inlineRubrics);
       if (rubricEvaluator) {
@@ -22961,25 +23054,25 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     }
     warnUnconsumedCriteria(outcome, evaluators, id ?? "unknown");
     const userFilePaths = collectResolvedInputFilePaths(inputMessages);
-    const caseWorkspace = await resolveWorkspaceConfig(testCaseConfig.workspace, evalFileDir);
+    const caseWorkspace = await resolveWorkspaceConfig(renderedCase.workspace, evalFileDir);
     const mergedWorkspace = mergeWorkspaceConfigs(suiteWorkspace, caseWorkspace);
-    const rawCaseMetadata = isJsonObject(testCaseConfig.metadata) ? testCaseConfig.metadata : void 0;
+    const rawCaseMetadata = isJsonObject(renderedCase.metadata) ? renderedCase.metadata : void 0;
     const suitePayload = suiteGovernance !== void 0 ? { governance: suiteGovernance } : void 0;
     const metadata = mergeSuiteMetadataPayload(rawCaseMetadata, suitePayload);
-    const caseTargets = extractTargetsFromTestCase(testCaseConfig);
-    const dependsOn = Array.isArray(testCaseConfig.depends_on) ? testCaseConfig.depends_on.filter(
+    const caseTargets = extractTargetsFromTestCase(renderedCase);
+    const dependsOn = Array.isArray(renderedCase.depends_on) ? renderedCase.depends_on.filter(
       (v) => typeof v === "string"
     ) : void 0;
-    const onDependencyFailureRaw = asString5(testCaseConfig.on_dependency_failure);
+    const onDependencyFailureRaw = asString5(renderedCase.on_dependency_failure);
     const onDependencyFailure = onDependencyFailureRaw === "skip" || onDependencyFailureRaw === "fail" || onDependencyFailureRaw === "run" ? onDependencyFailureRaw : void 0;
-    const modeRaw = asString5(testCaseConfig.mode);
+    const modeRaw = asString5(renderedCase.mode);
     const mode = modeRaw === "conversation" ? "conversation" : void 0;
-    const turns = Array.isArray(testCaseConfig.turns) ? parseTurns(testCaseConfig.turns) : void 0;
-    const aggregationRaw = asString5(testCaseConfig.aggregation);
+    const turns = Array.isArray(renderedCase.turns) ? parseTurns(renderedCase.turns) : void 0;
+    const aggregationRaw = asString5(renderedCase.aggregation);
     const aggregation = aggregationRaw === "mean" || aggregationRaw === "min" || aggregationRaw === "max" ? aggregationRaw : void 0;
-    const onTurnFailureRaw = asString5(testCaseConfig.on_turn_failure);
+    const onTurnFailureRaw = asString5(renderedCase.on_turn_failure);
     const onTurnFailure = onTurnFailureRaw === "continue" || onTurnFailureRaw === "stop" ? onTurnFailureRaw : void 0;
-    const windowSize = typeof testCaseConfig.window_size === "number" && testCaseConfig.window_size >= 1 ? testCaseConfig.window_size : void 0;
+    const windowSize = typeof renderedCase.window_size === "number" && renderedCase.window_size >= 1 ? renderedCase.window_size : void 0;
     const testCase = {
       id,
       suite: suiteName,
@@ -24111,7 +24204,7 @@ var init_rpc_metadata = __esm({
 });
 
 // ../../node_modules/.bun/@opentelemetry+core@2.5.1+460773ef8ff1e07c/node_modules/@opentelemetry/core/build/esm/utils/lodash.merge.js
-function isPlainObject(value) {
+function isPlainObject2(value) {
   if (!isObjectLike(value) || baseGetTag(value) !== objectTag) {
     return false;
   }
@@ -24270,7 +24363,7 @@ function isPrimitive(value) {
   return typeof value === "string" || typeof value === "number" || typeof value === "boolean" || typeof value === "undefined" || value instanceof Date || value instanceof RegExp || value === null;
 }
 function shouldMerge(one, two) {
-  if (!isPlainObject(one) || !isPlainObject(two)) {
+  if (!isPlainObject2(one) || !isPlainObject2(two)) {
     return false;
   }
   return true;