agentv 0.25.0 → 1.0.0

package/README.md

@@ -253,9 +253,9 @@ Code evaluators receive input via stdin and write output to stdout as JSON.
   "expected_outcome": "expected outcome description",
   "reference_answer": "gold standard answer (optional)",
   "candidate_answer": "generated code/text from the agent",
-  "guideline_paths": ["path1", "path2"],
-  "input_files": ["file1", "file2"],
-  "input_segments": [{"type": "text", "value": "..."}]
+  "guideline_files": ["path/to/guideline1.md", "path/to/guideline2.md"],
+  "input_files": ["path/to/data.json", "path/to/config.yaml"],
+  "input_messages": [{"role": "user", "content": "..."}]
 }
 ```
 

package/dist/{chunk-ZVSFP6NK.js → chunk-RIJO5WBF.js}

@@ -164,7 +164,7 @@ import { access as access6, mkdir as mkdir7 } from "node:fs/promises";
 import path19 from "node:path";
 import { pathToFileURL } from "node:url";
 
-// ../../packages/core/dist/chunk-OYTL3LNN.js
+// ../../packages/core/dist/chunk-V3JCB3HI.js
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
 import path from "node:path";
@@ -4211,7 +4211,7 @@ var coerce = {
 };
 var NEVER = INVALID;
 
-// ../../packages/core/dist/chunk-OYTL3LNN.js
+// ../../packages/core/dist/chunk-V3JCB3HI.js
 async function fileExists(filePath) {
   try {
     await access(filePath, constants.F_OK);
@@ -4470,7 +4470,10 @@ function resolveAzureConfig(target, env) {
   const apiKey = resolveString(apiKeySource, env, `${target.name} api key`);
   const deploymentName = resolveString(deploymentSource, env, `${target.name} deployment`);
   const version2 = normalizeAzureApiVersion(
-    resolveOptionalString(versionSource, env, `${target.name} api version`)
+    resolveOptionalString(versionSource, env, `${target.name} api version`, {
+      allowLiteral: true,
+      optionalEnv: true
+    })
   );
   const temperature = resolveOptionalNumber(temperatureSource, `${target.name} temperature`);
   const maxOutputTokens = resolveOptionalNumber(
@@ -34575,7 +34578,7 @@ var EVALUATOR_KIND_VALUES = [
   "rubric",
   "composite",
   "tool_trajectory",
-  "expected_messages"
+  "expected_tool_calls"
 ];
 var EVALUATOR_KIND_SET = new Set(EVALUATOR_KIND_VALUES);
 function isEvaluatorKind(value) {
@@ -34928,6 +34931,7 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
         logWarning2(`Skipping code_judge evaluator '${name16}' in '${evalId}': missing script`);
         continue;
       }
+      const weight2 = validateWeight(rawEvaluator.weight, name16, evalId);
       const cwd = asString2(rawEvaluator.cwd);
       let resolvedCwd;
       if (cwd) {
@@ -34948,7 +34952,8 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
         type: "code",
         script,
         cwd,
-        resolvedCwd
+        resolvedCwd,
+        ...weight2 !== void 0 ? { weight: weight2 } : {}
       });
       continue;
     }
@@ -35043,18 +35048,22 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
           ...promptPath2 ? { promptPath: promptPath2 } : {}
         };
       }
+      const weight2 = validateWeight(rawEvaluator.weight, name16, evalId);
       evaluators.push({
         name: name16,
         type: "composite",
         evaluators: memberEvaluators,
-        aggregator
+        aggregator,
+        ...weight2 !== void 0 ? { weight: weight2 } : {}
       });
       continue;
     }
-    if (typeValue === "expected_messages") {
+    if (typeValue === "expected_tool_calls") {
+      const weight2 = validateWeight(rawEvaluator.weight, name16, evalId);
       evaluators.push({
         name: name16,
-        type: "expected_messages"
+        type: "expected_tool_calls",
+        ...weight2 !== void 0 ? { weight: weight2 } : {}
       });
       continue;
     }
@@ -35110,12 +35119,14 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
         );
         continue;
       }
+      const weight2 = validateWeight(rawEvaluator.weight, name16, evalId);
       const config2 = {
         name: name16,
         type: "tool_trajectory",
         mode,
         ...minimums ? { minimums } : {},
-        ...expected ? { expected } : {}
+        ...expected ? { expected } : {},
+        ...weight2 !== void 0 ? { weight: weight2 } : {}
       };
       evaluators.push(config2);
       continue;
@@ -35156,19 +35167,23 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
         logWarning2(`Skipping rubric evaluator '${name16}' in '${evalId}': no valid rubrics found`);
         continue;
       }
+      const weight2 = validateWeight(rawEvaluator.weight, name16, evalId);
       evaluators.push({
         name: name16,
         type: "llm_judge",
-        rubrics: parsedRubrics
+        rubrics: parsedRubrics,
+        ...weight2 !== void 0 ? { weight: weight2 } : {}
       });
       continue;
     }
+    const weight = validateWeight(rawEvaluator.weight, name16, evalId);
     evaluators.push({
       name: name16,
       type: "llm_judge",
       prompt,
       promptPath,
-      ...parsedRubrics && parsedRubrics.length > 0 ? { rubrics: parsedRubrics } : {}
+      ...parsedRubrics && parsedRubrics.length > 0 ? { rubrics: parsedRubrics } : {},
+      ...weight !== void 0 ? { weight } : {}
     });
   }
   return evaluators.length > 0 ? evaluators : void 0;
@@ -35198,6 +35213,27 @@ ${detailBlock}${ANSI_RESET3}`);
     console.warn(`${ANSI_YELLOW3}Warning: ${message}${ANSI_RESET3}`);
   }
 }
+function validateWeight(rawWeight, evaluatorName, evalId) {
+  if (rawWeight === void 0) {
+    return void 0;
+  }
+  if (typeof rawWeight !== "number") {
+    throw new Error(
+      `Invalid weight for evaluator '${evaluatorName}' in '${evalId}': must be a number`
+    );
+  }
+  if (!Number.isFinite(rawWeight)) {
+    throw new Error(
+      `Invalid weight for evaluator '${evaluatorName}' in '${evalId}': must be finite (got ${rawWeight})`
+    );
+  }
+  if (rawWeight < 0) {
+    throw new Error(
+      `Invalid weight for evaluator '${evaluatorName}' in '${evalId}': must be non-negative (got ${rawWeight})`
+    );
+  }
+  return rawWeight;
+}
 var ANSI_YELLOW4 = "\x1B[33m";
 var ANSI_RESET4 = "\x1B[0m";
 async function processMessages(options) {
@@ -37906,9 +37942,11 @@ var CodeEvaluator = class {
         expected_outcome: context.evalCase.expected_outcome,
         reference_answer: context.evalCase.reference_answer,
         candidate_answer: context.candidate,
-        guideline_paths: context.evalCase.guideline_paths,
-        input_files: context.evalCase.file_paths,
-        input_segments: context.evalCase.input_segments
+        guideline_files: context.evalCase.guideline_paths,
+        input_files: context.evalCase.file_paths.filter(
+          (path132) => !context.evalCase.guideline_paths.includes(path132)
+        ),
+        input_messages: context.evalCase.input_messages
       },
       null,
       2
@@ -38174,8 +38212,8 @@ var ToolTrajectoryEvaluator = class {
     };
   }
 };
-var ExpectedMessagesEvaluator = class {
-  kind = "expected_messages";
+var ExpectedToolCallsEvaluator = class {
+  kind = "expected_tool_calls";
   evaluate(context) {
     const { candidateTrace, evalCase } = context;
     const expectedSegments = evalCase.expected_segments;
@@ -39070,14 +39108,12 @@ async function evaluateCandidate(options) {
   } else {
     if (promptInputs.chatPrompt) {
       lmProviderRequest = {
-        chat_prompt: promptInputs.chatPrompt,
-        guideline_paths: evalCase.guideline_paths
+        chat_prompt: promptInputs.chatPrompt
       };
     } else {
       lmProviderRequest = {
         question: promptInputs.question,
-        guidelines: promptInputs.guidelines,
-        guideline_paths: evalCase.guideline_paths
+        guidelines: promptInputs.guidelines
       };
     }
   }
@@ -39184,11 +39220,13 @@ async function runEvaluatorList(options) {
           now,
           judgeProvider
         });
-        scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
+        const weight = evaluator.weight ?? 1;
+        scored.push({ score: score2, name: evaluator.name, type: evaluator.type, weight });
         evaluatorResults.push({
           name: evaluator.name,
           type: evaluator.type,
           score: score2.score,
+          weight,
           verdict: score2.verdict,
           hits: score2.hits,
           misses: score2.misses,
@@ -39211,11 +39249,13 @@ async function runEvaluatorList(options) {
           promptInputs,
           now
         });
-        scored.push({ score: score2, name: evaluator.name, type: "code_judge" });
+        const weight = evaluator.weight ?? 1;
+        scored.push({ score: score2, name: evaluator.name, type: "code_judge", weight });
         evaluatorResults.push({
           name: evaluator.name,
           type: "code_judge",
           score: score2.score,
+          weight,
           verdict: score2.verdict,
           hits: score2.hits,
           misses: score2.misses,
@@ -39245,8 +39285,8 @@ async function runEvaluatorList(options) {
               return new ToolTrajectoryEvaluator({
                 config: memberConfig
               });
-            case "expected_messages":
-              return new ExpectedMessagesEvaluator();
+            case "expected_tool_calls":
+              return new ExpectedToolCallsEvaluator();
             default: {
               const unknownConfig = memberConfig;
               throw new Error(`Unsupported evaluator type in composite: ${unknownConfig.type}`);
@@ -39268,11 +39308,13 @@ async function runEvaluatorList(options) {
           now,
           judgeProvider
         });
-        scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
+        const weight = evaluator.weight ?? 1;
+        scored.push({ score: score2, name: evaluator.name, type: evaluator.type, weight });
         evaluatorResults.push({
           name: evaluator.name,
           type: evaluator.type,
           score: score2.score,
+          weight,
           verdict: score2.verdict,
           hits: score2.hits,
           misses: score2.misses,
@@ -39296,20 +39338,22 @@ async function runEvaluatorList(options) {
           candidateTrace,
           candidateTraceSummary
         });
-        scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
+        const weight = evaluator.weight ?? 1;
+        scored.push({ score: score2, name: evaluator.name, type: evaluator.type, weight });
         evaluatorResults.push({
           name: evaluator.name,
           type: evaluator.type,
           score: score2.score,
+          weight,
           verdict: score2.verdict,
           hits: score2.hits,
           misses: score2.misses,
           reasoning: score2.reasoning
         });
       }
-      if (evaluator.type === "expected_messages") {
-        const expectedMessagesEvaluator = new ExpectedMessagesEvaluator();
-        const score2 = expectedMessagesEvaluator.evaluate({
+      if (evaluator.type === "expected_tool_calls") {
+        const expectedToolCallsEvaluator = new ExpectedToolCallsEvaluator();
+        const score2 = expectedToolCallsEvaluator.evaluate({
           evalCase,
           candidate,
           target,
@@ -39320,11 +39364,13 @@ async function runEvaluatorList(options) {
           candidateTrace,
           candidateTraceSummary
         });
-        scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
+        const weight = evaluator.weight ?? 1;
+        scored.push({ score: score2, name: evaluator.name, type: evaluator.type, weight });
         evaluatorResults.push({
           name: evaluator.name,
           type: evaluator.type,
           score: score2.score,
+          weight,
           verdict: score2.verdict,
           hits: score2.hits,
           misses: score2.misses,
@@ -39342,15 +39388,18 @@ async function runEvaluatorList(options) {
         reasoning: message
       };
       const resultType = evaluator.type === "code" ? "code_judge" : evaluator.type;
+      const weight = evaluator.weight ?? 1;
       scored.push({
         score: fallbackScore,
         name: evaluator.name ?? "unknown",
-        type: resultType ?? "llm_judge"
+        type: resultType ?? "llm_judge",
+        weight
       });
       evaluatorResults.push({
         name: evaluator.name ?? "unknown",
         type: resultType ?? "llm_judge",
         score: 0,
+        weight,
         verdict: "fail",
         hits: [],
         misses: [`Evaluator '${evaluator.name ?? "unknown"}' failed: ${message}`],
@@ -39358,7 +39407,9 @@ async function runEvaluatorList(options) {
       });
     }
   }
-  const aggregateScore = scored.length > 0 ? scored.reduce((total, entry) => total + entry.score.score, 0) / scored.length : 0;
+  const aggregateScore = scored.length > 0 ? computeWeightedMean(
+    scored.map((entry) => ({ score: entry.score.score, weight: entry.weight }))
+  ) : 0;
   const hits = scored.flatMap((entry) => entry.score.hits);
   const misses = scored.flatMap((entry) => entry.score.misses);
   const expectedAspectCount = scored.reduce(
@@ -39584,6 +39635,16 @@ function mapChildResults(children) {
     evaluator_results: mapChildResults(child.evaluatorResults)
   }));
 }
+function computeWeightedMean(entries) {
+  let totalWeight = 0;
+  let weightedSum = 0;
+  for (const entry of entries) {
+    const weight = entry.weight ?? 1;
+    totalWeight += weight;
+    weightedSum += entry.score * weight;
+  }
+  return totalWeight > 0 ? weightedSum / totalWeight : 0;
+}
 var rubricItemSchema = external_exports.object({
   id: external_exports.string().describe("Short identifier for this rubric (e.g., clarity, completeness)"),
   description: external_exports.string().describe("What this rubric checks for"),
@@ -42647,4 +42708,4 @@ export {
   app,
   runCli
 };
-//# sourceMappingURL=chunk-ZVSFP6NK.js.map
+//# sourceMappingURL=chunk-RIJO5WBF.js.map