@voltagent/scorers 2.0.4 → 2.1.0

package/README.md

@@ -3,5 +3,9 @@
 Re-export of the prebuilt scorer utilities used by Viteval. These scorers originate from the Viteval project and are surfaced here so VoltAgent components can depend on them without pulling the full Viteval toolchain.
 
 ```ts
-import { scorers, AnswerCorrectness } from "@voltagent/scorers";
+import {
+  scorers,
+  createAnswerCorrectnessScorer,
+  createToolCallAccuracyScorerCode,
+} from "@voltagent/scorers";
 ```

package/dist/index.cjs

@@ -26,7 +26,7 @@ __export(index_exports, {
   ListContains: () => import_autoevals.ListContains,
   NumericDiff: () => import_autoevals.NumericDiff,
   adaptScorerForAgentEval: () => adaptScorerForAgentEval,
-  buildSamplingMetadata: () => import_core9.buildSamplingMetadata,
+  buildSamplingMetadata: () => import_core10.buildSamplingMetadata,
   createAnswerCorrectnessScorer: () => createAnswerCorrectnessScorer,
   createAnswerRelevancyScorer: () => createAnswerRelevancyScorer,
   createContextPrecisionScorer: () => createContextPrecisionScorer,
@@ -37,12 +37,13 @@ __export(index_exports, {
   createModerationScorer: () => createModerationScorer,
   createPossibleScorer: () => createPossibleScorer,
   createSummaryScorer: () => createSummaryScorer,
+  createToolCallAccuracyScorerCode: () => createToolCallAccuracyScorerCode,
   createTranslationScorer: () => createTranslationScorer,
-  normalizeScorerResult: () => import_core9.normalizeScorerResult,
+  normalizeScorerResult: () => import_core10.normalizeScorerResult,
   rawAutoEvalScorers: () => rawAutoEvalScorers,
-  runLocalScorers: () => import_core9.runLocalScorers,
+  runLocalScorers: () => import_core10.runLocalScorers,
   scorers: () => scorers,
-  shouldSample: () => import_core9.shouldSample
+  shouldSample: () => import_core10.shouldSample
 });
 module.exports = __toCommonJS(index_exports);
 var import_autoevals = require("autoevals");
@@ -285,7 +286,7 @@ function normalizeMetadata(value) {
 }
 
 // src/index.ts
-var import_core9 = require("@voltagent/core");
+var import_core10 = require("@voltagent/core");
 
 // src/llm/moderation.ts
 var import_core2 = require("@voltagent/core");
@@ -1561,6 +1562,349 @@ function mergeMetadata5(base, additional) {
   return { ...base, ...additional };
 }
 
+// src/tool-call-accuracy.ts
+var import_core9 = require("@voltagent/core");
+var import_utils9 = require("@voltagent/internal/utils");
+function createToolCallAccuracyScorerCode({
+  id = "toolCallAccuracyCode",
+  name = "Tool Call Accuracy (Code)",
+  expectedTool,
+  expectedToolOrder,
+  strictMode = false,
+  metadata,
+  buildPayload
+}) {
+  const normalizedExpectedTool = normalizeToolName(expectedTool);
+  const normalizedExpectedToolOrder = normalizeExpectedToolOrder(expectedToolOrder);
+  if (!normalizedExpectedTool && normalizedExpectedToolOrder.length === 0) {
+    throw new Error(
+      "createToolCallAccuracyScorerCode requires either expectedTool or expectedToolOrder"
+    );
+  }
+  const mode = normalizedExpectedToolOrder.length > 0 ? "tool_order" : "single_tool";
+  return (0, import_core9.buildScorer)({
+    id,
+    label: name,
+    metadata: mergeMetadata6(metadata, {
+      voltAgent: {
+        scorer: id,
+        category: "tool_call_accuracy",
+        mode: "code"
+      }
+    })
+  }).score((context) => {
+    const payload = resolvePayload6(context, buildPayload);
+    const actualTools = extractToolNames(payload);
+    const hasToolCalls = actualTools.length > 0;
+    let correctToolCalled = false;
+    let correctOrderCalled = null;
+    if (mode === "tool_order") {
+      correctOrderCalled = checkToolOrder(actualTools, normalizedExpectedToolOrder, strictMode);
+    } else if (normalizedExpectedTool) {
+      correctToolCalled = strictMode ? actualTools.length === 1 && actualTools[0] === normalizedExpectedTool : actualTools.includes(normalizedExpectedTool);
+    }
+    const score = mode === "tool_order" ? correctOrderCalled ? 1 : 0 : correctToolCalled ? 1 : 0;
+    const evaluation = {
+      score,
+      actualTools,
+      expectedTool: normalizedExpectedTool,
+      expectedToolOrder: normalizedExpectedToolOrder.length > 0 ? normalizedExpectedToolOrder : void 0,
+      strictMode,
+      hasToolCalls,
+      correctToolCalled,
+      correctOrderCalled,
+      mode
+    };
+    context.results.raw.toolCallAccuracyEvaluation = evaluation;
+    return score;
+  }).reason(({ results }) => {
+    const evaluation = results.raw.toolCallAccuracyEvaluation;
+    if (!evaluation) {
+      return { reason: "Tool call accuracy evaluation was not available." };
+    }
+    return {
+      reason: buildReason(evaluation),
+      metadata: {
+        toolCallAccuracy: {
+          mode: evaluation.mode,
+          strictMode: evaluation.strictMode,
+          hasToolCalls: evaluation.hasToolCalls,
+          actualTools: evaluation.actualTools,
+          expectedTool: evaluation.expectedTool,
+          expectedToolOrder: evaluation.expectedToolOrder,
+          correctToolCalled: evaluation.correctToolCalled,
+          correctOrderCalled: evaluation.correctOrderCalled,
+          score: evaluation.score
+        }
+      }
+    };
+  }).build();
+}
+function resolvePayload6(context, buildPayload) {
+  if (buildPayload) {
+    return buildPayload(context);
+  }
+  const payload = context.payload;
+  return {
+    toolCalls: payload.toolCalls,
+    toolResults: payload.toolResults,
+    messages: payload.messages,
+    output: payload.output,
+    rawOutput: payload.rawOutput
+  };
+}
+function extractToolNames(payload) {
+  const directToolCalls = extractToolNamesFromList(payload.toolCalls);
+  if (directToolCalls.length > 0) {
+    return directToolCalls;
+  }
+  const messageToolCalls = extractToolNamesFromMessages(payload.messages);
+  if (messageToolCalls.length > 0) {
+    return messageToolCalls;
+  }
+  const rawOutputToolCalls = extractToolNamesFromOutput(payload.rawOutput);
+  if (rawOutputToolCalls.length > 0) {
+    return rawOutputToolCalls;
+  }
+  return extractToolNamesFromOutput(payload.output);
+}
+function extractToolNamesFromOutput(value) {
+  if (Array.isArray(value)) {
+    return extractToolNamesFromMessages(value);
+  }
+  if (!isPlainRecord(value)) {
+    return [];
+  }
+  const fromToolCalls = extractToolNamesFromList(value.toolCalls);
+  if (fromToolCalls.length > 0) {
+    return fromToolCalls;
+  }
+  const fromMessages = extractToolNamesFromMessages(value.messages);
+  if (fromMessages.length > 0) {
+    return fromMessages;
+  }
+  if (Array.isArray(value.steps)) {
+    const fromSteps = extractToolNamesFromMessages(value.steps);
+    if (fromSteps.length > 0) {
+      return fromSteps;
+    }
+  }
+  return [];
+}
+function extractToolNamesFromMessages(value) {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  const toolNames = [];
+  for (const message of value) {
+    if (!isPlainRecord(message)) {
+      continue;
+    }
+    const messageType = normalizeMessageType(message.type);
+    if (messageType === "tool_call") {
+      const directToolName = extractToolName(message);
+      if (directToolName) {
+        toolNames.push(directToolName);
+      }
+    }
+    if (Array.isArray(message.toolInvocations)) {
+      toolNames.push(...extractToolNamesFromList(message.toolInvocations));
+    }
+    if (Array.isArray(message.parts)) {
+      toolNames.push(...extractToolNamesFromParts(message.parts));
+    }
+    if (isPlainRecord(message.content)) {
+      if (Array.isArray(message.content.toolInvocations)) {
+        toolNames.push(...extractToolNamesFromList(message.content.toolInvocations));
+      }
+      if (Array.isArray(message.content.parts)) {
+        toolNames.push(...extractToolNamesFromParts(message.content.parts));
+      }
+    } else if (Array.isArray(message.content)) {
+      toolNames.push(...extractToolNamesFromParts(message.content));
+    }
+  }
+  return toolNames;
+}
+function extractToolNamesFromParts(parts) {
+  const toolNames = [];
+  for (const part of parts) {
+    if (!isPlainRecord(part)) {
+      continue;
+    }
+    const partType = normalizeMessageType(part.type);
+    if (partType !== "tool_call") {
+      continue;
+    }
+    const toolName = extractToolName(part);
+    if (toolName) {
+      toolNames.push(toolName);
+    }
+  }
+  return toolNames;
+}
+function extractToolNamesFromList(value) {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  const toolNames = [];
+  for (const entry of value) {
+    const toolName = extractToolName(entry);
+    if (toolName) {
+      toolNames.push(toolName);
+    }
+  }
+  return toolNames;
+}
+function extractToolName(value) {
+  if (typeof value === "string") {
+    return normalizeToolName(value);
+  }
+  if (!isPlainRecord(value)) {
+    return void 0;
+  }
+  if (typeof value.toolName === "string") {
+    return normalizeToolName(value.toolName);
+  }
+  if (typeof value.name === "string") {
+    return normalizeToolName(value.name);
+  }
+  if (isPlainRecord(value.payload)) {
+    if (typeof value.payload.toolName === "string") {
+      return normalizeToolName(value.payload.toolName);
+    }
+    if (typeof value.payload.name === "string") {
+      return normalizeToolName(value.payload.name);
+    }
+  }
+  if (typeof value.type === "string") {
+    const normalizedType = normalizeMessageType(value.type);
+    if (shouldExtractToolNameFromType(normalizedType)) {
+      return normalizeToolTypeName(value.type);
+    }
+  }
+  return void 0;
+}
+function shouldExtractToolNameFromType(normalizedType) {
+  if (!normalizedType) {
+    return false;
+  }
+  if (!normalizedType.startsWith("tool")) {
+    return false;
+  }
+  if (normalizedType === "tool" || normalizedType === "tool_call" || normalizedType === "tool_result") {
+    return false;
+  }
+  const streamEventPrefixes = [
+    "tool_input_",
+    "tool_output_",
+    "tool_call_",
+    "tool_result_",
+    "tool_invocation_",
+    "tool_execution_"
+  ];
+  if (streamEventPrefixes.some((prefix) => normalizedType.startsWith(prefix))) {
+    return false;
+  }
+  const streamEventSuffixes = ["_start", "_end", "_delta", "_chunk", "_done"];
+  if (streamEventSuffixes.some((suffix) => normalizedType.endsWith(suffix))) {
+    return false;
+  }
+  return normalizedType.startsWith("tool_");
+}
+function normalizeToolName(value) {
+  if (typeof value !== "string") {
+    return void 0;
+  }
+  const trimmed = value.trim();
+  if (!trimmed) {
+    return void 0;
+  }
+  return trimmed;
+}
+function normalizeExpectedToolOrder(value) {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  const normalized = [];
+  for (const item of value) {
+    const toolName = normalizeToolName(item);
+    if (toolName) {
+      normalized.push(toolName);
+    }
+  }
+  return normalized;
+}
+function normalizeMessageType(value) {
+  if (typeof value !== "string") {
+    return void 0;
+  }
+  return value.toLowerCase().replace(/-/g, "_");
+}
+function normalizeToolTypeName(value) {
+  const normalized = value.toLowerCase();
+  if (normalized.startsWith("tool-") && normalized !== "tool-call" && normalized !== "tool-result") {
+    return normalizeToolName(value.slice(5));
+  }
+  if (normalized.startsWith("tool_") && normalized !== "tool_call" && normalized !== "tool_result") {
+    return normalizeToolName(value.slice(5));
+  }
+  return void 0;
+}
+function checkToolOrder(actualTools, expectedOrder, strictMode) {
+  if (strictMode) {
+    if (actualTools.length !== expectedOrder.length) {
+      return false;
+    }
+    for (let i = 0; i < expectedOrder.length; i++) {
+      if (actualTools[i] !== expectedOrder[i]) {
+        return false;
+      }
+    }
+    return true;
+  }
+  let searchIndex = 0;
+  for (const expectedTool of expectedOrder) {
+    const foundIndex = actualTools.indexOf(expectedTool, searchIndex);
+    if (foundIndex === -1) {
+      return false;
+    }
+    searchIndex = foundIndex + 1;
+  }
+  return true;
+}
+function buildReason(evaluation) {
+  const actualTools = (0, import_utils9.safeStringify)(evaluation.actualTools);
+  if (evaluation.mode === "tool_order") {
+    const expectedOrder = (0, import_utils9.safeStringify)(evaluation.expectedToolOrder ?? []);
+    if (!evaluation.hasToolCalls) {
+      return `No tool calls were made. Expected order: ${expectedOrder}.`;
+    }
+    if (evaluation.correctOrderCalled) {
+      return `Tool calls matched expected order ${expectedOrder}. Actual tools: ${actualTools}.`;
+    }
+    return `Tool call order mismatch. Expected order ${expectedOrder}, actual tools ${actualTools}.`;
+  }
+  const expectedTool = evaluation.expectedTool ?? "";
+  if (!evaluation.hasToolCalls) {
+    return `No tool calls were made. Expected tool: "${expectedTool}".`;
+  }
+  if (evaluation.correctToolCalled) {
+    return `Expected tool "${expectedTool}" was called. Actual tools: ${actualTools}.`;
+  }
+  return `Expected tool "${expectedTool}" was not called. Actual tools: ${actualTools}.`;
+}
+function isPlainRecord(value) {
+  if (value === null || typeof value !== "object") {
+    return false;
+  }
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+}
+function mergeMetadata6(base, additional) {
+  return { ...base, ...additional };
+}
+
 // src/index.ts
 var rawAutoEvalScorers = {
   listContains: import_autoevals.ListContains,
@@ -1626,6 +1970,7 @@ function adaptScorerForAgentEval(definition, options) {
   createModerationScorer,
   createPossibleScorer,
   createSummaryScorer,
+  createToolCallAccuracyScorerCode,
   createTranslationScorer,
   normalizeScorerResult,
   rawAutoEvalScorers,