@mastra/evals 1.1.2-alpha.0 → 1.2.0-alpha.0

package/dist/chunk-EVBNIL5M.js

@@ -0,0 +1,606 @@
+import { RequestContext } from '@mastra/core/request-context';
+export { extractTrajectory } from '@mastra/core/evals';
+
+// src/scorers/utils.ts
+function getTextContentFromMastraDBMessage(message) {
+  if (typeof message.content.content === "string" && message.content.content !== "") {
+    return message.content.content;
+  }
+  if (message.content.parts && Array.isArray(message.content.parts)) {
+    const textParts = message.content.parts.filter((p) => p.type === "text");
+    return textParts.length > 0 ? textParts[textParts.length - 1]?.text || "" : "";
+  }
+  return "";
+}
+var roundToTwoDecimals = (num) => {
+  return Math.round((num + Number.EPSILON) * 100) / 100;
+};
+function isCloserTo(value, target1, target2) {
+  return Math.abs(value - target1) < Math.abs(value - target2);
+}
+var createTestRun = (input, output, additionalContext, requestContext) => {
+  return {
+    input: [{ role: "user", content: input }],
+    output: { role: "assistant", text: output },
+    additionalContext: additionalContext ?? {},
+    requestContext: requestContext ?? {}
+  };
+};
+var getUserMessageFromRunInput = (input) => {
+  const message = input?.inputMessages.find(({ role }) => role === "user");
+  return message ? getTextContentFromMastraDBMessage(message) : void 0;
+};
+var getSystemMessagesFromRunInput = (input) => {
+  const systemMessages = [];
+  if (input?.systemMessages) {
+    systemMessages.push(
+      ...input.systemMessages.map((msg) => {
+        if (typeof msg.content === "string") {
+          return msg.content;
+        } else if (Array.isArray(msg.content)) {
+          return msg.content.filter((part) => part.type === "text").map((part) => part.text || "").join(" ");
+        }
+        return "";
+      }).filter((content) => content)
+    );
+  }
+  if (input?.taggedSystemMessages) {
+    Object.values(input.taggedSystemMessages).forEach((messages) => {
+      messages.forEach((msg) => {
+        if (typeof msg.content === "string") {
+          systemMessages.push(msg.content);
+        }
+      });
+    });
+  }
+  return systemMessages;
+};
+var getCombinedSystemPrompt = (input) => {
+  const systemMessages = getSystemMessagesFromRunInput(input);
+  return systemMessages.join("\n\n");
+};
+var getAssistantMessageFromRunOutput = (output) => {
+  const message = output?.find(({ role }) => role === "assistant");
+  return message ? getTextContentFromMastraDBMessage(message) : void 0;
+};
+var getReasoningFromRunOutput = (output) => {
+  if (!output) return void 0;
+  const message = output.find(({ role }) => role === "assistant");
+  if (!message) return void 0;
+  if (message.content.reasoning) {
+    return message.content.reasoning;
+  }
+  const reasoningParts = message.content.parts?.filter((p) => p.type === "reasoning");
+  if (reasoningParts && reasoningParts.length > 0) {
+    const reasoningTexts = reasoningParts.map((p) => {
+      if (p.details && Array.isArray(p.details)) {
+        return p.details.filter((d) => d.type === "text").map((d) => d.text).join("");
+      }
+      return p.reasoning || "";
+    }).filter(Boolean);
+    return reasoningTexts.length > 0 ? reasoningTexts.join("\n") : void 0;
+  }
+  return void 0;
+};
+var createToolInvocation = ({
+  toolCallId,
+  toolName,
+  args,
+  result,
+  state = "result"
+}) => {
+  return {
+    toolCallId,
+    toolName,
+    args,
+    result,
+    state
+  };
+};
+function createTestMessage({
+  content,
+  role,
+  id = "test-message",
+  toolInvocations = []
+}) {
+  return {
+    id,
+    role,
+    content: {
+      format: 2,
+      parts: [{ type: "text", text: content }],
+      content,
+      ...toolInvocations.length > 0 && {
+        toolInvocations: toolInvocations.map((ti) => ({
+          toolCallId: ti.toolCallId,
+          toolName: ti.toolName,
+          args: ti.args,
+          result: ti.result,
+          state: ti.state
+        }))
+      }
+    },
+    createdAt: /* @__PURE__ */ new Date()
+  };
+}
+var createAgentTestRun = ({
+  inputMessages = [],
+  output,
+  rememberedMessages = [],
+  systemMessages = [],
+  taggedSystemMessages = {},
+  requestContext = new RequestContext(),
+  runId = crypto.randomUUID()
+}) => {
+  return {
+    input: {
+      inputMessages,
+      rememberedMessages,
+      systemMessages,
+      taggedSystemMessages
+    },
+    output,
+    requestContext,
+    runId
+  };
+};
+var createTrajectoryTestRun = ({
+  inputMessages = [],
+  trajectory,
+  rememberedMessages = [],
+  systemMessages = [],
+  taggedSystemMessages = {},
+  requestContext = new RequestContext(),
+  runId = crypto.randomUUID(),
+  expectedTrajectory
+}) => {
+  return {
+    input: {
+      inputMessages,
+      rememberedMessages,
+      systemMessages,
+      taggedSystemMessages
+    },
+    output: trajectory,
+    expectedTrajectory,
+    requestContext,
+    runId
+  };
+};
+function extractToolCalls(output) {
+  const toolCalls = [];
+  const toolCallInfos = [];
+  for (let messageIndex = 0; messageIndex < output.length; messageIndex++) {
+    const message = output[messageIndex];
+    if (message?.content?.toolInvocations) {
+      for (let invocationIndex = 0; invocationIndex < message.content.toolInvocations.length; invocationIndex++) {
+        const invocation = message.content.toolInvocations[invocationIndex];
+        if (invocation && invocation.toolName && (invocation.state === "result" || invocation.state === "call")) {
+          toolCalls.push(invocation.toolName);
+          toolCallInfos.push({
+            toolName: invocation.toolName,
+            toolCallId: invocation.toolCallId || `${messageIndex}-${invocationIndex}`,
+            messageIndex,
+            invocationIndex
+          });
+        }
+      }
+    }
+  }
+  return { tools: toolCalls, toolCallInfos };
+}
+var extractInputMessages = (runInput) => {
+  return runInput?.inputMessages?.map((msg) => getTextContentFromMastraDBMessage(msg)) || [];
+};
+var extractAgentResponseMessages = (runOutput) => {
+  return runOutput.filter((msg) => msg.role === "assistant").map((msg) => getTextContentFromMastraDBMessage(msg));
+};
+function extractToolResults(output) {
+  const results = [];
+  for (const message of output) {
+    const toolInvocations = message?.content?.toolInvocations;
+    if (!toolInvocations) continue;
+    for (const invocation of toolInvocations) {
+      if (invocation.state === "result" && invocation.result !== void 0) {
+        results.push({
+          toolName: invocation.toolName,
+          toolCallId: invocation.toolCallId || "",
+          args: invocation.args || {},
+          result: invocation.result
+        });
+      }
+    }
+  }
+  return results;
+}
+function compareTrajectories(actual, expected, options = {}) {
+  const { compareStepData = false, allowRepeatedSteps = true } = options;
+  const trajectoryStepKeys = [
+    "toolArgs",
+    "toolResult",
+    "agentId",
+    "modelId",
+    "durationMs",
+    "success",
+    "promptTokens",
+    "completionTokens"
+  ];
+  const hasTrajectorySteps = expected.steps.length > 0 && expected.steps.some((s) => trajectoryStepKeys.some((k) => k in s));
+  let normalizedExpected;
+  if (hasTrajectorySteps) {
+    normalizedExpected = {
+      steps: expected.steps.map((s) => {
+        const stepData = getStepData(s);
+        const data = {};
+        if (stepData.input !== void 0) data.input = stepData.input;
+        if (stepData.output !== void 0) data.output = stepData.output;
+        return {
+          name: s.name,
+          stepType: s.stepType,
+          ...Object.keys(data).length > 0 ? { data } : {}
+        };
+      })
+    };
+  } else {
+    normalizedExpected = expected;
+  }
+  let ordering = "relaxed";
+  if (options.ordering) {
+    ordering = options.ordering;
+  } else if (options.strictOrder) {
+    ordering = "strict";
+  }
+  if (normalizedExpected.steps.length === 0) {
+    return {
+      score: actual.steps.length === 0 ? 1 : 0,
+      matchedSteps: 0,
+      totalExpectedSteps: 0,
+      totalActualSteps: actual.steps.length,
+      missingSteps: [],
+      extraSteps: actual.steps.map((s) => s.name),
+      outOfOrderSteps: [],
+      repeatedSteps: []
+    };
+  }
+  const actualNames = actual.steps.map((s) => s.name);
+  const nameCounts = /* @__PURE__ */ new Map();
+  for (const name of actualNames) {
+    nameCounts.set(name, (nameCounts.get(name) || 0) + 1);
+  }
+  const repeatedSteps = [...nameCounts.entries()].filter(([_, count]) => count > 1).map(([name]) => name);
+  if (ordering === "strict") {
+    return compareStrictOrder(actual, normalizedExpected, { compareStepData, allowRepeatedSteps, repeatedSteps });
+  }
+  if (ordering === "unordered") {
+    return compareUnorderedPresence(actual, normalizedExpected, {
+      compareStepData,
+      allowRepeatedSteps,
+      repeatedSteps
+    });
+  }
+  return compareRelaxedOrder(actual, normalizedExpected, { compareStepData, allowRepeatedSteps, repeatedSteps });
+}
+function compareStrictOrder(actual, expected, opts) {
+  const actualNames = actual.steps.map((s) => s.name);
+  const expectedNames = expected.steps.map((s) => s.name);
+  let matchedSteps = 0;
+  const outOfOrderSteps = [];
+  const matchedExpectedIndices = /* @__PURE__ */ new Set();
+  const maxLen = Math.max(actualNames.length, expectedNames.length);
+  for (let i = 0; i < maxLen; i++) {
+    const actualName = actualNames[i];
+    const expectedName = expectedNames[i];
+    if (actualName === expectedName) {
+      if (opts.compareStepData && actual.steps[i] && expected.steps[i]) {
+        if (expectedStepMatches(actual.steps[i], expected.steps[i], true)) {
+          matchedSteps++;
+          matchedExpectedIndices.add(i);
+        }
+      } else if (actual.steps[i] && expected.steps[i]) {
+        if (expectedStepMatches(actual.steps[i], expected.steps[i], false)) {
+          matchedSteps++;
+          matchedExpectedIndices.add(i);
+        }
+      } else {
+        matchedSteps++;
+        matchedExpectedIndices.add(i);
+      }
+    } else if (actualName && expectedNames.includes(actualName)) {
+      outOfOrderSteps.push(actualName);
+    }
+  }
+  const missingSteps = expectedNames.filter((_, i) => !matchedExpectedIndices.has(i));
+  const extraSteps = actualNames.filter((name) => !expectedNames.includes(name));
+  let score = matchedSteps / expected.steps.length;
+  if (actualNames.length > expectedNames.length) {
+    const extraPenalty = (actualNames.length - expectedNames.length) / expectedNames.length;
+    score = Math.max(0, score - extraPenalty * 0.5);
+  }
+  if (!opts.allowRepeatedSteps && opts.repeatedSteps.length > 0) {
+    score = Math.max(0, score - opts.repeatedSteps.length * 0.1);
+  }
+  return {
+    score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
+    matchedSteps,
+    totalExpectedSteps: expected.steps.length,
+    totalActualSteps: actual.steps.length,
+    missingSteps,
+    extraSteps,
+    outOfOrderSteps,
+    repeatedSteps: opts.repeatedSteps
+  };
+}
+function compareRelaxedOrder(actual, expected, opts) {
+  const actualNames = actual.steps.map((s) => s.name);
+  const expectedNames = expected.steps.map((s) => s.name);
+  let matchedSteps = 0;
+  let lastMatchedIndex = -1;
+  const outOfOrderSteps = [];
+  const matchedExpectedIndices = /* @__PURE__ */ new Set();
+  for (let i = 0; i < expectedNames.length; i++) {
+    const expectedName = expectedNames[i];
+    let found = false;
+    for (let j = lastMatchedIndex + 1; j < actualNames.length; j++) {
+      if (actualNames[j] === expectedName) {
+        if (actual.steps[j] && expected.steps[i]) {
+          if (expectedStepMatches(actual.steps[j], expected.steps[i], opts.compareStepData)) {
+            matchedSteps++;
+            lastMatchedIndex = j;
+            matchedExpectedIndices.add(i);
+            found = true;
+            break;
+          }
+        } else {
+          matchedSteps++;
+          lastMatchedIndex = j;
+          matchedExpectedIndices.add(i);
+          found = true;
+          break;
+        }
+      }
+    }
+    if (!found) {
+      if (actualNames.includes(expectedName)) {
+        outOfOrderSteps.push(expectedName);
+      }
+    }
+  }
+  const missingSteps = expectedNames.filter((_, i) => !matchedExpectedIndices.has(i));
+  const expectedSet = new Set(expectedNames);
+  const extraSteps = actualNames.filter((name) => !expectedSet.has(name));
+  let score = matchedSteps / expected.steps.length;
+  if (!opts.allowRepeatedSteps && opts.repeatedSteps.length > 0) {
+    score = Math.max(0, score - opts.repeatedSteps.length * 0.1);
+  }
+  return {
+    score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
+    matchedSteps,
+    totalExpectedSteps: expected.steps.length,
+    totalActualSteps: actual.steps.length,
+    missingSteps,
+    extraSteps,
+    outOfOrderSteps,
+    repeatedSteps: opts.repeatedSteps
+  };
+}
+function getStepData(step) {
+  switch (step.stepType) {
+    case "tool_call":
+    case "mcp_tool_call":
+      return { input: step.toolArgs, output: step.toolResult };
+    case "workflow_step":
+      return { output: step.output };
+    default:
+      return {};
+  }
+}
+function expectedStepMatches(actual, expected, compareData) {
+  if (actual.name !== expected.name) return false;
+  if (expected.stepType && actual.stepType !== expected.stepType) return false;
+  if (compareData && expected.data) {
+    const actualData = getStepData(actual);
+    for (const [key, value] of Object.entries(expected.data)) {
+      const actualField = key === "input" ? actualData.input : key === "output" ? actualData.output : void 0;
+      if (actualField === void 0) return false;
+      try {
+        if (JSON.stringify(actualField) !== JSON.stringify(value)) return false;
+      } catch {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+function compareUnorderedPresence(actual, expected, opts) {
+  const actualNames = actual.steps.map((s) => s.name);
+  const expectedNames = expected.steps.map((s) => s.name);
+  let matchedSteps = 0;
+  const matchedExpectedIndices = /* @__PURE__ */ new Set();
+  if (opts.compareStepData) {
+    const usedIndices = /* @__PURE__ */ new Set();
+    for (let i = 0; i < expected.steps.length; i++) {
+      const expectedStep = expected.steps[i];
+      for (let j = 0; j < actual.steps.length; j++) {
+        if (!usedIndices.has(j) && expectedStepMatches(actual.steps[j], expectedStep, true)) {
+          matchedSteps++;
+          matchedExpectedIndices.add(i);
+          usedIndices.add(j);
+          break;
+        }
+      }
+    }
+  } else {
+    const usedIndices = /* @__PURE__ */ new Set();
+    for (let i = 0; i < expected.steps.length; i++) {
+      const expectedStep = expected.steps[i];
+      for (let j = 0; j < actual.steps.length; j++) {
+        if (!usedIndices.has(j) && expectedStepMatches(actual.steps[j], expectedStep, false)) {
+          matchedSteps++;
+          matchedExpectedIndices.add(i);
+          usedIndices.add(j);
+          break;
+        }
+      }
+    }
+  }
+  const missingSteps = expectedNames.filter((_, i) => !matchedExpectedIndices.has(i));
+  const expectedSet = new Set(expectedNames);
+  const extraSteps = actualNames.filter((name) => !expectedSet.has(name));
+  let score = matchedSteps / expected.steps.length;
+  if (!opts.allowRepeatedSteps && opts.repeatedSteps.length > 0) {
+    score = Math.max(0, score - opts.repeatedSteps.length * 0.1);
+  }
+  return {
+    score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
+    matchedSteps,
+    totalExpectedSteps: expected.steps.length,
+    totalActualSteps: actual.steps.length,
+    missingSteps,
+    extraSteps,
+    outOfOrderSteps: [],
+    // ordering not checked in unordered mode
+    repeatedSteps: opts.repeatedSteps
+  };
+}
+function checkTrajectoryEfficiency(trajectory, options = {}) {
+  const { maxSteps, maxTotalTokens, maxTotalDurationMs, noRedundantCalls = true } = options;
+  const totalSteps = trajectory.steps.length;
+  let totalTokens = 0;
+  for (const step of trajectory.steps) {
+    if (step.stepType === "model_generation") {
+      totalTokens += (step.promptTokens ?? 0) + (step.completionTokens ?? 0);
+    }
+  }
+  const totalDurationMs = trajectory.totalDurationMs ?? trajectory.steps.reduce((sum, s) => sum + (s.durationMs ?? 0), 0);
+  const redundantCalls = [];
+  if (noRedundantCalls) {
+    for (let i = 1; i < trajectory.steps.length; i++) {
+      const prev = trajectory.steps[i - 1];
+      const curr = trajectory.steps[i];
+      if (prev.name === curr.name && prev.stepType === curr.stepType && (prev.stepType === "tool_call" || prev.stepType === "mcp_tool_call")) {
+        const prevArgs = prev.toolArgs;
+        const currArgs = curr.toolArgs;
+        try {
+          if (JSON.stringify(prevArgs) === JSON.stringify(currArgs)) {
+            redundantCalls.push({ name: curr.name, index: i });
+          }
+        } catch {
+        }
+      }
+    }
+  }
+  const overStepBudget = maxSteps !== void 0 && totalSteps > maxSteps;
+  const overTokenBudget = maxTotalTokens !== void 0 && totalTokens > maxTotalTokens;
+  const overDurationBudget = maxTotalDurationMs !== void 0 && totalDurationMs > maxTotalDurationMs;
+  const dimensions = [];
+  if (maxSteps !== void 0) {
+    dimensions.push(overStepBudget ? Math.max(0, 1 - (totalSteps - maxSteps) / maxSteps) : 1);
+  }
+  if (maxTotalTokens !== void 0) {
+    dimensions.push(overTokenBudget ? Math.max(0, 1 - (totalTokens - maxTotalTokens) / maxTotalTokens) : 1);
+  }
+  if (maxTotalDurationMs !== void 0) {
+    dimensions.push(
+      overDurationBudget ? Math.max(0, 1 - (totalDurationMs - maxTotalDurationMs) / maxTotalDurationMs) : 1
+    );
+  }
+  if (noRedundantCalls) {
+    dimensions.push(redundantCalls.length === 0 ? 1 : Math.max(0, 1 - redundantCalls.length * 0.2));
+  }
+  const score = dimensions.length > 0 ? dimensions.reduce((a, b) => a + b, 0) / dimensions.length : 1;
+  return {
+    score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
+    totalSteps,
+    overStepBudget,
+    totalTokens,
+    overTokenBudget,
+    totalDurationMs,
+    overDurationBudget,
+    redundantCalls
+  };
+}
+function checkTrajectoryBlacklist(trajectory, options = {}) {
+  const { blacklistedTools = [], blacklistedSequences = [] } = options;
+  const violatedTools = [];
+  const violatedSequences = [];
+  const stepNames = trajectory.steps.map((s) => s.name);
+  for (const forbidden of blacklistedTools) {
+    if (stepNames.includes(forbidden)) {
+      violatedTools.push(forbidden);
+    }
+  }
+  for (const sequence of blacklistedSequences) {
+    if (sequence.length === 0) continue;
+    for (let i = 0; i <= stepNames.length - sequence.length; i++) {
+      let match = true;
+      for (let j = 0; j < sequence.length; j++) {
+        if (stepNames[i + j] !== sequence[j]) {
+          match = false;
+          break;
+        }
+      }
+      if (match) {
+        violatedSequences.push(sequence);
+        break;
+      }
+    }
+  }
+  const hasViolations = violatedTools.length > 0 || violatedSequences.length > 0;
+  return {
+    score: hasViolations ? 0 : 1,
+    violatedTools,
+    violatedSequences
+  };
+}
+function analyzeToolFailures(trajectory, options = {}) {
+  const { maxRetriesPerTool = 2 } = options;
+  const patterns = [];
+  let totalRetries = 0;
+  const toolCallSteps = trajectory.steps.filter((s) => s.stepType === "tool_call" || s.stepType === "mcp_tool_call");
+  if (toolCallSteps.length === 0) {
+    return { score: 1, patterns: [], totalRetries: 0, excessiveRetryTools: [] };
+  }
+  let i = 0;
+  while (i < toolCallSteps.length) {
+    const currentTool = toolCallSteps[i];
+    let retryCount = 0;
+    let j = i + 1;
+    while (j < toolCallSteps.length && toolCallSteps[j].name === currentTool.name) {
+      const prevStep = toolCallSteps[j - 1];
+      if (prevStep.success === false) {
+        retryCount++;
+      }
+      j++;
+    }
+    if (retryCount > 0) {
+      const nextDifferentTool = j < toolCallSteps.length ? toolCallSteps[j] : void 0;
+      const lastRetry = toolCallSteps[j - 1];
+      const lastSuccess = lastRetry.success !== false;
+      patterns.push({
+        toolName: currentTool.name,
+        retryCount,
+        fellBackToAlternative: nextDifferentTool !== void 0 && !lastSuccess,
+        alternativeTool: nextDifferentTool !== void 0 && !lastSuccess ? nextDifferentTool.name : void 0,
+        eventuallySucceeded: lastSuccess
+      });
+      totalRetries += retryCount;
+    }
+    i = j;
+  }
+  const excessiveRetryTools = patterns.filter((p) => p.retryCount > maxRetriesPerTool).map((p) => p.toolName);
+  let score = 1;
+  if (toolCallSteps.length > 0) {
+    const excessRetries = patterns.reduce((sum, p) => sum + Math.max(0, p.retryCount - maxRetriesPerTool), 0);
+    score = Math.max(0, 1 - excessRetries * 0.2);
+  }
+  return {
+    score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
+    patterns,
+    totalRetries,
+    excessiveRetryTools
+  };
+}
+
+export { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, createAgentTestRun, createTestMessage, createTestRun, createToolInvocation, createTrajectoryTestRun, extractAgentResponseMessages, extractInputMessages, extractToolCalls, extractToolResults, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getReasoningFromRunOutput, getSystemMessagesFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isCloserTo, roundToTwoDecimals };
+//# sourceMappingURL=chunk-EVBNIL5M.js.map
+//# sourceMappingURL=chunk-EVBNIL5M.js.map