@mastra/core 1.31.0-alpha.3 → 1.31.0-alpha.5

package/dist/{chunk-LEXSJDD5.cjs → chunk-3CBQ4FAZ.cjs}

@@ -1,7 +1,8 @@
 'use strict';
 
-var chunkIKXZNMTB_cjs = require('./chunk-IKXZNMTB.cjs');
-var chunkQYT6OF4I_cjs = require('./chunk-QYT6OF4I.cjs');
+var chunkATJ2RA5L_cjs = require('./chunk-ATJ2RA5L.cjs');
+var chunkO4NY7VXW_cjs = require('./chunk-O4NY7VXW.cjs');
+var chunkAAGXQKWU_cjs = require('./chunk-AAGXQKWU.cjs');
 var chunkIBQB7RSW_cjs = require('./chunk-IBQB7RSW.cjs');
 var chunk5K2PRY26_cjs = require('./chunk-5K2PRY26.cjs');
 var chunk4U7ZLI36_cjs = require('./chunk-4U7ZLI36.cjs');
@@ -107,7 +108,7 @@ async function executeAgent(agent, item, signal, requestContext, experimentId, v
   const input = item.input;
   const reqCtx = requestContext ? new chunkKF4FCWWL_cjs.RequestContext(Object.entries(requestContext)) : void 0;
   const tracingOptions = experimentId ? { metadata: { experimentId } } : void 0;
-  const rawResult = chunkQYT6OF4I_cjs.isSupportedLanguageModel(model) ? await agent.generate(input, {
+  const rawResult = chunkO4NY7VXW_cjs.isSupportedLanguageModel(model) ? await agent.generate(input, {
     scorers: {},
     returnScorerData: true,
     abortSignal: signal,
@@ -240,8 +241,26 @@ function resolveScorers(mastra, scorers) {
     return scorer;
   }).filter((s) => s !== null);
 }
+async function extractTrajectoryFromStorage(storage, traceId) {
+  if (!storage || !traceId) return void 0;
+  try {
+    const observabilityStore = await storage.getStore("observability");
+    if (!observabilityStore) return void 0;
+    const trace = await observabilityStore.getTrace({ traceId });
+    if (!trace?.spans?.length) return void 0;
+    return chunkAAGXQKWU_cjs.extractTrajectoryFromTrace(trace.spans);
+  } catch {
+    return void 0;
+  }
+}
 async function runScorersForItem(scorers, item, output, storage, runId, targetType, targetId, itemId, scorerInput, scorerOutput, traceId, workflowData) {
   if (scorers.length === 0) return [];
+  const hasTrajectoryScorer = scorers.some((s) => s.type === "trajectory");
+  let trajectoryOutput;
+  if (hasTrajectoryScorer) {
+    const traceTrajectory = await extractTrajectoryFromStorage(storage, traceId);
+    trajectoryOutput = traceTrajectory ?? (scorerOutput ? chunkAAGXQKWU_cjs.extractTrajectory(scorerOutput) : { steps: [] });
+  }
   const targetCorrelationContext = {
     ...traceId ? { traceId } : {},
     entityType: toScorerTargetEntityType(targetType),
@@ -260,11 +279,12 @@ async function runScorersForItem(scorers, item, output, storage, runId, targetTy
         targetType,
         traceId,
         targetCorrelationContext,
+        scorer.type === "trajectory" ? trajectoryOutput : void 0,
         workflowData
       );
       if (storage && result.score !== null) {
         try {
-          await chunkIKXZNMTB_cjs.validateAndSaveScore(storage, {
+          await chunkATJ2RA5L_cjs.validateAndSaveScore(storage, {
             scorerId: scorer.id,
             score: result.score,
             reason: result.reason ?? void 0,
@@ -295,22 +315,33 @@ async function runScorersForItem(scorers, item, output, storage, runId, targetTy
       return result;
     })
   );
-  return settled.map(
-    (s, i) => s.status === "fulfilled" ? s.value : { scorerId: scorers[i].id, scorerName: scorers[i].name, score: null, reason: null, error: String(s.reason) }
-  );
+  return settled.map((s, i) => {
+    if (s.status === "fulfilled") return s.value;
+    const scorer = scorers[i];
+    return {
+      scorerId: scorer.id,
+      scorerName: scorer.name,
+      score: null,
+      reason: null,
+      error: String(s.reason),
+      targetScope: scorer.type === "trajectory" ? "trajectory" : "span"
+    };
+  });
 }
-async function runScorerSafe(scorer, item, output, scorerInput, scorerOutput, targetType, targetTraceId, targetCorrelationContext, workflowData) {
+async function runScorerSafe(scorer, item, output, scorerInput, scorerOutput, targetType, targetTraceId, targetCorrelationContext, trajectoryOutput, workflowData) {
   try {
-    const targetMetadata = workflowData && (workflowData.stepResults || workflowData.stepExecutionPath) ? {
+    const effectiveOutput = trajectoryOutput ?? scorerOutput ?? output;
+    const effectiveScope = trajectoryOutput ? "trajectory" : "span";
+    const targetMetadata = !trajectoryOutput && workflowData && (workflowData.stepResults || workflowData.stepExecutionPath) ? {
       ...workflowData.stepResults ? { stepResults: workflowData.stepResults } : {},
       ...workflowData.stepExecutionPath ? { stepExecutionPath: workflowData.stepExecutionPath } : {}
     } : void 0;
     const scoreResult = await scorer.run({
       input: scorerInput ?? item.input,
-      output: scorerOutput ?? output,
+      output: effectiveOutput,
       groundTruth: item.groundTruth,
       scoreSource: "experiment",
-      targetScope: "span",
+      targetScope: effectiveScope,
       targetEntityType: toScorerTargetEntityType(targetType),
       targetTraceId,
       ...workflowData?.spanId ? { targetSpanId: workflowData.spanId } : {},
@@ -343,7 +374,8 @@ async function runScorerSafe(scorer, item, output, scorerInput, scorerOutput, ta
         scorerName: scorer.name,
         score,
         reason,
-        error: null
+        error: null,
+        targetScope: effectiveScope
       },
       promptMetadata: {
         generateScorePrompt: str("generateScorePrompt"),
@@ -361,12 +393,151 @@ async function runScorerSafe(scorer, item, output, scorerInput, scorerOutput, ta
         scorerName: scorer.name,
         score: null,
         reason: null,
-        error: error instanceof Error ? error.message : String(error)
+        error: error instanceof Error ? error.message : String(error),
+        targetScope: trajectoryOutput ? "trajectory" : "span"
       },
       promptMetadata: {}
     };
   }
 }
+function resolveStepScorers(mastra, stepsConfig) {
+  if (!stepsConfig) return {};
+  const resolved = {};
+  for (const [stepId, scorers] of Object.entries(stepsConfig)) {
+    const stepScorers = resolveScorers(mastra, scorers);
+    if (stepScorers.length > 0) resolved[stepId] = stepScorers;
+  }
+  return resolved;
+}
+async function runStepScorersForItem(stepScorers, item, workflowData, storage, runId, targetType, targetId, itemId, traceId) {
+  const stepIds = Object.keys(stepScorers);
+  if (stepIds.length === 0) return [];
+  const results = [];
+  const stepResults = workflowData?.stepResults;
+  for (const stepId of stepIds) {
+    const scorers = stepScorers[stepId];
+    const stepResult = stepResults?.[stepId];
+    if (!stepResult || stepResult.status !== "success" || stepResult.output === void 0) {
+      for (const scorer of scorers) {
+        results.push({
+          scorerId: scorer.id,
+          scorerName: scorer.name,
+          score: null,
+          reason: null,
+          error: `Step "${stepId}" did not produce a successful output (status: ${stepResult?.status ?? "missing"})`,
+          targetScope: "span",
+          stepId
+        });
+      }
+      continue;
+    }
+    const stepInput = stepResult.payload !== void 0 ? stepResult.payload : item.input;
+    const stepOutput = stepResult.output;
+    const targetCorrelationContext = {
+      ...traceId ? { traceId } : {},
+      entityType: chunk5K2PRY26_cjs.EntityType.WORKFLOW_STEP,
+      entityId: stepId,
+      entityName: stepId,
+      experimentId: runId
+    };
+    const settled = await Promise.allSettled(
+      scorers.map(async (scorer) => {
+        try {
+          const scoreResult = await scorer.run({
+            input: stepInput,
+            output: stepOutput,
+            groundTruth: item.groundTruth,
+            scoreSource: "experiment",
+            targetScope: "span",
+            targetEntityType: chunk5K2PRY26_cjs.EntityType.WORKFLOW_STEP,
+            targetTraceId: traceId,
+            ...targetCorrelationContext ? { targetCorrelationContext } : {}
+          });
+          if (typeof scoreResult !== "object" || scoreResult === null) {
+            return {
+              scorerId: scorer.id,
+              scorerName: scorer.name,
+              score: null,
+              reason: null,
+              error: `Scorer ${scorer.name} (${scorer.id}) returned invalid result on step ${stepId}`,
+              targetScope: "span",
+              stepId
+            };
+          }
+          const fields = scoreResult;
+          const score = typeof fields.score === "number" ? fields.score : null;
+          const reason = typeof fields.reason === "string" ? fields.reason : null;
+          if (storage && score !== null) {
+            try {
+              await chunkATJ2RA5L_cjs.validateAndSaveScore(storage, {
+                scorerId: scorer.id,
+                score,
+                reason: reason ?? void 0,
+                input: stepInput,
+                output: stepOutput,
+                additionalContext: { ...item.metadata, stepId },
+                entityType: "WORKFLOW_STEP",
+                entityId: itemId,
+                source: "TEST",
+                runId,
+                traceId,
+                scorer: {
+                  id: scorer.id,
+                  name: scorer.name,
+                  description: scorer.description ?? "",
+                  hasJudge: !!scorer.judge
+                },
+                entity: {
+                  id: targetId,
+                  name: targetId
+                }
+              });
+            } catch (saveError) {
+              console.warn(`Failed to save score for step scorer ${scorer.id} on ${stepId}:`, saveError);
+            }
+          }
+          return {
+            scorerId: scorer.id,
+            scorerName: scorer.name,
+            score,
+            reason,
+            error: null,
+            targetScope: "span",
+            stepId
+          };
+        } catch (error) {
+          return {
+            scorerId: scorer.id,
+            scorerName: scorer.name,
+            score: null,
+            reason: null,
+            error: error instanceof Error ? error.message : String(error),
+            targetScope: "span",
+            stepId
+          };
+        }
+      })
+    );
+    for (let i = 0; i < settled.length; i++) {
+      const s = settled[i];
+      if (s.status === "fulfilled") {
+        results.push(s.value);
+      } else {
+        const scorer = scorers[i];
+        results.push({
+          scorerId: scorer.id,
+          scorerName: scorer.name,
+          score: null,
+          reason: null,
+          error: String(s.reason),
+          targetScope: "span",
+          stepId
+        });
+      }
+    }
+  }
+  return results;
+}
 
 // src/datasets/experiment/analytics/aggregate.ts
 function computeMean(values) {
@@ -719,10 +890,23 @@ async function runExperiment(mastra, config) {
     await markFailedOnSetupError(err);
     throw err;
   }
-  let mergedScorerInput = scorerInput;
+  let stepsConfigInput;
+  const flatScorerInput = (() => {
+    if (!scorerInput) return void 0;
+    if (Array.isArray(scorerInput)) return scorerInput;
+    const flat = [];
+    if ("agent" in scorerInput && scorerInput.agent) flat.push(...scorerInput.agent);
+    if ("workflow" in scorerInput && scorerInput.workflow) flat.push(...scorerInput.workflow);
+    if ("trajectory" in scorerInput && scorerInput.trajectory) flat.push(...scorerInput.trajectory);
+    if ("steps" in scorerInput && scorerInput.steps) {
+      stepsConfigInput = scorerInput.steps;
+    }
+    return flat;
+  })();
+  let mergedScorerInput = flatScorerInput;
   const datasetScorerIds = datasetRecord?.scorerIds ?? [];
   if (datasetScorerIds.length > 0) {
-    mergedScorerInput = [...scorerInput ?? [], ...datasetScorerIds];
+    mergedScorerInput = [...flatScorerInput ?? [], ...datasetScorerIds];
   }
   if (mergedScorerInput && mergedScorerInput.length > 0) {
     const seen = /* @__PURE__ */ new Set();
@@ -736,6 +920,7 @@ async function runExperiment(mastra, config) {
     });
   }
   const scorers = resolveScorers(mastra, mergedScorerInput);
+  const stepScorers = resolveStepScorers(mastra, stepsConfigInput);
   if (experimentsStore) {
     if (!providedExperimentId) {
       await experimentsStore.createExperiment({
@@ -812,7 +997,7 @@ async function runExperiment(mastra, config) {
           stepExecutionPath: execResult.stepExecutionPath,
           spanId: execResult.spanId
         } : void 0;
-        const itemScores = await runScorersForItem(
+        const flatScores = await runScorersForItem(
           scorers,
           item,
           execResult.output,
@@ -826,6 +1011,18 @@ async function runExperiment(mastra, config) {
           execResult.traceId ?? void 0,
           workflowData
         );
+        const stepScores = await runStepScorersForItem(
+          stepScorers,
+          item,
+          workflowData,
+          storage ?? null,
+          experimentId,
+          targetType ?? "agent",
+          targetId ?? "inline",
+          item.id,
+          execResult.traceId ?? void 0
+        );
+        const itemScores = [...flatScores, ...stepScores];
         if (experimentsStore) {
           try {
             await experimentsStore.addExperimentResult({
@@ -1447,5 +1644,5 @@ exports.isRegression = isRegression;
 exports.resolveScorers = resolveScorers;
 exports.runExperiment = runExperiment;
 exports.runScorersForItem = runScorersForItem;
-//# sourceMappingURL=chunk-LEXSJDD5.cjs.map
-//# sourceMappingURL=chunk-LEXSJDD5.cjs.map
+//# sourceMappingURL=chunk-3CBQ4FAZ.cjs.map
+//# sourceMappingURL=chunk-3CBQ4FAZ.cjs.map