@mastra/core 0.15.2 → 0.15.3-alpha.0

package/dist/scores/index.cjs

@@ -1,6 +1,6 @@
 'use strict';
 
-var chunkSNYSVGIU_cjs = require('../chunk-SNYSVGIU.cjs');
+var chunkI2YGYG3Y_cjs = require('../chunk-I2YGYG3Y.cjs');
 var chunkC73WLCY3_cjs = require('../chunk-C73WLCY3.cjs');
 var zod = require('zod');
 var crypto = require('crypto');
@@ -161,7 +161,7 @@ var MastraScorer = class _MastraScorer {
   }
   toMastraWorkflow() {
     const workflowSteps = this.steps.map((scorerStep) => {
-      return chunkSNYSVGIU_cjs.createStep({
+      return chunkI2YGYG3Y_cjs.createStep({
         id: scorerStep.name,
         description: `Scorer step: ${scorerStep.name}`,
         inputSchema: zod.z.any(),
@@ -194,7 +194,7 @@ var MastraScorer = class _MastraScorer {
         }
       });
     });
-    const workflow = chunkSNYSVGIU_cjs.createWorkflow({
+    const workflow = chunkI2YGYG3Y_cjs.createWorkflow({
       id: `scorer-${this.config.name}`,
       description: this.config.description,
       inputSchema: zod.z.object({
@@ -249,7 +249,7 @@ var MastraScorer = class _MastraScorer {
         }
       });
     }
-    const judge = new chunkSNYSVGIU_cjs.Agent({ name: "judge", model, instructions });
+    const judge = new chunkI2YGYG3Y_cjs.Agent({ name: "judge", model, instructions });
     if (scorerStep.name === "generateScore") {
       let result;
       if (model.specificationVersion === "v2") {
@@ -313,16 +313,132 @@ function createScorer(config) {
   });
 }
 
-// src/scores/run-experiment.ts
-var runExperiment = async ({
-  data,
-  scorers,
-  target,
-  onItemComplete,
-  concurrency = 1
-}) => {
+// src/scores/run-experiment/scorerAccumulator.ts
+var ScoreAccumulator = class {
+  flatScores = {};
+  workflowScores = {};
+  stepScores = {};
+  addScores(scorerResults) {
+    const isTargetWorkflowAndHasStepScores = "steps" in scorerResults;
+    if (isTargetWorkflowAndHasStepScores) {
+      this.addNestedScores(scorerResults);
+    } else {
+      this.addFlatScores(scorerResults);
+    }
+  }
+  addFlatScores(scorerResults) {
+    for (const [scorerName, result] of Object.entries(scorerResults)) {
+      if (!this.flatScores[scorerName]) {
+        this.flatScores[scorerName] = [];
+      }
+      this.flatScores[scorerName].push(result.score);
+    }
+  }
+  addNestedScores(scorerResults) {
+    if ("workflow" in scorerResults && scorerResults.workflow) {
+      for (const [scorerName, result] of Object.entries(scorerResults.workflow)) {
+        if (!this.workflowScores[scorerName]) {
+          this.workflowScores[scorerName] = [];
+        }
+        this.workflowScores[scorerName].push(result.score);
+      }
+    }
+    if ("steps" in scorerResults && scorerResults.steps) {
+      for (const [stepId, stepResults] of Object.entries(scorerResults.steps)) {
+        if (!this.stepScores[stepId]) {
+          this.stepScores[stepId] = {};
+        }
+        for (const [scorerName, result] of Object.entries(stepResults)) {
+          if (!this.stepScores[stepId][scorerName]) {
+            this.stepScores[stepId][scorerName] = [];
+          }
+          this.stepScores[stepId][scorerName].push(result.score);
+        }
+      }
+    }
+  }
+  addStepScores(stepScorerResults) {
+    for (const [stepId, stepResults] of Object.entries(stepScorerResults)) {
+      if (!this.stepScores[stepId]) {
+        this.stepScores[stepId] = {};
+      }
+      for (const [scorerName, result] of Object.entries(stepResults)) {
+        if (!this.stepScores[stepId][scorerName]) {
+          this.stepScores[stepId][scorerName] = [];
+        }
+        this.stepScores[stepId][scorerName].push(result.score);
+      }
+    }
+  }
+  getAverageScores() {
+    const result = {};
+    for (const [scorerName, scoreArray] of Object.entries(this.flatScores)) {
+      result[scorerName] = this.getAverageScore(scoreArray);
+    }
+    if (Object.keys(this.workflowScores).length > 0) {
+      result.workflow = {};
+      for (const [scorerName, scoreArray] of Object.entries(this.workflowScores)) {
+        result.workflow[scorerName] = this.getAverageScore(scoreArray);
+      }
+    }
+    if (Object.keys(this.stepScores).length > 0) {
+      result.steps = {};
+      for (const [stepId, stepScorers] of Object.entries(this.stepScores)) {
+        result.steps[stepId] = {};
+        for (const [scorerName, scoreArray] of Object.entries(stepScorers)) {
+          result.steps[stepId][scorerName] = this.getAverageScore(scoreArray);
+        }
+      }
+    }
+    return result;
+  }
+  getAverageScore(scoreArray) {
+    if (scoreArray.length > 0) {
+      return scoreArray.reduce((a, b) => a + b, 0) / scoreArray.length;
+    } else {
+      return 0;
+    }
+  }
+};
+
+// src/scores/run-experiment/index.ts
+async function runExperiment(config) {
+  const { data, scorers, target, onItemComplete, concurrency = 1 } = config;
+  validateExperimentInputs(data, scorers, target);
   let totalItems = 0;
-  const scoreAccumulators = {};
+  const scoreAccumulator = new ScoreAccumulator();
+  const pMap = (await import('p-map')).default;
+  await pMap(
+    data,
+    async (item) => {
+      const targetResult = await executeTarget(target, item);
+      const scorerResults = await runScorers(scorers, targetResult, item);
+      scoreAccumulator.addScores(scorerResults);
+      if (onItemComplete) {
+        await onItemComplete({
+          item,
+          targetResult,
+          scorerResults
+        });
+      }
+      totalItems++;
+    },
+    { concurrency }
+  );
+  return {
+    scores: scoreAccumulator.getAverageScores(),
+    summary: {
+      totalItems
+    }
+  };
+}
+function isWorkflow(target) {
+  return target instanceof chunkI2YGYG3Y_cjs.Workflow;
+}
+function isWorkflowScorerConfig(scorers) {
+  return typeof scorers === "object" && !Array.isArray(scorers) && ("workflow" in scorers || "steps" in scorers);
+}
+function validateExperimentInputs(data, scorers, target) {
   if (data.length === 0) {
     throw new chunkC73WLCY3_cjs.MastraError({
       domain: "SCORER",
@@ -331,106 +447,184 @@ var runExperiment = async ({
       text: "Failed to run experiment: Data array is empty"
     });
   }
-  if (scorers.length === 0) {
+  for (let i = 0; i < data.length; i++) {
+    const item = data[i];
+    if (!item || typeof item !== "object" || !("input" in item)) {
+      throw new chunkC73WLCY3_cjs.MastraError({
+        domain: "SCORER",
+        id: "INVALID_DATA_ITEM",
+        category: "USER",
+        text: `Invalid data item at index ${i}: must have 'input' properties`
+      });
+    }
+  }
+  if (Array.isArray(scorers)) {
+    if (scorers.length === 0) {
+      throw new chunkC73WLCY3_cjs.MastraError({
+        domain: "SCORER",
+        id: "NO_SCORERS_PROVIDED",
+        category: "USER",
+        text: "At least one scorer must be provided"
+      });
+    }
+  } else if (isWorkflow(target) && isWorkflowScorerConfig(scorers)) {
+    const hasScorers = scorers.workflow && scorers.workflow.length > 0 || scorers.steps && Object.keys(scorers.steps).length > 0;
+    if (!hasScorers) {
+      throw new chunkC73WLCY3_cjs.MastraError({
+        domain: "SCORER",
+        id: "NO_SCORERS_PROVIDED",
+        category: "USER",
+        text: "At least one workflow or step scorer must be provided"
+      });
+    }
+  } else if (!isWorkflow(target) && !Array.isArray(scorers)) {
     throw new chunkC73WLCY3_cjs.MastraError({
       domain: "SCORER",
-      id: "RUN_EXPERIMENT_FAILED_NO_SCORERS_PROVIDED",
+      id: "INVALID_AGENT_SCORERS",
       category: "USER",
-      text: "Failed to run experiment: No scorers provided"
+      text: "Agent scorers must be an array of scorers"
     });
   }
-  if (!target) {
-    throw new chunkC73WLCY3_cjs.MastraError({
-      domain: "SCORER",
-      id: "RUN_EXPERIMENT_FAILED_NO_TARGET_PROVIDED",
-      category: "USER",
-      text: "Failed to run experiment: No target provided"
+}
+async function executeTarget(target, item) {
+  try {
+    if (isWorkflow(target)) {
+      return await executeWorkflow(target, item);
+    } else {
+      return await executeAgent(target, item);
+    }
+  } catch (error) {
+    throw new chunkC73WLCY3_cjs.MastraError(
+      {
+        domain: "SCORER",
+        id: "RUN_EXPERIMENT_TARGET_FAILED_TO_GENERATE_RESULT",
+        category: "USER",
+        text: "Failed to run experiment: Error generating result from target",
+        details: {
+          item: JSON.stringify(item)
+        }
+      },
+      error
+    );
+  }
+}
+async function executeWorkflow(target, item) {
+  const run = target.createRun({ disableScorers: true });
+  const workflowResult = await run.start({
+    inputData: item.input,
+    runtimeContext: item.runtimeContext
+  });
+  return {
+    scoringData: {
+      input: item.input,
+      output: workflowResult.status === "success" ? workflowResult.result : void 0,
+      stepResults: workflowResult.steps
+    }
+  };
+}
+async function executeAgent(agent, item) {
+  const model = await agent.getModel();
+  if (model.specificationVersion === "v2") {
+    return await agent.generateVNext(item.input, {
+      scorers: {},
+      returnScorerData: true,
+      runtimeContext: item.runtimeContext
+    });
+  } else {
+    return await agent.generate(item.input, {
+      scorers: {},
+      returnScorerData: true,
+      runtimeContext: item.runtimeContext
     });
   }
-  const pMap = (await import('p-map')).default;
-  await pMap(
-    data,
-    async (item) => {
-      let targetResult;
+}
+async function runScorers(scorers, targetResult, item) {
+  const scorerResults = {};
+  if (Array.isArray(scorers)) {
+    for (const scorer of scorers) {
       try {
-        const model = await target.getModel();
-        if (model.specificationVersion === "v2") {
-          targetResult = await target.generateVNext(item.input, {
-            scorers: {},
-            returnScorerData: true,
-            runtimeContext: item.runtimeContext
-          });
-        } else {
-          targetResult = await target.generate(item.input, {
-            scorers: {},
-            returnScorerData: true,
-            runtimeContext: item.runtimeContext
-          });
-        }
+        const score = await scorer.run({
+          input: targetResult.scoringData?.input,
+          output: targetResult.scoringData?.output,
+          groundTruth: item.groundTruth,
+          runtimeContext: item.runtimeContext
+        });
+        scorerResults[scorer.name] = score;
       } catch (error) {
         throw new chunkC73WLCY3_cjs.MastraError(
           {
             domain: "SCORER",
-            id: "RUN_EXPERIMENT_TARGET_FAILED_TO_GENERATE_RESULT",
+            id: "RUN_EXPERIMENT_SCORER_FAILED_TO_SCORE_RESULT",
             category: "USER",
-            text: "Failed to run experiment: Error generating result from target",
+            text: `Failed to run experiment: Error running scorer ${scorer.name}`,
             details: {
+              scorerName: scorer.name,
               item: JSON.stringify(item)
             }
           },
           error
         );
       }
-      const scorerResults = {};
-      for (const scorer of scorers) {
-        try {
-          const score = await scorer.run({
-            input: targetResult.scoringData?.input,
-            output: targetResult.scoringData?.output,
-            groundTruth: item.groundTruth,
-            runtimeContext: item.runtimeContext
-          });
-          scorerResults[scorer.name] = score;
-        } catch (error) {
-          throw new chunkC73WLCY3_cjs.MastraError(
-            {
-              domain: "SCORER",
-              id: "RUN_EXPERIMENT_SCORER_FAILED_TO_SCORE_RESULT",
-              category: "USER",
-              text: `Failed to run experiment: Error running scorer ${scorer.name}`,
-              details: {
-                scorerName: scorer.name,
-                item: JSON.stringify(item)
-              }
-            },
-            error
-          );
-        }
+    }
+  } else {
+    if (scorers.workflow) {
+      const workflowScorerResults = {};
+      for (const scorer of scorers.workflow) {
+        const score = await scorer.run({
+          input: targetResult.scoringData.input,
+          output: targetResult.scoringData.output,
+          groundTruth: item.groundTruth,
+          runtimeContext: item.runtimeContext
+        });
+        workflowScorerResults[scorer.name] = score;
+      }
+      if (Object.keys(workflowScorerResults).length > 0) {
+        scorerResults.workflow = workflowScorerResults;
       }
-      for (const [scorerName, result] of Object.entries(scorerResults)) {
-        if (!scoreAccumulators[scorerName]) {
-          scoreAccumulators[scorerName] = [];
+    }
+    if (scorers.steps) {
+      const stepScorerResults = {};
+      for (const [stepId, stepScorers] of Object.entries(scorers.steps)) {
+        const stepResult = targetResult.scoringData.stepResults?.[stepId];
+        if (stepResult?.status === "success" && stepResult.payload && stepResult.output) {
+          const stepResults = {};
+          for (const scorer of stepScorers) {
+            try {
+              const score = await scorer.run({
+                input: stepResult.payload,
+                output: stepResult.output,
+                groundTruth: item.groundTruth,
+                runtimeContext: item.runtimeContext
+              });
+              stepResults[scorer.name] = score;
+            } catch (error) {
+              throw new chunkC73WLCY3_cjs.MastraError(
+                {
+                  domain: "SCORER",
+                  id: "RUN_EXPERIMENT_SCORER_FAILED_TO_SCORE_STEP_RESULT",
+                  category: "USER",
+                  text: `Failed to run experiment: Error running scorer ${scorer.name} on step ${stepId}`,
+                  details: {
+                    scorerName: scorer.name,
+                    stepId
+                  }
+                },
+                error
+              );
+            }
+          }
+          if (Object.keys(stepResults).length > 0) {
+            stepScorerResults[stepId] = stepResults;
+          }
         }
-        scoreAccumulators[scorerName].push(result.score);
       }
-      if (onItemComplete) {
-        onItemComplete({ item, targetResult, scorerResults });
+      if (Object.keys(stepScorerResults).length > 0) {
+        scorerResults.steps = stepScorerResults;
       }
-      totalItems++;
-    },
-    { concurrency }
-  );
-  const averageScores = {};
-  for (const [scorerName, scores] of Object.entries(scoreAccumulators)) {
-    averageScores[scorerName] = scores.reduce((a, b) => a + b, 0) / scores.length;
-  }
-  return {
-    scores: averageScores,
-    summary: {
-      totalItems
     }
-  };
-};
+  }
+  return scorerResults;
+}
 
 exports.MastraScorer = MastraScorer;
 exports.createScorer = createScorer;